fix(?! needs testing) a critical bug in SHAKE XOF: the state would never transition to SQUEEZING, resulting in incorrect outputs

aux/decaffeinate_ed25519_too.sage

@@ -5,12 +5,15 @@ ii = sqrt(F(-1))
 def lobit(x): return int(x) & 1
 def hibit(x): return lobit(2*x)
 
-magic = sqrt(F(-121666))
+magic = sqrt(dM-1)
 if lobit(magic): magic = -magic
 
-def eddsa_decode(y):
-    hi = int(y) & 2^255
-    y = F(y-hi)
+def eddsa_decode(y_ser):
+    """
+    Recover x,y corresponding to the eddsa standard representation.
+    """
+    hi = int(y_ser) & 2^255
+    y = F(y_ser-hi)
     
     x = sqrt((y^2-1)/(d*y^2+1))                                                               
     if int(x) & 1: x = -x
@@ -37,6 +40,11 @@ def eddsa_to_decaf(x,y):
     return s
 
 def isqrt_trick(to_isr,to_inv):
+    """
+    The "inverse square root" trick:
+    Return 1/sqrt(to_isr), 1/to_inv.
+    Also return their product because that turns out to be useful.
+    """
     to_sqrt = to_isr*to_inv^2
     
     if to_sqrt == 0: return 0,0,0 # This happens automatically in C; just to avoid problems in SAGE
@@ -58,7 +66,7 @@ def eddsa_to_decaf_opt(x,y,z=None):
     but whatever.
     """
     if z is None:
-        # Pretend that we're in projective
+        # Pretend that we're in projective coordinates
         z = F.random_element()
         x *= z
         y *= z
@@ -105,9 +113,9 @@ def decaf_to_eddsa(s):
 def decaf_to_eddsa_opt(s):
     """
     Convert a Decaf representation to an EdDSA point, in a manner compatible
-    with libdecaf.
+    with libdecaf.  Optimized to use only one invsqrt.
     
-    This function is slightly less horrible if we don't want to decode to affine.
+    This function would be slightly simpler if we didn't want to decode to affine.
     """
     if s < 0 or s >= F.modulus(): raise Exception("out of field!")
     s = F(s)
@@ -118,13 +126,13 @@ def decaf_to_eddsa_opt(s):
     isr,inv,isr_times_inv = isqrt_trick(curve_eqn,s*(1-s^2)*(1+s^2))
     if isr == 0: raise Exception("Invalid: nonstandard encoding of zero")
     
-    t = isr_times_inv * curve_eqn * (1-s^2) * (1+s^2)
     x = 2 * magic * s * isr
     y = (1-s^2)^2 * s * inv
     
-    hibit_t = hibit(t)
+    tmp = isr_times_inv * curve_eqn * (1+s^2) # = sqrt(curve_eqn) / s / (1-s^2)
+    hibit_t = hibit(tmp * (1-s^2))
     if hibit_t: x = -x
-    if lobit(t * (1+s^2)^2 * s * inv) != hibit_t: raise Exception("invalid: t/y has high bit")
+    if lobit(tmp * (1+s^2)) != hibit_t: raise Exception("invalid: bits don't match")
     
     assert y^2 - x^2 == 1+d*x^2*y^2
     return (x,y)

src/GENERATED/include/decaf/shake.h

@@ -53,8 +53,10 @@ void decaf_sponge_init (
  * @param [inout] sponge The context.
  * @param [in] in The input data.
  * @param [in] len The input data's length in bytes.
+ * @return DECAF_FAILURE if the sponge has already been used for output.
+ * @return DECAF_SUCCESS otherwise.
  */
-void decaf_sha3_update (
+decaf_error_t decaf_sha3_update (
     struct decaf_keccak_sponge_s * __restrict__ sponge,
     const uint8_t *in,
     size_t len
@@ -156,8 +158,8 @@ decaf_error_t decaf_sponge_hash (
     static inline void DECAF_NONNULL decaf_shake##n##_gen_init(decaf_keccak_sponge_t sponge) { \
         decaf_sponge_init(sponge, &DECAF_SHAKE##n##_params_s); \
     } \
-    static inline void DECAF_NONNULL decaf_shake##n##_update(decaf_shake##n##_ctx_t sponge, const uint8_t *in, size_t inlen ) { \
-        decaf_sha3_update(sponge->s, in, inlen); \
+    static inline decaf_error_t DECAF_NONNULL decaf_shake##n##_update(decaf_shake##n##_ctx_t sponge, const uint8_t *in, size_t inlen ) { \
+        return decaf_sha3_update(sponge->s, in, inlen); \
     } \
     static inline void  DECAF_NONNULL decaf_shake##n##_final(decaf_shake##n##_ctx_t sponge, uint8_t *out, size_t outlen ) { \
         decaf_sha3_output(sponge->s, out, outlen); \
@@ -182,8 +184,8 @@ decaf_error_t decaf_sponge_hash (
     static inline void DECAF_NONNULL decaf_sha3_##n##_gen_init(decaf_keccak_sponge_t sponge) { \
         decaf_sponge_init(sponge, &DECAF_SHA3_##n##_params_s); \
     } \
-    static inline void DECAF_NONNULL decaf_sha3_##n##_update(decaf_sha3_##n##_ctx_t sponge, const uint8_t *in, size_t inlen ) { \
-        decaf_sha3_update(sponge->s, in, inlen); \
+    static inline decaf_error_t DECAF_NONNULL decaf_sha3_##n##_update(decaf_sha3_##n##_ctx_t sponge, const uint8_t *in, size_t inlen ) { \
+        return decaf_sha3_update(sponge->s, in, inlen); \
     } \
     static inline decaf_error_t DECAF_NONNULL decaf_sha3_##n##_final(decaf_sha3_##n##_ctx_t sponge, uint8_t *out, size_t outlen ) { \
         decaf_error_t ret = decaf_sha3_output(sponge->s, out, outlen); \

src/public_include/decaf/shake.h

@@ -53,8 +53,10 @@ void decaf_sponge_init (
  * @param [inout] sponge The context.
  * @param [in] in The input data.
  * @param [in] len The input data's length in bytes.
+ * @return DECAF_FAILURE if the sponge has already been used for output.
+ * @return DECAF_SUCCESS otherwise.
  */
-void decaf_sha3_update (
+decaf_error_t decaf_sha3_update (
     struct decaf_keccak_sponge_s * __restrict__ sponge,
     const uint8_t *in,
     size_t len
@@ -156,8 +158,8 @@ decaf_error_t decaf_sponge_hash (
     static inline void DECAF_NONNULL decaf_shake##n##_gen_init(decaf_keccak_sponge_t sponge) { \
         decaf_sponge_init(sponge, &DECAF_SHAKE##n##_params_s); \
     } \
-    static inline void DECAF_NONNULL decaf_shake##n##_update(decaf_shake##n##_ctx_t sponge, const uint8_t *in, size_t inlen ) { \
-        decaf_sha3_update(sponge->s, in, inlen); \
+    static inline decaf_error_t DECAF_NONNULL decaf_shake##n##_update(decaf_shake##n##_ctx_t sponge, const uint8_t *in, size_t inlen ) { \
+        return decaf_sha3_update(sponge->s, in, inlen); \
     } \
     static inline void  DECAF_NONNULL decaf_shake##n##_final(decaf_shake##n##_ctx_t sponge, uint8_t *out, size_t outlen ) { \
         decaf_sha3_output(sponge->s, out, outlen); \
@@ -182,8 +184,8 @@ decaf_error_t decaf_sponge_hash (
     static inline void DECAF_NONNULL decaf_sha3_##n##_gen_init(decaf_keccak_sponge_t sponge) { \
         decaf_sponge_init(sponge, &DECAF_SHA3_##n##_params_s); \
     } \
-    static inline void DECAF_NONNULL decaf_sha3_##n##_update(decaf_sha3_##n##_ctx_t sponge, const uint8_t *in, size_t inlen ) { \
-        decaf_sha3_update(sponge->s, in, inlen); \
+    static inline decaf_error_t DECAF_NONNULL decaf_sha3_##n##_update(decaf_sha3_##n##_ctx_t sponge, const uint8_t *in, size_t inlen ) { \
+        return decaf_sha3_update(sponge->s, in, inlen); \
     } \
     static inline decaf_error_t DECAF_NONNULL decaf_sha3_##n##_final(decaf_sha3_##n##_ctx_t sponge, uint8_t *out, size_t outlen ) { \
         decaf_error_t ret = decaf_sha3_output(sponge->s, out, outlen); \

src/shake.c

@@ -85,12 +85,11 @@ void keccakf(kdomain_t state, uint8_t start_round) {
     for (i=0; i<25; i++) a[i] = htole64(a[i]);
 }
 
-void decaf_sha3_update (
+decaf_error_t decaf_sha3_update (
     struct decaf_keccak_sponge_s * __restrict__ decaf_sponge,
     const uint8_t *in,
     size_t len
 ) {
-    if (!len) return;
     assert(decaf_sponge->params->position < decaf_sponge->params->rate);
     assert(decaf_sponge->params->rate < sizeof(decaf_sponge->state));
     assert(decaf_sponge->params->flags == FLAG_ABSORBING);
@@ -100,7 +99,7 @@ void decaf_sha3_update (
         if (cando > len) {
             for (i = 0; i < len; i += 1) state[i] ^= in[i];
             decaf_sponge->params->position += len;
-            return;
+            break;
         } else {
             for (i = 0; i < cando; i += 1) state[i] ^= in[i];
             dokeccak(decaf_sponge);
@@ -108,6 +107,7 @@ void decaf_sha3_update (
             in += cando;
         }
     }
+    return (decaf_sponge->params->flags == FLAG_ABSORBING) ? DECAF_SUCCESS : DECAF_FAILURE;
 }
 
 decaf_error_t decaf_sha3_output (
@@ -136,6 +136,7 @@ decaf_error_t decaf_sha3_output (
             state[decaf_sponge->params->position] ^= decaf_sponge->params->pad;
             state[decaf_sponge->params->rate - 1] ^= decaf_sponge->params->rate_pad;
             dokeccak(decaf_sponge);
+            decaf_sponge->params->flags = FLAG_SQUEEZING;
             break;
         }
     default:
@@ -173,6 +174,7 @@ void decaf_sha3_reset (
     decaf_keccak_sponge_t decaf_sponge
 ) {
     decaf_sponge_init(decaf_sponge, decaf_sponge->params);
+    decaf_sponge->params->flags = FLAG_ABSORBING;
     decaf_sponge->params->remaining = decaf_sponge->params->max_out;
 }