Also fix int overflow for systems with 4-byte ULs

xmss_fast.c

@@ -945,13 +945,13 @@ int xmssmt_sign(unsigned char *sk, bds_state *states, unsigned char *wots_sigs,
 
   SET_TREE_ADDRESS(addr, (idx_tree + 1));
   // mandatory update for NEXT_0 (does not count towards h-k/2) if NEXT_0 exists
-  if ((1 + idx_tree) * (1 << tree_h) + idx_leaf < (1UL << h)) {
+  if ((1 + idx_tree) * (1 << tree_h) + idx_leaf < (1ULL << h)) {
     bds_state_update(&states[params->d], sk_seed, &(params->xmss_par), pub_seed, addr);
   }
 
   for (i = 0; i < params->d; i++) {
     // check if we're not at the end of a tree
-    if (! (((idx + 1) & ((1UL << ((i+1)*tree_h)) - 1)) == 0)) {
+    if (! (((idx + 1) & ((1ULL << ((i+1)*tree_h)) - 1)) == 0)) {
       idx_leaf = (idx >> (tree_h * i)) & ((1 << tree_h)-1);
       idx_tree = (idx >> (tree_h * (i+1)));
       SET_LAYER_ADDRESS(addr, i);
@@ -962,14 +962,14 @@ int xmssmt_sign(unsigned char *sk, bds_state *states, unsigned char *wots_sigs,
       updates = bds_treehash_update(&states[i], updates, sk_seed, &(params->xmss_par), pub_seed, addr);
       SET_TREE_ADDRESS(addr, (idx_tree + 1));
       // if a NEXT-tree exists for this level;
-      if ((1 + idx_tree) * (1 << tree_h) + idx_leaf < (1UL << (h - tree_h * i))) {
-        if (i > 0 && updates > 0 && states[params->d + i].next_leaf < (1UL << h)) {
+      if ((1 + idx_tree) * (1 << tree_h) + idx_leaf < (1ULL << (h - tree_h * i))) {
+        if (i > 0 && updates > 0 && states[params->d + i].next_leaf < (1ULL << h)) {
           bds_state_update(&states[params->d + i], sk_seed, &(params->xmss_par), pub_seed, addr);
           updates--;
         }
       }
     }
-    else if (idx < (1UL << h) - 1) {
+    else if (idx < (1ULL << h) - 1) {
       memcpy(&tmp, states+params->d + i, sizeof(bds_state));
       memcpy(states+params->d + i, states + i, sizeof(bds_state));
       memcpy(states + i, &tmp, sizeof(bds_state));