Change the order of GCD and trial division.

RSA key generation currently does the GCD check before the primality
test, in hopes of discarding things invalid by other means before
running the expensive primality check.

However, GCD is about to get a bit more expensive to clear the timing
leak, and the trial division part of primality testing is quite fast.
Thus, split that portion out via a new bn_is_obviously_composite and
call it before GCD.

Median of 29 RSA keygens: 0m0.252s -> 0m0.207s
(Accuracy beyond 0.1s is questionable.)

Bug: 238
Change-Id: I3999771fb73cca16797cab9332d14c4ebeb02046
Reviewed-on: https://boringssl-review.googlesource.com/26366
Reviewed-by: Adam Langley <alangley@gmail.com>

crypto/fipsmodule/bn/internal.h

@@ -375,6 +375,10 @@ int bn_less_than_montgomery_R(const BIGNUM *bn, const BN_MONT_CTX *mont);
 // in time independent of the value of |bn|, but it treats |d| as public.
 OPENSSL_EXPORT uint16_t bn_mod_u16_consttime(const BIGNUM *bn, uint16_t d);
 
+// bn_odd_number_is_obviously_composite returns one if |bn| is divisible by one
+// of the first several odd primes and zero otherwise.
+int bn_odd_number_is_obviously_composite(const BIGNUM *bn);
+
 // bn_rshift_secret_shift behaves like |BN_rshift| but runs in time independent
 // of both |a| and |n|.
 OPENSSL_EXPORT int bn_rshift_secret_shift(BIGNUM *r, const BIGNUM *a,

crypto/fipsmodule/bn/prime.c

@@ -590,6 +590,21 @@ uint16_t bn_mod_u16_consttime(const BIGNUM *bn, uint16_t d) {
   return ret;
 }
 
+static int bn_trial_division(uint16_t *out, const BIGNUM *bn) {
+  for (int i = 1; i < NUMPRIMES; i++) {
+    if (bn_mod_u16_consttime(bn, primes[i]) == 0) {
+      *out = primes[i];
+      return 1;
+    }
+  }
+  return 0;
+}
+
+int bn_odd_number_is_obviously_composite(const BIGNUM *bn) {
+  uint16_t prime;
+  return bn_trial_division(&prime, bn) && !BN_is_word(bn, prime);
+}
+
 int BN_primality_test(int *is_probably_prime, const BIGNUM *w,
                       int iterations, BN_CTX *ctx, int do_trial_division,
                       BN_GENCB *cb) {
@@ -617,11 +632,10 @@ int BN_primality_test(int *is_probably_prime, const BIGNUM *w,
 
   if (do_trial_division) {
     // Perform additional trial division checks to discard small primes.
-    for (int i = 1; i < NUMPRIMES; i++) {
-      if (bn_mod_u16_consttime(w, primes[i]) == 0) {
-        *is_probably_prime = BN_is_word(w, primes[i]);
-        return 1;
-      }
+    uint16_t prime;
+    if (bn_trial_division(&prime, w)) {
+      *is_probably_prime = BN_is_word(w, prime);
+      return 1;
     }
     if (!BN_GENCB_call(cb, 1, -1)) {
       return 0;

crypto/fipsmodule/rsa/rsa_impl.c

@@ -993,21 +993,25 @@ static int generate_prime(BIGNUM *out, int bits, const BIGNUM *e,
       continue;
     }
 
-    // Check gcd(out-1, e) is one (steps 4.5 and 5.6).
-    if (!BN_sub(tmp, out, BN_value_one()) ||
-        !BN_gcd(tmp, tmp, e, ctx)) {
-      goto err;
-    }
-    if (BN_is_one(tmp)) {
-      // Test |out| for primality (steps 4.5.1 and 5.6.1).
-      int is_probable_prime;
-      if (!BN_primality_test(&is_probable_prime, out, BN_prime_checks, ctx, 1,
-                             cb)) {
+    // RSA key generation's bottleneck is discarding composites. If it fails
+    // trial division, do not bother computing a GCD or performing Rabin-Miller.
+    if (!bn_odd_number_is_obviously_composite(out)) {
+      // Check gcd(out-1, e) is one (steps 4.5 and 5.6).
+      if (!BN_sub(tmp, out, BN_value_one()) ||
+          !BN_gcd(tmp, tmp, e, ctx)) {
         goto err;
       }
-      if (is_probable_prime) {
-        ret = 1;
-        goto err;
+      if (BN_is_one(tmp)) {
+        // Test |out| for primality (steps 4.5.1 and 5.6.1).
+        int is_probable_prime;
+        if (!BN_primality_test(&is_probable_prime, out, BN_prime_checks, ctx, 0,
+                               cb)) {
+          goto err;
+        }
+        if (is_probable_prime) {
+          ret = 1;
+          goto err;
+        }
       }
     }