Add RSA_check_fips to support public key validation checks.

Change-Id: I0e00f099a17d88f56b49970e612b0911afd9661e
Reviewed-on: https://boringssl-review.googlesource.com/14866
Reviewed-by: Steven Valdez <svaldez@google.com>
Commit-Queue: Steven Valdez <svaldez@google.com>
CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org>

crypto/err/rsa.errordata

@@ -36,6 +36,7 @@ RSA,134,ONLY_ONE_OF_P_Q_GIVEN
 RSA,135,OUTPUT_BUFFER_TOO_SMALL
 RSA,136,PADDING_CHECK_FAILED
 RSA,137,PKCS_DECODING_ERROR
+RSA,146,PUBLIC_KEY_VALIDATION_FAILED
 RSA,138,SLEN_CHECK_FAILED
 RSA,139,SLEN_RECOVERY_FAILED
 RSA,140,TOO_LONG

crypto/rsa/rsa.c

@@ -69,6 +69,7 @@
 
 #include "internal.h"
 #include "../internal.h"
+#include "../bn/internal.h"
 
 
 static CRYPTO_EX_DATA_CLASS g_ex_data_class = CRYPTO_EX_DATA_CLASS_INIT;
@@ -669,6 +670,62 @@ out:
   return ok;
 }
 
+
+/* This is the product of the 132 smallest odd primes, from 3 to 751. */
+static const BN_ULONG kSmallFactorsLimbs[] = {
+    TOBN(0xc4309333, 0x3ef4e3e1), TOBN(0x71161eb6, 0xcd2d655f),
+    TOBN(0x95e2238c, 0x0bf94862), TOBN(0x3eb233d3, 0x24f7912b),
+    TOBN(0x6b55514b, 0xbf26c483), TOBN(0x0a84d817, 0x5a144871),
+    TOBN(0x77d12fee, 0x9b82210a), TOBN(0xdb5b93c2, 0x97f050b3),
+    TOBN(0x4acad6b9, 0x4d6c026b), TOBN(0xeb7751f3, 0x54aec893),
+    TOBN(0xdba53368, 0x36bc85c4), TOBN(0xd85a1b28, 0x7f5ec78e),
+    TOBN(0x2eb072d8, 0x6b322244), TOBN(0xbba51112, 0x5e2b3aea),
+    TOBN(0x36ed1a6c, 0x0e2486bf), TOBN(0x5f270460, 0xec0c5727),
+    0x000017b1
+};
+static const BIGNUM kSmallFactors = STATIC_BIGNUM(kSmallFactorsLimbs);
+
+int RSA_check_fips(const RSA *key) {
+  if (RSA_is_opaque(key)) {
+    /* Opaque keys can't be checked. */
+    OPENSSL_PUT_ERROR(RSA, RSA_R_PUBLIC_KEY_VALIDATION_FAILED);
+    return 0;
+  }
+
+  if (!RSA_check_key(key)) {
+    return 0;
+  }
+
+  BN_CTX *ctx = BN_CTX_new();
+  if (ctx == NULL) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
+    return 0;
+  }
+
+  BIGNUM small_gcd;
+  BN_init(&small_gcd);
+
+  int ret = 1;
+
+  /* Perform partial public key validation of RSA keys (SP 800-89 5.3.3). */
+  /* TODO(svaldez): Check that n is composite and not a power of a prime using
+   * extended Miller-Rabin. */
+  if (BN_num_bits(key->e) < 16 ||
+      BN_num_bits(key->e) > 256 ||
+      !BN_is_odd(key->n) ||
+      !BN_is_odd(key->e) ||
+      !BN_gcd(&small_gcd, key->n, &kSmallFactors, ctx) ||
+      !BN_is_one(&small_gcd)) {
+    OPENSSL_PUT_ERROR(RSA, RSA_R_PUBLIC_KEY_VALIDATION_FAILED);
+    ret = 0;
+  }
+
+  BN_free(&small_gcd);
+  BN_CTX_free(ctx);
+
+  return ret;
+}
+
 int RSA_recover_crt_params(RSA *rsa) {
   BN_CTX *ctx;
   BIGNUM *totient, *rem, *multiple, *p_plus_q, *p_minus_q;

crypto/rsa/rsa_test.cc

@@ -654,6 +654,7 @@ TEST(RSATest, BadKey) {
 
   // Bad keys are detected.
   EXPECT_FALSE(RSA_check_key(key.get()));
+  EXPECT_FALSE(RSA_check_fips(key.get()));
 
   // Bad keys may not be parsed.
   uint8_t *der;

include/openssl/rsa.h

@@ -299,6 +299,10 @@ OPENSSL_EXPORT RSA *RSAPrivateKey_dup(const RSA *rsa);
  * returns zero then a more detailed error is available on the error queue. */
 OPENSSL_EXPORT int RSA_check_key(const RSA *rsa);
 
+/* RSA_check_fips performs public key validatity tests on |key|. It returns one
+ * if they pass and zero otherwise. Opaque keys always fail. */
+OPENSSL_EXPORT int RSA_check_fips(const RSA *key);
+
 /* RSA_recover_crt_params uses |rsa->n|, |rsa->d| and |rsa->e| in order to
  * calculate the two primes used and thus the precomputed, CRT values. These
  * values are set in the |p|, |q|, |dmp1|, |dmq1| and |iqmp| members of |rsa|,
@@ -687,5 +691,6 @@ BORINGSSL_MAKE_DELETER(RSA, RSA_free)
 #define RSA_R_UNKNOWN_PADDING_TYPE 143
 #define RSA_R_VALUE_MISSING 144
 #define RSA_R_WRONG_SIGNATURE_LENGTH 145
+#define RSA_R_PUBLIC_KEY_VALIDATION_FAILED 146
 
 #endif  /* OPENSSL_HEADER_RSA_H */