Fix check_fips for public keys and synchronize the EC and RSA versions.

Change-Id: Ibebf787445578608845df8861d67cd1e65ed0b35
Reviewed-on: https://boringssl-review.googlesource.com/15004
Reviewed-by: Steven Valdez <svaldez@google.com>
Commit-Queue: Steven Valdez <svaldez@google.com>
CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org>

crypto/ec/ec_key.c

@@ -347,18 +347,32 @@ err:
 }
 
 int EC_KEY_check_fips(const EC_KEY *key) {
+  if (EC_KEY_is_opaque(key)) {
+    /* Opaque keys can't be checked. */
+    OPENSSL_PUT_ERROR(EC, EC_R_PUBLIC_KEY_VALIDATION_FAILED);
+    return 0;
+  }
+
+  if (!EC_KEY_check_key(key)) {
+    return 0;
+  }
+
+  if (!key->priv_key) {
+    return 1;
+  }
+
   uint8_t data[16] = {0};
   unsigned sig_len = ECDSA_size(key);
   uint8_t *sig = OPENSSL_malloc(sig_len);
   if (sig == NULL) {
-    OPENSSL_PUT_ERROR(EVP, ERR_R_INTERNAL_ERROR);
+    OPENSSL_PUT_ERROR(EC, ERR_R_MALLOC_FAILURE);
     return 0;
   }
 
   int ret = 1;
   if (!ECDSA_sign(0, data, sizeof(data), sig, &sig_len, key) ||
       !ECDSA_verify(0, data, sizeof(data), sig, sig_len, key)) {
-    OPENSSL_PUT_ERROR(EVP, ERR_R_INTERNAL_ERROR);
+    OPENSSL_PUT_ERROR(EC, EC_R_PUBLIC_KEY_VALIDATION_FAILED);
     ret = 0;
   }
 

crypto/err/ec.errordata

@@ -24,6 +24,7 @@ EC,117,NOT_INITIALIZED
 EC,118,PKPARAMETERS2GROUP_FAILURE
 EC,119,POINT_AT_INFINITY
 EC,120,POINT_IS_NOT_ON_CURVE
+EC,132,PUBLIC_KEY_VALIDATION_FAILED
 EC,121,SLOT_FULL
 EC,122,UNDEFINED_GENERATOR
 EC,123,UNKNOWN_GROUP

crypto/rsa/rsa.c

@@ -682,7 +682,9 @@ int RSA_check_fips(RSA *key) {
   BN_free(&small_gcd);
   BN_CTX_free(ctx);
 
-  if (!ret) {
+  if (!ret || key->d == NULL || key->p == NULL) {
+    /* On a failure or on only a public key, there's nothing else can be
+     * checked. */
     return ret;
   }
 
@@ -694,13 +696,13 @@ int RSA_check_fips(RSA *key) {
   unsigned sig_len = RSA_size(key);
   uint8_t *sig = OPENSSL_malloc(sig_len);
   if (sig == NULL) {
-    OPENSSL_PUT_ERROR(EVP, ERR_R_INTERNAL_ERROR);
+    OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
     return 0;
   }
 
   if (!RSA_sign(NID_sha256, data, sizeof(data), sig, &sig_len, key) ||
       !RSA_verify(NID_sha256, data, sizeof(data), sig, sig_len, key)) {
-    OPENSSL_PUT_ERROR(EVP, ERR_R_INTERNAL_ERROR);
+    OPENSSL_PUT_ERROR(RSA, ERR_R_INTERNAL_ERROR);
     ret = 0;
   }
 

crypto/rsa/rsa_test.cc

@@ -135,6 +135,16 @@ static const uint8_t kFIPSKey[] =
     "\xb3\xf5\x9a\x6c\x3d\x5a\x72\xb1\x2d\xfe\xac\x09\x4f\xdd\xe5\x44\xd1\x4e"
     "\xf8\x59\x85\x3a\x65\xe2\xcd\xbc\x27\x1d\x9b\x48\x9f\xb9";
 
+static const uint8_t kFIPSPublicKey[] =
+    "\x30\x81\x89\x02\x81\x81\x00\xa1\x71\x90\x77\x86\x8a\xc7\xb8\xfc\x2a\x45"
+    "\x82\x6d\xee\xeb\x35\x3a\x18\x3f\xb6\xb0\x1e\xb1\xd3\x09\x6b\x05\x4d\xec"
+    "\x1c\x37\x6f\x09\x31\x32\xda\x21\x8a\x49\x0e\x16\x28\xed\x9a\x30\xf3\x14"
+    "\x53\xfd\x5b\xb0\xf6\x4a\x5d\x52\xe1\xda\xe1\x40\x6e\x65\xbf\xca\x45\xd9"
+    "\x62\x96\x4a\x1e\x11\xc4\x61\x83\x1f\x58\x8d\x5e\xd0\x12\xaf\xa5\xec\x9b"
+    "\x97\x2f\x6c\xb2\x82\x4a\x73\xd0\xd3\x9a\xc9\x69\x6b\x24\x3c\x82\x6f\xee"
+    "\x4d\x0c\x7e\xdf\xd7\xae\xea\x3a\xeb\x04\x27\x8d\x43\x81\x59\xa7\x90\x56"
+    "\xc1\x69\x42\xb3\xaf\x1c\x8d\x4e\xbf\x02\x03\x01\x00\x01";
+
 // kOAEPCiphertext1 is a sample encryption of |kPlaintext| with |kKey1| using
 // RSA OAEP.
 static const uint8_t kOAEPCiphertext1[] =
@@ -467,9 +477,13 @@ TEST(RSATest, CheckFIPS) {
   bssl::UniquePtr<RSA> rsa(
       RSA_private_key_from_bytes(kFIPSKey, sizeof(kFIPSKey) - 1));
   ASSERT_TRUE(rsa);
-
-  EXPECT_TRUE(RSA_check_key(rsa.get()));
   EXPECT_TRUE(RSA_check_fips(rsa.get()));
+
+  // Check that RSA_check_fips works on a public key.
+  bssl::UniquePtr<RSA> pub(
+      RSA_public_key_from_bytes(kFIPSPublicKey, sizeof(kFIPSPublicKey) - 1));
+  ASSERT_TRUE(pub);
+  EXPECT_TRUE(RSA_check_fips(pub.get()));
 }
 
 TEST(RSATest, BadKey) {

include/openssl/ec.h

@@ -402,5 +402,6 @@ BORINGSSL_MAKE_DELETER(EC_GROUP, EC_GROUP_free)
 #define EC_R_ENCODE_ERROR 129
 #define EC_R_GROUP_MISMATCH 130
 #define EC_R_INVALID_COFACTOR 131
+#define EC_R_PUBLIC_KEY_VALIDATION_FAILED 132
 
 #endif  /* OPENSSL_HEADER_EC_H */