Introduce EVP_PKEY_is_opaque to replace RSA_METHOD_FLAG_NO_CHECK.

Custom RSA and ECDSA keys may not expose the key material. Plumb and "opaque"
bit out of the *_METHOD up to EVP_PKEY. Query that in ssl_rsa.c to skip the
sanity checks for certificate and key matching.

Change-Id: I362a2d5116bfd1803560dfca1d69a91153e895fc
Reviewed-on: https://boringssl-review.googlesource.com/1255
Reviewed-by: Adam Langley <agl@google.com>

crypto/ec/ec_key.c

@@ -243,6 +243,10 @@ int EC_KEY_up_ref(EC_KEY *r) {
   return CRYPTO_add(&r->references, 1, CRYPTO_LOCK_EC) > 1;
 }
 
+int EC_KEY_is_opaque(const EC_KEY *key) {
+  return key->ecdsa_meth && (key->ecdsa_meth->flags & ECDSA_FLAG_OPAQUE);
+}
+
 const EC_GROUP *EC_KEY_get0_group(const EC_KEY *key) { return key->group; }
 
 int EC_KEY_set_group(EC_KEY *key, const EC_GROUP *group) {

crypto/evp/evp.c

@@ -117,6 +117,13 @@ void EVP_PKEY_free(EVP_PKEY *pkey) {
   OPENSSL_free(pkey);
 }
 
+int EVP_PKEY_is_opaque(const EVP_PKEY *pkey) {
+  if (pkey->ameth && pkey->ameth->pkey_opaque) {
+    return pkey->ameth->pkey_opaque(pkey);
+  }
+  return 0;
+}
+
 int EVP_PKEY_cmp(const EVP_PKEY *a, const EVP_PKEY *b) {
   if (a->type != b->type) {
     return -1;

crypto/evp/internal.h

@@ -87,6 +87,10 @@ struct evp_pkey_asn1_method_st {
   int (*priv_print)(BIO *out, const EVP_PKEY *pkey, int indent,
                     ASN1_PCTX *pctx);
 
+  /* pkey_opaque returns 1 if the |pk| is opaque. Opaque keys are backed by
+   * custom implementations which do not expose key material and parameters.*/
+  int (*pkey_opaque)(const EVP_PKEY *pk);
+
   int (*pkey_size)(const EVP_PKEY *pk);
   int (*pkey_bits)(const EVP_PKEY *pk);
 

crypto/evp/p_ec_asn1.c

@@ -519,6 +519,10 @@ static int eckey_priv_print(BIO *bp, const EVP_PKEY *pkey, int indent,
   return do_EC_KEY_print(bp, pkey->pkey.ec, indent, 2);
 }
 
+static int eckey_opaque(const EVP_PKEY *pkey) {
+  return EC_KEY_is_opaque(pkey->pkey.ec);
+}
+
 static int old_ec_priv_decode(EVP_PKEY *pkey, const uint8_t **pder,
                               int derlen) {
   EC_KEY *ec;
@@ -561,6 +565,8 @@ const EVP_PKEY_ASN1_METHOD ec_asn1_meth = {
   eckey_priv_encode,
   eckey_priv_print,
 
+  eckey_opaque,
+
   int_ec_size,
   ec_bits,
 

crypto/evp/p_hmac_asn1.c

@@ -91,6 +91,7 @@ const EVP_PKEY_ASN1_METHOD hmac_asn1_meth = {
     "HMAC",              "OpenSSL HMAC method", 0 /* pub_decode */,
     0 /* pub_encode */,  0 /* pub_cmp */,       0 /* pub_print */,
     0 /*priv_decode */,  0 /* priv_encode */,   0 /* priv_print */,
+    0 /* pkey_opaque */,
     hmac_size,           0 /* pkey_bits */,     0 /* param_decode */,
     0 /* param_encode*/, 0 /* param_missing*/,  0 /* param_copy*/,
     0 /* param_cmp*/,    0 /* param_print*/,    0 /* sig_print*/,

crypto/evp/p_rsa_asn1.c

@@ -149,6 +149,10 @@ static int rsa_priv_decode(EVP_PKEY *pkey, PKCS8_PRIV_KEY_INFO *p8) {
   return 1;
 }
 
+static int rsa_opaque(const EVP_PKEY *pkey) {
+  return RSA_is_opaque(pkey->pkey.rsa);
+}
+
 static int int_rsa_size(const EVP_PKEY *pkey) {
   return RSA_size(pkey->pkey.rsa);
 }
@@ -728,6 +732,8 @@ const EVP_PKEY_ASN1_METHOD rsa_asn1_meth = {
   rsa_priv_encode,
   rsa_priv_print,
 
+  rsa_opaque,
+
   int_rsa_size,
   rsa_bits,
 

crypto/rsa/rsa.c

@@ -256,6 +256,10 @@ unsigned RSA_size(const RSA *rsa) {
   return RSA_default_method.size(rsa);
 }
 
+int RSA_is_opaque(const RSA *rsa) {
+  return rsa->meth && (rsa->meth->flags & RSA_FLAG_OPAQUE);
+}
+
 int RSA_get_ex_new_index(long argl, void *argp, CRYPTO_EX_new *new_func,
                          CRYPTO_EX_dup *dup_func, CRYPTO_EX_free *free_func) {
   return CRYPTO_get_ex_new_index(CRYPTO_EX_INDEX_RSA, argl, argp, new_func,

include/openssl/ec_key.h

@@ -109,6 +109,10 @@ EC_KEY *EC_KEY_dup(const EC_KEY *src);
  * success and zero otherwise. */
 int EC_KEY_up_ref(EC_KEY *key);
 
+/* EC_KEY_is_opaque returns one if |key| is opaque and doesn't expose its key
+ * material. Otherwise it return zero. */
+int EC_KEY_is_opaque(const EC_KEY *key);
+
 /* EC_KEY_get0_group returns a pointer to the |EC_GROUP| object inside |key|. */
 const EC_GROUP *EC_KEY_get0_group(const EC_KEY *key);
 
@@ -229,6 +233,11 @@ void *EC_KEY_get_ex_data(const EC_KEY *r, int idx);
 
 /* ECDSA method. */
 
+/* ECDSA_FLAG_OPAQUE specifies that this ECDSA_METHOD does not expose its key
+ * material. This may be set if, for instance, it is wrapping some other crypto
+ * API, like a platform key store. */
+#define ECDSA_FLAG_OPAQUE 1
+
 /* ecdsa_method_st is a structure of function pointers for implementing ECDSA.
  * See engine.h. */
 struct ecdsa_method_st {
@@ -251,6 +260,8 @@ struct ecdsa_method_st {
   /* verify matches the arguments and behaviour of |ECDSA_verify|. */
   int (*verify)(const uint8_t *digest, size_t digest_len, const uint8_t *sig,
                 size_t sig_len, EC_KEY *eckey);
+
+  int flags;
 };
 
 

include/openssl/evp.h

@@ -89,6 +89,11 @@ EVP_PKEY *EVP_PKEY_new();
  * itself. */
 void EVP_PKEY_free(EVP_PKEY *pkey);
 
+/* EVP_PKEY_is_opaque returns one if |pkey| is opaque. Opaque keys are backed by
+ * custom implementations which do not expose key material and parameters. It is
+ * an error to attempt to duplicate, export, or compare an opaque key. */
+int EVP_PKEY_is_opaque(const EVP_PKEY *pkey);
+
 /* EVP_PKEY_cmp compares |a| and |b| and returns one if they are equal, zero if
  * not and a negative number on error.
  *

include/openssl/rsa.h

@@ -235,6 +235,10 @@ int RSA_public_decrypt(int flen, const uint8_t *from, uint8_t *to, RSA *rsa,
  * of a signature of encrypted value using |rsa|. */
 unsigned RSA_size(const RSA *rsa);
 
+/* RSA_is_opaque returns one if |rsa| is opaque and doesn't expose its key
+ * material. Otherwise it return zero. */
+int RSA_is_opaque(const RSA *rsa);
+
 /* RSAPublicKey_dup allocates a fresh |RSA| and copies the private key from
  * |rsa| into it. It returns the fresh |RSA| object, or NULL on error. */
 RSA *RSAPublicKey_dup(const RSA *rsa);
@@ -291,6 +295,10 @@ int RSA_get_ex_new_index(long argl, void *argp, CRYPTO_EX_new *new_func,
 int RSA_set_ex_data(RSA *r, int idx, void *arg);
 void *RSA_get_ex_data(const RSA *r, int idx);
 
+/* RSA_FLAG_OPAQUE specifies that this RSA_METHOD does not expose its key
+ * material. This may be set if, for instance, it is wrapping some other crypto
+ * API, like a platform key store. */
+#define RSA_FLAG_OPAQUE 1
 
 /* RSA_FLAG_CACHE_PUBLIC causes a precomputed Montgomery context to be created,
  * on demand, for the public key operations. */

ssl/ssl_rsa.c

@@ -213,16 +213,10 @@ static int ssl_set_pkey(CERT *c, EVP_PKEY *pkey)
 		EVP_PKEY_free(pktmp);
 		ERR_clear_error();
 
-                /* TODO(fork): remove this? */
-#if 0
-		/* Don't check the public/private key, this is mostly
-		 * for smart cards. */
-		if ((pkey->type == EVP_PKEY_RSA) &&
-			(RSA_flags(pkey->pkey.rsa) & RSA_METHOD_FLAG_NO_CHECK))
-			;
-		else
-#endif
-		if (!X509_check_private_key(c->pkeys[i].x509,pkey))
+		/* Sanity-check that the private key and the certificate match,
+		 * unless the key is opaque (in case of, say, a smartcard). */
+		if (!EVP_PKEY_is_opaque(pkey) &&
+			!X509_check_private_key(c->pkeys[i].x509,pkey))
 			{
 			X509_free(c->pkeys[i].x509);
 			c->pkeys[i].x509 = NULL;
@@ -430,17 +424,10 @@ static int ssl_set_cert(CERT *c, X509 *x)
 		EVP_PKEY_copy_parameters(pkey,c->pkeys[i].privatekey);
 		ERR_clear_error();
 
-                /* TODO(fork): remove this? */
-#if 0
-		/* Don't check the public/private key, this is mostly
-		 * for smart cards. */
-		if ((c->pkeys[i].privatekey->type == EVP_PKEY_RSA) &&
-			(RSA_flags(c->pkeys[i].privatekey->pkey.rsa) &
-			 RSA_METHOD_FLAG_NO_CHECK))
-			 ;
-		else
-#endif
-		if (!X509_check_private_key(x,c->pkeys[i].privatekey))
+		/* Sanity-check that the private key and the certificate match,
+		 * unless the key is opaque (in case of, say, a smartcard). */
+		if (!EVP_PKEY_is_opaque(c->pkeys[i].privatekey) &&
+			!X509_check_private_key(x,c->pkeys[i].privatekey))
 			{
 			/* don't fail for a cert/key mismatch, just free
 			 * current private key (when switching to a different