ssl3_cert_verify_hash should take the EVP_PKEY type.

After the custom key method support, the EVP_PKEY parameter is somewhat
confusing (to be resolved with the certificate slots removal) as it must
always refer to a private key. ssl3_cert_verify_hash is sometimes used
with the peer's public key. If custom keys were supported on the server,
this would break.

Fix this by passing a pkey_type parameter and letting the caller decide
whether this uses SSL_PRIVATE_KEY_METHOD or not.

Change-Id: I673b92579a84b4561f28026ec0b1c78a6bfee440
Reviewed-on: https://boringssl-review.googlesource.com/5341
Reviewed-by: Adam Langley <agl@google.com>

ssl/internal.h

@@ -907,11 +907,11 @@ int ssl3_hash_current_message(SSL *s);
 /* ssl3_cert_verify_hash writes the CertificateVerify hash into the bytes
  * pointed to by |out| and writes the number of bytes to |*out_len|. |out| must
  * have room for EVP_MAX_MD_SIZE bytes. For TLS 1.2 and up, |*out_md| is used
- * for the hash function, otherwise the hash function depends on the type of
- * |pkey| and is written to |*out_md|. It returns one on success and zero on
+ * for the hash function, otherwise the hash function depends on |pkey_type|
+ * and is written to |*out_md|. It returns one on success and zero on
  * failure. */
 int ssl3_cert_verify_hash(SSL *s, uint8_t *out, size_t *out_len,
-                          const EVP_MD **out_md, EVP_PKEY *pkey);
+                          const EVP_MD **out_md, int pkey_type);
 
 int ssl3_send_finished(SSL *s, int a, int b, const char *sender, int slen);
 int ssl3_supports_cipher(const SSL_CIPHER *cipher);

ssl/s3_both.c

@@ -457,9 +457,7 @@ OPENSSL_COMPILE_ASSERT(EVP_MAX_MD_SIZE > MD5_DIGEST_LENGTH + SHA_DIGEST_LENGTH,
                        combined_tls_hash_fits_in_max);
 
 int ssl3_cert_verify_hash(SSL *s, uint8_t *out, size_t *out_len,
-                          const EVP_MD **out_md, EVP_PKEY *pkey) {
-  const int type = ssl_private_key_type(s, pkey);
-
+                          const EVP_MD **out_md, int pkey_type) {
   /* For TLS v1.2 send signature algorithm and signature using
    * agreed digest and cached handshake records. Otherwise, use
    * SHA1 or MD5 + SHA1 depending on key type.  */
@@ -482,7 +480,7 @@ int ssl3_cert_verify_hash(SSL *s, uint8_t *out, size_t *out_len,
       return 0;
     }
     *out_len = len;
-  } else if (type == EVP_PKEY_RSA) {
+  } else if (pkey_type == EVP_PKEY_RSA) {
     if (s->enc_method->cert_verify_mac(s, NID_md5, out) == 0 ||
         s->enc_method->cert_verify_mac(s, NID_sha1, out + MD5_DIGEST_LENGTH) ==
             0) {
@@ -490,7 +488,7 @@ int ssl3_cert_verify_hash(SSL *s, uint8_t *out, size_t *out_len,
     }
     *out_len = MD5_DIGEST_LENGTH + SHA_DIGEST_LENGTH;
     *out_md = EVP_md5_sha1();
-  } else if (type == EVP_PKEY_EC) {
+  } else if (pkey_type == EVP_PKEY_EC) {
     if (s->enc_method->cert_verify_mac(s, NID_sha1, out) == 0) {
       return 0;
     }

ssl/s3_clnt.c

@@ -2041,7 +2041,8 @@ int ssl3_send_cert_verify(SSL *s) {
       }
 
       /* Compute the digest. */
-      if (!ssl3_cert_verify_hash(s, digest, &digest_length, &md, pkey)) {
+      const int pkey_type = ssl_private_key_type(s, pkey);
+      if (!ssl3_cert_verify_hash(s, digest, &digest_length, &md, pkey_type)) {
         return -1;
       }
 

ssl/s3_srvr.c

@@ -2077,7 +2077,7 @@ int ssl3_get_cert_verify(SSL *s) {
   }
 
   /* Compute the digest. */
-  if (!ssl3_cert_verify_hash(s, digest, &digest_length, &md, pkey)) {
+  if (!ssl3_cert_verify_hash(s, digest, &digest_length, &md, pkey->type)) {
     goto err;
   }