Slightly simplify and deprecate i2d_{Public,Private}Key.

There are all the type-specific serializations rather than something
tagged with a type. i2d_PrivateKey's PKCS#8 codepath was unreachable
because every EVP_PKEY type has an old_priv_encode function.

To prune EVP_PKEY_ASN1_METHOD further, replace i2d_PrivateKey into a
switch case so we don't need to keep old_priv_encode around. This cuts
down on a case of outside modules reaching into crypto/evp method
tables.

Change-Id: I30db2eed836d560056ba9d1425b960d0602c3cf2
Reviewed-on: https://boringssl-review.googlesource.com/6865
Reviewed-by: Adam Langley <agl@google.com>

crypto/evp/internal.h

@@ -139,7 +139,6 @@ struct evp_pkey_asn1_method_st {
 
   int (*old_priv_decode)(EVP_PKEY *pkey, const uint8_t **pder,
                          int derlen);
-  int (*old_priv_encode)(const EVP_PKEY *pkey, uint8_t **pder);
 
   /* Converting parameters to/from AlgorithmIdentifier (X509_ALGOR). */
   int (*digest_verify_init_from_algorithm)(EVP_MD_CTX *ctx,

crypto/evp/p_dsa_asn1.c

@@ -447,10 +447,6 @@ static int old_dsa_priv_decode(EVP_PKEY *pkey, const uint8_t **pder,
   return 1;
 }
 
-static int old_dsa_priv_encode(const EVP_PKEY *pkey, uint8_t **pder) {
-  return i2d_DSAPrivateKey(pkey->pkey.dsa, pder);
-}
-
 static int dsa_sig_print(BIO *bp, const X509_ALGOR *sigalg,
                          const ASN1_STRING *sig, int indent, ASN1_PCTX *pctx) {
   DSA_SIG *dsa_sig;
@@ -520,7 +516,6 @@ const EVP_PKEY_ASN1_METHOD dsa_asn1_meth = {
 
   int_dsa_free,
   old_dsa_priv_decode,
-  old_dsa_priv_encode,
 
   NULL  /* digest_verify_init_from_algorithm */,
   NULL  /* digest_sign_algorithm */,

crypto/evp/p_ec_asn1.c

@@ -461,10 +461,6 @@ static int old_ec_priv_decode(EVP_PKEY *pkey, const uint8_t **pder,
   return 1;
 }
 
-static int old_ec_priv_encode(const EVP_PKEY *pkey, uint8_t **pder) {
-  return i2d_ECPrivateKey(pkey->pkey.ec, pder);
-}
-
 const EVP_PKEY_ASN1_METHOD ec_asn1_meth = {
   EVP_PKEY_EC,
   0,
@@ -493,7 +489,6 @@ const EVP_PKEY_ASN1_METHOD ec_asn1_meth = {
 
   int_ec_free,
   old_ec_priv_decode,
-  old_ec_priv_encode,
 
   NULL /* digest_verify_init_from_algorithm */,
   NULL /* digest_sign_algorithm */,

crypto/evp/p_rsa_asn1.c

@@ -461,10 +461,6 @@ static int old_rsa_priv_decode(EVP_PKEY *pkey, const uint8_t **pder,
   return 1;
 }
 
-static int old_rsa_priv_encode(const EVP_PKEY *pkey, uint8_t **pder) {
-  return i2d_RSAPrivateKey(pkey->pkey.rsa, pder);
-}
-
 /* allocate and set algorithm ID from EVP_MD, default SHA1 */
 static int rsa_md_to_algor(X509_ALGOR **palg, const EVP_MD *md) {
   if (EVP_MD_type(md) == NID_sha1) {
@@ -734,7 +730,6 @@ const EVP_PKEY_ASN1_METHOD rsa_asn1_meth = {
   int_rsa_free,
 
   old_rsa_priv_decode,
-  old_rsa_priv_encode,
 
   rsa_digest_verify_init_from_algorithm,
   rsa_digest_sign_algorithm,

crypto/x509/i2d_pr.c

@@ -55,29 +55,29 @@
  * copied and put under another distribution licence
  * [including the GNU Public Licence.] */
 
-#include <openssl/x509.h>
-
 #include <openssl/asn1.h>
+#include <openssl/ec_key.h>
 #include <openssl/err.h>
 #include <openssl/evp.h>
+#include <openssl/rsa.h>
+#include <openssl/dsa.h>
 
-#include "../evp/internal.h"
 
-int i2d_PrivateKey(const EVP_PKEY *a, unsigned char **pp)
+int i2d_PrivateKey(const EVP_PKEY *a, uint8_t **pp)
 {
-    if (a->ameth && a->ameth->old_priv_encode) {
-        return a->ameth->old_priv_encode(a, pp);
-    }
-    if (a->ameth && a->ameth->priv_encode) {
-        PKCS8_PRIV_KEY_INFO *p8 = EVP_PKEY2PKCS8((EVP_PKEY *)a);
-        int ret = i2d_PKCS8_PRIV_KEY_INFO(p8, pp);
-        PKCS8_PRIV_KEY_INFO_free(p8);
-        return ret;
+    switch (EVP_PKEY_id(a)) {
+    case EVP_PKEY_RSA:
+        return i2d_RSAPrivateKey(a->pkey.rsa, pp);
+    case EVP_PKEY_EC:
+        return i2d_ECPrivateKey(a->pkey.ec, pp);
+    case EVP_PKEY_DSA:
+        return i2d_DSAPrivateKey(a->pkey.dsa, pp);
+    default:
+        /*
+         * Although this file is in crypto/x509 for layering reasons, it emits
+         * an error code from ASN1 for OpenSSL compatibility.
+         */
+        OPENSSL_PUT_ERROR(ASN1, ASN1_R_UNSUPPORTED_PUBLIC_KEY_TYPE);
+        return -1;
     }
-    /*
-     * Although this file is in crypto/x509 for layering reasons, it emits an
-     * error code from ASN1 for OpenSSL compatibility.
-     */
-    OPENSSL_PUT_ERROR(ASN1, ASN1_R_UNSUPPORTED_PUBLIC_KEY_TYPE);
-    return -1;
 }

include/openssl/evp.h

@@ -217,21 +217,6 @@ OPENSSL_EXPORT EVP_PKEY *d2i_PrivateKey(int type, EVP_PKEY **out,
 OPENSSL_EXPORT EVP_PKEY *d2i_AutoPrivateKey(EVP_PKEY **out, const uint8_t **inp,
                                             long len);
 
-/* i2d_PrivateKey marshals a private key from |key| to an ASN.1, DER
- * structure. If |outp| is not NULL then the result is written to |*outp| and
- * |*outp| is advanced just past the output. It returns the number of bytes in
- * the result, whether written or not, or a negative value on error. */
-OPENSSL_EXPORT int i2d_PrivateKey(const EVP_PKEY *key, uint8_t **outp);
-
-/* i2d_PublicKey marshals a public key from |key| to a type-specific format.
- * If |outp| is not NULL then the result is written to |*outp| and
- * |*outp| is advanced just past the output. It returns the number of bytes in
- * the result, whether written or not, or a negative value on error.
- *
- * RSA keys are serialized as a DER-encoded RSAPublicKey (RFC 3447) structure.
- * EC keys are serialized as an EC point per SEC 1. */
-OPENSSL_EXPORT int i2d_PublicKey(EVP_PKEY *key, uint8_t **outp);
-
 
 /* Signing */
 
@@ -707,6 +692,29 @@ OPENSSL_EXPORT void EVP_MD_do_all_sorted(void (*callback)(const EVP_MD *cipher,
                                                           void *arg),
                                          void *arg);
 
+/* i2d_PrivateKey marshals a private key from |key| to an ASN.1, DER
+ * structure. If |outp| is not NULL then the result is written to |*outp| and
+ * |*outp| is advanced just past the output. It returns the number of bytes in
+ * the result, whether written or not, or a negative value on error.
+ *
+ * RSA keys are serialized as a DER-encoded RSAPublicKey (RFC 3447) structure.
+ * EC keys are serialized as a DER-encoded ECPrivateKey (RFC 5915) structure.
+ *
+ * Use |RSA_marshal_private_key| or |EC_marshal_private_key| instead. */
+OPENSSL_EXPORT int i2d_PrivateKey(const EVP_PKEY *key, uint8_t **outp);
+
+/* i2d_PublicKey marshals a public key from |key| to a type-specific format.
+ * If |outp| is not NULL then the result is written to |*outp| and
+ * |*outp| is advanced just past the output. It returns the number of bytes in
+ * the result, whether written or not, or a negative value on error.
+ *
+ * RSA keys are serialized as a DER-encoded RSAPublicKey (RFC 3447) structure.
+ * EC keys are serialized as an EC point per SEC 1.
+ *
+ * Use |RSA_marshal_public_key| or |EC_POINT_point2cbb| instead. */
+OPENSSL_EXPORT int i2d_PublicKey(EVP_PKEY *key, uint8_t **outp);
+
+
 /* Private functions */
 
 /* EVP_PKEY_asn1_find returns the ASN.1 method table for the given |nid|, which