Tweak RSA errors for compatibility.

cryptography.io wants RSA_R_BLOCK_TYPE_IS_NOT_02, only used by the
ancient RSA_padding_check_SSLv23 function. Define it but never emit it.

Additionally, it's rather finicky about RSA_R_TOO_LARGE* errors. We
merged them in BoringSSL because having RSA_R_TOO_LARGE,
RSA_R_TOO_LARGE_FOR_MODULUS, and RSA_R_TOO_LARGE_FOR_KEY_SIZE is a
little silly. But since we don't expect well-behaved code to condition
on error codes anyway, perhaps that wasn't worth it.  Split them back
up.

Looking through OpenSSL, there is a vague semantic difference:

RSA_R_DIGEST_TOO_BIG_FOR_RSA_KEY - Specifically emitted if a digest is
too big for PKCS#1 signing with this key.

RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE - You asked me to sign or encrypt a
digest/plaintext, but it's too big for this key.

RSA_R_DATA_TOO_LARGE_FOR_MODULUS - You gave me an RSA ciphertext or
signature and it is not fully reduced modulo N.
-OR-
The padding functions produced something that isn't reduced, but I
believe this is unreachable outside of RSA_NO_PADDING.

RSA_R_DATA_TOO_LARGE - Some low-level padding function was told to copy
a digest/plaintext into some buffer, but the buffer was too small. I
think this is basically unreachable.
-OR-
You asked me to verify a PSS signature, but I didn't need to bother
because the digest/salt parameters you picked were too big.

Update-Note: This depends on cl/196566462.
Change-Id: I2e539e075eff8bfcd52ccde365e975ebcee72567
Reviewed-on: https://boringssl-review.googlesource.com/28547
Reviewed-by: Adam Langley <agl@google.com>

crypto/err/rsa.errordata

@@ -6,6 +6,7 @@ RSA,104,BAD_RSA_PARAMETERS
 RSA,105,BAD_SIGNATURE
 RSA,106,BAD_VERSION
 RSA,107,BLOCK_TYPE_IS_NOT_01
+RSA,148,BLOCK_TYPE_IS_NOT_02
 RSA,108,BN_NOT_INITIALIZED
 RSA,109,CANNOT_RECOVER_MULTI_PRIME_KEY
 RSA,110,CRT_PARAMS_ALREADY_GIVEN

crypto/evp/evp_tests.txt

@@ -374,7 +374,7 @@ RSAPadding = PSS
 PSSSaltLength = 223
 Digest = SHA256
 Input = "0123456789ABCDEF0123456789ABCDEF"
-Error = DATA_TOO_LARGE
+Error = DATA_TOO_LARGE_FOR_KEY_SIZE
 
 # The salt length is too large for the modulus (verifying).
 Verify = RSA-2048
@@ -391,14 +391,14 @@ RSAPadding = PSS
 PSSSaltLength = 0
 Digest = SHA512
 Input = "0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF"
-Error = DATA_TOO_LARGE
+Error = DATA_TOO_LARGE_FOR_KEY_SIZE
 
 Sign = RSA-512
 RSAPadding = PSS
 PSSSaltLength = -2
 Digest = SHA512
 Input = "0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF0123456789ABCDEF"
-Error = DATA_TOO_LARGE
+Error = DATA_TOO_LARGE_FOR_KEY_SIZE
 
 # The hash is too large for the modulus (verifying).
 Verify = RSA-512

crypto/fipsmodule/rsa/padding.c

@@ -170,7 +170,7 @@ int RSA_padding_add_PKCS1_type_2(uint8_t *to, size_t to_len,
   }
 
   if (from_len > to_len - RSA_PKCS1_PADDING_SIZE) {
-    OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE);
+    OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
     return 0;
   }
 
@@ -254,7 +254,7 @@ int RSA_padding_check_PKCS1_type_2(uint8_t *out, size_t *out_len,
 int RSA_padding_add_none(uint8_t *to, size_t to_len, const uint8_t *from,
                          size_t from_len) {
   if (from_len > to_len) {
-    OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE);
+    OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
     return 0;
   }
 
@@ -330,7 +330,7 @@ int RSA_padding_add_PKCS1_OAEP_mgf1(uint8_t *to, size_t to_len,
 
   size_t emlen = to_len - 1;
   if (from_len > emlen - 2 * mdlen - 1) {
-    OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE);
+    OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
     return 0;
   }
 
@@ -608,7 +608,7 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
   }
 
   if (emLen < hLen + 2) {
-    OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE);
+    OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
     goto err;
   }
 
@@ -629,7 +629,7 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
   }
 
   if (emLen - hLen - 2 < sLen) {
-    OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE);
+    OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE_FOR_KEY_SIZE);
     goto err;
   }
 

crypto/fipsmodule/rsa/rsa.c

@@ -76,6 +76,10 @@
 #include "internal.h"
 
 
+// RSA_R_BLOCK_TYPE_IS_NOT_02 is part of the legacy SSLv23 padding scheme.
+// Cryptography.io depends on this error code.
+OPENSSL_DECLARE_ERROR_REASON(RSA, BLOCK_TYPE_IS_NOT_02)
+
 DEFINE_STATIC_EX_DATA_CLASS(g_rsa_ex_data_class);
 
 RSA *RSA_new(void) { return RSA_new_method(NULL); }

crypto/fipsmodule/rsa/rsa_impl.c

@@ -303,7 +303,7 @@ int RSA_encrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
 
   if (BN_ucmp(f, rsa->n) >= 0) {
     // usually the padding functions would catch this
-    OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE);
+    OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
     goto err;
   }
 
@@ -609,7 +609,7 @@ int RSA_verify_raw(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
   }
 
   if (BN_ucmp(f, rsa->n) >= 0) {
-    OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE);
+    OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
     goto err;
   }
 
@@ -683,7 +683,7 @@ int rsa_default_private_transform(RSA *rsa, uint8_t *out, const uint8_t *in,
 
   if (BN_ucmp(f, rsa->n) >= 0) {
     // Usually the padding functions would catch this.
-    OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE);
+    OPENSSL_PUT_ERROR(RSA, RSA_R_DATA_TOO_LARGE_FOR_MODULUS);
     goto err;
   }
 

include/openssl/rsa.h

@@ -752,5 +752,6 @@ BORINGSSL_MAKE_DELETER(RSA, RSA_free)
 #define RSA_R_WRONG_SIGNATURE_LENGTH 145
 #define RSA_R_PUBLIC_KEY_VALIDATION_FAILED 146
 #define RSA_R_D_OUT_OF_RANGE 147
+#define RSA_R_BLOCK_TYPE_IS_NOT_02 148
 
 #endif  // OPENSSL_HEADER_RSA_H