size_t RSA_private_decrypt's input.

Change-Id: If05761052e235b38d9798b2fe4d8ba44293af891
Reviewed-on: https://boringssl-review.googlesource.com/5944
Reviewed-by: Adam Langley <agl@google.com>

crypto/rsa/rsa.c

@@ -56,6 +56,7 @@
 
 #include <openssl/rsa.h>
 
+#include <limits.h>
 #include <string.h>
 
 #include <openssl/bn.h>
@@ -240,7 +241,7 @@ int RSA_decrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out,
                                     padding);
 }
 
-int RSA_private_decrypt(int flen, const uint8_t *from, uint8_t *to, RSA *rsa,
+int RSA_private_decrypt(size_t flen, const uint8_t *from, uint8_t *to, RSA *rsa,
                         int padding) {
   size_t out_len;
 
@@ -248,6 +249,10 @@ int RSA_private_decrypt(int flen, const uint8_t *from, uint8_t *to, RSA *rsa,
     return -1;
   }
 
+  if (out_len > INT_MAX) {
+    OPENSSL_PUT_ERROR(RSA, ERR_R_OVERFLOW);
+    return -1;
+  }
   return out_len;
 }
 

include/openssl/rsa.h

@@ -164,7 +164,7 @@ OPENSSL_EXPORT int RSA_public_encrypt(int flen, const uint8_t *from,
  *
  * WARNING: this function is dangerous because it breaks the usual return value
  * convention. Use |RSA_decrypt| instead. */
-OPENSSL_EXPORT int RSA_private_decrypt(int flen, const uint8_t *from,
+OPENSSL_EXPORT int RSA_private_decrypt(size_t flen, const uint8_t *from,
                                        uint8_t *to, RSA *rsa, int padding);
 
 /* RSA_message_index_PKCS1_type_2 performs the first step of a PKCS #1 padding