From 978f16ea08a24cb740741c9956a934544c767db3 Mon Sep 17 00:00:00 2001 From: Matt Braithwaite Date: Mon, 19 Oct 2015 13:38:36 -0700 Subject: [PATCH] size_t RSA functions. This extends 79c59a30 to |RSA_public_encrypt|, |RSA_private_encrypt|, and |RSA_public_decrypt|. It benefits Conscrypt, which expects these functions to have the same signature as |RSA_public_private_decrypt|. Change-Id: Id1ce3118e8f20a9f43fd4f7bfc478c72a0c64e4b Reviewed-on: https://boringssl-review.googlesource.com/6286 Reviewed-by: David Benjamin Reviewed-by: Adam Langley --- crypto/rsa/rsa.c | 18 +++++++++++++++--- include/openssl/rsa.h | 6 +++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/crypto/rsa/rsa.c b/crypto/rsa/rsa.c index 63eb170c..2a0b4ce2 100644 --- a/crypto/rsa/rsa.c +++ b/crypto/rsa/rsa.c @@ -199,7 +199,7 @@ int RSA_encrypt(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out, padding); } -int RSA_public_encrypt(int flen, const uint8_t *from, uint8_t *to, RSA *rsa, +int RSA_public_encrypt(size_t flen, const uint8_t *from, uint8_t *to, RSA *rsa, int padding) { size_t out_len; @@ -207,6 +207,10 @@ int RSA_public_encrypt(int flen, const uint8_t *from, uint8_t *to, RSA *rsa, return -1; } + if (out_len > INT_MAX) { + OPENSSL_PUT_ERROR(RSA, ERR_R_OVERFLOW); + return -1; + } return out_len; } @@ -220,7 +224,7 @@ int RSA_sign_raw(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out, padding); } -int RSA_private_encrypt(int flen, const uint8_t *from, uint8_t *to, RSA *rsa, +int RSA_private_encrypt(size_t flen, const uint8_t *from, uint8_t *to, RSA *rsa, int padding) { size_t out_len; @@ -228,6 +232,10 @@ int RSA_private_encrypt(int flen, const uint8_t *from, uint8_t *to, RSA *rsa, return -1; } + if (out_len > INT_MAX) { + OPENSSL_PUT_ERROR(RSA, ERR_R_OVERFLOW); + return -1; + } return out_len; } @@ -266,7 +274,7 @@ int RSA_verify_raw(RSA *rsa, size_t *out_len, uint8_t *out, size_t max_out, padding); } -int RSA_public_decrypt(int flen, const uint8_t *from, uint8_t *to, RSA *rsa, +int RSA_public_decrypt(size_t flen, const uint8_t *from, uint8_t *to, RSA *rsa, int padding) { size_t out_len; @@ -274,6 +282,10 @@ int RSA_public_decrypt(int flen, const uint8_t *from, uint8_t *to, RSA *rsa, return -1; } + if (out_len > INT_MAX) { + OPENSSL_PUT_ERROR(RSA, ERR_R_OVERFLOW); + return -1; + } return out_len; } diff --git a/include/openssl/rsa.h b/include/openssl/rsa.h index e44e228e..e0c43687 100644 --- a/include/openssl/rsa.h +++ b/include/openssl/rsa.h @@ -152,7 +152,7 @@ OPENSSL_EXPORT int RSA_decrypt(RSA *rsa, size_t *out_len, uint8_t *out, * * WARNING: this function is dangerous because it breaks the usual return value * convention. Use |RSA_encrypt| instead. */ -OPENSSL_EXPORT int RSA_public_encrypt(int flen, const uint8_t *from, +OPENSSL_EXPORT int RSA_public_encrypt(size_t flen, const uint8_t *from, uint8_t *to, RSA *rsa, int padding); /* RSA_private_decrypt decrypts |flen| bytes from |from| with the public key in @@ -244,7 +244,7 @@ OPENSSL_EXPORT int RSA_verify_raw(RSA *rsa, size_t *out_len, uint8_t *out, * * WARNING: this function is dangerous because it breaks the usual return value * convention. Use |RSA_sign_raw| instead. */ -OPENSSL_EXPORT int RSA_private_encrypt(int flen, const uint8_t *from, +OPENSSL_EXPORT int RSA_private_encrypt(size_t flen, const uint8_t *from, uint8_t *to, RSA *rsa, int padding); /* RSA_public_decrypt verifies |flen| bytes of signature from |from| using the @@ -255,7 +255,7 @@ OPENSSL_EXPORT int RSA_private_encrypt(int flen, const uint8_t *from, * * WARNING: this function is dangerous because it breaks the usual return value * convention. Use |RSA_verify_raw| instead. */ -OPENSSL_EXPORT int RSA_public_decrypt(int flen, const uint8_t *from, +OPENSSL_EXPORT int RSA_public_decrypt(size_t flen, const uint8_t *from, uint8_t *to, RSA *rsa, int padding);