Tidy up PKCS1_MGF1.

Fix non-standard variable names, return value convention, unsigned vs
size_t, etc. This also fixes one size_t truncation warning.

BUG=22

Change-Id: Ibe083db90e8dac45d64da9ead8f519dd2fea96ea
Reviewed-on: https://boringssl-review.googlesource.com/9133
Commit-Queue: David Benjamin <davidben@google.com>
Commit-Queue: Adam Langley <agl@google.com>
Reviewed-by: Adam Langley <agl@google.com>
CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org>
This commit is contained in:
David Benjamin 2016-08-04 17:48:42 -04:00 committed by CQ bot account: commit-bot@chromium.org
parent ea655fa33f
commit 9305a13252

View File

@ -274,47 +274,46 @@ int RSA_padding_add_none(uint8_t *to, unsigned to_len, const uint8_t *from,
return 1; return 1;
} }
static int PKCS1_MGF1(uint8_t *mask, unsigned len, const uint8_t *seed, static int PKCS1_MGF1(uint8_t *out, size_t len, const uint8_t *seed,
unsigned seedlen, const EVP_MD *dgst) { size_t seed_len, const EVP_MD *md) {
unsigned outlen = 0; int ret = 0;
uint32_t i; EVP_MD_CTX ctx;
uint8_t cnt[4]; EVP_MD_CTX_init(&ctx);
EVP_MD_CTX c;
uint8_t md[EVP_MAX_MD_SIZE];
unsigned mdlen;
int ret = -1;
EVP_MD_CTX_init(&c); size_t md_len = EVP_MD_size(md);
mdlen = EVP_MD_size(dgst);
for (i = 0; outlen < len; i++) { for (uint32_t i = 0; len > 0; i++) {
cnt[0] = (uint8_t)((i >> 24) & 255); uint8_t counter[4];
cnt[1] = (uint8_t)((i >> 16) & 255); counter[0] = (uint8_t)(i >> 24);
cnt[2] = (uint8_t)((i >> 8)) & 255; counter[1] = (uint8_t)(i >> 16);
cnt[3] = (uint8_t)(i & 255); counter[2] = (uint8_t)(i >> 8);
if (!EVP_DigestInit_ex(&c, dgst, NULL) || counter[3] = (uint8_t)i;
!EVP_DigestUpdate(&c, seed, seedlen) || if (!EVP_DigestInit_ex(&ctx, md, NULL) ||
!EVP_DigestUpdate(&c, cnt, 4)) { !EVP_DigestUpdate(&ctx, seed, seed_len) ||
!EVP_DigestUpdate(&ctx, counter, sizeof(counter))) {
goto err; goto err;
} }
if (outlen + mdlen <= len) { if (md_len <= len) {
if (!EVP_DigestFinal_ex(&c, mask + outlen, NULL)) { if (!EVP_DigestFinal_ex(&ctx, out, NULL)) {
goto err; goto err;
} }
outlen += mdlen; out += md_len;
len -= md_len;
} else { } else {
if (!EVP_DigestFinal_ex(&c, md, NULL)) { uint8_t digest[EVP_MAX_MD_SIZE];
if (!EVP_DigestFinal_ex(&ctx, digest, NULL)) {
goto err; goto err;
} }
memcpy(mask + outlen, md, len - outlen); memcpy(out, digest, len);
outlen = len; len = 0;
} }
} }
ret = 0;
ret = 1;
err: err:
EVP_MD_CTX_cleanup(&c); EVP_MD_CTX_cleanup(&ctx);
return ret; return ret;
} }
@ -372,14 +371,14 @@ int RSA_padding_add_PKCS1_OAEP_mgf1(uint8_t *to, unsigned to_len,
return 0; return 0;
} }
if (PKCS1_MGF1(dbmask, emlen - mdlen, seed, mdlen, mgf1md) < 0) { if (!PKCS1_MGF1(dbmask, emlen - mdlen, seed, mdlen, mgf1md)) {
goto out; goto out;
} }
for (i = 0; i < emlen - mdlen; i++) { for (i = 0; i < emlen - mdlen; i++) {
db[i] ^= dbmask[i]; db[i] ^= dbmask[i];
} }
if (PKCS1_MGF1(seedmask, mdlen, db, emlen - mdlen, mgf1md) < 0) { if (!PKCS1_MGF1(seedmask, mdlen, db, emlen - mdlen, mgf1md)) {
goto out; goto out;
} }
for (i = 0; i < mdlen; i++) { for (i = 0; i < mdlen; i++) {
@ -428,14 +427,14 @@ int RSA_padding_check_PKCS1_OAEP_mgf1(uint8_t *to, unsigned to_len,
maskedseed = from + 1; maskedseed = from + 1;
maskeddb = from + 1 + mdlen; maskeddb = from + 1 + mdlen;
if (PKCS1_MGF1(seed, mdlen, maskeddb, dblen, mgf1md)) { if (!PKCS1_MGF1(seed, mdlen, maskeddb, dblen, mgf1md)) {
goto err; goto err;
} }
for (i = 0; i < mdlen; i++) { for (i = 0; i < mdlen; i++) {
seed[i] ^= maskedseed[i]; seed[i] ^= maskedseed[i];
} }
if (PKCS1_MGF1(db, dblen, seed, mdlen, mgf1md)) { if (!PKCS1_MGF1(db, dblen, seed, mdlen, mgf1md)) {
goto err; goto err;
} }
for (i = 0; i < dblen; i++) { for (i = 0; i < dblen; i++) {
@ -547,7 +546,7 @@ int RSA_verify_PKCS1_PSS_mgf1(RSA *rsa, const uint8_t *mHash,
OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE); OPENSSL_PUT_ERROR(RSA, ERR_R_MALLOC_FAILURE);
goto err; goto err;
} }
if (PKCS1_MGF1(DB, maskedDBLen, H, hLen, mgf1Hash) < 0) { if (!PKCS1_MGF1(DB, maskedDBLen, H, hLen, mgf1Hash)) {
goto err; goto err;
} }
for (i = 0; i < maskedDBLen; i++) { for (i = 0; i < maskedDBLen; i++) {
@ -673,7 +672,7 @@ int RSA_padding_add_PKCS1_PSS_mgf1(RSA *rsa, unsigned char *EM,
EVP_MD_CTX_cleanup(&ctx); EVP_MD_CTX_cleanup(&ctx);
/* Generate dbMask in place then perform XOR on it */ /* Generate dbMask in place then perform XOR on it */
if (PKCS1_MGF1(EM, maskedDBLen, H, hLen, mgf1Hash)) { if (!PKCS1_MGF1(EM, maskedDBLen, H, hLen, mgf1Hash)) {
goto err; goto err;
} }