Promise more accurate bounds than EVP_MD_MAX_SIZE.

A caller using EVP_Digest* which a priori knows tighter bounds on the
hash function used (perhaps because it is always a particular hash) can
assume the function will not write more bytes than the size of the hash.

The letter of the rules before vaguely[*] allowed for more than
EVP_MD_MAX_SIZE bytes written which made for some unreasonable code in
Chromium. Officially clarify this and add tests which, when paired with
valgrind and ASan prove it.

BUG=59

[*] Not really. I think it already promised the output length will be
both the number of bytes written and the size of the hash and the size
of the hash is given by what the function promises to compute. Meh.

Change-Id: I736d526e81cca30475c90897bca896293ff30278
Reviewed-on: https://boringssl-review.googlesource.com/9066
Reviewed-by: Eric Roman <ericroman@google.com>
Reviewed-by: David Benjamin <davidben@google.com>
Commit-Queue: David Benjamin <davidben@google.com>
CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org>

crypto/digest/digest_test.cc

@@ -171,13 +171,13 @@ static int TestDigest(const TestVector *test) {
       return false;
     }
   }
-  uint8_t digest[EVP_MAX_MD_SIZE];
+  std::unique_ptr<uint8_t[]> digest(new uint8_t[EVP_MD_size(test->md.func())]);
   unsigned digest_len;
-  if (!EVP_DigestFinal_ex(ctx.get(), digest, &digest_len)) {
+  if (!EVP_DigestFinal_ex(ctx.get(), digest.get(), &digest_len)) {
     fprintf(stderr, "EVP_DigestFinal_ex failed\n");
     return false;
   }
-  if (!CompareDigest(test, digest, digest_len)) {
+  if (!CompareDigest(test, digest.get(), digest_len)) {
     return false;
   }
 
@@ -198,7 +198,7 @@ static int TestDigest(const TestVector *test) {
       }
     }
   }
-  if (!EVP_DigestFinal_ex(ctx.get(), digest, &digest_len)) {
+  if (!EVP_DigestFinal_ex(ctx.get(), digest.get(), &digest_len)) {
     fprintf(stderr, "EVP_DigestFinal_ex failed\n");
     return false;
   }
@@ -206,19 +206,19 @@ static int TestDigest(const TestVector *test) {
     fprintf(stderr, "EVP_MD_size output incorrect\n");
     return false;
   }
-  if (!CompareDigest(test, digest, digest_len)) {
+  if (!CompareDigest(test, digest.get(), digest_len)) {
     return false;
   }
 
   // Test the one-shot function.
   if (test->md.one_shot_func && test->repeat == 1) {
     uint8_t *out = test->md.one_shot_func((const uint8_t *)test->input,
-                                          strlen(test->input), digest);
-    if (out != digest) {
+                                          strlen(test->input), digest.get());
+    if (out != digest.get()) {
       fprintf(stderr, "one_shot_func gave incorrect return\n");
       return false;
     }
-    if (!CompareDigest(test, digest, EVP_MD_size(test->md.func()))) {
+    if (!CompareDigest(test, digest.get(), EVP_MD_size(test->md.func()))) {
       return false;
     }
 

include/openssl/digest.h

@@ -147,10 +147,11 @@ OPENSSL_EXPORT int EVP_DigestUpdate(EVP_MD_CTX *ctx, const void *data,
 #define EVP_MAX_MD_BLOCK_SIZE 128 /* SHA-512 is the longest so far. */
 
 /* EVP_DigestFinal_ex finishes the digest in |ctx| and writes the output to
- * |md_out|. At most |EVP_MAX_MD_SIZE| bytes are written. If |out_size| is not
- * NULL then |*out_size| is set to the number of bytes written. It returns one.
- * After this call, the hash cannot be updated or finished again until
- * |EVP_DigestInit_ex| is called to start another hashing operation. */
+ * |md_out|. |EVP_MD_CTX_size| bytes are written, which is at most
+ * |EVP_MAX_MD_SIZE|. If |out_size| is not NULL then |*out_size| is set to the
+ * number of bytes written. It returns one. After this call, the hash cannot be
+ * updated or finished again until |EVP_DigestInit_ex| is called to start
+ * another hashing operation. */
 OPENSSL_EXPORT int EVP_DigestFinal_ex(EVP_MD_CTX *ctx, uint8_t *md_out,
                                       unsigned int *out_size);
 
@@ -159,11 +160,11 @@ OPENSSL_EXPORT int EVP_DigestFinal_ex(EVP_MD_CTX *ctx, uint8_t *md_out,
 OPENSSL_EXPORT int EVP_DigestFinal(EVP_MD_CTX *ctx, uint8_t *md_out,
                                    unsigned int *out_size);
 
-/* EVP_Digest performs a complete hashing operation in one call. It hashes
- * |len| bytes from |data| and writes the digest to |md_out|. At most
- * |EVP_MAX_MD_SIZE| bytes are written. If |out_size| is not NULL then
- * |*out_size| is set to the number of bytes written. It returns one on success
- * and zero otherwise. */
+/* EVP_Digest performs a complete hashing operation in one call. It hashes |len|
+ * bytes from |data| and writes the digest to |md_out|. |EVP_MD_CTX_size| bytes
+ * are written, which is at most |EVP_MAX_MD_SIZE|. If |out_size| is not NULL
+ * then |*out_size| is set to the number of bytes written. It returns one on
+ * success and zero otherwise. */
 OPENSSL_EXPORT int EVP_Digest(const void *data, size_t len, uint8_t *md_out,
                               unsigned int *md_out_size, const EVP_MD *type,
                               ENGINE *impl);