Check for i2d_name_canon failures in x509_name_canon.

Change-Id: I0e1d79e85a2d20ab4105b81d39cdbbd692ba67da
Reviewed-on: https://boringssl-review.googlesource.com/12221
Reviewed-by: David Benjamin <davidben@google.com>
Commit-Queue: David Benjamin <davidben@google.com>
CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org>

crypto/x509/x_name.c

@@ -346,7 +346,7 @@ static int x509_name_canon(X509_NAME *a)
     STACK_OF(STACK_OF_X509_NAME_ENTRY) *intname = NULL;
     STACK_OF(X509_NAME_ENTRY) *entries = NULL;
     X509_NAME_ENTRY *entry, *tmpentry = NULL;
-    int set = -1, ret = 0;
+    int set = -1, ret = 0, len;
     size_t i;
 
     if (a->canon_enc) {
@@ -386,7 +386,11 @@ static int x509_name_canon(X509_NAME *a)
 
     /* Finally generate encoding */
 
-    a->canon_enclen = i2d_name_canon(intname, NULL);
+    len = i2d_name_canon(intname, NULL);
+    if (len < 0) {
+        goto err;
+    }
+    a->canon_enclen = len;
 
     p = OPENSSL_malloc(a->canon_enclen);