Check for overflows in ASN1_object_size().

(Imported from upstream's b10c10422a9ec4db426be3ef99031f0807d2ded0,
ff8b6b92f44c682ad78f60c32ec154e0bfabebb2, and
134ab5139a8d41455a81d9fcc31b3edb8a4b2f5c.)

Change-Id: Icf1661a4d0249ae5af72cda15b12822b86e35a82
Reviewed-on: https://boringssl-review.googlesource.com/10361
CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org>
Commit-Queue: David Benjamin <davidben@google.com>
Commit-Queue: Adam Langley <agl@google.com>
Reviewed-by: Adam Langley <agl@google.com>

crypto/asn1/a_object.c

@@ -72,7 +72,7 @@ int i2d_ASN1_OBJECT(ASN1_OBJECT *a, unsigned char **pp)
         return (0);
 
     objsize = ASN1_object_size(0, a->length, V_ASN1_OBJECT);
-    if (pp == NULL)
+    if (pp == NULL || objsize == -1)
         return objsize;
 
     p = *pp;

crypto/asn1/asn1_lib.c

@@ -298,26 +298,30 @@ static void asn1_put_length(unsigned char **pp, int length)
 
 int ASN1_object_size(int constructed, int length, int tag)
 {
-    int ret;
-
-    ret = length;
-    ret++;
+    int ret = 1;
+    if (length < 0)
+        return -1;
     if (tag >= 31) {
         while (tag > 0) {
             tag >>= 7;
             ret++;
         }
     }
-    if (constructed == 2)
-        return ret + 3;
-    ret++;
-    if (length > 127) {
-        while (length > 0) {
-            length >>= 8;
-            ret++;
+    if (constructed == 2) {
+        ret += 3;
+    } else {
+        ret++;
+        if (length > 127) {
+            int tmplen = length;
+            while (tmplen > 0) {
+                tmplen >>= 8;
+                ret++;
+            }
         }
     }
-    return (ret);
+    if (ret >= INT_MAX - length)
+        return -1;
+    return ret + length;
 }
 
 static int _asn1_Finish(ASN1_const_CTX *c)

crypto/asn1/tasn_enc.c

@@ -56,6 +56,7 @@
 
 #include <openssl/asn1.h>
 
+#include <limits.h>
 #include <string.h>
 
 #include <openssl/asn1t.h>
@@ -213,17 +214,19 @@ int ASN1_item_ex_i2d(ASN1_VALUE **pval, unsigned char **out,
         for (i = 0, tt = it->templates; i < it->tcount; tt++, i++) {
             const ASN1_TEMPLATE *seqtt;
             ASN1_VALUE **pseqval;
+            int tmplen;
             seqtt = asn1_do_adb(pval, tt, 1);
             if (!seqtt)
                 return 0;
             pseqval = asn1_get_field_ptr(pval, seqtt);
-            /* FIXME: check for errors in enhanced version */
-            seqcontlen += asn1_template_ex_i2d(pseqval, NULL, seqtt,
-                                               -1, aclass);
+            tmplen = asn1_template_ex_i2d(pseqval, NULL, seqtt, -1, aclass);
+            if (tmplen == -1 || (tmplen > INT_MAX - seqcontlen))
+                return -1;
+            seqcontlen += tmplen;
         }
 
         seqlen = ASN1_object_size(ndef, seqcontlen, tag);
-        if (!out)
+        if (!out || seqlen == -1)
             return seqlen;
         /* Output SEQUENCE header */
         ASN1_put_object(out, ndef, seqcontlen, tag, aclass);
@@ -337,19 +340,24 @@ static int asn1_template_ex_i2d(ASN1_VALUE **pval, unsigned char **out,
         /* Determine total length of items */
         skcontlen = 0;
         for (j = 0; j < sk_ASN1_VALUE_num(sk); j++) {
+            int tmplen;
             skitem = sk_ASN1_VALUE_value(sk, j);
-            skcontlen += ASN1_item_ex_i2d(&skitem, NULL,
-                                          ASN1_ITEM_ptr(tt->item),
-                                          -1, iclass);
+            tmplen = ASN1_item_ex_i2d(&skitem, NULL, ASN1_ITEM_ptr(tt->item),
+                                      -1, iclass);
+            if (tmplen == -1 || (skcontlen > INT_MAX - tmplen))
+                return -1;
+            skcontlen += tmplen;
         }
         sklen = ASN1_object_size(ndef, skcontlen, sktag);
+        if (sklen == -1)
+            return -1;
         /* If EXPLICIT need length of surrounding tag */
         if (flags & ASN1_TFLG_EXPTAG)
             ret = ASN1_object_size(ndef, sklen, ttag);
         else
             ret = sklen;
 
-        if (!out)
+        if (!out || ret == -1)
             return ret;
 
         /* Now encode this lot... */
@@ -378,7 +386,7 @@ static int asn1_template_ex_i2d(ASN1_VALUE **pval, unsigned char **out,
             return 0;
         /* Find length of EXPLICIT tag */
         ret = ASN1_object_size(ndef, i, ttag);
-        if (out) {
+        if (out && ret != -1) {
             /* Output tag and item */
             ASN1_put_object(out, ndef, i, ttag, tclass);
             ASN1_item_ex_i2d(pval, out, ASN1_ITEM_ptr(tt->item), -1, iclass);