Fix EVP_Cipher error-handling.

Turns out the EVP_CIPH_FLAG_CUSTOM_CIPHER ciphers (i.e. legacy EVP_CIPHER
AES-GCM) have a completely different return value setup than the normal ones
which are the standard one/zero. (Except that they never return zero; see
TODO.)

Fix checks in ssl/ and remove remnants of EVP_CIPH_FLAG_CUSTOM_CIPHER in ssl/
as we're using EVP_AEAD now.

See CHANGES entry added in upstream's 3da0ca796c.

Change-Id: Ia4d0ff59b03c35fab3a08141c60b9534cb7172e2
Reviewed-on: https://boringssl-review.googlesource.com/2606
Reviewed-by: Adam Langley <agl@google.com>

include/openssl/cipher.h

@@ -80,11 +80,14 @@ OPENSSL_EXPORT const EVP_CIPHER *EVP_des_ede3_cbc(void);
 OPENSSL_EXPORT const EVP_CIPHER *EVP_aes_128_ecb(void);
 OPENSSL_EXPORT const EVP_CIPHER *EVP_aes_128_cbc(void);
 OPENSSL_EXPORT const EVP_CIPHER *EVP_aes_128_ctr(void);
-OPENSSL_EXPORT const EVP_CIPHER *EVP_aes_128_gcm(void);
 
 OPENSSL_EXPORT const EVP_CIPHER *EVP_aes_256_ecb(void);
 OPENSSL_EXPORT const EVP_CIPHER *EVP_aes_256_cbc(void);
 OPENSSL_EXPORT const EVP_CIPHER *EVP_aes_256_ctr(void);
+
+/* Deprecated AES-GCM implementations that set |EVP_CIPH_FLAG_CUSTOM_CIPHER|.
+ * Use |EVP_aead_aes_128_gcm| and |EVP_aead_aes_256_gcm| instead. */
+OPENSSL_EXPORT const EVP_CIPHER *EVP_aes_128_gcm(void);
 OPENSSL_EXPORT const EVP_CIPHER *EVP_aes_256_gcm(void);
 
 /* EVP_enc_null returns a 'cipher' that passes plaintext through as
@@ -190,10 +193,17 @@ OPENSSL_EXPORT int EVP_DecryptFinal_ex(EVP_CIPHER_CTX *ctx, unsigned char *out,
                                        int *out_len);
 
 /* EVP_Cipher performs a one-shot encryption/decryption operation. No partial
- * blocks etc are maintained between calls. It returns the number of bytes
- * written or -1 on error.
+ * blocks etc are maintained between calls. It returns one on success and zero
+ * otherwise, unless |EVP_CIPHER_flags| has |EVP_CIPH_FLAG_CUSTOM_CIPHER|
+ * set. Then it returns the number of bytes written or -1 on error.
+ *
+ * WARNING: this differs from the usual return value convention when using
+ * |EVP_CIPH_FLAG_CUSTOM_CIPHER|.
  *
- * WARNING: this differs from the usual return value convention. */
+ * TODO(davidben): The normal ciphers currently never fail, even if, e.g.,
+ * |in_len| is not a multiple of the block size for CBC-mode decryption. The
+ * input just gets rounded up while the output gets truncated. This should
+ * either be officially documented or fail. */
 OPENSSL_EXPORT int EVP_Cipher(EVP_CIPHER_CTX *ctx, uint8_t *out,
                               const uint8_t *in, size_t in_len);
 

ssl/s3_enc.c

@@ -469,7 +469,7 @@ int ssl3_enc(SSL *s, int send)
 			/* otherwise, rec->length >= bs */
 			}
 		
-		if (EVP_Cipher(ds,rec->data, rec->input, l) < 0)
+		if (!EVP_Cipher(ds, rec->data, rec->input, l))
 			return -1;
 
 		if (EVP_MD_CTX_md(s->read_hash) != NULL)

ssl/t1_enc.c

@@ -941,11 +941,8 @@ int tls1_enc(SSL *s, int send)
 				return 0;
 			}
 		
-		i = EVP_Cipher(ds,rec->data,rec->input,l);
-		if ((EVP_CIPHER_flags(ds->cipher)&EVP_CIPH_FLAG_CUSTOM_CIPHER)
-						?(i<0)
-						:(i==0))
-			return -1;	/* AEAD can fail to verify MAC */
+		if (!EVP_Cipher(ds, rec->data, rec->input, l))
+			return -1;
 
 		ret = 1;
 		if (EVP_MD_CTX_md(s->read_hash) != NULL)