From 26ba48a6fbbf7b25bbfb521d3f5591e2d5a0b4bd Mon Sep 17 00:00:00 2001 From: Adam Langley Date: Thu, 3 May 2018 11:44:22 -0700 Subject: [PATCH] Make x86(-64) use the same aes_hw_* infrastructure as POWER and the ARMs. This also happens to make the AES_[en|de]crypt functions use AES-NI (where available) on Intel. Update-Note: this substantially changes how AES-NI is triggered. Worth running bssl speed (on both k8 and ppc), before and after, to confirm that there are no regressions. Change-Id: I5f22c1975236bbc1633c24ab60d683bca8ddd4c3 Reviewed-on: https://boringssl-review.googlesource.com/28026 Reviewed-by: David Benjamin --- crypto/fipsmodule/aes/aes.c | 8 + crypto/fipsmodule/aes/asm/aesni-x86_64.pl | 120 ++++---- crypto/fipsmodule/aes/internal.h | 25 +- crypto/fipsmodule/aes/mode_wrappers.c | 31 +- crypto/fipsmodule/cipher/e_aes.c | 338 ++-------------------- 5 files changed, 136 insertions(+), 386 deletions(-) diff --git a/crypto/fipsmodule/aes/aes.c b/crypto/fipsmodule/aes/aes.c index 43e2e167..fa92a81e 100644 --- a/crypto/fipsmodule/aes/aes.c +++ b/crypto/fipsmodule/aes/aes.c @@ -810,6 +810,14 @@ void AES_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) { // control asm symbol visibility with command line flags and such so they are // always hidden and wrapped by these C functions, which can be so // controlled. +// +// Be aware that on x86(-64), the asm_AES_* functions are incompatible with the +// aes_hw_* functions. The latter set |AES_KEY.rounds| to one less than the true +// value, which breaks the former. Therefore the two functions cannot mix. +// +// On AArch64, we don't have asm_AES_* functions and so must use the generic +// versions when hardware support isn't provided. However, the Aarch64 assembly +// doesn't have the same compatibility problem. void asm_AES_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key); void AES_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) { diff --git a/crypto/fipsmodule/aes/asm/aesni-x86_64.pl b/crypto/fipsmodule/aes/asm/aesni-x86_64.pl index a1869418..65452742 100644 --- a/crypto/fipsmodule/aes/asm/aesni-x86_64.pl +++ b/crypto/fipsmodule/aes/asm/aesni-x86_64.pl @@ -188,7 +188,7 @@ # incurred by operations on %xmm8-15. As ECB is not considered # critical, nothing was done to mitigate the problem. -$PREFIX="aesni"; # if $PREFIX is set to "AES", the script +$PREFIX="aes_hw"; # if $PREFIX is set to "AES", the script # generates drop-in replacement for # crypto/aes/asm/aes-x86_64.pl:-) @@ -206,7 +206,7 @@ die "can't locate x86_64-xlate.pl"; open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\""; *STDOUT=*OUT; -$movkey = $PREFIX eq "aesni" ? "movups" : "movups"; +$movkey = $PREFIX eq "aes_hw" ? "movups" : "movups"; @_4args=$win64? ("%rcx","%rdx","%r8", "%r9") : # Win64 order ("%rdi","%rsi","%rdx","%rcx"); # Unix order @@ -577,27 +577,27 @@ _aesni_${dir}rypt8: .size _aesni_${dir}rypt8,.-_aesni_${dir}rypt8 ___ } -&aesni_generate2("enc") if ($PREFIX eq "aesni"); +&aesni_generate2("enc") if ($PREFIX eq "aes_hw"); &aesni_generate2("dec"); -&aesni_generate3("enc") if ($PREFIX eq "aesni"); +&aesni_generate3("enc") if ($PREFIX eq "aes_hw"); &aesni_generate3("dec"); -&aesni_generate4("enc") if ($PREFIX eq "aesni"); +&aesni_generate4("enc") if ($PREFIX eq "aes_hw"); &aesni_generate4("dec"); -&aesni_generate6("enc") if ($PREFIX eq "aesni"); +&aesni_generate6("enc") if ($PREFIX eq "aes_hw"); &aesni_generate6("dec"); -&aesni_generate8("enc") if ($PREFIX eq "aesni"); +&aesni_generate8("enc") if ($PREFIX eq "aes_hw"); &aesni_generate8("dec"); -if ($PREFIX eq "aesni") { +if ($PREFIX eq "aes_hw") { ######################################################################## # void aesni_ecb_encrypt (const void *in, void *out, # size_t length, const AES_KEY *key, # int enc); $code.=<<___; -.globl aesni_ecb_encrypt -.type aesni_ecb_encrypt,\@function,5 +.globl ${PREFIX}_ecb_encrypt +.type ${PREFIX}_ecb_encrypt,\@function,5 .align 16 -aesni_ecb_encrypt: +${PREFIX}_ecb_encrypt: ___ $code.=<<___ if ($win64); lea -0x58(%rsp),%rsp @@ -943,7 +943,7 @@ $code.=<<___ if ($win64); ___ $code.=<<___; ret -.size aesni_ecb_encrypt,.-aesni_ecb_encrypt +.size ${PREFIX}_ecb_encrypt,.-${PREFIX}_ecb_encrypt ___ { @@ -964,10 +964,10 @@ my $iv="%xmm6"; my $bswap_mask="%xmm7"; $code.=<<___; -.globl aesni_ccm64_encrypt_blocks -.type aesni_ccm64_encrypt_blocks,\@function,6 +.globl ${PREFIX}_ccm64_encrypt_blocks +.type ${PREFIX}_ccm64_encrypt_blocks,\@function,6 .align 16 -aesni_ccm64_encrypt_blocks: +${PREFIX}_ccm64_encrypt_blocks: ___ $code.=<<___ if ($win64); lea -0x58(%rsp),%rsp @@ -1050,14 +1050,14 @@ $code.=<<___ if ($win64); ___ $code.=<<___; ret -.size aesni_ccm64_encrypt_blocks,.-aesni_ccm64_encrypt_blocks +.size ${PREFIX}_ccm64_encrypt_blocks,.-${PREFIX}_ccm64_encrypt_blocks ___ ###################################################################### $code.=<<___; -.globl aesni_ccm64_decrypt_blocks -.type aesni_ccm64_decrypt_blocks,\@function,6 +.globl ${PREFIX}_ccm64_decrypt_blocks +.type ${PREFIX}_ccm64_decrypt_blocks,\@function,6 .align 16 -aesni_ccm64_decrypt_blocks: +${PREFIX}_ccm64_decrypt_blocks: ___ $code.=<<___ if ($win64); lea -0x58(%rsp),%rsp @@ -1157,7 +1157,7 @@ $code.=<<___ if ($win64); ___ $code.=<<___; ret -.size aesni_ccm64_decrypt_blocks,.-aesni_ccm64_decrypt_blocks +.size ${PREFIX}_ccm64_decrypt_blocks,.-${PREFIX}_ccm64_decrypt_blocks ___ } ###################################################################### @@ -1178,10 +1178,10 @@ my ($key0,$ctr)=("%ebp","${ivp}d"); my $frame_size = 0x80 + ($win64?160:0); $code.=<<___; -.globl aesni_ctr32_encrypt_blocks -.type aesni_ctr32_encrypt_blocks,\@function,5 +.globl ${PREFIX}_ctr32_encrypt_blocks +.type ${PREFIX}_ctr32_encrypt_blocks,\@function,5 .align 16 -aesni_ctr32_encrypt_blocks: +${PREFIX}_ctr32_encrypt_blocks: .cfi_startproc cmp \$1,$len jne .Lctr32_bulk @@ -1734,7 +1734,7 @@ $code.=<<___; .Lctr32_epilogue: ret .cfi_endproc -.size aesni_ctr32_encrypt_blocks,.-aesni_ctr32_encrypt_blocks +.size ${PREFIX}_ctr32_encrypt_blocks,.-${PREFIX}_ctr32_encrypt_blocks ___ } @@ -1751,10 +1751,10 @@ my $frame_size = 0x70 + ($win64?160:0); my $key_ = "%rbp"; # override so that we can use %r11 as FP $code.=<<___; -.globl aesni_xts_encrypt -.type aesni_xts_encrypt,\@function,6 +.globl ${PREFIX}_xts_encrypt +.type ${PREFIX}_xts_encrypt,\@function,6 .align 16 -aesni_xts_encrypt: +${PREFIX}_xts_encrypt: .cfi_startproc lea (%rsp),%r11 # frame pointer .cfi_def_cfa_register %r11 @@ -2230,14 +2230,14 @@ $code.=<<___; .Lxts_enc_epilogue: ret .cfi_endproc -.size aesni_xts_encrypt,.-aesni_xts_encrypt +.size ${PREFIX}_xts_encrypt,.-${PREFIX}_xts_encrypt ___ $code.=<<___; -.globl aesni_xts_decrypt -.type aesni_xts_decrypt,\@function,6 +.globl ${PREFIX}_xts_decrypt +.type ${PREFIX}_xts_decrypt,\@function,6 .align 16 -aesni_xts_decrypt: +${PREFIX}_xts_decrypt: .cfi_startproc lea (%rsp),%r11 # frame pointer .cfi_def_cfa_register %r11 @@ -2739,7 +2739,7 @@ $code.=<<___; .Lxts_dec_epilogue: ret .cfi_endproc -.size aesni_xts_decrypt,.-aesni_xts_decrypt +.size ${PREFIX}_xts_decrypt,.-${PREFIX}_xts_decrypt ___ } @@ -2759,10 +2759,10 @@ my $seventh_arg = $win64 ? 56 : 8; my $blocks = $len; $code.=<<___; -.globl aesni_ocb_encrypt -.type aesni_ocb_encrypt,\@function,6 +.globl ${PREFIX}_ocb_encrypt +.type ${PREFIX}_ocb_encrypt,\@function,6 .align 32 -aesni_ocb_encrypt: +${PREFIX}_ocb_encrypt: .cfi_startproc lea (%rsp),%rax push %rbx @@ -3011,7 +3011,7 @@ $code.=<<___; .Locb_enc_epilogue: ret .cfi_endproc -.size aesni_ocb_encrypt,.-aesni_ocb_encrypt +.size ${PREFIX}_ocb_encrypt,.-${PREFIX}_ocb_encrypt .type __ocb_encrypt6,\@abi-omnipotent .align 32 @@ -3219,10 +3219,10 @@ __ocb_encrypt1: ret .size __ocb_encrypt1,.-__ocb_encrypt1 -.globl aesni_ocb_decrypt -.type aesni_ocb_decrypt,\@function,6 +.globl ${PREFIX}_ocb_decrypt +.type ${PREFIX}_ocb_decrypt,\@function,6 .align 32 -aesni_ocb_decrypt: +${PREFIX}_ocb_decrypt: .cfi_startproc lea (%rsp),%rax push %rbx @@ -3493,7 +3493,7 @@ $code.=<<___; .Locb_dec_epilogue: ret .cfi_endproc -.size aesni_ocb_decrypt,.-aesni_ocb_decrypt +.size ${PREFIX}_ocb_decrypt,.-${PREFIX}_ocb_decrypt .type __ocb_decrypt6,\@abi-omnipotent .align 32 @@ -4736,7 +4736,7 @@ $disp="%r9"; $code.=<<___; .extern __imp_RtlVirtualUnwind ___ -$code.=<<___ if ($PREFIX eq "aesni"); +$code.=<<___ if ($PREFIX eq "aes_hw"); .type ecb_ccm64_se_handler,\@abi-omnipotent .align 16 ecb_ccm64_se_handler: @@ -4776,7 +4776,7 @@ ecb_ccm64_se_handler: lea 0x58(%rax),%rax # adjust stack pointer jmp .Lcommon_seh_tail -.size ecb_ccm64_se_handler,.-ecb_ccm64_se_handler +.size ${PREFIX}_ccm64_se_handler,.-${PREFIX}_ccm64_se_handler .type ctr_xts_se_handler,\@abi-omnipotent .align 16 @@ -4968,37 +4968,37 @@ cbc_se_handler: .section .pdata .align 4 ___ -$code.=<<___ if ($PREFIX eq "aesni"); - .rva .LSEH_begin_aesni_ecb_encrypt - .rva .LSEH_end_aesni_ecb_encrypt +$code.=<<___ if ($PREFIX eq "aes_hw"); + .rva .LSEH_begin_${PREFIX}_ecb_encrypt + .rva .LSEH_end_${PREFIX}_ecb_encrypt .rva .LSEH_info_ecb - .rva .LSEH_begin_aesni_ccm64_encrypt_blocks - .rva .LSEH_end_aesni_ccm64_encrypt_blocks + .rva .LSEH_begin_${PREFIX}_ccm64_encrypt_blocks + .rva .LSEH_end_${PREFIX}_ccm64_encrypt_blocks .rva .LSEH_info_ccm64_enc - .rva .LSEH_begin_aesni_ccm64_decrypt_blocks - .rva .LSEH_end_aesni_ccm64_decrypt_blocks + .rva .LSEH_begin_${PREFIX}_ccm64_decrypt_blocks + .rva .LSEH_end_${PREFIX}_ccm64_decrypt_blocks .rva .LSEH_info_ccm64_dec - .rva .LSEH_begin_aesni_ctr32_encrypt_blocks - .rva .LSEH_end_aesni_ctr32_encrypt_blocks + .rva .LSEH_begin_${PREFIX}_ctr32_encrypt_blocks + .rva .LSEH_end_${PREFIX}_ctr32_encrypt_blocks .rva .LSEH_info_ctr32 - .rva .LSEH_begin_aesni_xts_encrypt - .rva .LSEH_end_aesni_xts_encrypt + .rva .LSEH_begin_${PREFIX}_xts_encrypt + .rva .LSEH_end_${PREFIX}_xts_encrypt .rva .LSEH_info_xts_enc - .rva .LSEH_begin_aesni_xts_decrypt - .rva .LSEH_end_aesni_xts_decrypt + .rva .LSEH_begin_${PREFIX}_xts_decrypt + .rva .LSEH_end_${PREFIX}_xts_decrypt .rva .LSEH_info_xts_dec - .rva .LSEH_begin_aesni_ocb_encrypt - .rva .LSEH_end_aesni_ocb_encrypt + .rva .LSEH_begin_${PREFIX}_ocb_encrypt + .rva .LSEH_end_${PREFIX}_ocb_encrypt .rva .LSEH_info_ocb_enc - .rva .LSEH_begin_aesni_ocb_decrypt - .rva .LSEH_end_aesni_ocb_decrypt + .rva .LSEH_begin_${PREFIX}_ocb_decrypt + .rva .LSEH_end_${PREFIX}_ocb_decrypt .rva .LSEH_info_ocb_dec ___ $code.=<<___; @@ -5016,7 +5016,7 @@ $code.=<<___; .section .xdata .align 8 ___ -$code.=<<___ if ($PREFIX eq "aesni"); +$code.=<<___ if ($PREFIX eq "aes_hw"); .LSEH_info_ecb: .byte 9,0,0,0 .rva ecb_ccm64_se_handler diff --git a/crypto/fipsmodule/aes/internal.h b/crypto/fipsmodule/aes/internal.h index 45db9eec..7c19b9c5 100644 --- a/crypto/fipsmodule/aes/internal.h +++ b/crypto/fipsmodule/aes/internal.h @@ -24,21 +24,30 @@ extern "C" { #endif -#if !defined(OPENSSL_NO_ASM) && (defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64)) +#if !defined(OPENSSL_NO_ASM) + +#if defined(OPENSSL_X86_64) #define HWAES +#define HWAES_ECB static int hwaes_capable(void) { - return CRYPTO_is_ARMv8_AES_capable(); + return (OPENSSL_ia32cap_P[1] & (1 << (57 - 32))) != 0; } -#endif // !NO_ASM && (AES || AARCH64) +#elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64) +#define HWAES -#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_PPC64LE) +static int hwaes_capable(void) { + return CRYPTO_is_ARMv8_AES_capable(); +} +#elif defined(OPENSSL_PPC64LE) #define HWAES static int hwaes_capable(void) { return CRYPTO_is_PPC64LE_vcrypto_capable(); } -#endif // !NO_ASM && PPC64LE +#endif + +#endif // !NO_ASM #if defined(HWAES) @@ -93,6 +102,12 @@ static void aes_hw_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, #endif // !HWAES + +#if defined(HWAES_ECB) +void aes_hw_ecb_encrypt(const uint8_t *in, uint8_t *out, size_t length, + const AES_KEY *key, const int enc); +#endif + #if defined(__cplusplus) } // extern C #endif diff --git a/crypto/fipsmodule/aes/mode_wrappers.c b/crypto/fipsmodule/aes/mode_wrappers.c index 34514db5..60f3774b 100644 --- a/crypto/fipsmodule/aes/mode_wrappers.c +++ b/crypto/fipsmodule/aes/mode_wrappers.c @@ -6,7 +6,7 @@ * are met: * * 1. Redistributions of source code must retain the above copyright - * notice, this list of conditions and the following disclaimer. + * notice, this list of conditions and the following disclaimer. * * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in @@ -50,6 +50,7 @@ #include +#include "../aes/internal.h" #include "../modes/internal.h" @@ -72,27 +73,31 @@ void AES_ecb_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key, } } -#if defined(OPENSSL_NO_ASM) || \ - (!defined(OPENSSL_X86_64) && !defined(OPENSSL_X86)) +#if !defined(OPENSSL_NO_ASM) && (defined(OPENSSL_X86_64) || defined(OPENSSL_X86)) +void asm_AES_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t len, + const AES_KEY *key, uint8_t *ivec, const int enc); +#endif + void AES_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t len, const AES_KEY *key, uint8_t *ivec, const int enc) { +#if defined(HWAES) + if (hwaes_capable()) { + aes_hw_cbc_encrypt(in, out, len, key, ivec, enc); + return; + } +#endif +#if !defined(OPENSSL_NO_ASM) && \ + (defined(OPENSSL_X86_64) || defined(OPENSSL_X86)) + asm_AES_cbc_encrypt(in, out, len, key, ivec, enc); +#else if (enc) { CRYPTO_cbc128_encrypt(in, out, len, key, ivec, (block128_f)AES_encrypt); } else { CRYPTO_cbc128_decrypt(in, out, len, key, ivec, (block128_f)AES_decrypt); } +#endif } -#else - -void asm_AES_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t len, - const AES_KEY *key, uint8_t *ivec, const int enc); -void AES_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t len, - const AES_KEY *key, uint8_t *ivec, const int enc) { - asm_AES_cbc_encrypt(in, out, len, key, ivec, enc); -} - -#endif // OPENSSL_NO_ASM || (!OPENSSL_X86_64 && !OPENSSL_X86) void AES_ofb128_encrypt(const uint8_t *in, uint8_t *out, size_t length, const AES_KEY *key, uint8_t *ivec, int *num) { diff --git a/crypto/fipsmodule/cipher/e_aes.c b/crypto/fipsmodule/cipher/e_aes.c index 3617e484..9c482ad5 100644 --- a/crypto/fipsmodule/cipher/e_aes.c +++ b/crypto/fipsmodule/cipher/e_aes.c @@ -189,38 +189,6 @@ static void vpaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length, } #endif -#if !defined(OPENSSL_NO_ASM) && \ - (defined(OPENSSL_X86_64) || defined(OPENSSL_X86)) -int aesni_set_encrypt_key(const uint8_t *userKey, int bits, AES_KEY *key); -int aesni_set_decrypt_key(const uint8_t *userKey, int bits, AES_KEY *key); - -void aesni_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key); -void aesni_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key); - -void aesni_ecb_encrypt(const uint8_t *in, uint8_t *out, size_t length, - const AES_KEY *key, int enc); -void aesni_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length, - const AES_KEY *key, uint8_t *ivec, int enc); - -#else - -// On other platforms, aesni_capable() will always return false and so the -// following will never be called. -static void aesni_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) { - abort(); -} -static int aesni_set_encrypt_key(const uint8_t *userKey, int bits, - AES_KEY *key) { - abort(); -} -static void aesni_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, - size_t blocks, const void *key, - const uint8_t *ivec) { - abort(); -} - -#endif - static int aes_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key, const uint8_t *iv, int enc) { int ret, mode; @@ -337,22 +305,9 @@ static int aes_ofb_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out, const uint8_t *in, return 1; } -static char aesni_capable(void); - ctr128_f aes_ctr_set_key(AES_KEY *aes_key, GCM128_CONTEXT *gcm_ctx, block128_f *out_block, const uint8_t *key, size_t key_bytes) { - if (aesni_capable()) { - aesni_set_encrypt_key(key, key_bytes * 8, aes_key); - if (gcm_ctx != NULL) { - CRYPTO_gcm128_init(gcm_ctx, aes_key, (block128_f)aesni_encrypt, 1); - } - if (out_block) { - *out_block = (block128_f) aesni_encrypt; - } - return (ctr128_f)aesni_ctr32_encrypt_blocks; - } - if (hwaes_capable()) { aes_hw_set_encrypt_key(key, key_bytes * 8, aes_key); if (gcm_ctx != NULL) { @@ -811,126 +766,22 @@ DEFINE_LOCAL_DATA(EVP_CIPHER, aes_256_gcm_generic) { out->ctrl = aes_gcm_ctrl; } -#if !defined(OPENSSL_NO_ASM) && \ - (defined(OPENSSL_X86_64) || defined(OPENSSL_X86)) - -// AES-NI section. - -static char aesni_capable(void) { - return (OPENSSL_ia32cap_P[1] & (1 << (57 - 32))) != 0; -} - -static int aesni_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key, - const uint8_t *iv, int enc) { - int ret, mode; - EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data; - - mode = ctx->cipher->flags & EVP_CIPH_MODE_MASK; - if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE) && !enc) { - ret = aesni_set_decrypt_key(key, ctx->key_len * 8, ctx->cipher_data); - dat->block = (block128_f)aesni_decrypt; - dat->stream.cbc = - mode == EVP_CIPH_CBC_MODE ? (cbc128_f)aesni_cbc_encrypt : NULL; - } else { - ret = aesni_set_encrypt_key(key, ctx->key_len * 8, ctx->cipher_data); - dat->block = (block128_f)aesni_encrypt; - if (mode == EVP_CIPH_CBC_MODE) { - dat->stream.cbc = (cbc128_f)aesni_cbc_encrypt; - } else if (mode == EVP_CIPH_CTR_MODE) { - dat->stream.ctr = (ctr128_f)aesni_ctr32_encrypt_blocks; - } else { - dat->stream.cbc = NULL; - } - } - - if (ret < 0) { - OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_AES_KEY_SETUP_FAILED); - return 0; - } - - return 1; -} - -static int aesni_cbc_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out, - const uint8_t *in, size_t len) { - aesni_cbc_encrypt(in, out, len, ctx->cipher_data, ctx->iv, ctx->encrypt); - - return 1; -} +#if defined(HWAES_ECB) -static int aesni_ecb_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out, - const uint8_t *in, size_t len) { +static int aes_hw_ecb_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out, + const uint8_t *in, size_t len) { size_t bl = ctx->cipher->block_size; if (len < bl) { return 1; } - aesni_ecb_encrypt(in, out, len, ctx->cipher_data, ctx->encrypt); - - return 1; -} + aes_hw_ecb_encrypt(in, out, len, ctx->cipher_data, ctx->encrypt); -static int aesni_gcm_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key, - const uint8_t *iv, int enc) { - EVP_AES_GCM_CTX *gctx = ctx->cipher_data; - if (!iv && !key) { - return 1; - } - if (key) { - aesni_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks); - CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, (block128_f)aesni_encrypt, 1); - gctx->ctr = (ctr128_f)aesni_ctr32_encrypt_blocks; - // If we have an iv can set it directly, otherwise use - // saved IV. - if (iv == NULL && gctx->iv_set) { - iv = gctx->iv; - } - if (iv) { - CRYPTO_gcm128_setiv(&gctx->gcm, &gctx->ks.ks, iv, gctx->ivlen); - gctx->iv_set = 1; - } - gctx->key_set = 1; - } else { - // If key set use IV, otherwise copy - if (gctx->key_set) { - CRYPTO_gcm128_setiv(&gctx->gcm, &gctx->ks.ks, iv, gctx->ivlen); - } else { - OPENSSL_memcpy(gctx->iv, iv, gctx->ivlen); - } - gctx->iv_set = 1; - gctx->iv_gen = 0; - } return 1; } -DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_cbc) { - memset(out, 0, sizeof(EVP_CIPHER)); - - out->nid = NID_aes_128_cbc; - out->block_size = 16; - out->key_len = 16; - out->iv_len = 16; - out->ctx_size = sizeof(EVP_AES_KEY); - out->flags = EVP_CIPH_CBC_MODE; - out->init = aesni_init_key; - out->cipher = aesni_cbc_cipher; -} - -DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_ctr) { - memset(out, 0, sizeof(EVP_CIPHER)); - - out->nid = NID_aes_128_ctr; - out->block_size = 1; - out->key_len = 16; - out->iv_len = 16; - out->ctx_size = sizeof(EVP_AES_KEY); - out->flags = EVP_CIPH_CTR_MODE; - out->init = aesni_init_key; - out->cipher = aes_ctr_cipher; -} - -DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_ecb) { +DEFINE_LOCAL_DATA(EVP_CIPHER, aes_hw_128_ecb) { memset(out, 0, sizeof(EVP_CIPHER)); out->nid = NID_aes_128_ecb; @@ -938,67 +789,11 @@ DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_ecb) { out->key_len = 16; out->ctx_size = sizeof(EVP_AES_KEY); out->flags = EVP_CIPH_ECB_MODE; - out->init = aesni_init_key; - out->cipher = aesni_ecb_cipher; -} - -DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_ofb) { - memset(out, 0, sizeof(EVP_CIPHER)); - - out->nid = NID_aes_128_ofb128; - out->block_size = 1; - out->key_len = 16; - out->iv_len = 16; - out->ctx_size = sizeof(EVP_AES_KEY); - out->flags = EVP_CIPH_OFB_MODE; - out->init = aesni_init_key; - out->cipher = aes_ofb_cipher; -} - -DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_gcm) { - memset(out, 0, sizeof(EVP_CIPHER)); - - out->nid = NID_aes_128_gcm; - out->block_size = 1; - out->key_len = 16; - out->iv_len = 12; - out->ctx_size = sizeof(EVP_AES_GCM_CTX); - out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV | - EVP_CIPH_FLAG_CUSTOM_CIPHER | EVP_CIPH_ALWAYS_CALL_INIT | - EVP_CIPH_CTRL_INIT | EVP_CIPH_FLAG_AEAD_CIPHER; - out->init = aesni_gcm_init_key; - out->cipher = aes_gcm_cipher; - out->cleanup = aes_gcm_cleanup; - out->ctrl = aes_gcm_ctrl; -} - -DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_192_cbc) { - memset(out, 0, sizeof(EVP_CIPHER)); - - out->nid = NID_aes_192_cbc; - out->block_size = 16; - out->key_len = 24; - out->iv_len = 16; - out->ctx_size = sizeof(EVP_AES_KEY); - out->flags = EVP_CIPH_CBC_MODE; - out->init = aesni_init_key; - out->cipher = aesni_cbc_cipher; -} - -DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_192_ctr) { - memset(out, 0, sizeof(EVP_CIPHER)); - - out->nid = NID_aes_192_ctr; - out->block_size = 1; - out->key_len = 24; - out->iv_len = 16; - out->ctx_size = sizeof(EVP_AES_KEY); - out->flags = EVP_CIPH_CTR_MODE; - out->init = aesni_init_key; - out->cipher = aes_ctr_cipher; + out->init = aes_init_key; + out->cipher = aes_hw_ecb_cipher; } -DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_192_ecb) { +DEFINE_LOCAL_DATA(EVP_CIPHER, aes_hw_192_ecb) { memset(out, 0, sizeof(EVP_CIPHER)); out->nid = NID_aes_192_ecb; @@ -1006,54 +801,11 @@ DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_192_ecb) { out->key_len = 24; out->ctx_size = sizeof(EVP_AES_KEY); out->flags = EVP_CIPH_ECB_MODE; - out->init = aesni_init_key; - out->cipher = aesni_ecb_cipher; -} - -DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_192_gcm) { - memset(out, 0, sizeof(EVP_CIPHER)); - - out->nid = NID_aes_192_gcm; - out->block_size = 1; - out->key_len = 24; - out->iv_len = 12; - out->ctx_size = sizeof(EVP_AES_GCM_CTX); - out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV | - EVP_CIPH_FLAG_CUSTOM_CIPHER | EVP_CIPH_ALWAYS_CALL_INIT | - EVP_CIPH_CTRL_INIT | EVP_CIPH_FLAG_AEAD_CIPHER; - out->init = aesni_gcm_init_key; - out->cipher = aes_gcm_cipher; - out->cleanup = aes_gcm_cleanup; - out->ctrl = aes_gcm_ctrl; -} - -DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_cbc) { - memset(out, 0, sizeof(EVP_CIPHER)); - - out->nid = NID_aes_256_cbc; - out->block_size = 16; - out->key_len = 32; - out->iv_len = 16; - out->ctx_size = sizeof(EVP_AES_KEY); - out->flags = EVP_CIPH_CBC_MODE; - out->init = aesni_init_key; - out->cipher = aesni_cbc_cipher; -} - -DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_ctr) { - memset(out, 0, sizeof(EVP_CIPHER)); - - out->nid = NID_aes_256_ctr; - out->block_size = 1; - out->key_len = 32; - out->iv_len = 16; - out->ctx_size = sizeof(EVP_AES_KEY); - out->flags = EVP_CIPH_CTR_MODE; - out->init = aesni_init_key; - out->cipher = aes_ctr_cipher; + out->init = aes_init_key; + out->cipher = aes_hw_ecb_cipher; } -DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_ecb) { +DEFINE_LOCAL_DATA(EVP_CIPHER, aes_hw_256_ecb) { memset(out, 0, sizeof(EVP_CIPHER)); out->nid = NID_aes_256_ecb; @@ -1061,80 +813,50 @@ DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_ecb) { out->key_len = 32; out->ctx_size = sizeof(EVP_AES_KEY); out->flags = EVP_CIPH_ECB_MODE; - out->init = aesni_init_key; - out->cipher = aesni_ecb_cipher; -} - -DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_ofb) { - memset(out, 0, sizeof(EVP_CIPHER)); - - out->nid = NID_aes_256_ofb128; - out->block_size = 1; - out->key_len = 32; - out->iv_len = 16; - out->ctx_size = sizeof(EVP_AES_KEY); - out->flags = EVP_CIPH_OFB_MODE; - out->init = aesni_init_key; - out->cipher = aes_ofb_cipher; + out->init = aes_init_key; + out->cipher = aes_hw_ecb_cipher; } -DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_gcm) { - memset(out, 0, sizeof(EVP_CIPHER)); +#define EVP_ECB_CIPHER_FUNCTION(keybits) \ + const EVP_CIPHER *EVP_aes_##keybits##_ecb(void) { \ + if (hwaes_capable()) { \ + return aes_hw_##keybits##_ecb(); \ + } \ + return aes_##keybits##_ecb_generic(); \ + } - out->nid = NID_aes_256_gcm; - out->block_size = 1; - out->key_len = 32; - out->iv_len = 12; - out->ctx_size = sizeof(EVP_AES_GCM_CTX); - out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV | - EVP_CIPH_FLAG_CUSTOM_CIPHER | EVP_CIPH_ALWAYS_CALL_INIT | - EVP_CIPH_CTRL_INIT | EVP_CIPH_CUSTOM_COPY | - EVP_CIPH_FLAG_AEAD_CIPHER; - out->init = aesni_gcm_init_key; - out->cipher = aes_gcm_cipher; - out->cleanup = aes_gcm_cleanup; - out->ctrl = aes_gcm_ctrl; -} +#else -#define EVP_CIPHER_FUNCTION(keybits, mode) \ - const EVP_CIPHER *EVP_aes_##keybits##_##mode(void) { \ - if (aesni_capable()) { \ - return aesni_##keybits##_##mode(); \ - } else { \ - return aes_##keybits##_##mode##_generic(); \ - } \ +#define EVP_ECB_CIPHER_FUNCTION(keybits) \ + const EVP_CIPHER *EVP_aes_##keybits##_ecb(void) { \ + return aes_##keybits##_ecb_generic(); \ } -#else // ^^^ OPENSSL_X86_64 || OPENSSL_X86 - -static char aesni_capable(void) { - return 0; -} +#endif // HWAES_ECB #define EVP_CIPHER_FUNCTION(keybits, mode) \ const EVP_CIPHER *EVP_aes_##keybits##_##mode(void) { \ return aes_##keybits##_##mode##_generic(); \ } -#endif - EVP_CIPHER_FUNCTION(128, cbc) EVP_CIPHER_FUNCTION(128, ctr) -EVP_CIPHER_FUNCTION(128, ecb) EVP_CIPHER_FUNCTION(128, ofb) EVP_CIPHER_FUNCTION(128, gcm) EVP_CIPHER_FUNCTION(192, cbc) EVP_CIPHER_FUNCTION(192, ctr) -EVP_CIPHER_FUNCTION(192, ecb) EVP_CIPHER_FUNCTION(192, gcm) EVP_CIPHER_FUNCTION(256, cbc) EVP_CIPHER_FUNCTION(256, ctr) -EVP_CIPHER_FUNCTION(256, ecb) EVP_CIPHER_FUNCTION(256, ofb) EVP_CIPHER_FUNCTION(256, gcm) +EVP_ECB_CIPHER_FUNCTION(128) +EVP_ECB_CIPHER_FUNCTION(192) +EVP_ECB_CIPHER_FUNCTION(256) + #define EVP_AEAD_AES_GCM_TAG_LEN 16 @@ -1429,7 +1151,7 @@ DEFINE_METHOD_FUNCTION(EVP_AEAD, EVP_aead_aes_256_gcm_tls12) { int EVP_has_aes_hardware(void) { #if defined(OPENSSL_X86) || defined(OPENSSL_X86_64) - return aesni_capable() && crypto_gcm_clmul_enabled(); + return hwaes_capable() && crypto_gcm_clmul_enabled(); #elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64) return hwaes_capable() && CRYPTO_is_ARMv8_PMULL_capable(); #else