diff --git a/crypto/fipsmodule/CMakeLists.txt b/crypto/fipsmodule/CMakeLists.txt index 04b2ffb2..09d210bf 100644 --- a/crypto/fipsmodule/CMakeLists.txt +++ b/crypto/fipsmodule/CMakeLists.txt @@ -69,6 +69,7 @@ if(${ARCH} STREQUAL "aarch64") sha1-armv8.${ASM_EXT} sha256-armv8.${ASM_EXT} sha512-armv8.${ASM_EXT} + vpaes-armv8.${ASM_EXT} ) endif() @@ -120,6 +121,7 @@ perlasm(sha512-586.${ASM_EXT} sha/asm/sha512-586.pl) perlasm(sha512-armv4.${ASM_EXT} sha/asm/sha512-armv4.pl) perlasm(sha512-armv8.${ASM_EXT} sha/asm/sha512-armv8.pl) perlasm(sha512-x86_64.${ASM_EXT} sha/asm/sha512-x86_64.pl) +perlasm(vpaes-armv8.${ASM_EXT} aes/asm/vpaes-armv8.pl) perlasm(vpaes-x86_64.${ASM_EXT} aes/asm/vpaes-x86_64.pl) perlasm(vpaes-x86.${ASM_EXT} aes/asm/vpaes-x86.pl) perlasm(x86_64-mont5.${ASM_EXT} bn/asm/x86_64-mont5.pl) diff --git a/crypto/fipsmodule/aes/aes_test.cc b/crypto/fipsmodule/aes/aes_test.cc index a0c94114..2222b63d 100644 --- a/crypto/fipsmodule/aes/aes_test.cc +++ b/crypto/fipsmodule/aes/aes_test.cc @@ -250,6 +250,9 @@ TEST(AESTest, ABI) { SCOPED_TRACE(blocks); CHECK_ABI(vpaes_cbc_encrypt, buf, buf, AES_BLOCK_SIZE * blocks, &key, block, AES_ENCRYPT); +#if defined(VPAES_CTR32) + CHECK_ABI(vpaes_ctr32_encrypt_blocks, buf, buf, blocks, &key, block); +#endif } CHECK_ABI(vpaes_set_decrypt_key, kKey, bits, &key); diff --git a/crypto/fipsmodule/aes/asm/vpaes-armv8.pl b/crypto/fipsmodule/aes/asm/vpaes-armv8.pl index 5131e13a..49eaf0d3 100755 --- a/crypto/fipsmodule/aes/asm/vpaes-armv8.pl +++ b/crypto/fipsmodule/aes/asm/vpaes-armv8.pl @@ -42,7 +42,7 @@ while (($output=shift) && ($output!~/\w[\w\-]*\.\w+$/)) {} $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1; ( $xlate="${dir}arm-xlate.pl" and -f $xlate ) or -( $xlate="${dir}../../perlasm/arm-xlate.pl" and -f $xlate) or +( $xlate="${dir}../../../perlasm/arm-xlate.pl" and -f $xlate) or die "can't locate arm-xlate.pl"; open OUT,"| \"$^X\" $xlate $flavour $output"; @@ -1171,7 +1171,8 @@ vpaes_cbc_decrypt: ret .size vpaes_cbc_decrypt,.-vpaes_cbc_decrypt ___ -if (1) { +# We omit vpaes_ecb_* in BoringSSL. They are unused. +if (0) { $code.=<<___; .globl vpaes_ecb_encrypt .type vpaes_ecb_encrypt,%function @@ -1253,7 +1254,89 @@ vpaes_ecb_decrypt: ret .size vpaes_ecb_decrypt,.-vpaes_ecb_decrypt ___ -} } +} + +my ($ctr, $ctr_tmp) = ("w6", "w7"); + +# void vpaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, size_t len, +# const AES_KEY *key, const uint8_t ivec[16]); +$code.=<<___; +.globl vpaes_ctr32_encrypt_blocks +.type vpaes_ctr32_encrypt_blocks,%function +.align 4 +vpaes_ctr32_encrypt_blocks: + stp x29,x30,[sp,#-16]! + add x29,sp,#0 + stp d8,d9,[sp,#-16]! // ABI spec says so + stp d10,d11,[sp,#-16]! + stp d12,d13,[sp,#-16]! + stp d14,d15,[sp,#-16]! + + cbz $len, .Lctr32_done + + // Note, unlike the other functions, $len here is measured in blocks, + // not bytes. + mov x17, $len + mov x2, $key + + // Load the IV and counter portion. + ldr $ctr, [$ivec, #12] + ld1 {v7.16b}, [$ivec] + + bl _vpaes_encrypt_preheat + tst x17, #1 + rev $ctr, $ctr // The counter is big-endian. + b.eq .Lctr32_prep_loop + + // Handle one block so the remaining block count is even for + // _vpaes_encrypt_2x. + ld1 {v6.16b}, [$inp], #16 // Load input ahead of time + bl _vpaes_encrypt_core + eor v0.16b, v0.16b, v6.16b // XOR input and result + st1 {v0.16b}, [$out], #16 + subs x17, x17, #1 + // Update the counter. + add $ctr, $ctr, #1 + rev $ctr_tmp, $ctr + mov v7.s[3], $ctr_tmp + b.ls .Lctr32_done + +.Lctr32_prep_loop: + // _vpaes_encrypt_core takes its input from v7, while _vpaes_encrypt_2x + // uses v14 and v15. + mov v15.16b, v7.16b + mov v14.16b, v7.16b + add $ctr, $ctr, #1 + rev $ctr_tmp, $ctr + mov v15.s[3], $ctr_tmp + +.Lctr32_loop: + ld1 {v6.16b,v7.16b}, [$inp], #32 // Load input ahead of time + bl _vpaes_encrypt_2x + eor v0.16b, v0.16b, v6.16b // XOR input and result + eor v1.16b, v1.16b, v7.16b // XOR input and result (#2) + st1 {v0.16b,v1.16b}, [$out], #32 + subs x17, x17, #2 + // Update the counter. + add $ctr_tmp, $ctr, #1 + add $ctr, $ctr, #2 + rev $ctr_tmp, $ctr_tmp + mov v14.s[3], $ctr_tmp + rev $ctr_tmp, $ctr + mov v15.s[3], $ctr_tmp + b.hi .Lctr32_loop + +.Lctr32_done: + ldp d14,d15,[sp],#16 + ldp d12,d13,[sp],#16 + ldp d10,d11,[sp],#16 + ldp d8,d9,[sp],#16 + ldp x29,x30,[sp],#16 + ret +.size vpaes_ctr32_encrypt_blocks,.-vpaes_ctr32_encrypt_blocks +___ +} + print $code; close STDOUT; diff --git a/crypto/fipsmodule/aes/internal.h b/crypto/fipsmodule/aes/internal.h index 0df30d9f..a05abcbf 100644 --- a/crypto/fipsmodule/aes/internal.h +++ b/crypto/fipsmodule/aes/internal.h @@ -35,13 +35,13 @@ OPENSSL_INLINE int hwaes_capable(void) { } #define VPAES -OPENSSL_INLINE char vpaes_capable(void) { +OPENSSL_INLINE int vpaes_capable(void) { return (OPENSSL_ia32cap_get()[1] & (1 << (41 - 32))) != 0; } #if defined(OPENSSL_X86_64) #define BSAES -OPENSSL_INLINE char bsaes_capable(void) { return vpaes_capable(); } +OPENSSL_INLINE int bsaes_capable(void) { return vpaes_capable(); } #endif // X86_64 #elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64) @@ -51,7 +51,13 @@ OPENSSL_INLINE int hwaes_capable(void) { return CRYPTO_is_ARMv8_AES_capable(); } #if defined(OPENSSL_ARM) #define BSAES -OPENSSL_INLINE char bsaes_capable(void) { return CRYPTO_is_NEON_capable(); } +OPENSSL_INLINE int bsaes_capable(void) { return CRYPTO_is_NEON_capable(); } +#endif + +#if defined(OPENSSL_AARCH64) +#define VPAES +#define VPAES_CTR32 +OPENSSL_INLINE int vpaes_capable(void) { return CRYPTO_is_NEON_capable(); } #endif #elif defined(OPENSSL_PPC64LE) @@ -162,6 +168,10 @@ void vpaes_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key); void vpaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length, const AES_KEY *key, uint8_t *ivec, int enc); +#if defined(VPAES_CTR32) +void vpaes_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out, size_t len, + const AES_KEY *key, const uint8_t ivec[16]); +#endif #else OPENSSL_INLINE char vpaes_capable(void) { return 0; } diff --git a/crypto/fipsmodule/cipher/e_aes.c b/crypto/fipsmodule/cipher/e_aes.c index 460deedd..51a1fb1c 100644 --- a/crypto/fipsmodule/cipher/e_aes.c +++ b/crypto/fipsmodule/cipher/e_aes.c @@ -143,7 +143,15 @@ static int aes_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key, } else if (vpaes_capable()) { ret = vpaes_set_encrypt_key(key, ctx->key_len * 8, &dat->ks.ks); dat->block = vpaes_encrypt; - dat->stream.cbc = mode == EVP_CIPH_CBC_MODE ? vpaes_cbc_encrypt : NULL; + dat->stream.cbc = NULL; + if (mode == EVP_CIPH_CBC_MODE) { + dat->stream.cbc = vpaes_cbc_encrypt; + } +#if defined(VPAES_CTR32) + if (mode == EVP_CIPH_CTR_MODE) { + dat->stream.ctr = vpaes_ctr32_encrypt_blocks; + } +#endif } else { ret = aes_nohw_set_encrypt_key(key, ctx->key_len * 8, &dat->ks.ks); dat->block = aes_nohw_encrypt; @@ -253,7 +261,11 @@ ctr128_f aes_ctr_set_key(AES_KEY *aes_key, GCM128_KEY *gcm_key, if (gcm_key != NULL) { CRYPTO_gcm128_init_key(gcm_key, aes_key, vpaes_encrypt, 0); } +#if defined(VPAES_CTR32) + return vpaes_ctr32_encrypt_blocks; +#else return NULL; +#endif } aes_nohw_set_encrypt_key(key, key_bytes * 8, aes_key);