kris
/
boringssl
1
0
フォーク 0
コード 課題 0 プルリクエスト 0 リリース 0 Wiki アクティビティ
ソースを参照

Revert "Make x86(-64) use the same aes_hw_* infrastructure as POWER and the ARMs." 

Broke Aarch64 on the main builders (but not the trybots, somehow.)

Change-Id: I53eb09c99ef42a59628b0506b5ddb125299b554a
Reviewed-on: https://boringssl-review.googlesource.com/28364
Reviewed-by: Adam Langley <agl@google.com>
kris/onging/CECPQ3_patch15
Adam Langley 6年前
親
5f001d1423
コミット
aca24c8724
5個のファイルの変更、386行の追加、136行の削除
分割表示
差分オプション
統計情報を表示 Patchファイルをダウンロード Diffファイルをダウンロード
  1. +0
    -8
      crypto/fipsmodule/aes/aes.c
  2. +60
    -60
      crypto/fipsmodule/aes/asm/aesni-x86_64.pl
  3. +5
    -20
      crypto/fipsmodule/aes/internal.h
  4. +13
    -18
      crypto/fipsmodule/aes/mode_wrappers.c
  5. +308
    -30
      crypto/fipsmodule/cipher/e_aes.c

+ 0
- 8
crypto/fipsmodule/aes/aes.c ファイルの表示

@@ -810,14 +810,6 @@ void AES_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) {
// control asm symbol visibility with command line flags and such so they are
// always hidden and wrapped by these C functions, which can be so
// controlled.
//
// Be aware that on x86(-64), the asm_AES_* functions are incompatible with the
// aes_hw_* functions. The latter set |AES_KEY.rounds| to one less than the true
// value, which breaks the former. Therefore the two functions cannot mix.
//
// On AArch64, we don't have asm_AES_* functions and so must use the generic
// versions when hardware support isn't provided. However, the Aarch64 assembly
// doesn't have the same compatibility problem.

void asm_AES_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key);
void AES_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) {


+ 60
- 60
crypto/fipsmodule/aes/asm/aesni-x86_64.pl ファイルの表示

@@ -188,7 +188,7 @@
# incurred by operations on %xmm8-15. As ECB is not considered
# critical, nothing was done to mitigate the problem.

$PREFIX="aes_hw"; # if $PREFIX is set to "AES", the script
$PREFIX="aesni"; # if $PREFIX is set to "AES", the script
# generates drop-in replacement for
# crypto/aes/asm/aes-x86_64.pl:-)

@@ -206,7 +206,7 @@ die "can't locate x86_64-xlate.pl";
open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
*STDOUT=*OUT;

$movkey = $PREFIX eq "aes_hw" ? "movups" : "movups";
$movkey = $PREFIX eq "aesni" ? "movups" : "movups";
@_4args=$win64? ("%rcx","%rdx","%r8", "%r9") : # Win64 order
("%rdi","%rsi","%rdx","%rcx"); # Unix order

@@ -577,27 +577,27 @@ _aesni_${dir}rypt8:
.size _aesni_${dir}rypt8,.-_aesni_${dir}rypt8
___
}
&aesni_generate2("enc") if ($PREFIX eq "aes_hw");
&aesni_generate2("enc") if ($PREFIX eq "aesni");
&aesni_generate2("dec");
&aesni_generate3("enc") if ($PREFIX eq "aes_hw");
&aesni_generate3("enc") if ($PREFIX eq "aesni");
&aesni_generate3("dec");
&aesni_generate4("enc") if ($PREFIX eq "aes_hw");
&aesni_generate4("enc") if ($PREFIX eq "aesni");
&aesni_generate4("dec");
&aesni_generate6("enc") if ($PREFIX eq "aes_hw");
&aesni_generate6("enc") if ($PREFIX eq "aesni");
&aesni_generate6("dec");
&aesni_generate8("enc") if ($PREFIX eq "aes_hw");
&aesni_generate8("enc") if ($PREFIX eq "aesni");
&aesni_generate8("dec");
if ($PREFIX eq "aes_hw") {
if ($PREFIX eq "aesni") {
########################################################################
# void aesni_ecb_encrypt (const void *in, void *out,
# size_t length, const AES_KEY *key,
# int enc);
$code.=<<___;
.globl ${PREFIX}_ecb_encrypt
.type ${PREFIX}_ecb_encrypt,\@function,5
.globl aesni_ecb_encrypt
.type aesni_ecb_encrypt,\@function,5
.align 16
${PREFIX}_ecb_encrypt:
aesni_ecb_encrypt:
___
$code.=<<___ if ($win64);
lea -0x58(%rsp),%rsp
@@ -943,7 +943,7 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
ret
.size ${PREFIX}_ecb_encrypt,.-${PREFIX}_ecb_encrypt
.size aesni_ecb_encrypt,.-aesni_ecb_encrypt
___
{
@@ -964,10 +964,10 @@ my $iv="%xmm6";
my $bswap_mask="%xmm7";

$code.=<<___;
.globl ${PREFIX}_ccm64_encrypt_blocks
.type ${PREFIX}_ccm64_encrypt_blocks,\@function,6
.globl aesni_ccm64_encrypt_blocks
.type aesni_ccm64_encrypt_blocks,\@function,6
.align 16
${PREFIX}_ccm64_encrypt_blocks:
aesni_ccm64_encrypt_blocks:
___
$code.=<<___ if ($win64);
lea -0x58(%rsp),%rsp
@@ -1050,14 +1050,14 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
ret
.size ${PREFIX}_ccm64_encrypt_blocks,.-${PREFIX}_ccm64_encrypt_blocks
.size aesni_ccm64_encrypt_blocks,.-aesni_ccm64_encrypt_blocks
___
######################################################################
$code.=<<___;
.globl ${PREFIX}_ccm64_decrypt_blocks
.type ${PREFIX}_ccm64_decrypt_blocks,\@function,6
.globl aesni_ccm64_decrypt_blocks
.type aesni_ccm64_decrypt_blocks,\@function,6
.align 16
${PREFIX}_ccm64_decrypt_blocks:
aesni_ccm64_decrypt_blocks:
___
$code.=<<___ if ($win64);
lea -0x58(%rsp),%rsp
@@ -1157,7 +1157,7 @@ $code.=<<___ if ($win64);
___
$code.=<<___;
ret
.size ${PREFIX}_ccm64_decrypt_blocks,.-${PREFIX}_ccm64_decrypt_blocks
.size aesni_ccm64_decrypt_blocks,.-aesni_ccm64_decrypt_blocks
___
}
######################################################################
@@ -1178,10 +1178,10 @@ my ($key0,$ctr)=("%ebp","${ivp}d");
my $frame_size = 0x80 + ($win64?160:0);

$code.=<<___;
.globl ${PREFIX}_ctr32_encrypt_blocks
.type ${PREFIX}_ctr32_encrypt_blocks,\@function,5
.globl aesni_ctr32_encrypt_blocks
.type aesni_ctr32_encrypt_blocks,\@function,5
.align 16
${PREFIX}_ctr32_encrypt_blocks:
aesni_ctr32_encrypt_blocks:
.cfi_startproc
cmp \$1,$len
jne .Lctr32_bulk
@@ -1734,7 +1734,7 @@ $code.=<<___;
.Lctr32_epilogue:
ret
.cfi_endproc
.size ${PREFIX}_ctr32_encrypt_blocks,.-${PREFIX}_ctr32_encrypt_blocks
.size aesni_ctr32_encrypt_blocks,.-aesni_ctr32_encrypt_blocks
___
}
@@ -1751,10 +1751,10 @@ my $frame_size = 0x70 + ($win64?160:0);
my $key_ = "%rbp"; # override so that we can use %r11 as FP

$code.=<<___;
.globl ${PREFIX}_xts_encrypt
.type ${PREFIX}_xts_encrypt,\@function,6
.globl aesni_xts_encrypt
.type aesni_xts_encrypt,\@function,6
.align 16
${PREFIX}_xts_encrypt:
aesni_xts_encrypt:
.cfi_startproc
lea (%rsp),%r11 # frame pointer
.cfi_def_cfa_register %r11
@@ -2230,14 +2230,14 @@ $code.=<<___;
.Lxts_enc_epilogue:
ret
.cfi_endproc
.size ${PREFIX}_xts_encrypt,.-${PREFIX}_xts_encrypt
.size aesni_xts_encrypt,.-aesni_xts_encrypt
___

$code.=<<___;
.globl ${PREFIX}_xts_decrypt
.type ${PREFIX}_xts_decrypt,\@function,6
.globl aesni_xts_decrypt
.type aesni_xts_decrypt,\@function,6
.align 16
${PREFIX}_xts_decrypt:
aesni_xts_decrypt:
.cfi_startproc
lea (%rsp),%r11 # frame pointer
.cfi_def_cfa_register %r11
@@ -2739,7 +2739,7 @@ $code.=<<___;
.Lxts_dec_epilogue:
ret
.cfi_endproc
.size ${PREFIX}_xts_decrypt,.-${PREFIX}_xts_decrypt
.size aesni_xts_decrypt,.-aesni_xts_decrypt
___
}
@@ -2759,10 +2759,10 @@ my $seventh_arg = $win64 ? 56 : 8;
my $blocks = $len;

$code.=<<___;
.globl ${PREFIX}_ocb_encrypt
.type ${PREFIX}_ocb_encrypt,\@function,6
.globl aesni_ocb_encrypt
.type aesni_ocb_encrypt,\@function,6
.align 32
${PREFIX}_ocb_encrypt:
aesni_ocb_encrypt:
.cfi_startproc
lea (%rsp),%rax
push %rbx
@@ -3011,7 +3011,7 @@ $code.=<<___;
.Locb_enc_epilogue:
ret
.cfi_endproc
.size ${PREFIX}_ocb_encrypt,.-${PREFIX}_ocb_encrypt
.size aesni_ocb_encrypt,.-aesni_ocb_encrypt

.type __ocb_encrypt6,\@abi-omnipotent
.align 32
@@ -3219,10 +3219,10 @@ __ocb_encrypt1:
ret
.size __ocb_encrypt1,.-__ocb_encrypt1

.globl ${PREFIX}_ocb_decrypt
.type ${PREFIX}_ocb_decrypt,\@function,6
.globl aesni_ocb_decrypt
.type aesni_ocb_decrypt,\@function,6
.align 32
${PREFIX}_ocb_decrypt:
aesni_ocb_decrypt:
.cfi_startproc
lea (%rsp),%rax
push %rbx
@@ -3493,7 +3493,7 @@ $code.=<<___;
.Locb_dec_epilogue:
ret
.cfi_endproc
.size ${PREFIX}_ocb_decrypt,.-${PREFIX}_ocb_decrypt
.size aesni_ocb_decrypt,.-aesni_ocb_decrypt

.type __ocb_decrypt6,\@abi-omnipotent
.align 32
@@ -4736,7 +4736,7 @@ $disp="%r9";
$code.=<<___;
.extern __imp_RtlVirtualUnwind
___
$code.=<<___ if ($PREFIX eq "aes_hw");
$code.=<<___ if ($PREFIX eq "aesni");
.type ecb_ccm64_se_handler,\@abi-omnipotent
.align 16
ecb_ccm64_se_handler:
@@ -4776,7 +4776,7 @@ ecb_ccm64_se_handler:
lea 0x58(%rax),%rax # adjust stack pointer

jmp .Lcommon_seh_tail
.size ${PREFIX}_ccm64_se_handler,.-${PREFIX}_ccm64_se_handler
.size ecb_ccm64_se_handler,.-ecb_ccm64_se_handler

.type ctr_xts_se_handler,\@abi-omnipotent
.align 16
@@ -4968,37 +4968,37 @@ cbc_se_handler:
.section .pdata
.align 4
___
$code.=<<___ if ($PREFIX eq "aes_hw");
.rva .LSEH_begin_${PREFIX}_ecb_encrypt
.rva .LSEH_end_${PREFIX}_ecb_encrypt
$code.=<<___ if ($PREFIX eq "aesni");
.rva .LSEH_begin_aesni_ecb_encrypt
.rva .LSEH_end_aesni_ecb_encrypt
.rva .LSEH_info_ecb

.rva .LSEH_begin_${PREFIX}_ccm64_encrypt_blocks
.rva .LSEH_end_${PREFIX}_ccm64_encrypt_blocks
.rva .LSEH_begin_aesni_ccm64_encrypt_blocks
.rva .LSEH_end_aesni_ccm64_encrypt_blocks
.rva .LSEH_info_ccm64_enc

.rva .LSEH_begin_${PREFIX}_ccm64_decrypt_blocks
.rva .LSEH_end_${PREFIX}_ccm64_decrypt_blocks
.rva .LSEH_begin_aesni_ccm64_decrypt_blocks
.rva .LSEH_end_aesni_ccm64_decrypt_blocks
.rva .LSEH_info_ccm64_dec

.rva .LSEH_begin_${PREFIX}_ctr32_encrypt_blocks
.rva .LSEH_end_${PREFIX}_ctr32_encrypt_blocks
.rva .LSEH_begin_aesni_ctr32_encrypt_blocks
.rva .LSEH_end_aesni_ctr32_encrypt_blocks
.rva .LSEH_info_ctr32

.rva .LSEH_begin_${PREFIX}_xts_encrypt
.rva .LSEH_end_${PREFIX}_xts_encrypt
.rva .LSEH_begin_aesni_xts_encrypt
.rva .LSEH_end_aesni_xts_encrypt
.rva .LSEH_info_xts_enc

.rva .LSEH_begin_${PREFIX}_xts_decrypt
.rva .LSEH_end_${PREFIX}_xts_decrypt
.rva .LSEH_begin_aesni_xts_decrypt
.rva .LSEH_end_aesni_xts_decrypt
.rva .LSEH_info_xts_dec

.rva .LSEH_begin_${PREFIX}_ocb_encrypt
.rva .LSEH_end_${PREFIX}_ocb_encrypt
.rva .LSEH_begin_aesni_ocb_encrypt
.rva .LSEH_end_aesni_ocb_encrypt
.rva .LSEH_info_ocb_enc

.rva .LSEH_begin_${PREFIX}_ocb_decrypt
.rva .LSEH_end_${PREFIX}_ocb_decrypt
.rva .LSEH_begin_aesni_ocb_decrypt
.rva .LSEH_end_aesni_ocb_decrypt
.rva .LSEH_info_ocb_dec
___
$code.=<<___;
@@ -5016,7 +5016,7 @@ $code.=<<___;
.section .xdata
.align 8
___
$code.=<<___ if ($PREFIX eq "aes_hw");
$code.=<<___ if ($PREFIX eq "aesni");
.LSEH_info_ecb:
.byte 9,0,0,0
.rva ecb_ccm64_se_handler


+ 5
- 20
crypto/fipsmodule/aes/internal.h ファイルの表示

@@ -24,30 +24,21 @@ extern "C" {
#endif


#if !defined(OPENSSL_NO_ASM)

#if defined(OPENSSL_X86_64)
#define HWAES
#define HWAES_ECB

static int hwaes_capable(void) {
return (OPENSSL_ia32cap_P[1] & (1 << (57 - 32))) != 0;
}
#elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64)
#if !defined(OPENSSL_NO_ASM) && (defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64))
#define HWAES

static int hwaes_capable(void) {
return CRYPTO_is_ARMv8_AES_capable();
}
#elif defined(OPENSSL_PPC64LE)
#endif // !NO_ASM && (AES || AARCH64)

#if !defined(OPENSSL_NO_ASM) && defined(OPENSSL_PPC64LE)
#define HWAES

static int hwaes_capable(void) {
return CRYPTO_is_PPC64LE_vcrypto_capable();
}
#endif

#endif // !NO_ASM
#endif // !NO_ASM && PPC64LE


#if defined(HWAES)
@@ -102,12 +93,6 @@ static void aes_hw_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out,

#endif // !HWAES


#if defined(HWAES_ECB)
void aes_hw_ecb_encrypt(const uint8_t *in, uint8_t *out, size_t length,
const AES_KEY *key, const int enc);
#endif

#if defined(__cplusplus)
} // extern C
#endif


+ 13
- 18
crypto/fipsmodule/aes/mode_wrappers.c ファイルの表示

@@ -6,7 +6,7 @@
* are met:
*
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* notice, this list of conditions and the following disclaimer.
*
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in
@@ -50,7 +50,6 @@

#include <assert.h>

#include "../aes/internal.h"
#include "../modes/internal.h"


@@ -73,31 +72,27 @@ void AES_ecb_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key,
}
}

#if !defined(OPENSSL_NO_ASM) && (defined(OPENSSL_X86_64) || defined(OPENSSL_X86))
void asm_AES_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t len,
const AES_KEY *key, uint8_t *ivec, const int enc);
#endif

#if defined(OPENSSL_NO_ASM) || \
(!defined(OPENSSL_X86_64) && !defined(OPENSSL_X86))
void AES_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t len,
const AES_KEY *key, uint8_t *ivec, const int enc) {
#if defined(HWAES)
if (hwaes_capable()) {
aes_hw_cbc_encrypt(in, out, len, key, ivec, enc);
return;
}
#endif

#if !defined(OPENSSL_NO_ASM) && \
(defined(OPENSSL_X86_64) || defined(OPENSSL_X86))
asm_AES_cbc_encrypt(in, out, len, key, ivec, enc);
#else
if (enc) {
CRYPTO_cbc128_encrypt(in, out, len, key, ivec, (block128_f)AES_encrypt);
} else {
CRYPTO_cbc128_decrypt(in, out, len, key, ivec, (block128_f)AES_decrypt);
}
#endif
}
#else

void asm_AES_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t len,
const AES_KEY *key, uint8_t *ivec, const int enc);
void AES_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t len,
const AES_KEY *key, uint8_t *ivec, const int enc) {
asm_AES_cbc_encrypt(in, out, len, key, ivec, enc);
}

#endif // OPENSSL_NO_ASM || (!OPENSSL_X86_64 && !OPENSSL_X86)

void AES_ofb128_encrypt(const uint8_t *in, uint8_t *out, size_t length,
const AES_KEY *key, uint8_t *ivec, int *num) {


+ 308
- 30
crypto/fipsmodule/cipher/e_aes.c ファイルの表示

@@ -189,6 +189,38 @@ static void vpaes_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length,
}
#endif

#if !defined(OPENSSL_NO_ASM) && \
(defined(OPENSSL_X86_64) || defined(OPENSSL_X86))
int aesni_set_encrypt_key(const uint8_t *userKey, int bits, AES_KEY *key);
int aesni_set_decrypt_key(const uint8_t *userKey, int bits, AES_KEY *key);

void aesni_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key);
void aesni_decrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key);

void aesni_ecb_encrypt(const uint8_t *in, uint8_t *out, size_t length,
const AES_KEY *key, int enc);
void aesni_cbc_encrypt(const uint8_t *in, uint8_t *out, size_t length,
const AES_KEY *key, uint8_t *ivec, int enc);

#else

// On other platforms, aesni_capable() will always return false and so the
// following will never be called.
static void aesni_encrypt(const uint8_t *in, uint8_t *out, const AES_KEY *key) {
abort();
}
static int aesni_set_encrypt_key(const uint8_t *userKey, int bits,
AES_KEY *key) {
abort();
}
static void aesni_ctr32_encrypt_blocks(const uint8_t *in, uint8_t *out,
size_t blocks, const void *key,
const uint8_t *ivec) {
abort();
}

#endif

static int aes_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key,
const uint8_t *iv, int enc) {
int ret, mode;
@@ -305,9 +337,22 @@ static int aes_ofb_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out, const uint8_t *in,
return 1;
}

static char aesni_capable(void);

ctr128_f aes_ctr_set_key(AES_KEY *aes_key, GCM128_CONTEXT *gcm_ctx,
block128_f *out_block, const uint8_t *key,
size_t key_bytes) {
if (aesni_capable()) {
aesni_set_encrypt_key(key, key_bytes * 8, aes_key);
if (gcm_ctx != NULL) {
CRYPTO_gcm128_init(gcm_ctx, aes_key, (block128_f)aesni_encrypt, 1);
}
if (out_block) {
*out_block = (block128_f) aesni_encrypt;
}
return (ctr128_f)aesni_ctr32_encrypt_blocks;
}

if (hwaes_capable()) {
aes_hw_set_encrypt_key(key, key_bytes * 8, aes_key);
if (gcm_ctx != NULL) {
@@ -766,22 +811,126 @@ DEFINE_LOCAL_DATA(EVP_CIPHER, aes_256_gcm_generic) {
out->ctrl = aes_gcm_ctrl;
}

#if defined(HWAES_ECB)
#if !defined(OPENSSL_NO_ASM) && \
(defined(OPENSSL_X86_64) || defined(OPENSSL_X86))

// AES-NI section.

static char aesni_capable(void) {
return (OPENSSL_ia32cap_P[1] & (1 << (57 - 32))) != 0;
}

static int aesni_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key,
const uint8_t *iv, int enc) {
int ret, mode;
EVP_AES_KEY *dat = (EVP_AES_KEY *)ctx->cipher_data;

mode = ctx->cipher->flags & EVP_CIPH_MODE_MASK;
if ((mode == EVP_CIPH_ECB_MODE || mode == EVP_CIPH_CBC_MODE) && !enc) {
ret = aesni_set_decrypt_key(key, ctx->key_len * 8, ctx->cipher_data);
dat->block = (block128_f)aesni_decrypt;
dat->stream.cbc =
mode == EVP_CIPH_CBC_MODE ? (cbc128_f)aesni_cbc_encrypt : NULL;
} else {
ret = aesni_set_encrypt_key(key, ctx->key_len * 8, ctx->cipher_data);
dat->block = (block128_f)aesni_encrypt;
if (mode == EVP_CIPH_CBC_MODE) {
dat->stream.cbc = (cbc128_f)aesni_cbc_encrypt;
} else if (mode == EVP_CIPH_CTR_MODE) {
dat->stream.ctr = (ctr128_f)aesni_ctr32_encrypt_blocks;
} else {
dat->stream.cbc = NULL;
}
}

if (ret < 0) {
OPENSSL_PUT_ERROR(CIPHER, CIPHER_R_AES_KEY_SETUP_FAILED);
return 0;
}

return 1;
}

static int aesni_cbc_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out,
const uint8_t *in, size_t len) {
aesni_cbc_encrypt(in, out, len, ctx->cipher_data, ctx->iv, ctx->encrypt);

return 1;
}

static int aes_hw_ecb_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out,
const uint8_t *in, size_t len) {
static int aesni_ecb_cipher(EVP_CIPHER_CTX *ctx, uint8_t *out,
const uint8_t *in, size_t len) {
size_t bl = ctx->cipher->block_size;

if (len < bl) {
return 1;
}

aes_hw_ecb_encrypt(in, out, len, ctx->cipher_data, ctx->encrypt);
aesni_ecb_encrypt(in, out, len, ctx->cipher_data, ctx->encrypt);

return 1;
}

static int aesni_gcm_init_key(EVP_CIPHER_CTX *ctx, const uint8_t *key,
const uint8_t *iv, int enc) {
EVP_AES_GCM_CTX *gctx = ctx->cipher_data;
if (!iv && !key) {
return 1;
}
if (key) {
aesni_set_encrypt_key(key, ctx->key_len * 8, &gctx->ks.ks);
CRYPTO_gcm128_init(&gctx->gcm, &gctx->ks, (block128_f)aesni_encrypt, 1);
gctx->ctr = (ctr128_f)aesni_ctr32_encrypt_blocks;
// If we have an iv can set it directly, otherwise use
// saved IV.
if (iv == NULL && gctx->iv_set) {
iv = gctx->iv;
}
if (iv) {
CRYPTO_gcm128_setiv(&gctx->gcm, &gctx->ks.ks, iv, gctx->ivlen);
gctx->iv_set = 1;
}
gctx->key_set = 1;
} else {
// If key set use IV, otherwise copy
if (gctx->key_set) {
CRYPTO_gcm128_setiv(&gctx->gcm, &gctx->ks.ks, iv, gctx->ivlen);
} else {
OPENSSL_memcpy(gctx->iv, iv, gctx->ivlen);
}
gctx->iv_set = 1;
gctx->iv_gen = 0;
}
return 1;
}

DEFINE_LOCAL_DATA(EVP_CIPHER, aes_hw_128_ecb) {
DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_cbc) {
memset(out, 0, sizeof(EVP_CIPHER));

out->nid = NID_aes_128_cbc;
out->block_size = 16;
out->key_len = 16;
out->iv_len = 16;
out->ctx_size = sizeof(EVP_AES_KEY);
out->flags = EVP_CIPH_CBC_MODE;
out->init = aesni_init_key;
out->cipher = aesni_cbc_cipher;
}

DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_ctr) {
memset(out, 0, sizeof(EVP_CIPHER));

out->nid = NID_aes_128_ctr;
out->block_size = 1;
out->key_len = 16;
out->iv_len = 16;
out->ctx_size = sizeof(EVP_AES_KEY);
out->flags = EVP_CIPH_CTR_MODE;
out->init = aesni_init_key;
out->cipher = aes_ctr_cipher;
}

DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_ecb) {
memset(out, 0, sizeof(EVP_CIPHER));

out->nid = NID_aes_128_ecb;
@@ -789,11 +938,67 @@ DEFINE_LOCAL_DATA(EVP_CIPHER, aes_hw_128_ecb) {
out->key_len = 16;
out->ctx_size = sizeof(EVP_AES_KEY);
out->flags = EVP_CIPH_ECB_MODE;
out->init = aes_init_key;
out->cipher = aes_hw_ecb_cipher;
out->init = aesni_init_key;
out->cipher = aesni_ecb_cipher;
}

DEFINE_LOCAL_DATA(EVP_CIPHER, aes_hw_192_ecb) {
DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_ofb) {
memset(out, 0, sizeof(EVP_CIPHER));

out->nid = NID_aes_128_ofb128;
out->block_size = 1;
out->key_len = 16;
out->iv_len = 16;
out->ctx_size = sizeof(EVP_AES_KEY);
out->flags = EVP_CIPH_OFB_MODE;
out->init = aesni_init_key;
out->cipher = aes_ofb_cipher;
}

DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_128_gcm) {
memset(out, 0, sizeof(EVP_CIPHER));

out->nid = NID_aes_128_gcm;
out->block_size = 1;
out->key_len = 16;
out->iv_len = 12;
out->ctx_size = sizeof(EVP_AES_GCM_CTX);
out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV |
EVP_CIPH_FLAG_CUSTOM_CIPHER | EVP_CIPH_ALWAYS_CALL_INIT |
EVP_CIPH_CTRL_INIT | EVP_CIPH_FLAG_AEAD_CIPHER;
out->init = aesni_gcm_init_key;
out->cipher = aes_gcm_cipher;
out->cleanup = aes_gcm_cleanup;
out->ctrl = aes_gcm_ctrl;
}

DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_192_cbc) {
memset(out, 0, sizeof(EVP_CIPHER));

out->nid = NID_aes_192_cbc;
out->block_size = 16;
out->key_len = 24;
out->iv_len = 16;
out->ctx_size = sizeof(EVP_AES_KEY);
out->flags = EVP_CIPH_CBC_MODE;
out->init = aesni_init_key;
out->cipher = aesni_cbc_cipher;
}

DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_192_ctr) {
memset(out, 0, sizeof(EVP_CIPHER));

out->nid = NID_aes_192_ctr;
out->block_size = 1;
out->key_len = 24;
out->iv_len = 16;
out->ctx_size = sizeof(EVP_AES_KEY);
out->flags = EVP_CIPH_CTR_MODE;
out->init = aesni_init_key;
out->cipher = aes_ctr_cipher;
}

DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_192_ecb) {
memset(out, 0, sizeof(EVP_CIPHER));

out->nid = NID_aes_192_ecb;
@@ -801,11 +1006,54 @@ DEFINE_LOCAL_DATA(EVP_CIPHER, aes_hw_192_ecb) {
out->key_len = 24;
out->ctx_size = sizeof(EVP_AES_KEY);
out->flags = EVP_CIPH_ECB_MODE;
out->init = aes_init_key;
out->cipher = aes_hw_ecb_cipher;
out->init = aesni_init_key;
out->cipher = aesni_ecb_cipher;
}

DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_192_gcm) {
memset(out, 0, sizeof(EVP_CIPHER));

out->nid = NID_aes_192_gcm;
out->block_size = 1;
out->key_len = 24;
out->iv_len = 12;
out->ctx_size = sizeof(EVP_AES_GCM_CTX);
out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV |
EVP_CIPH_FLAG_CUSTOM_CIPHER | EVP_CIPH_ALWAYS_CALL_INIT |
EVP_CIPH_CTRL_INIT | EVP_CIPH_FLAG_AEAD_CIPHER;
out->init = aesni_gcm_init_key;
out->cipher = aes_gcm_cipher;
out->cleanup = aes_gcm_cleanup;
out->ctrl = aes_gcm_ctrl;
}

DEFINE_LOCAL_DATA(EVP_CIPHER, aes_hw_256_ecb) {
DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_cbc) {
memset(out, 0, sizeof(EVP_CIPHER));

out->nid = NID_aes_256_cbc;
out->block_size = 16;
out->key_len = 32;
out->iv_len = 16;
out->ctx_size = sizeof(EVP_AES_KEY);
out->flags = EVP_CIPH_CBC_MODE;
out->init = aesni_init_key;
out->cipher = aesni_cbc_cipher;
}

DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_ctr) {
memset(out, 0, sizeof(EVP_CIPHER));

out->nid = NID_aes_256_ctr;
out->block_size = 1;
out->key_len = 32;
out->iv_len = 16;
out->ctx_size = sizeof(EVP_AES_KEY);
out->flags = EVP_CIPH_CTR_MODE;
out->init = aesni_init_key;
out->cipher = aes_ctr_cipher;
}

DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_ecb) {
memset(out, 0, sizeof(EVP_CIPHER));

out->nid = NID_aes_256_ecb;
@@ -813,50 +1061,80 @@ DEFINE_LOCAL_DATA(EVP_CIPHER, aes_hw_256_ecb) {
out->key_len = 32;
out->ctx_size = sizeof(EVP_AES_KEY);
out->flags = EVP_CIPH_ECB_MODE;
out->init = aes_init_key;
out->cipher = aes_hw_ecb_cipher;
out->init = aesni_init_key;
out->cipher = aesni_ecb_cipher;
}

#define EVP_ECB_CIPHER_FUNCTION(keybits) \
const EVP_CIPHER *EVP_aes_##keybits##_ecb(void) { \
if (hwaes_capable()) { \
return aes_hw_##keybits##_ecb(); \
} \
return aes_##keybits##_ecb_generic(); \
}
DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_ofb) {
memset(out, 0, sizeof(EVP_CIPHER));

#else
out->nid = NID_aes_256_ofb128;
out->block_size = 1;
out->key_len = 32;
out->iv_len = 16;
out->ctx_size = sizeof(EVP_AES_KEY);
out->flags = EVP_CIPH_OFB_MODE;
out->init = aesni_init_key;
out->cipher = aes_ofb_cipher;
}

DEFINE_LOCAL_DATA(EVP_CIPHER, aesni_256_gcm) {
memset(out, 0, sizeof(EVP_CIPHER));

#define EVP_ECB_CIPHER_FUNCTION(keybits) \
const EVP_CIPHER *EVP_aes_##keybits##_ecb(void) { \
return aes_##keybits##_ecb_generic(); \
out->nid = NID_aes_256_gcm;
out->block_size = 1;
out->key_len = 32;
out->iv_len = 12;
out->ctx_size = sizeof(EVP_AES_GCM_CTX);
out->flags = EVP_CIPH_GCM_MODE | EVP_CIPH_CUSTOM_IV |
EVP_CIPH_FLAG_CUSTOM_CIPHER | EVP_CIPH_ALWAYS_CALL_INIT |
EVP_CIPH_CTRL_INIT | EVP_CIPH_CUSTOM_COPY |
EVP_CIPH_FLAG_AEAD_CIPHER;
out->init = aesni_gcm_init_key;
out->cipher = aes_gcm_cipher;
out->cleanup = aes_gcm_cleanup;
out->ctrl = aes_gcm_ctrl;
}

#define EVP_CIPHER_FUNCTION(keybits, mode) \
const EVP_CIPHER *EVP_aes_##keybits##_##mode(void) { \
if (aesni_capable()) { \
return aesni_##keybits##_##mode(); \
} else { \
return aes_##keybits##_##mode##_generic(); \
} \
}

#endif // HWAES_ECB
#else // ^^^ OPENSSL_X86_64 || OPENSSL_X86

static char aesni_capable(void) {
return 0;
}

#define EVP_CIPHER_FUNCTION(keybits, mode) \
const EVP_CIPHER *EVP_aes_##keybits##_##mode(void) { \
return aes_##keybits##_##mode##_generic(); \
}

#endif

EVP_CIPHER_FUNCTION(128, cbc)
EVP_CIPHER_FUNCTION(128, ctr)
EVP_CIPHER_FUNCTION(128, ecb)
EVP_CIPHER_FUNCTION(128, ofb)
EVP_CIPHER_FUNCTION(128, gcm)

EVP_CIPHER_FUNCTION(192, cbc)
EVP_CIPHER_FUNCTION(192, ctr)
EVP_CIPHER_FUNCTION(192, ecb)
EVP_CIPHER_FUNCTION(192, gcm)

EVP_CIPHER_FUNCTION(256, cbc)
EVP_CIPHER_FUNCTION(256, ctr)
EVP_CIPHER_FUNCTION(256, ecb)
EVP_CIPHER_FUNCTION(256, ofb)
EVP_CIPHER_FUNCTION(256, gcm)

EVP_ECB_CIPHER_FUNCTION(128)
EVP_ECB_CIPHER_FUNCTION(192)
EVP_ECB_CIPHER_FUNCTION(256)


#define EVP_AEAD_AES_GCM_TAG_LEN 16

@@ -1151,7 +1429,7 @@ DEFINE_METHOD_FUNCTION(EVP_AEAD, EVP_aead_aes_256_gcm_tls12) {

int EVP_has_aes_hardware(void) {
#if defined(OPENSSL_X86) || defined(OPENSSL_X86_64)
return hwaes_capable() && crypto_gcm_clmul_enabled();
return aesni_capable() && crypto_gcm_clmul_enabled();
#elif defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64)
return hwaes_capable() && CRYPTO_is_ARMv8_PMULL_capable();
#else


書き込み プレビュー
読み込み中…
キャンセル
保存