Import chacha-x86.pl fix.
Patch from https://mta.openssl.org/pipermail/openssl-dev/2016-March/005625.html. Upstream has yet to make a decision on aliasing requirements for their assembly. If they choose to go with the stricter aliasing requirement rather than land this patch, we'll probably want to tweak EVP_AEAD's API guarantees accordingly and then undiverge. In the meantime, import this to avoid a regression on x86 from when we had compiler-vectorized code on GCC platforms. Per our assembly coverage tools and pending multi-CPU-variant tests, we have good coverage here. Unlike Poly1305 (which is currently waiting on yet another upstream bugfix), where there is risk of missed carries everywhere, it is much more difficult to accidentally make a ChaCha20 implementation that fails based on the data passed into it. This restores a sizeable speed improvement on x86. Before: Did 1131000 ChaCha20-Poly1305 (16 bytes) seal operations in 1000205us (1130768.2 ops/sec): 18.1 MB/s Did 161000 ChaCha20-Poly1305 (1350 bytes) seal operations in 1006136us (160018.1 ops/sec): 216.0 MB/s Did 28000 ChaCha20-Poly1305 (8192 bytes) seal operations in 1023264us (27363.4 ops/sec): 224.2 MB/s Did 1166000 ChaCha20-Poly1305-Old (16 bytes) seal operations in 1000447us (1165479.0 ops/sec): 18.6 MB/s Did 160000 ChaCha20-Poly1305-Old (1350 bytes) seal operations in 1004818us (159232.8 ops/sec): 215.0 MB/s Did 30000 ChaCha20-Poly1305-Old (8192 bytes) seal operations in 1016977us (29499.2 ops/sec): 241.7 MB/s After: Did 2208000 ChaCha20-Poly1305 (16 bytes) seal operations in 1000031us (2207931.6 ops/sec): 35.3 MB/s Did 402000 ChaCha20-Poly1305 (1350 bytes) seal operations in 1001717us (401310.9 ops/sec): 541.8 MB/s Did 97000 ChaCha20-Poly1305 (8192 bytes) seal operations in 1005394us (96479.6 ops/sec): 790.4 MB/s Did 2444000 ChaCha20-Poly1305-Old (16 bytes) seal operations in 1000089us (2443782.5 ops/sec): 39.1 MB/s Did 459000 ChaCha20-Poly1305-Old (1350 bytes) seal operations in 1000563us (458741.7 ops/sec): 619.3 MB/s Did 97000 ChaCha20-Poly1305-Old (8192 bytes) seal operations in 1007942us (96235.7 ops/sec): 788.4 MB/s Change-Id: I976da606dae062a776e0cc01229ec03a074035d1 Reviewed-on: https://boringssl-review.googlesource.com/7561 Reviewed-by: Steven Valdez <svaldez@google.com> Reviewed-by: David Benjamin <davidben@google.com>
This commit is contained in:
David Benjamin
parent
17d729e61b
commit
762e1d039c
@ -19,13 +19,13 @@
|
||||
# P4 18.6/+84%
|
||||
# Core2 9.56/+89% 4.83
|
||||
# Westmere 9.50/+45% 3.35
|
||||
# Sandy Bridge 10.5/+47% 3.20
|
||||
# Haswell 8.15/+50% 2.83
|
||||
# Silvermont 17.4/+36% 8.35
|
||||
# Sandy Bridge 10.7/+47% 3.24
|
||||
# Haswell 8.22/+50% 2.89
|
||||
# Silvermont 17.8/+36% 8.53
|
||||
# Sledgehammer 10.2/+54%
|
||||
# Bulldozer 13.4/+50% 4.38(*)
|
||||
# Bulldozer 13.5/+50% 4.39(*)
|
||||
#
|
||||
# (*) Bulldozer actually executes 4xXOP code path that delivers 3.55;
|
||||
# (*) Bulldozer actually executes 4xXOP code path that delivers 3.50;
|
||||
#
|
||||
# Modified from upstream OpenSSL to remove the XOP code.
|
||||
|
||||
@ -224,18 +224,20 @@ if ($xmm) {
|
||||
|
||||
&xor ($a, &DWP(4*0,$b)); # xor with input
|
||||
&xor ($b_,&DWP(4*4,$b));
|
||||
&mov (&DWP(4*0,"esp"),$a);
|
||||
&mov (&DWP(4*0,"esp"),$a); # off-load for later write
|
||||
&mov ($a,&wparam(0)); # load output pointer
|
||||
&xor ($c, &DWP(4*8,$b));
|
||||
&xor ($c_,&DWP(4*9,$b));
|
||||
&xor ($d, &DWP(4*12,$b));
|
||||
&xor ($d_,&DWP(4*14,$b));
|
||||
&mov (&DWP(4*4,$a),$b_); # write output
|
||||
&mov (&DWP(4*8,$a),$c);
|
||||
&mov (&DWP(4*9,$a),$c_);
|
||||
&mov (&DWP(4*12,$a),$d);
|
||||
&mov (&DWP(4*14,$a),$d_);
|
||||
&mov (&DWP(4*4,"esp"),$b_);
|
||||
&mov ($b_,&DWP(4*0,"esp"));
|
||||
&mov (&DWP(4*8,"esp"),$c);
|
||||
&mov (&DWP(4*9,"esp"),$c_);
|
||||
&mov (&DWP(4*12,"esp"),$d);
|
||||
&mov (&DWP(4*14,"esp"),$d_);
|
||||
|
||||
&mov (&DWP(4*0,$a),$b_); # write output in order
|
||||
&mov ($b_,&DWP(4*1,"esp"));
|
||||
&mov ($c, &DWP(4*2,"esp"));
|
||||
&mov ($c_,&DWP(4*3,"esp"));
|
||||
@ -252,35 +254,45 @@ if ($xmm) {
|
||||
&xor ($d, &DWP(4*5,$b));
|
||||
&xor ($d_,&DWP(4*6,$b));
|
||||
&mov (&DWP(4*1,$a),$b_);
|
||||
&mov ($b_,&DWP(4*4,"esp"));
|
||||
&mov (&DWP(4*2,$a),$c);
|
||||
&mov (&DWP(4*3,$a),$c_);
|
||||
&mov (&DWP(4*4,$a),$b_);
|
||||
&mov (&DWP(4*5,$a),$d);
|
||||
&mov (&DWP(4*6,$a),$d_);
|
||||
|
||||
&mov ($b_,&DWP(4*7,"esp"));
|
||||
&mov ($c, &DWP(4*10,"esp"));
|
||||
&mov ($c,&DWP(4*7,"esp"));
|
||||
&mov ($d,&DWP(4*8,"esp"));
|
||||
&mov ($d_,&DWP(4*9,"esp"));
|
||||
&add ($c,&DWP(64+4*7,"esp"));
|
||||
&mov ($b_, &DWP(4*10,"esp"));
|
||||
&xor ($c,&DWP(4*7,$b));
|
||||
&mov ($c_,&DWP(4*11,"esp"));
|
||||
&mov (&DWP(4*7,$a),$c);
|
||||
&mov (&DWP(4*8,$a),$d);
|
||||
&mov (&DWP(4*9,$a),$d_);
|
||||
|
||||
&add ($b_, &DWP(64+4*10,"esp"));
|
||||
&add ($c_,&DWP(64+4*11,"esp"));
|
||||
&xor ($b_, &DWP(4*10,$b));
|
||||
&xor ($c_,&DWP(4*11,$b));
|
||||
&mov (&DWP(4*10,$a),$b_);
|
||||
&mov (&DWP(4*11,$a),$c_);
|
||||
|
||||
&mov ($c,&DWP(4*12,"esp"));
|
||||
&mov ($c_,&DWP(4*14,"esp"));
|
||||
&mov ($d, &DWP(4*13,"esp"));
|
||||
&mov ($d_,&DWP(4*15,"esp"));
|
||||
&add ($b_,&DWP(64+4*7,"esp"));
|
||||
&add ($c, &DWP(64+4*10,"esp"));
|
||||
&add ($c_,&DWP(64+4*11,"esp"));
|
||||
&add ($d, &DWP(64+4*13,"esp"));
|
||||
&add ($d_,&DWP(64+4*15,"esp"));
|
||||
&xor ($b_,&DWP(4*7,$b));
|
||||
&xor ($c, &DWP(4*10,$b));
|
||||
&xor ($c_,&DWP(4*11,$b));
|
||||
&xor ($d, &DWP(4*13,$b));
|
||||
&xor ($d_,&DWP(4*15,$b));
|
||||
&lea ($b,&DWP(4*16,$b));
|
||||
&mov (&DWP(4*7,$a),$b_);
|
||||
&mov ($b_,&DWP(4*0,"esp"));
|
||||
&mov (&DWP(4*10,$a),$c);
|
||||
&mov (&DWP(4*12,$a),$c);
|
||||
&mov ($c,&wparam(2)); # len
|
||||
&mov (&DWP(4*11,$a),$c_);
|
||||
&mov (&DWP(4*13,$a),$d);
|
||||
&mov (&DWP(4*14,$a),$c_);
|
||||
&mov (&DWP(4*15,$a),$d_);
|
||||
&mov (&DWP(4*0,$a),$b_);
|
||||
&lea ($a,&DWP(4*16,$a));
|
||||
&sub ($c,64);
|
||||
&jnz (&label("outer_loop"));
|
||||
@ -555,12 +567,12 @@ my ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-1)&3),($ai,$bi,$ci,$di)); # previous
|
||||
|
||||
my ($xa0,$xa1,$xa2,$xa3,$xt0,$xt1,$xt2,$xt3)=map("xmm$_",(0..7));
|
||||
|
||||
#&movdqa ($xa0,&QWP(16*0-128,"ebx")); # it's there
|
||||
&movdqa ($xa1,&QWP(16*1-128,"ebx"));
|
||||
&movdqa ($xa2,&QWP(16*2-128,"ebx"));
|
||||
&movdqa ($xa3,&QWP(16*3-128,"ebx"));
|
||||
|
||||
for($i=0;$i<256;$i+=64) {
|
||||
#&movdqa ($xa0,&QWP($i+16*0-128,"ebx")); # it's there
|
||||
&movdqa ($xa1,&QWP($i+16*1-128,"ebx"));
|
||||
&movdqa ($xa2,&QWP($i+16*2-128,"ebx"));
|
||||
&movdqa ($xa3,&QWP($i+16*3-128,"ebx"));
|
||||
|
||||
&paddd ($xa0,&QWP($i+16*0-128,"ebp")); # accumulate key material
|
||||
&paddd ($xa1,&QWP($i+16*1-128,"ebp"));
|
||||
&paddd ($xa2,&QWP($i+16*2-128,"ebp"));
|
||||
@ -581,25 +593,29 @@ my ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-1)&3),($ai,$bi,$ci,$di)); # previous
|
||||
|
||||
#($xa2,$xt2)=($xt2,$xa2);
|
||||
|
||||
&movdqu ($xt0,&QWP(64*0-128,$inp)); # load input
|
||||
&movdqu ($xt1,&QWP(64*1-128,$inp));
|
||||
&movdqu ($xa2,&QWP(64*2-128,$inp));
|
||||
&movdqu ($xt3,&QWP(64*3-128,$inp));
|
||||
&lea ($inp,&QWP($i<192?16:(64*4-16*3),$inp));
|
||||
&pxor ($xt0,$xa0);
|
||||
&movdqa (&QWP($i+16*0-128,"ebx"),$xa0);
|
||||
&movdqa ($xa0,&QWP($i+16*4-128,"ebx")) if ($i<192);
|
||||
&pxor ($xt1,$xa1);
|
||||
&movdqa ($xa1,&QWP($i+16*5-128,"ebx")) if ($i<192);
|
||||
&pxor ($xt2,$xa2);
|
||||
&movdqa ($xa2,&QWP($i+16*6-128,"ebx")) if ($i<192);
|
||||
&pxor ($xt3,$xa3);
|
||||
&movdqa ($xa3,&QWP($i+16*7-128,"ebx")) if ($i<192);
|
||||
&movdqu (&QWP(64*0-128,$out),$xt0); # store output
|
||||
&movdqu (&QWP(64*1-128,$out),$xt1);
|
||||
&movdqu (&QWP(64*2-128,$out),$xt2);
|
||||
&movdqu (&QWP(64*3-128,$out),$xt3);
|
||||
&lea ($out,&QWP($i<192?16:(64*4-16*3),$out));
|
||||
&movdqa (&QWP($i+16*1-128,"ebx"),$xa1);
|
||||
&movdqa (&QWP($i+16*2-128,"ebx"),$xt2);
|
||||
&movdqa (&QWP($i+16*3-128,"ebx"),$xa3);
|
||||
}
|
||||
for($i=0;$i<256;$i+=64) {
|
||||
my $j = 16*($i/64);
|
||||
&movdqu ($xa0,&QWP($i+16*0-128,$inp)); # load input
|
||||
&movdqu ($xa1,&QWP($i+16*1-128,$inp));
|
||||
&movdqu ($xa2,&QWP($i+16*2-128,$inp));
|
||||
&movdqu ($xa3,&QWP($i+16*3-128,$inp));
|
||||
&pxor ($xa0,&QWP($j+64*0-128,"ebx"));
|
||||
&pxor ($xa1,&QWP($j+64*1-128,"ebx"));
|
||||
&pxor ($xa2,&QWP($j+64*2-128,"ebx"));
|
||||
&pxor ($xa3,&QWP($j+64*3-128,"ebx"));
|
||||
&movdqu (&QWP($i+16*0-128,$out),$xa0); # write output
|
||||
&movdqu (&QWP($i+16*1-128,$out),$xa1);
|
||||
&movdqu (&QWP($i+16*2-128,$out),$xa2);
|
||||
&movdqu (&QWP($i+16*3-128,$out),$xa3);
|
||||
}
|
||||
&lea ($inp,&DWP(256,$inp));
|
||||
&lea ($out,&DWP(256,$out));
|
||||
&sub ($len,64*4);
|
||||
&jnc (&label("outer_loop"));
|
||||
|
||||
|
||||
@ -25,11 +25,9 @@
|
||||
(((uint32_t)((p)[0])) | ((uint32_t)((p)[1]) << 8) | \
|
||||
((uint32_t)((p)[2]) << 16) | ((uint32_t)((p)[3]) << 24))
|
||||
|
||||
/* TODO(davidben): Re-enable the ChaCha20 assembly for OPENSSL_X86 once they
|
||||
* pass the in-place tests. */
|
||||
#if !defined(OPENSSL_NO_ASM) && \
|
||||
(defined(OPENSSL_X86_64) || defined(OPENSSL_ARM) || \
|
||||
defined(OPENSSL_AARCH64))
|
||||
(defined(OPENSSL_X86) || defined(OPENSSL_X86_64) || \
|
||||
defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64))
|
||||
|
||||
/* ChaCha20_ctr32 is defined in asm/chacha-*.pl. */
|
||||
void ChaCha20_ctr32(uint8_t *out, const uint8_t *in, size_t in_len,
|
||||
|
||||
Loading…
Reference in New Issue
Block a user