kris/boringssl
1
0
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Import chacha-x86.pl fix.

Browse Source 
Patch from https://mta.openssl.org/pipermail/openssl-dev/2016-March/005625.html.

Upstream has yet to make a decision on aliasing requirements for their
assembly. If they choose to go with the stricter aliasing requirement rather
than land this patch, we'll probably want to tweak EVP_AEAD's API guarantees
accordingly and then undiverge.

In the meantime, import this to avoid a regression on x86 from when we had
compiler-vectorized code on GCC platforms.  Per our assembly coverage tools and
pending multi-CPU-variant tests, we have good coverage here. Unlike Poly1305
(which is currently waiting on yet another upstream bugfix), where there is
risk of missed carries everywhere, it is much more difficult to accidentally
make a ChaCha20 implementation that fails based on the data passed into it.

This restores a sizeable speed improvement on x86.

Before:
Did 1131000 ChaCha20-Poly1305 (16 bytes) seal operations in 1000205us (1130768.2 ops/sec): 18.1 MB/s
Did 161000 ChaCha20-Poly1305 (1350 bytes) seal operations in 1006136us (160018.1 ops/sec): 216.0 MB/s
Did 28000 ChaCha20-Poly1305 (8192 bytes) seal operations in 1023264us (27363.4 ops/sec): 224.2 MB/s
Did 1166000 ChaCha20-Poly1305-Old (16 bytes) seal operations in 1000447us (1165479.0 ops/sec): 18.6 MB/s
Did 160000 ChaCha20-Poly1305-Old (1350 bytes) seal operations in 1004818us (159232.8 ops/sec): 215.0 MB/s
Did 30000 ChaCha20-Poly1305-Old (8192 bytes) seal operations in 1016977us (29499.2 ops/sec): 241.7 MB/s

After:
Did 2208000 ChaCha20-Poly1305 (16 bytes) seal operations in 1000031us (2207931.6 ops/sec): 35.3 MB/s
Did 402000 ChaCha20-Poly1305 (1350 bytes) seal operations in 1001717us (401310.9 ops/sec): 541.8 MB/s
Did 97000 ChaCha20-Poly1305 (8192 bytes) seal operations in 1005394us (96479.6 ops/sec): 790.4 MB/s
Did 2444000 ChaCha20-Poly1305-Old (16 bytes) seal operations in 1000089us (2443782.5 ops/sec): 39.1 MB/s
Did 459000 ChaCha20-Poly1305-Old (1350 bytes) seal operations in 1000563us (458741.7 ops/sec): 619.3 MB/s
Did 97000 ChaCha20-Poly1305-Old (8192 bytes) seal operations in 1007942us (96235.7 ops/sec): 788.4 MB/s

Change-Id: I976da606dae062a776e0cc01229ec03a074035d1
Reviewed-on: https://boringssl-review.googlesource.com/7561
Reviewed-by: Steven Valdez <svaldez@google.com>
Reviewed-by: David Benjamin <davidben@google.com>
This commit is contained in:
David Benjamin 2016-03-24 20:40:22 -04:00
parent 17d729e61b
commit 762e1d039c
2 changed files with 64 additions and 50 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

108
crypto/chacha/asm/chacha-x86.pl
View File

@ -19,13 +19,13 @@
# P4 18.6/+84% # P4 18.6/+84%
# Core2 9.56/+89% 4.83 # Core2 9.56/+89% 4.83
# Westmere 9.50/+45% 3.35 # Westmere 9.50/+45% 3.35
# Sandy Bridge 10.5/+47% 3.20 # Sandy Bridge 10.7/+47% 3.24
# Haswell 8.15/+50% 2.83 # Haswell 8.22/+50% 2.89
# Silvermont 17.4/+36% 8.35 # Silvermont 17.8/+36% 8.53
# Sledgehammer 10.2/+54% # Sledgehammer 10.2/+54%
# Bulldozer 13.4/+50% 4.38(*) # Bulldozer 13.5/+50% 4.39(*)
# #
# (*) Bulldozer actually executes 4xXOP code path that delivers 3.55; # (*) Bulldozer actually executes 4xXOP code path that delivers 3.50;
# #
# Modified from upstream OpenSSL to remove the XOP code. # Modified from upstream OpenSSL to remove the XOP code.
@ -224,18 +224,20 @@ if ($xmm) {
&xor ($a, &DWP(4*0,$b)); # xor with input &xor ($a, &DWP(4*0,$b)); # xor with input
&xor ($b_,&DWP(4*4,$b)); &xor ($b_,&DWP(4*4,$b));
&mov (&DWP(4*0,"esp"),$a); &mov (&DWP(4*0,"esp"),$a); # off-load for later write
&mov ($a,&wparam(0)); # load output pointer &mov ($a,&wparam(0)); # load output pointer
&xor ($c, &DWP(4*8,$b)); &xor ($c, &DWP(4*8,$b));
&xor ($c_,&DWP(4*9,$b)); &xor ($c_,&DWP(4*9,$b));
&xor ($d, &DWP(4*12,$b)); &xor ($d, &DWP(4*12,$b));
&xor ($d_,&DWP(4*14,$b)); &xor ($d_,&DWP(4*14,$b));
&mov (&DWP(4*4,$a),$b_); # write output &mov (&DWP(4*4,"esp"),$b_);
&mov (&DWP(4*8,$a),$c); &mov ($b_,&DWP(4*0,"esp"));
&mov (&DWP(4*9,$a),$c_); &mov (&DWP(4*8,"esp"),$c);
&mov (&DWP(4*12,$a),$d); &mov (&DWP(4*9,"esp"),$c_);
&mov (&DWP(4*14,$a),$d_); &mov (&DWP(4*12,"esp"),$d);
&mov (&DWP(4*14,"esp"),$d_);
&mov (&DWP(4*0,$a),$b_); # write output in order
&mov ($b_,&DWP(4*1,"esp")); &mov ($b_,&DWP(4*1,"esp"));
&mov ($c, &DWP(4*2,"esp")); &mov ($c, &DWP(4*2,"esp"));
&mov ($c_,&DWP(4*3,"esp")); &mov ($c_,&DWP(4*3,"esp"));
@ -252,35 +254,45 @@ if ($xmm) {
&xor ($d, &DWP(4*5,$b)); &xor ($d, &DWP(4*5,$b));
&xor ($d_,&DWP(4*6,$b)); &xor ($d_,&DWP(4*6,$b));
&mov (&DWP(4*1,$a),$b_); &mov (&DWP(4*1,$a),$b_);
&mov ($b_,&DWP(4*4,"esp"));
&mov (&DWP(4*2,$a),$c); &mov (&DWP(4*2,$a),$c);
&mov (&DWP(4*3,$a),$c_); &mov (&DWP(4*3,$a),$c_);
&mov (&DWP(4*4,$a),$b_);
&mov (&DWP(4*5,$a),$d); &mov (&DWP(4*5,$a),$d);
&mov (&DWP(4*6,$a),$d_); &mov (&DWP(4*6,$a),$d_);
&mov ($b_,&DWP(4*7,"esp")); &mov ($c,&DWP(4*7,"esp"));
&mov ($c, &DWP(4*10,"esp")); &mov ($d,&DWP(4*8,"esp"));
&mov ($d_,&DWP(4*9,"esp"));
&add ($c,&DWP(64+4*7,"esp"));
&mov ($b_, &DWP(4*10,"esp"));
&xor ($c,&DWP(4*7,$b));
&mov ($c_,&DWP(4*11,"esp")); &mov ($c_,&DWP(4*11,"esp"));
&mov (&DWP(4*7,$a),$c);
&mov (&DWP(4*8,$a),$d);
&mov (&DWP(4*9,$a),$d_);
&add ($b_, &DWP(64+4*10,"esp"));
&add ($c_,&DWP(64+4*11,"esp"));
&xor ($b_, &DWP(4*10,$b));
&xor ($c_,&DWP(4*11,$b));
&mov (&DWP(4*10,$a),$b_);
&mov (&DWP(4*11,$a),$c_);
&mov ($c,&DWP(4*12,"esp"));
&mov ($c_,&DWP(4*14,"esp"));
&mov ($d, &DWP(4*13,"esp")); &mov ($d, &DWP(4*13,"esp"));
&mov ($d_,&DWP(4*15,"esp")); &mov ($d_,&DWP(4*15,"esp"));
&add ($b_,&DWP(64+4*7,"esp"));
&add ($c, &DWP(64+4*10,"esp"));
&add ($c_,&DWP(64+4*11,"esp"));
&add ($d, &DWP(64+4*13,"esp")); &add ($d, &DWP(64+4*13,"esp"));
&add ($d_,&DWP(64+4*15,"esp")); &add ($d_,&DWP(64+4*15,"esp"));
&xor ($b_,&DWP(4*7,$b));
&xor ($c, &DWP(4*10,$b));
&xor ($c_,&DWP(4*11,$b));
&xor ($d, &DWP(4*13,$b)); &xor ($d, &DWP(4*13,$b));
&xor ($d_,&DWP(4*15,$b)); &xor ($d_,&DWP(4*15,$b));
&lea ($b,&DWP(4*16,$b)); &lea ($b,&DWP(4*16,$b));
&mov (&DWP(4*7,$a),$b_); &mov (&DWP(4*12,$a),$c);
&mov ($b_,&DWP(4*0,"esp"));
&mov (&DWP(4*10,$a),$c);
&mov ($c,&wparam(2)); # len &mov ($c,&wparam(2)); # len
&mov (&DWP(4*11,$a),$c_);
&mov (&DWP(4*13,$a),$d); &mov (&DWP(4*13,$a),$d);
&mov (&DWP(4*14,$a),$c_);
&mov (&DWP(4*15,$a),$d_); &mov (&DWP(4*15,$a),$d_);
&mov (&DWP(4*0,$a),$b_);
&lea ($a,&DWP(4*16,$a)); &lea ($a,&DWP(4*16,$a));
&sub ($c,64); &sub ($c,64);
&jnz (&label("outer_loop")); &jnz (&label("outer_loop"));
@ -555,12 +567,12 @@ my ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-1)&3),($ai,$bi,$ci,$di)); # previous
my ($xa0,$xa1,$xa2,$xa3,$xt0,$xt1,$xt2,$xt3)=map("xmm$_",(0..7)); my ($xa0,$xa1,$xa2,$xa3,$xt0,$xt1,$xt2,$xt3)=map("xmm$_",(0..7));
#&movdqa ($xa0,&QWP(16*0-128,"ebx")); # it's there
&movdqa ($xa1,&QWP(16*1-128,"ebx"));
&movdqa ($xa2,&QWP(16*2-128,"ebx"));
&movdqa ($xa3,&QWP(16*3-128,"ebx"));
for($i=0;$i<256;$i+=64) { for($i=0;$i<256;$i+=64) {
#&movdqa ($xa0,&QWP($i+16*0-128,"ebx")); # it's there
&movdqa ($xa1,&QWP($i+16*1-128,"ebx"));
&movdqa ($xa2,&QWP($i+16*2-128,"ebx"));
&movdqa ($xa3,&QWP($i+16*3-128,"ebx"));
&paddd ($xa0,&QWP($i+16*0-128,"ebp")); # accumulate key material &paddd ($xa0,&QWP($i+16*0-128,"ebp")); # accumulate key material
&paddd ($xa1,&QWP($i+16*1-128,"ebp")); &paddd ($xa1,&QWP($i+16*1-128,"ebp"));
&paddd ($xa2,&QWP($i+16*2-128,"ebp")); &paddd ($xa2,&QWP($i+16*2-128,"ebp"));
@ -581,25 +593,29 @@ my ($ap,$bp,$cp,$dp)=map(($_&~3)+(($_-1)&3),($ai,$bi,$ci,$di)); # previous
#($xa2,$xt2)=($xt2,$xa2); #($xa2,$xt2)=($xt2,$xa2);
&movdqu ($xt0,&QWP(64*0-128,$inp)); # load input &movdqa (&QWP($i+16*0-128,"ebx"),$xa0);
&movdqu ($xt1,&QWP(64*1-128,$inp));
&movdqu ($xa2,&QWP(64*2-128,$inp));
&movdqu ($xt3,&QWP(64*3-128,$inp));
&lea ($inp,&QWP($i<192?16:(64*4-16*3),$inp));
&pxor ($xt0,$xa0);
&movdqa ($xa0,&QWP($i+16*4-128,"ebx")) if ($i<192); &movdqa ($xa0,&QWP($i+16*4-128,"ebx")) if ($i<192);
&pxor ($xt1,$xa1); &movdqa (&QWP($i+16*1-128,"ebx"),$xa1);
&movdqa ($xa1,&QWP($i+16*5-128,"ebx")) if ($i<192); &movdqa (&QWP($i+16*2-128,"ebx"),$xt2);
&pxor ($xt2,$xa2); &movdqa (&QWP($i+16*3-128,"ebx"),$xa3);
&movdqa ($xa2,&QWP($i+16*6-128,"ebx")) if ($i<192);
&pxor ($xt3,$xa3);
&movdqa ($xa3,&QWP($i+16*7-128,"ebx")) if ($i<192);
&movdqu (&QWP(64*0-128,$out),$xt0); # store output
&movdqu (&QWP(64*1-128,$out),$xt1);
&movdqu (&QWP(64*2-128,$out),$xt2);
&movdqu (&QWP(64*3-128,$out),$xt3);
&lea ($out,&QWP($i<192?16:(64*4-16*3),$out));
} }
for($i=0;$i<256;$i+=64) {
my $j = 16*($i/64);
&movdqu ($xa0,&QWP($i+16*0-128,$inp)); # load input
&movdqu ($xa1,&QWP($i+16*1-128,$inp));
&movdqu ($xa2,&QWP($i+16*2-128,$inp));
&movdqu ($xa3,&QWP($i+16*3-128,$inp));
&pxor ($xa0,&QWP($j+64*0-128,"ebx"));
&pxor ($xa1,&QWP($j+64*1-128,"ebx"));
&pxor ($xa2,&QWP($j+64*2-128,"ebx"));
&pxor ($xa3,&QWP($j+64*3-128,"ebx"));
&movdqu (&QWP($i+16*0-128,$out),$xa0); # write output
&movdqu (&QWP($i+16*1-128,$out),$xa1);
&movdqu (&QWP($i+16*2-128,$out),$xa2);
&movdqu (&QWP($i+16*3-128,$out),$xa3);
}
&lea ($inp,&DWP(256,$inp));
&lea ($out,&DWP(256,$out));
&sub ($len,64*4); &sub ($len,64*4);
&jnc (&label("outer_loop")); &jnc (&label("outer_loop"));

6
crypto/chacha/chacha.c
View File

@ -25,11 +25,9 @@
(((uint32_t)((p)[0])) | ((uint32_t)((p)[1]) << 8) | \ (((uint32_t)((p)[0])) | ((uint32_t)((p)[1]) << 8) | \
((uint32_t)((p)[2]) << 16) | ((uint32_t)((p)[3]) << 24)) ((uint32_t)((p)[2]) << 16) | ((uint32_t)((p)[3]) << 24))
/* TODO(davidben): Re-enable the ChaCha20 assembly for OPENSSL_X86 once they
* pass the in-place tests. */
#if !defined(OPENSSL_NO_ASM) && \ #if !defined(OPENSSL_NO_ASM) && \
(defined(OPENSSL_X86_64) || defined(OPENSSL_ARM) || \ (defined(OPENSSL_X86) || defined(OPENSSL_X86_64) || \
defined(OPENSSL_AARCH64)) defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64))
/* ChaCha20_ctr32 is defined in asm/chacha-*.pl. */ /* ChaCha20_ctr32 is defined in asm/chacha-*.pl. */
void ChaCha20_ctr32(uint8_t *out, const uint8_t *in, size_t in_len, void ChaCha20_ctr32(uint8_t *out, const uint8_t *in, size_t in_len,