2016-02-23 21:25:40 +00:00
|
|
|
#!/usr/bin/env perl
|
|
|
|
|
#
|
|
|
|
|
# ====================================================================
|
|
|
|
|
# Written by Andy Polyakov <appro@openssl.org> for the OpenSSL
|
|
|
|
|
# project. The module is, however, dual licensed under OpenSSL and
|
|
|
|
|
# CRYPTOGAMS licenses depending on where you obtain it. For further
|
|
|
|
|
# details see http://www.openssl.org/~appro/cryptogams/.
|
|
|
|
|
# ====================================================================
|
|
|
|
|
#
|
|
|
|
|
# This module implements Poly1305 hash for x86.
|
|
|
|
|
#
|
|
|
|
|
# April 2015
|
|
|
|
|
#
|
|
|
|
|
# Numbers are cycles per processed byte with poly1305_blocks alone,
|
|
|
|
|
# measured with rdtsc at fixed clock frequency.
|
|
|
|
|
#
|
|
|
|
|
# IALU/gcc-3.4(*) SSE2(**) AVX2
|
|
|
|
|
# Pentium 15.7/+80% -
|
|
|
|
|
# PIII 6.21/+90% -
|
|
|
|
|
# P4 19.8/+40% 3.24
|
|
|
|
|
# Core 2 4.85/+90% 1.80
|
|
|
|
|
# Westmere 4.58/+100% 1.43
|
|
|
|
|
# Sandy Bridge 3.90/+100% 1.36
|
|
|
|
|
# Haswell 3.88/+70% 1.18 0.72
|
|
|
|
|
# Silvermont 11.0/+40% 4.80
|
|
|
|
|
# VIA Nano 6.71/+90% 2.47
|
|
|
|
|
# Sledgehammer 3.51/+180% 4.27
|
|
|
|
|
# Bulldozer 4.53/+140% 1.31
|
|
|
|
|
#
|
|
|
|
|
# (*) gcc 4.8 for some reason generated worse code;
|
|
|
|
|
# (**) besides SSE2 there are floating-point and AVX options; FP
|
|
|
|
|
# is deemed unnecessary, because pre-SSE2 processor are too
|
|
|
|
|
# old to care about, while it's not the fastest option on
|
|
|
|
|
# SSE2-capable ones; AVX is omitted, because it doesn't give
|
|
|
|
|
# a lot of improvement, 5-10% depending on processor;
|
|
|
|
|
|
|
|
|
|
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
|
|
|
|
|
push(@INC,"${dir}","${dir}../../perlasm");
|
|
|
|
|
require "x86asm.pl";
|
|
|
|
|
|
|
|
|
|
&asm_init($ARGV[0],"poly1305-x86.pl",$ARGV[$#ARGV] eq "386");
|
|
|
|
|
|
|
|
|
|
$sse2=$avx=0;
|
|
|
|
|
for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }
|
|
|
|
|
|
|
|
|
|
if ($sse2) {
|
|
|
|
|
&static_label("const_sse2");
|
|
|
|
|
&static_label("enter_blocks");
|
|
|
|
|
&static_label("enter_emit");
|
|
|
|
|
&external_label("OPENSSL_ia32cap_P");
|
|
|
|
|
|
2016-03-03 20:36:33 +00:00
|
|
|
# This may be set to 2, but valgrind can't do AVX2 on 32-bit. Without a
|
|
|
|
|
# way to verify test coverage, keep it disabled.
|
Enable upstream's Poly1305 code.
The C implementation is still our existing C implementation, but slightly
tweaked to fit with upstream's init/block/emits convention.
I've tested this by looking at code coverage in kcachegrind and
valgrind --tool=callgrind --dump-instr=yes --collect-jumps=yes
(NB: valgrind 3.11.0 is needed for AVX2. And even that only does 64-bit AVX2,
so we can't get coverage for the 32-bit code yet. But I had to disable that
anyway.)
This was paired with a hacked up version of poly1305_test that would repeat
tests with different ia32cap and armcap values. This isn't checked in, but we
badly need a story for testing all the different variants.
I'm not happy with upstream's code in either the C/asm boundary or how it
dispatches between different versions, but just debugging the code has been a
significant time investment. I'd hoped to extract the SIMD parts and do the
rest in C, but I think we need to focus on testing first (and use that to
guide what modifications would help). For now, this version seems to work at
least.
The x86 (not x86_64) AVX2 code needs to be disabled because it's broken. It
also seems pretty unnecessary.
https://rt.openssl.org/Ticket/Display.html?id=4346
Otherwise it seems to work and buys us a decent performance improvement.
Notably, my Nexus 6P is finally faster at ChaCha20-Poly1305 than my Nexus 4!
bssl speed numbers follow:
x86
---
Old:
Did 1554000 ChaCha20-Poly1305 (16 bytes) seal operations in 1000536us (1553167.5 ops/sec): 24.9 MB/s
Did 136000 ChaCha20-Poly1305 (1350 bytes) seal operations in 1003947us (135465.3 ops/sec): 182.9 MB/s
Did 30000 ChaCha20-Poly1305 (8192 bytes) seal operations in 1022990us (29325.8 ops/sec): 240.2 MB/s
Did 1888000 ChaCha20-Poly1305-Old (16 bytes) seal operations in 1000206us (1887611.2 ops/sec): 30.2 MB/s
Did 173000 ChaCha20-Poly1305-Old (1350 bytes) seal operations in 1003036us (172476.4 ops/sec): 232.8 MB/s
Did 30000 ChaCha20-Poly1305-Old (8192 bytes) seal operations in 1027759us (29189.7 ops/sec): 239.1 MB/s
New:
Did 2030000 ChaCha20-Poly1305 (16 bytes) seal operations in 1000507us (2028971.3 ops/sec): 32.5 MB/s
Did 404000 ChaCha20-Poly1305 (1350 bytes) seal operations in 1000287us (403884.1 ops/sec): 545.2 MB/s
Did 83000 ChaCha20-Poly1305 (8192 bytes) seal operations in 1001258us (82895.7 ops/sec): 679.1 MB/s
Did 2018000 ChaCha20-Poly1305-Old (16 bytes) seal operations in 1000006us (2017987.9 ops/sec): 32.3 MB/s
Did 360000 ChaCha20-Poly1305-Old (1350 bytes) seal operations in 1001962us (359295.1 ops/sec): 485.0 MB/s
Did 85000 ChaCha20-Poly1305-Old (8192 bytes) seal operations in 1002479us (84789.8 ops/sec): 694.6 MB/s
x86_64, no AVX2
---
Old:
Did 2023000 ChaCha20-Poly1305 (16 bytes) seal operations in 1000258us (2022478.2 ops/sec): 32.4 MB/s
Did 466000 ChaCha20-Poly1305 (1350 bytes) seal operations in 1002619us (464782.7 ops/sec): 627.5 MB/s
Did 90000 ChaCha20-Poly1305 (8192 bytes) seal operations in 1001133us (89898.1 ops/sec): 736.4 MB/s
Did 2238000 ChaCha20-Poly1305-Old (16 bytes) seal operations in 1000175us (2237608.4 ops/sec): 35.8 MB/s
Did 483000 ChaCha20-Poly1305-Old (1350 bytes) seal operations in 1001348us (482349.8 ops/sec): 651.2 MB/s
Did 90000 ChaCha20-Poly1305-Old (8192 bytes) seal operations in 1003141us (89718.2 ops/sec): 735.0 MB/s
New:
Did 2558000 ChaCha20-Poly1305 (16 bytes) seal operations in 1000275us (2557296.7 ops/sec): 40.9 MB/s
Did 510000 ChaCha20-Poly1305 (1350 bytes) seal operations in 1001810us (509078.6 ops/sec): 687.3 MB/s
Did 115000 ChaCha20-Poly1305 (8192 bytes) seal operations in 1006457us (114262.2 ops/sec): 936.0 MB/s
Did 2818000 ChaCha20-Poly1305-Old (16 bytes) seal operations in 1000187us (2817473.1 ops/sec): 45.1 MB/s
Did 418000 ChaCha20-Poly1305-Old (1350 bytes) seal operations in 1001140us (417524.0 ops/sec): 563.7 MB/s
Did 91000 ChaCha20-Poly1305-Old (8192 bytes) seal operations in 1002539us (90769.5 ops/sec): 743.6 MB/s
x86_64, AVX2
---
Old:
Did 2516000 ChaCha20-Poly1305 (16 bytes) seal operations in 1000115us (2515710.7 ops/sec): 40.3 MB/s
Did 774000 ChaCha20-Poly1305 (1350 bytes) seal operations in 1000300us (773767.9 ops/sec): 1044.6 MB/s
Did 171000 ChaCha20-Poly1305 (8192 bytes) seal operations in 1004373us (170255.5 ops/sec): 1394.7 MB/s
Did 2580000 ChaCha20-Poly1305-Old (16 bytes) seal operations in 1000144us (2579628.5 ops/sec): 41.3 MB/s
Did 769000 ChaCha20-Poly1305-Old (1350 bytes) seal operations in 1000472us (768637.2 ops/sec): 1037.7 MB/s
Did 169000 ChaCha20-Poly1305-Old (8192 bytes) seal operations in 1000320us (168945.9 ops/sec): 1384.0 MB/s
New:
Did 3240000 ChaCha20-Poly1305 (16 bytes) seal operations in 1000114us (3239630.7 ops/sec): 51.8 MB/s
Did 932000 ChaCha20-Poly1305 (1350 bytes) seal operations in 1000059us (931945.0 ops/sec): 1258.1 MB/s
Did 217000 ChaCha20-Poly1305 (8192 bytes) seal operations in 1003282us (216290.1 ops/sec): 1771.8 MB/s
Did 3187000 ChaCha20-Poly1305-Old (16 bytes) seal operations in 1000100us (3186681.3 ops/sec): 51.0 MB/s
Did 926000 ChaCha20-Poly1305-Old (1350 bytes) seal operations in 1000071us (925934.3 ops/sec): 1250.0 MB/s
Did 215000 ChaCha20-Poly1305-Old (8192 bytes) seal operations in 1000479us (214897.1 ops/sec): 1760.4 MB/s
arm, Nexus 4
---
Old:
Did 430248 ChaCha20-Poly1305 (16 bytes) seal operations in 1000153us (430182.2 ops/sec): 6.9 MB/s
Did 115250 ChaCha20-Poly1305 (1350 bytes) seal operations in 1000549us (115186.8 ops/sec): 155.5 MB/s
Did 27000 ChaCha20-Poly1305 (8192 bytes) seal operations in 1030124us (26210.4 ops/sec): 214.7 MB/s
Did 451750 ChaCha20-Poly1305-Old (16 bytes) seal operations in 1000549us (451502.1 ops/sec): 7.2 MB/s
Did 118000 ChaCha20-Poly1305-Old (1350 bytes) seal operations in 1001557us (117816.6 ops/sec): 159.1 MB/s
Did 27000 ChaCha20-Poly1305-Old (8192 bytes) seal operations in 1024263us (26360.4 ops/sec): 215.9 MB/s
New:
Did 553644 ChaCha20-Poly1305 (16 bytes) seal operations in 1000183us (553542.7 ops/sec): 8.9 MB/s
Did 126000 ChaCha20-Poly1305 (1350 bytes) seal operations in 1000396us (125950.1 ops/sec): 170.0 MB/s
Did 27000 ChaCha20-Poly1305 (8192 bytes) seal operations in 1000336us (26990.9 ops/sec): 221.1 MB/s
Did 559000 ChaCha20-Poly1305-Old (16 bytes) seal operations in 1001465us (558182.3 ops/sec): 8.9 MB/s
Did 124000 ChaCha20-Poly1305-Old (1350 bytes) seal operations in 1000824us (123897.9 ops/sec): 167.3 MB/s
Did 28000 ChaCha20-Poly1305-Old (8192 bytes) seal operations in 1034854us (27057.0 ops/sec): 221.7 MB/s
aarch64, Nexus 6P
---
Old:
Did 358000 ChaCha20-Poly1305 (16 bytes) seal operations in 1000358us (357871.9 ops/sec): 5.7 MB/s
Did 45000 ChaCha20-Poly1305 (1350 bytes) seal operations in 1022386us (44014.7 ops/sec): 59.4 MB/s
Did 8657 ChaCha20-Poly1305 (8192 bytes) seal operations in 1063722us (8138.4 ops/sec): 66.7 MB/s
Did 350000 ChaCha20-Poly1305-Old (16 bytes) seal operations in 1000074us (349974.1 ops/sec): 5.6 MB/s
Did 44000 ChaCha20-Poly1305-Old (1350 bytes) seal operations in 1007907us (43654.8 ops/sec): 58.9 MB/s
Did 8525 ChaCha20-Poly1305-Old (8192 bytes) seal operations in 1042644us (8176.3 ops/sec): 67.0 MB/s
New:
Did 713000 ChaCha20-Poly1305 (16 bytes) seal operations in 1000190us (712864.6 ops/sec): 11.4 MB/s
Did 180000 ChaCha20-Poly1305 (1350 bytes) seal operations in 1004249us (179238.4 ops/sec): 242.0 MB/s
Did 41000 ChaCha20-Poly1305 (8192 bytes) seal operations in 1005811us (40763.1 ops/sec): 333.9 MB/s
Did 775000 ChaCha20-Poly1305-Old (16 bytes) seal operations in 1000719us (774443.2 ops/sec): 12.4 MB/s
Did 182000 ChaCha20-Poly1305-Old (1350 bytes) seal operations in 1003529us (181360.0 ops/sec): 244.8 MB/s
Did 41000 ChaCha20-Poly1305-Old (8192 bytes) seal operations in 1010576us (40570.9 ops/sec): 332.4 MB/s
Change-Id: Iaa4ab86ac1174b79833077963cc3616cfb08e686
Reviewed-on: https://boringssl-review.googlesource.com/7226
Reviewed-by: Adam Langley <agl@google.com>
2016-02-23 22:43:36 +00:00
|
|
|
$avx = 0;
|
2016-02-23 21:25:40 +00:00
|
|
|
}
|
|
|
|
|
|
|
|
|
|
########################################################################
|
|
|
|
|
# Layout of opaque area is following.
|
|
|
|
|
#
|
|
|
|
|
# unsigned __int32 h[5]; # current hash value base 2^32
|
|
|
|
|
# unsigned __int32 pad; # is_base2_26 in vector context
|
|
|
|
|
# unsigned __int32 r[4]; # key value base 2^32
|
|
|
|
|
|
|
|
|
|
&align(64);
|
|
|
|
|
&function_begin("poly1305_init");
|
|
|
|
|
&mov ("edi",&wparam(0)); # context
|
|
|
|
|
&mov ("esi",&wparam(1)); # key
|
|
|
|
|
&mov ("ebp",&wparam(2)); # function table
|
|
|
|
|
|
|
|
|
|
&xor ("eax","eax");
|
|
|
|
|
&mov (&DWP(4*0,"edi"),"eax"); # zero hash value
|
|
|
|
|
&mov (&DWP(4*1,"edi"),"eax");
|
|
|
|
|
&mov (&DWP(4*2,"edi"),"eax");
|
|
|
|
|
&mov (&DWP(4*3,"edi"),"eax");
|
|
|
|
|
&mov (&DWP(4*4,"edi"),"eax");
|
|
|
|
|
&mov (&DWP(4*5,"edi"),"eax"); # is_base2_26
|
|
|
|
|
|
|
|
|
|
&cmp ("esi",0);
|
|
|
|
|
&je (&label("nokey"));
|
|
|
|
|
|
|
|
|
|
if ($sse2) {
|
|
|
|
|
&call (&label("pic_point"));
|
|
|
|
|
&set_label("pic_point");
|
|
|
|
|
&blindpop("ebx");
|
|
|
|
|
|
|
|
|
|
&lea ("eax",&DWP("poly1305_blocks-".&label("pic_point"),"ebx"));
|
|
|
|
|
&lea ("edx",&DWP("poly1305_emit-".&label("pic_point"),"ebx"));
|
|
|
|
|
|
|
|
|
|
&picmeup("edi","OPENSSL_ia32cap_P","ebx",&label("pic_point"));
|
|
|
|
|
&mov ("ecx",&DWP(0,"edi"));
|
|
|
|
|
&and ("ecx",1<<26|1<<24);
|
|
|
|
|
&cmp ("ecx",1<<26|1<<24); # SSE2 and XMM?
|
|
|
|
|
&jne (&label("no_sse2"));
|
|
|
|
|
|
|
|
|
|
&lea ("eax",&DWP("_poly1305_blocks_sse2-".&label("pic_point"),"ebx"));
|
|
|
|
|
&lea ("edx",&DWP("_poly1305_emit_sse2-".&label("pic_point"),"ebx"));
|
|
|
|
|
|
|
|
|
|
if ($avx>1) {
|
|
|
|
|
&mov ("ecx",&DWP(8,"edi"));
|
|
|
|
|
&test ("ecx",1<<5); # AVX2?
|
|
|
|
|
&jz (&label("no_sse2"));
|
|
|
|
|
|
|
|
|
|
&lea ("eax",&DWP("_poly1305_blocks_avx2-".&label("pic_point"),"ebx"));
|
|
|
|
|
}
|
|
|
|
|
&set_label("no_sse2");
|
|
|
|
|
&mov ("edi",&wparam(0)); # reload context
|
|
|
|
|
&mov (&DWP(0,"ebp"),"eax"); # fill function table
|
|
|
|
|
&mov (&DWP(4,"ebp"),"edx");
|
|
|
|
|
}
|
|
|
|
|
|
|
|
|
|
&mov ("eax",&DWP(4*0,"esi")); # load input key
|
|
|
|
|
&mov ("ebx",&DWP(4*1,"esi"));
|
|
|
|
|
&mov ("ecx",&DWP(4*2,"esi"));
|
|
|
|
|
&mov ("edx",&DWP(4*3,"esi"));
|
|
|
|
|
&and ("eax",0x0fffffff);
|
|
|
|
|
&and ("ebx",0x0ffffffc);
|
|
|
|
|
&and ("ecx",0x0ffffffc);
|
|
|
|
|
&and ("edx",0x0ffffffc);
|
|
|
|
|
&mov (&DWP(4*6,"edi"),"eax");
|
|
|
|
|
&mov (&DWP(4*7,"edi"),"ebx");
|
|
|
|
|
&mov (&DWP(4*8,"edi"),"ecx");
|
|
|
|
|
&mov (&DWP(4*9,"edi"),"edx");
|
|
|
|
|
|
|
|
|
|
&mov ("eax",$sse2);
|
|
|
|
|
&set_label("nokey");
|
|
|
|
|
&function_end("poly1305_init");
|
|
|
|
|
|
|
|
|
|
($h0,$h1,$h2,$h3,$h4,
|
|
|
|
|
$d0,$d1,$d2,$d3,
|
|
|
|
|
$r0,$r1,$r2,$r3,
|
|
|
|
|
$s1,$s2,$s3)=map(4*$_,(0..15));
|
|
|
|
|
|
|
|
|
|
&function_begin("poly1305_blocks");
|
|
|
|
|
&mov ("edi",&wparam(0)); # ctx
|
|
|
|
|
&mov ("esi",&wparam(1)); # inp
|
|
|
|
|
&mov ("ecx",&wparam(2)); # len
|
|
|
|
|
&set_label("enter_blocks");
|
|
|
|
|
&and ("ecx",-15);
|
|
|
|
|
&jz (&label("nodata"));
|
|
|
|
|
|
|
|
|
|
&stack_push(16);
|
|
|
|
|
&mov ("eax",&DWP(4*6,"edi")); # r0
|
|
|
|
|
&mov ("ebx",&DWP(4*7,"edi")); # r1
|
|
|
|
|
&lea ("ebp",&DWP(0,"esi","ecx")); # end of input
|
|
|
|
|
&mov ("ecx",&DWP(4*8,"edi")); # r2
|
|
|
|
|
&mov ("edx",&DWP(4*9,"edi")); # r3
|
|
|
|
|
|
|
|
|
|
&mov (&wparam(2),"ebp");
|
|
|
|
|
&mov ("ebp","esi");
|
|
|
|
|
|
|
|
|
|
&mov (&DWP($r0,"esp"),"eax"); # r0
|
|
|
|
|
&mov ("eax","ebx");
|
|
|
|
|
&shr ("eax",2);
|
|
|
|
|
&mov (&DWP($r1,"esp"),"ebx"); # r1
|
|
|
|
|
&add ("eax","ebx"); # s1
|
|
|
|
|
&mov ("ebx","ecx");
|
|
|
|
|
&shr ("ebx",2);
|
|
|
|
|
&mov (&DWP($r2,"esp"),"ecx"); # r2
|
|
|
|
|
&add ("ebx","ecx"); # s2
|
|
|
|
|
&mov ("ecx","edx");
|
|
|
|
|
&shr ("ecx",2);
|
|
|
|
|
&mov (&DWP($r3,"esp"),"edx"); # r3
|
|
|
|
|
&add ("ecx","edx"); # s3
|
|
|
|
|
&mov (&DWP($s1,"esp"),"eax"); # s1
|
|
|
|
|
&mov (&DWP($s2,"esp"),"ebx"); # s2
|
|
|
|
|
&mov (&DWP($s3,"esp"),"ecx"); # s3
|
|
|
|
|
|
|
|
|
|
&mov ("eax",&DWP(4*0,"edi")); # load hash value
|
|
|
|
|
&mov ("ebx",&DWP(4*1,"edi"));
|
|
|
|
|
&mov ("ecx",&DWP(4*2,"edi"));
|
|
|
|
|
&mov ("esi",&DWP(4*3,"edi"));
|
|
|
|
|
&mov ("edi",&DWP(4*4,"edi"));
|
|
|
|
|
&jmp (&label("loop"));
|
|
|
|
|
|
|
|
|
|
&set_label("loop",32);
|
|
|
|
|
&add ("eax",&DWP(4*0,"ebp")); # accumulate input
|
|
|
|
|
&adc ("ebx",&DWP(4*1,"ebp"));
|
|
|
|
|
&adc ("ecx",&DWP(4*2,"ebp"));
|
|
|
|
|
&adc ("esi",&DWP(4*3,"ebp"));
|
|
|
|
|
&lea ("ebp",&DWP(4*4,"ebp"));
|
|
|
|
|
&adc ("edi",&wparam(3)); # padbit
|
|
|
|
|
|
|
|
|
|
&mov (&DWP($h0,"esp"),"eax"); # put aside hash[+inp]
|
|
|
|
|
&mov (&DWP($h3,"esp"),"esi");
|
|
|
|
|
|
|
|
|
|
&mul (&DWP($r0,"esp")); # h0*r0
|
|
|
|
|
&mov (&DWP($h4,"esp"),"edi");
|
|
|
|
|
&mov ("edi","eax");
|
|
|
|
|
&mov ("eax","ebx"); # h1
|
|
|
|
|
&mov ("esi","edx");
|
|
|
|
|
&mul (&DWP($s3,"esp")); # h1*s3
|
|
|
|
|
&add ("edi","eax");
|
|
|
|
|
&mov ("eax","ecx"); # h2
|
|
|
|
|
&adc ("esi","edx");
|
|
|
|
|
&mul (&DWP($s2,"esp")); # h2*s2
|
|
|
|
|
&add ("edi","eax");
|
|
|
|
|
&mov ("eax",&DWP($h3,"esp"));
|
|
|
|
|
&adc ("esi","edx");
|
|
|
|
|
&mul (&DWP($s1,"esp")); # h3*s1
|
|
|
|
|
&add ("edi","eax");
|
|
|
|
|
&mov ("eax",&DWP($h0,"esp"));
|
|
|
|
|
&adc ("esi","edx");
|
|
|
|
|
|
|
|
|
|
&mul (&DWP($r1,"esp")); # h0*r1
|
|
|
|
|
&mov (&DWP($d0,"esp"),"edi");
|
|
|
|
|
&xor ("edi","edi");
|
|
|
|
|
&add ("esi","eax");
|
|
|
|
|
&mov ("eax","ebx"); # h1
|
|
|
|
|
&adc ("edi","edx");
|
|
|
|
|
&mul (&DWP($r0,"esp")); # h1*r0
|
|
|
|
|
&add ("esi","eax");
|
|
|
|
|
&mov ("eax","ecx"); # h2
|
|
|
|
|
&adc ("edi","edx");
|
|
|
|
|
&mul (&DWP($s3,"esp")); # h2*s3
|
|
|
|
|
&add ("esi","eax");
|
|
|
|
|
&mov ("eax",&DWP($h3,"esp"));
|
|
|
|
|
&adc ("edi","edx");
|
|
|
|
|
&mul (&DWP($s2,"esp")); # h3*s2
|
|
|
|
|
&add ("esi","eax");
|
|
|
|
|
&mov ("eax",&DWP($h4,"esp"));
|
|
|
|
|
&adc ("edi","edx");
|
|
|
|
|
&imul ("eax",&DWP($s1,"esp")); # h4*s1
|
|
|
|
|
&add ("esi","eax");
|
|
|
|
|
&mov ("eax",&DWP($h0,"esp"));
|
|
|
|
|
&adc ("edi",0);
|
|
|
|
|
|
|
|
|
|
&mul (&DWP($r2,"esp")); # h0*r2
|
|
|
|
|
&mov (&DWP($d1,"esp"),"esi");
|
|
|
|
|
&xor ("esi","esi");
|
|
|
|
|
&add ("edi","eax");
|
|
|
|
|
&mov ("eax","ebx"); # h1
|
|
|
|
|
&adc ("esi","edx");
|
|
|
|
|
&mul (&DWP($r1,"esp")); # h1*r1
|
|
|
|
|
&add ("edi","eax");
|
|
|
|
|
&mov ("eax","ecx"); # h2
|
|
|
|
|
&adc ("esi","edx");
|
|
|
|
|
&mul (&DWP($r0,"esp")); # h2*r0
|
|
|
|
|
&add ("edi","eax");
|
|
|
|
|
&mov ("eax",&DWP($h3,"esp"));
|
|
|
|
|
&adc ("esi","edx");
|
|
|
|
|
&mul (&DWP($s3,"esp")); # h3*s3
|
|
|
|
|
&add ("edi","eax");
|
|
|
|
|
&mov ("eax",&DWP($h4,"esp"));
|
|
|
|
|
&adc ("esi","edx");
|
|
|
|
|
&imul ("eax",&DWP($s2,"esp")); # h4*s2
|
|
|
|
|
&add ("edi","eax");
|
|
|
|
|
&mov ("eax",&DWP($h0,"esp"));
|
|
|
|
|
&adc ("esi",0);
|
|
|
|
|
|
|
|
|
|
&mul (&DWP($r3,"esp")); # h0*r3
|
|
|
|
|
&mov (&DWP($d2,"esp"),"edi");
|
|
|
|
|
&xor ("edi","edi");
|
|
|
|
|
&add ("esi","eax");
|
|
|
|
|
&mov ("eax","ebx"); # h1
|
|
|
|
|
&adc ("edi","edx");
|
|
|
|
|
&mul (&DWP($r2,"esp")); # h1*r2
|
|
|
|
|
&add ("esi","eax");
|
|
|
|
|
&mov ("eax","ecx"); # h2
|
|
|
|
|
&adc ("edi","edx");
|
|
|
|
|
&mul (&DWP($r1,"esp")); # h2*r1
|
|
|
|
|
&add ("esi","eax");
|
|
|
|
|
&mov ("eax",&DWP($h3,"esp"));
|
|
|
|
|
&adc ("edi","edx");
|
|
|
|
|
&mul (&DWP($r0,"esp")); # h3*r0
|
|
|
|
|
&add ("esi","eax");
|
|
|
|
|
&mov ("ecx",&DWP($h4,"esp"));
|
|
|
|
|
&adc ("edi","edx");
|
|
|
|
|
|
|
|
|
|
&mov ("edx","ecx");
|
|
|
|
|
&imul ("ecx",&DWP($s3,"esp")); # h4*s3
|
|
|
|
|
&add ("esi","ecx");
|
|
|
|
|
&mov ("eax",&DWP($d0,"esp"));
|
|
|
|
|
&adc ("edi",0);
|
|
|
|
|
|
|
|
|
|
&imul ("edx",&DWP($r0,"esp")); # h4*r0
|
|
|
|
|
&add ("edx","edi");
|
|
|
|
|
|
|
|
|
|
&mov ("ebx",&DWP($d1,"esp"));
|
|
|
|
|
&mov ("ecx",&DWP($d2,"esp"));
|
|
|
|
|
|
|
|
|
|
&mov ("edi","edx"); # last reduction step
|
|
|
|
|
&shr ("edx",2);
|
|
|
|
|
&and ("edi",3);
|
|
|
|
|
&lea ("edx",&DWP(0,"edx","edx",4)); # *5
|
|
|
|
|
&add ("eax","edx");
|
|
|
|
|
&adc ("ebx",0);
|
|
|
|
|
&adc ("ecx",0);
|
|
|
|
|
&adc ("esi",0);
|
|
|
|
|
|
|
|
|
|
&cmp ("ebp",&wparam(2)); # done yet?
|
|
|
|
|
&jne (&label("loop"));
|
|
|
|
|
|
|
|
|
|
&mov ("edx",&wparam(0)); # ctx
|
|
|
|
|
&stack_pop(16);
|
|
|
|
|
&mov (&DWP(4*0,"edx"),"eax"); # store hash value
|
|
|
|
|
&mov (&DWP(4*1,"edx"),"ebx");
|
|
|
|
|
&mov (&DWP(4*2,"edx"),"ecx");
|
|
|
|
|
&mov (&DWP(4*3,"edx"),"esi");
|
|
|
|
|
&mov (&DWP(4*4,"edx"),"edi");
|
|
|
|
|
&set_label("nodata");
|
|
|
|
|
&function_end("poly1305_blocks");
|
|
|
|
|
|
|
|
|
|
&function_begin("poly1305_emit");
|
|
|
|
|
&mov ("ebp",&wparam(0)); # context
|
|
|
|
|
&set_label("enter_emit");
|
|
|
|
|
&mov ("edi",&wparam(1)); # output
|
|
|
|
|
&mov ("eax",&DWP(4*0,"ebp")); # load hash value
|
|
|
|
|
&mov ("ebx",&DWP(4*1,"ebp"));
|
|
|
|
|
&mov ("ecx",&DWP(4*2,"ebp"));
|
|
|
|
|
&mov ("edx",&DWP(4*3,"ebp"));
|
|
|
|
|
&mov ("esi",&DWP(4*4,"ebp"));
|
|
|
|
|
|
|
|
|
|
&add ("eax",5); # compare to modulus
|
|
|
|
|
&adc ("ebx",0);
|
|
|
|
|
&adc ("ecx",0);
|
|
|
|
|
&adc ("edx",0);
|
|
|
|
|
&adc ("esi",0);
|
|
|
|
|
&shr ("esi",2); # did it carry/borrow?
|
|
|
|
|
&neg ("esi"); # do we choose hash-modulus?
|
|
|
|
|
|
|
|
|
|
&and ("eax","esi");
|
|
|
|
|
&and ("ebx","esi");
|
|
|
|
|
&and ("ecx","esi");
|
|
|
|
|
&and ("edx","esi");
|
|
|
|
|
&mov (&DWP(4*0,"edi"),"eax");
|
|
|
|
|
&mov (&DWP(4*1,"edi"),"ebx");
|
|
|
|
|
&mov (&DWP(4*2,"edi"),"ecx");
|
|
|
|
|
&mov (&DWP(4*3,"edi"),"edx");
|
|
|
|
|
|
|
|
|
|
¬ ("esi"); # or original hash value?
|
|
|
|
|
&mov ("eax",&DWP(4*0,"ebp"));
|
|
|
|
|
&mov ("ebx",&DWP(4*1,"ebp"));
|
|
|
|
|
&mov ("ecx",&DWP(4*2,"ebp"));
|
|
|
|
|
&mov ("edx",&DWP(4*3,"ebp"));
|
|
|
|
|
&mov ("ebp",&wparam(2));
|
|
|
|
|
&and ("eax","esi");
|
|
|
|
|
&and ("ebx","esi");
|
|
|
|
|
&and ("ecx","esi");
|
|
|
|
|
&and ("edx","esi");
|
|
|
|
|
&or ("eax",&DWP(4*0,"edi"));
|
|
|
|
|
&or ("ebx",&DWP(4*1,"edi"));
|
|
|
|
|
&or ("ecx",&DWP(4*2,"edi"));
|
|
|
|
|
&or ("edx",&DWP(4*3,"edi"));
|
|
|
|
|
|
|
|
|
|
&add ("eax",&DWP(4*0,"ebp")); # accumulate key
|
|
|
|
|
&adc ("ebx",&DWP(4*1,"ebp"));
|
|
|
|
|
&adc ("ecx",&DWP(4*2,"ebp"));
|
|
|
|
|
&adc ("edx",&DWP(4*3,"ebp"));
|
|
|
|
|
|
|
|
|
|
&mov (&DWP(4*0,"edi"),"eax");
|
|
|
|
|
&mov (&DWP(4*1,"edi"),"ebx");
|
|
|
|
|
&mov (&DWP(4*2,"edi"),"ecx");
|
|
|
|
|
&mov (&DWP(4*3,"edi"),"edx");
|
|
|
|
|
&function_end("poly1305_emit");
|
|
|
|
|
|
|
|
|
|
if ($sse2) {
|
|
|
|
|
########################################################################
|
|
|
|
|
# Layout of opaque area is following.
|
|
|
|
|
#
|
|
|
|
|
# unsigned __int32 h[5]; # current hash value base 2^26
|
|
|
|
|
# unsigned __int32 is_base2_26;
|
|
|
|
|
# unsigned __int32 r[4]; # key value base 2^32
|
|
|
|
|
# unsigned __int32 pad[2];
|
|
|
|
|
# struct { unsigned __int32 r^4, r^3, r^2, r^1; } r[9];
|
|
|
|
|
#
|
|
|
|
|
# where r^n are base 2^26 digits of degrees of multiplier key. There are
|
|
|
|
|
# 5 digits, but last four are interleaved with multiples of 5, totalling
|
|
|
|
|
# in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4.
|
|
|
|
|
|
|
|
|
|
my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("xmm$_",(0..7));
|
|
|
|
|
my $MASK=$T2; # borrow and keep in mind
|
|
|
|
|
|
|
|
|
|
&align (32);
|
|
|
|
|
&function_begin_B("_poly1305_init_sse2");
|
|
|
|
|
&movdqu ($D4,&QWP(4*6,"edi")); # key base 2^32
|
|
|
|
|
&lea ("edi",&DWP(16*3,"edi")); # size optimization
|
|
|
|
|
&mov ("ebp","esp");
|
|
|
|
|
&sub ("esp",16*(9+5));
|
|
|
|
|
&and ("esp",-16);
|
|
|
|
|
|
|
|
|
|
#&pand ($D4,&QWP(96,"ebx")); # magic mask
|
|
|
|
|
&movq ($MASK,&QWP(64,"ebx"));
|
|
|
|
|
|
|
|
|
|
&movdqa ($D0,$D4);
|
|
|
|
|
&movdqa ($D1,$D4);
|
|
|
|
|
&movdqa ($D2,$D4);
|
|
|
|
|
|
|
|
|
|
&pand ($D0,$MASK); # -> base 2^26
|
|
|
|
|
&psrlq ($D1,26);
|
|
|
|
|
&psrldq ($D2,6);
|
|
|
|
|
&pand ($D1,$MASK);
|
|
|
|
|
&movdqa ($D3,$D2);
|
|
|
|
|
&psrlq ($D2,4)
|
|
|
|
|
&psrlq ($D3,30);
|
|
|
|
|
&pand ($D2,$MASK);
|
|
|
|
|
&pand ($D3,$MASK);
|
|
|
|
|
&psrldq ($D4,13);
|
|
|
|
|
|
|
|
|
|
&lea ("edx",&DWP(16*9,"esp")); # size optimization
|
|
|
|
|
&mov ("ecx",2);
|
|
|
|
|
&set_label("square");
|
|
|
|
|
&movdqa (&QWP(16*0,"esp"),$D0);
|
|
|
|
|
&movdqa (&QWP(16*1,"esp"),$D1);
|
|
|
|
|
&movdqa (&QWP(16*2,"esp"),$D2);
|
|
|
|
|
&movdqa (&QWP(16*3,"esp"),$D3);
|
|
|
|
|
&movdqa (&QWP(16*4,"esp"),$D4);
|
|
|
|
|
|
|
|
|
|
&movdqa ($T1,$D1);
|
|
|
|
|
&movdqa ($T0,$D2);
|
|
|
|
|
&pslld ($T1,2);
|
|
|
|
|
&pslld ($T0,2);
|
|
|
|
|
&paddd ($T1,$D1); # *5
|
|
|
|
|
&paddd ($T0,$D2); # *5
|
|
|
|
|
&movdqa (&QWP(16*5,"esp"),$T1);
|
|
|
|
|
&movdqa (&QWP(16*6,"esp"),$T0);
|
|
|
|
|
&movdqa ($T1,$D3);
|
|
|
|
|
&movdqa ($T0,$D4);
|
|
|
|
|
&pslld ($T1,2);
|
|
|
|
|
&pslld ($T0,2);
|
|
|
|
|
&paddd ($T1,$D3); # *5
|
|
|
|
|
&paddd ($T0,$D4); # *5
|
|
|
|
|
&movdqa (&QWP(16*7,"esp"),$T1);
|
|
|
|
|
&movdqa (&QWP(16*8,"esp"),$T0);
|
|
|
|
|
|
|
|
|
|
&pshufd ($T1,$D0,0b01000100);
|
|
|
|
|
&movdqa ($T0,$D1);
|
|
|
|
|
&pshufd ($D1,$D1,0b01000100);
|
|
|
|
|
&pshufd ($D2,$D2,0b01000100);
|
|
|
|
|
&pshufd ($D3,$D3,0b01000100);
|
|
|
|
|
&pshufd ($D4,$D4,0b01000100);
|
|
|
|
|
&movdqa (&QWP(16*0,"edx"),$T1);
|
|
|
|
|
&movdqa (&QWP(16*1,"edx"),$D1);
|
|
|
|
|
&movdqa (&QWP(16*2,"edx"),$D2);
|
|
|
|
|
&movdqa (&QWP(16*3,"edx"),$D3);
|
|
|
|
|
&movdqa (&QWP(16*4,"edx"),$D4);
|
|
|
|
|
|
|
|
|
|
################################################################
|
|
|
|
|
# d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
|
|
|
|
|
# d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
|
|
|
|
|
# d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
|
|
|
|
|
# d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
|
|
|
|
|
# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
|
|
|
|
|
|
|
|
|
|
&pmuludq ($D4,$D0); # h4*r0
|
|
|
|
|
&pmuludq ($D3,$D0); # h3*r0
|
|
|
|
|
&pmuludq ($D2,$D0); # h2*r0
|
|
|
|
|
&pmuludq ($D1,$D0); # h1*r0
|
|
|
|
|
&pmuludq ($D0,$T1); # h0*r0
|
|
|
|
|
|
|
|
|
|
sub pmuladd {
|
|
|
|
|
my $load = shift;
|
|
|
|
|
my $base = shift; $base = "esp" if (!defined($base));
|
|
|
|
|
|
|
|
|
|
################################################################
|
|
|
|
|
# As for choice to "rotate" $T0-$T2 in order to move paddq
|
|
|
|
|
# past next multiplication. While it makes code harder to read
|
|
|
|
|
# and doesn't have significant effect on most processors, it
|
|
|
|
|
# makes a lot of difference on Atom, up to 30% improvement.
|
|
|
|
|
|
|
|
|
|
&movdqa ($T1,$T0);
|
|
|
|
|
&pmuludq ($T0,&QWP(16*3,$base)); # r1*h3
|
|
|
|
|
&movdqa ($T2,$T1);
|
|
|
|
|
&pmuludq ($T1,&QWP(16*2,$base)); # r1*h2
|
|
|
|
|
&paddq ($D4,$T0);
|
|
|
|
|
&movdqa ($T0,$T2);
|
|
|
|
|
&pmuludq ($T2,&QWP(16*1,$base)); # r1*h1
|
|
|
|
|
&paddq ($D3,$T1);
|
|
|
|
|
&$load ($T1,5); # s1
|
|
|
|
|
&pmuludq ($T0,&QWP(16*0,$base)); # r1*h0
|
|
|
|
|
&paddq ($D2,$T2);
|
|
|
|
|
&pmuludq ($T1,&QWP(16*4,$base)); # s1*h4
|
|
|
|
|
&$load ($T2,2); # r2^n
|
|
|
|
|
&paddq ($D1,$T0);
|
|
|
|
|
|
|
|
|
|
&movdqa ($T0,$T2);
|
|
|
|
|
&pmuludq ($T2,&QWP(16*2,$base)); # r2*h2
|
|
|
|
|
&paddq ($D0,$T1);
|
|
|
|
|
&movdqa ($T1,$T0);
|
|
|
|
|
&pmuludq ($T0,&QWP(16*1,$base)); # r2*h1
|
|
|
|
|
&paddq ($D4,$T2);
|
|
|
|
|
&$load ($T2,6); # s2^n
|
|
|
|
|
&pmuludq ($T1,&QWP(16*0,$base)); # r2*h0
|
|
|
|
|
&paddq ($D3,$T0);
|
|
|
|
|
&movdqa ($T0,$T2);
|
|
|
|
|
&pmuludq ($T2,&QWP(16*4,$base)); # s2*h4
|
|
|
|
|
&paddq ($D2,$T1);
|
|
|
|
|
&pmuludq ($T0,&QWP(16*3,$base)); # s2*h3
|
|
|
|
|
&$load ($T1,3); # r3^n
|
|
|
|
|
&paddq ($D1,$T2);
|
|
|
|
|
|
|
|
|
|
&movdqa ($T2,$T1);
|
|
|
|
|
&pmuludq ($T1,&QWP(16*1,$base)); # r3*h1
|
|
|
|
|
&paddq ($D0,$T0);
|
|
|
|
|
&$load ($T0,7); # s3^n
|
|
|
|
|
&pmuludq ($T2,&QWP(16*0,$base)); # r3*h0
|
|
|
|
|
&paddq ($D4,$T1);
|
|
|
|
|
&movdqa ($T1,$T0);
|
|
|
|
|
&pmuludq ($T0,&QWP(16*4,$base)); # s3*h4
|
|
|
|
|
&paddq ($D3,$T2);
|
|
|
|
|
&movdqa ($T2,$T1);
|
|
|
|
|
&pmuludq ($T1,&QWP(16*3,$base)); # s3*h3
|
|
|
|
|
&paddq ($D2,$T0);
|
|
|
|
|
&pmuludq ($T2,&QWP(16*2,$base)); # s3*h2
|
|
|
|
|
&$load ($T0,4); # r4^n
|
|
|
|
|
&paddq ($D1,$T1);
|
|
|
|
|
|
|
|
|
|
&$load ($T1,8); # s4^n
|
|
|
|
|
&pmuludq ($T0,&QWP(16*0,$base)); # r4*h0
|
|
|
|
|
&paddq ($D0,$T2);
|
|
|
|
|
&movdqa ($T2,$T1);
|
|
|
|
|
&pmuludq ($T1,&QWP(16*4,$base)); # s4*h4
|
|
|
|
|
&paddq ($D4,$T0);
|
|
|
|
|
&movdqa ($T0,$T2);
|
|
|
|
|
&pmuludq ($T2,&QWP(16*1,$base)); # s4*h1
|
|
|
|
|
&paddq ($D3,$T1);
|
|
|
|
|
&movdqa ($T1,$T0);
|
|
|
|
|
&pmuludq ($T0,&QWP(16*2,$base)); # s4*h2
|
|
|
|
|
&paddq ($D0,$T2);
|
|
|
|
|
&pmuludq ($T1,&QWP(16*3,$base)); # s4*h3
|
|
|
|
|
&movdqa ($MASK,&QWP(64,"ebx"));
|
|
|
|
|
&paddq ($D1,$T0);
|
|
|
|
|
&paddq ($D2,$T1);
|
|
|
|
|
}
|
|
|
|
|
&pmuladd (sub { my ($reg,$i)=@_;
|
|
|
|
|
&movdqa ($reg,&QWP(16*$i,"esp"));
|
|
|
|
|
},"edx");
|
|
|
|
|
|
|
|
|
|
sub lazy_reduction {
|
2016-03-03 20:36:33 +00:00
|
|
|
my $extra = shift;
|
2016-03-17 20:27:41 +00:00
|
|
|
my $paddx = defined($extra) ? paddq : paddd;
|
2016-03-03 20:36:33 +00:00
|
|
|
|
2016-02-23 21:25:40 +00:00
|
|
|
################################################################
|
|
|
|
|
# lazy reduction as discussed in "NEON crypto" by D.J. Bernstein
|
|
|
|
|
# and P. Schwabe
|
|
|
|
|
|
|
|
|
|
&movdqa ($T0,$D3);
|
|
|
|
|
&pand ($D3,$MASK);
|
|
|
|
|
&psrlq ($T0,26);
|
2016-03-03 20:36:33 +00:00
|
|
|
&$extra () if (defined($extra));
|
2016-02-23 21:25:40 +00:00
|
|
|
&paddq ($T0,$D4); # h3 -> h4
|
|
|
|
|
&movdqa ($T1,$D0);
|
|
|
|
|
&pand ($D0,$MASK);
|
|
|
|
|
&psrlq ($T1,26);
|
|
|
|
|
&movdqa ($D4,$T0);
|
|
|
|
|
&paddq ($T1,$D1); # h0 -> h1
|
|
|
|
|
&psrlq ($T0,26);
|
|
|
|
|
&pand ($D4,$MASK);
|
|
|
|
|
&movdqa ($D1,$T1);
|
|
|
|
|
&psrlq ($T1,26);
|
|
|
|
|
&paddd ($D0,$T0); # favour paddd when
|
|
|
|
|
# possible, because
|
|
|
|
|
# paddq is "broken"
|
|
|
|
|
# on Atom
|
|
|
|
|
&psllq ($T0,2);
|
2016-03-17 20:27:41 +00:00
|
|
|
&paddq ($T1,$D2); # h1 -> h2
|
|
|
|
|
&$paddx ($T0,$D0); # h4 -> h0
|
|
|
|
|
&pand ($D1,$MASK);
|
2016-02-23 21:25:40 +00:00
|
|
|
&movdqa ($D2,$T1);
|
|
|
|
|
&psrlq ($T1,26);
|
|
|
|
|
&pand ($D2,$MASK);
|
|
|
|
|
&paddd ($T1,$D3); # h2 -> h3
|
|
|
|
|
&movdqa ($D0,$T0);
|
|
|
|
|
&psrlq ($T0,26);
|
|
|
|
|
&movdqa ($D3,$T1);
|
|
|
|
|
&psrlq ($T1,26);
|
|
|
|
|
&pand ($D0,$MASK);
|
|
|
|
|
&paddd ($D1,$T0); # h0 -> h1
|
|
|
|
|
&pand ($D3,$MASK);
|
|
|
|
|
&paddd ($D4,$T1); # h3 -> h4
|
|
|
|
|
}
|
|
|
|
|
&lazy_reduction ();
|
|
|
|
|
|
|
|
|
|
&dec ("ecx");
|
|
|
|
|
&jz (&label("square_break"));
|
|
|
|
|
|
|
|
|
|
&punpcklqdq ($D0,&QWP(16*0,"esp")); # 0:r^1:0:r^2
|
|
|
|
|
&punpcklqdq ($D1,&QWP(16*1,"esp"));
|
|
|
|
|
&punpcklqdq ($D2,&QWP(16*2,"esp"));
|
|
|
|
|
&punpcklqdq ($D3,&QWP(16*3,"esp"));
|
|
|
|
|
&punpcklqdq ($D4,&QWP(16*4,"esp"));
|
|
|
|
|
&jmp (&label("square"));
|
|
|
|
|
|
|
|
|
|
&set_label("square_break");
|
|
|
|
|
&psllq ($D0,32); # -> r^3:0:r^4:0
|
|
|
|
|
&psllq ($D1,32);
|
|
|
|
|
&psllq ($D2,32);
|
|
|
|
|
&psllq ($D3,32);
|
|
|
|
|
&psllq ($D4,32);
|
|
|
|
|
&por ($D0,&QWP(16*0,"esp")); # r^3:r^1:r^4:r^2
|
|
|
|
|
&por ($D1,&QWP(16*1,"esp"));
|
|
|
|
|
&por ($D2,&QWP(16*2,"esp"));
|
|
|
|
|
&por ($D3,&QWP(16*3,"esp"));
|
|
|
|
|
&por ($D4,&QWP(16*4,"esp"));
|
|
|
|
|
|
|
|
|
|
&pshufd ($D0,$D0,0b10001101); # -> r^1:r^2:r^3:r^4
|
|
|
|
|
&pshufd ($D1,$D1,0b10001101);
|
|
|
|
|
&pshufd ($D2,$D2,0b10001101);
|
|
|
|
|
&pshufd ($D3,$D3,0b10001101);
|
|
|
|
|
&pshufd ($D4,$D4,0b10001101);
|
|
|
|
|
|
|
|
|
|
&movdqu (&QWP(16*0,"edi"),$D0); # save the table
|
|
|
|
|
&movdqu (&QWP(16*1,"edi"),$D1);
|
|
|
|
|
&movdqu (&QWP(16*2,"edi"),$D2);
|
|
|
|
|
&movdqu (&QWP(16*3,"edi"),$D3);
|
|
|
|
|
&movdqu (&QWP(16*4,"edi"),$D4);
|
|
|
|
|
|
|
|
|
|
&movdqa ($T1,$D1);
|
|
|
|
|
&movdqa ($T0,$D2);
|
|
|
|
|
&pslld ($T1,2);
|
|
|
|
|
&pslld ($T0,2);
|
|
|
|
|
&paddd ($T1,$D1); # *5
|
|
|
|
|
&paddd ($T0,$D2); # *5
|
|
|
|
|
&movdqu (&QWP(16*5,"edi"),$T1);
|
|
|
|
|
&movdqu (&QWP(16*6,"edi"),$T0);
|
|
|
|
|
&movdqa ($T1,$D3);
|
|
|
|
|
&movdqa ($T0,$D4);
|
|
|
|
|
&pslld ($T1,2);
|
|
|
|
|
&pslld ($T0,2);
|
|
|
|
|
&paddd ($T1,$D3); # *5
|
|
|
|
|
&paddd ($T0,$D4); # *5
|
|
|
|
|
&movdqu (&QWP(16*7,"edi"),$T1);
|
|
|
|
|
&movdqu (&QWP(16*8,"edi"),$T0);
|
|
|
|
|
|
|
|
|
|
&mov ("esp","ebp");
|
|
|
|
|
&lea ("edi",&DWP(-16*3,"edi")); # size de-optimization
|
|
|
|
|
&ret ();
|
|
|
|
|
&function_end_B("_poly1305_init_sse2");
|
|
|
|
|
|
|
|
|
|
&align (32);
|
|
|
|
|
&function_begin("_poly1305_blocks_sse2");
|
|
|
|
|
&mov ("edi",&wparam(0)); # ctx
|
|
|
|
|
&mov ("esi",&wparam(1)); # inp
|
|
|
|
|
&mov ("ecx",&wparam(2)); # len
|
|
|
|
|
|
|
|
|
|
&mov ("eax",&DWP(4*5,"edi")); # is_base2_26
|
|
|
|
|
&and ("ecx",-16);
|
|
|
|
|
&jz (&label("nodata"));
|
|
|
|
|
&cmp ("ecx",64);
|
|
|
|
|
&jae (&label("enter_sse2"));
|
|
|
|
|
&test ("eax","eax"); # is_base2_26?
|
|
|
|
|
&jz (&label("enter_blocks"));
|
|
|
|
|
|
|
|
|
|
&set_label("enter_sse2",16);
|
|
|
|
|
&call (&label("pic_point"));
|
|
|
|
|
&set_label("pic_point");
|
|
|
|
|
&blindpop("ebx");
|
|
|
|
|
&lea ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
|
|
|
|
|
|
|
|
|
|
&test ("eax","eax"); # is_base2_26?
|
|
|
|
|
&jnz (&label("base2_26"));
|
|
|
|
|
|
|
|
|
|
&call ("_poly1305_init_sse2");
|
|
|
|
|
|
|
|
|
|
################################################# base 2^32 -> base 2^26
|
|
|
|
|
&mov ("eax",&DWP(0,"edi"));
|
|
|
|
|
&mov ("ecx",&DWP(3,"edi"));
|
|
|
|
|
&mov ("edx",&DWP(6,"edi"));
|
|
|
|
|
&mov ("esi",&DWP(9,"edi"));
|
|
|
|
|
&mov ("ebp",&DWP(13,"edi"));
|
|
|
|
|
&mov (&DWP(4*5,"edi"),1); # is_base2_26
|
|
|
|
|
|
|
|
|
|
&shr ("ecx",2);
|
|
|
|
|
&and ("eax",0x3ffffff);
|
|
|
|
|
&shr ("edx",4);
|
|
|
|
|
&and ("ecx",0x3ffffff);
|
|
|
|
|
&shr ("esi",6);
|
|
|
|
|
&and ("edx",0x3ffffff);
|
|
|
|
|
|
|
|
|
|
&movd ($D0,"eax");
|
|
|
|
|
&movd ($D1,"ecx");
|
|
|
|
|
&movd ($D2,"edx");
|
|
|
|
|
&movd ($D3,"esi");
|
|
|
|
|
&movd ($D4,"ebp");
|
|
|
|
|
|
|
|
|
|
&mov ("esi",&wparam(1)); # [reload] inp
|
|
|
|
|
&mov ("ecx",&wparam(2)); # [reload] len
|
|
|
|
|
&jmp (&label("base2_32"));
|
|
|
|
|
|
|
|
|
|
&set_label("base2_26",16);
|
|
|
|
|
&movd ($D0,&DWP(4*0,"edi")); # load hash value
|
|
|
|
|
&movd ($D1,&DWP(4*1,"edi"));
|
|
|
|
|
&movd ($D2,&DWP(4*2,"edi"));
|
|
|
|
|
&movd ($D3,&DWP(4*3,"edi"));
|
|
|
|
|
&movd ($D4,&DWP(4*4,"edi"));
|
|
|
|
|
&movdqa ($MASK,&QWP(64,"ebx"));
|
|
|
|
|
|
|
|
|
|
&set_label("base2_32");
|
|
|
|
|
&mov ("eax",&wparam(3)); # padbit
|
|
|
|
|
&mov ("ebp","esp");
|
|
|
|
|
|
|
|
|
|
&sub ("esp",16*(5+5+5+9+9));
|
|
|
|
|
&and ("esp",-16);
|
|
|
|
|
|
|
|
|
|
&lea ("edi",&DWP(16*3,"edi")); # size optimization
|
|
|
|
|
&shl ("eax",24); # padbit
|
|
|
|
|
|
|
|
|
|
&test ("ecx",31);
|
|
|
|
|
&jz (&label("even"));
|
|
|
|
|
|
|
|
|
|
################################################################
|
|
|
|
|
# process single block, with SSE2, because it's still faster
|
|
|
|
|
# even though half of result is discarded
|
|
|
|
|
|
|
|
|
|
&movdqu ($T1,&QWP(0,"esi")); # input
|
|
|
|
|
&lea ("esi",&DWP(16,"esi"));
|
|
|
|
|
|
|
|
|
|
&movdqa ($T0,$T1); # -> base 2^26 ...
|
|
|
|
|
&pand ($T1,$MASK);
|
|
|
|
|
&paddd ($D0,$T1); # ... and accumuate
|
|
|
|
|
|
|
|
|
|
&movdqa ($T1,$T0);
|
|
|
|
|
&psrlq ($T0,26);
|
|
|
|
|
&psrldq ($T1,6);
|
|
|
|
|
&pand ($T0,$MASK);
|
|
|
|
|
&paddd ($D1,$T0);
|
|
|
|
|
|
|
|
|
|
&movdqa ($T0,$T1);
|
|
|
|
|
&psrlq ($T1,4);
|
|
|
|
|
&pand ($T1,$MASK);
|
|
|
|
|
&paddd ($D2,$T1);
|
|
|
|
|
|
|
|
|
|
&movdqa ($T1,$T0);
|
|
|
|
|
&psrlq ($T0,30);
|
|
|
|
|
&pand ($T0,$MASK);
|
|
|
|
|
&psrldq ($T1,7);
|
|
|
|
|
&paddd ($D3,$T0);
|
|
|
|
|
|
|
|
|
|
&movd ($T0,"eax"); # padbit
|
|
|
|
|
&paddd ($D4,$T1);
|
|
|
|
|
&movd ($T1,&DWP(16*0+12,"edi")); # r0
|
|
|
|
|
&paddd ($D4,$T0);
|
|
|
|
|
|
|
|
|
|
&movdqa (&QWP(16*0,"esp"),$D0);
|
|
|
|
|
&movdqa (&QWP(16*1,"esp"),$D1);
|
|
|
|
|
&movdqa (&QWP(16*2,"esp"),$D2);
|
|
|
|
|
&movdqa (&QWP(16*3,"esp"),$D3);
|
|
|
|
|
&movdqa (&QWP(16*4,"esp"),$D4);
|
|
|
|
|
|
|
|
|
|
################################################################
|
|
|
|
|
# d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
|
|
|
|
|
# d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
|
|
|
|
|
# d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
|
|
|
|
|
# d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
|
|
|
|
|
# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
|
|
|
|
|
|
|
|
|
|
&pmuludq ($D0,$T1); # h4*r0
|
|
|
|
|
&pmuludq ($D1,$T1); # h3*r0
|
|
|
|
|
&pmuludq ($D2,$T1); # h2*r0
|
|
|
|
|
&movd ($T0,&DWP(16*1+12,"edi")); # r1
|
|
|
|
|
&pmuludq ($D3,$T1); # h1*r0
|
|
|
|
|
&pmuludq ($D4,$T1); # h0*r0
|
|
|
|
|
|
|
|
|
|
&pmuladd (sub { my ($reg,$i)=@_;
|
|
|
|
|
&movd ($reg,&DWP(16*$i+12,"edi"));
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
&lazy_reduction ();
|
|
|
|
|
|
|
|
|
|
&sub ("ecx",16);
|
|
|
|
|
&jz (&label("done"));
|
|
|
|
|
|
|
|
|
|
&set_label("even");
|
|
|
|
|
&lea ("edx",&DWP(16*(5+5+5+9),"esp"));# size optimization
|
|
|
|
|
&lea ("eax",&DWP(-16*2,"esi"));
|
|
|
|
|
&sub ("ecx",64);
|
|
|
|
|
|
|
|
|
|
################################################################
|
|
|
|
|
# expand and copy pre-calculated table to stack
|
|
|
|
|
|
|
|
|
|
&movdqu ($T0,&QWP(16*0,"edi")); # r^1:r^2:r^3:r^4
|
|
|
|
|
&pshufd ($T1,$T0,0b01000100); # duplicate r^3:r^4
|
|
|
|
|
&cmovb ("esi","eax");
|
|
|
|
|
&pshufd ($T0,$T0,0b11101110); # duplicate r^1:r^2
|
|
|
|
|
&movdqa (&QWP(16*0,"edx"),$T1);
|
|
|
|
|
&lea ("eax",&DWP(16*10,"esp"));
|
|
|
|
|
&movdqu ($T1,&QWP(16*1,"edi"));
|
|
|
|
|
&movdqa (&QWP(16*(0-9),"edx"),$T0);
|
|
|
|
|
&pshufd ($T0,$T1,0b01000100);
|
|
|
|
|
&pshufd ($T1,$T1,0b11101110);
|
|
|
|
|
&movdqa (&QWP(16*1,"edx"),$T0);
|
|
|
|
|
&movdqu ($T0,&QWP(16*2,"edi"));
|
|
|
|
|
&movdqa (&QWP(16*(1-9),"edx"),$T1);
|
|
|
|
|
&pshufd ($T1,$T0,0b01000100);
|
|
|
|
|
&pshufd ($T0,$T0,0b11101110);
|
|
|
|
|
&movdqa (&QWP(16*2,"edx"),$T1);
|
|
|
|
|
&movdqu ($T1,&QWP(16*3,"edi"));
|
|
|
|
|
&movdqa (&QWP(16*(2-9),"edx"),$T0);
|
|
|
|
|
&pshufd ($T0,$T1,0b01000100);
|
|
|
|
|
&pshufd ($T1,$T1,0b11101110);
|
|
|
|
|
&movdqa (&QWP(16*3,"edx"),$T0);
|
|
|
|
|
&movdqu ($T0,&QWP(16*4,"edi"));
|
|
|
|
|
&movdqa (&QWP(16*(3-9),"edx"),$T1);
|
|
|
|
|
&pshufd ($T1,$T0,0b01000100);
|
|
|
|
|
&pshufd ($T0,$T0,0b11101110);
|
|
|
|
|
&movdqa (&QWP(16*4,"edx"),$T1);
|
|
|
|
|
&movdqu ($T1,&QWP(16*5,"edi"));
|
|
|
|
|
&movdqa (&QWP(16*(4-9),"edx"),$T0);
|
|
|
|
|
&pshufd ($T0,$T1,0b01000100);
|
|
|
|
|
&pshufd ($T1,$T1,0b11101110);
|
|
|
|
|
&movdqa (&QWP(16*5,"edx"),$T0);
|
|
|
|
|
&movdqu ($T0,&QWP(16*6,"edi"));
|
|
|
|
|
&movdqa (&QWP(16*(5-9),"edx"),$T1);
|
|
|
|
|
&pshufd ($T1,$T0,0b01000100);
|
|
|
|
|
&pshufd ($T0,$T0,0b11101110);
|
|
|
|
|
&movdqa (&QWP(16*6,"edx"),$T1);
|
|
|
|
|
&movdqu ($T1,&QWP(16*7,"edi"));
|
|
|
|
|
&movdqa (&QWP(16*(6-9),"edx"),$T0);
|
|
|
|
|
&pshufd ($T0,$T1,0b01000100);
|
|
|
|
|
&pshufd ($T1,$T1,0b11101110);
|
|
|
|
|
&movdqa (&QWP(16*7,"edx"),$T0);
|
|
|
|
|
&movdqu ($T0,&QWP(16*8,"edi"));
|
|
|
|
|
&movdqa (&QWP(16*(7-9),"edx"),$T1);
|
|
|
|
|
&pshufd ($T1,$T0,0b01000100);
|
|
|
|
|
&pshufd ($T0,$T0,0b11101110);
|
|
|
|
|
&movdqa (&QWP(16*8,"edx"),$T1);
|
|
|
|
|
&movdqa (&QWP(16*(8-9),"edx"),$T0);
|
|
|
|
|
|
|
|
|
|
sub load_input {
|
|
|
|
|
my ($inpbase,$offbase)=@_;
|
|
|
|
|
|
|
|
|
|
&movdqu ($T0,&QWP($inpbase+0,"esi")); # load input
|
|
|
|
|
&movdqu ($T1,&QWP($inpbase+16,"esi"));
|
|
|
|
|
&lea ("esi",&DWP(16*2,"esi"));
|
|
|
|
|
|
|
|
|
|
&movdqa (&QWP($offbase+16*2,"esp"),$D2);
|
|
|
|
|
&movdqa (&QWP($offbase+16*3,"esp"),$D3);
|
|
|
|
|
&movdqa (&QWP($offbase+16*4,"esp"),$D4);
|
|
|
|
|
|
|
|
|
|
&movdqa ($D2,$T0); # splat input
|
|
|
|
|
&movdqa ($D3,$T1);
|
|
|
|
|
&psrldq ($D2,6);
|
|
|
|
|
&psrldq ($D3,6);
|
|
|
|
|
&movdqa ($D4,$T0);
|
|
|
|
|
&punpcklqdq ($D2,$D3); # 2:3
|
|
|
|
|
&punpckhqdq ($D4,$T1); # 4
|
|
|
|
|
&punpcklqdq ($T0,$T1); # 0:1
|
|
|
|
|
|
|
|
|
|
&movdqa ($D3,$D2);
|
|
|
|
|
&psrlq ($D2,4);
|
|
|
|
|
&psrlq ($D3,30);
|
|
|
|
|
&movdqa ($T1,$T0);
|
|
|
|
|
&psrlq ($D4,40); # 4
|
|
|
|
|
&psrlq ($T1,26);
|
|
|
|
|
&pand ($T0,$MASK); # 0
|
|
|
|
|
&pand ($T1,$MASK); # 1
|
|
|
|
|
&pand ($D2,$MASK); # 2
|
|
|
|
|
&pand ($D3,$MASK); # 3
|
|
|
|
|
&por ($D4,&QWP(0,"ebx")); # padbit, yes, always
|
|
|
|
|
|
|
|
|
|
&movdqa (&QWP($offbase+16*0,"esp"),$D0) if ($offbase);
|
|
|
|
|
&movdqa (&QWP($offbase+16*1,"esp"),$D1) if ($offbase);
|
|
|
|
|
}
|
|
|
|
|
&load_input (16*2,16*5);
|
|
|
|
|
|
|
|
|
|
&jbe (&label("skip_loop"));
|
|
|
|
|
&jmp (&label("loop"));
|
|
|
|
|
|
|
|
|
|
&set_label("loop",32);
|
|
|
|
|
################################################################
|
|
|
|
|
# ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2
|
|
|
|
|
# ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r
|
|
|
|
|
# \___________________/
|
|
|
|
|
# ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2
|
|
|
|
|
# ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r
|
|
|
|
|
# \___________________/ \____________________/
|
|
|
|
|
################################################################
|
|
|
|
|
|
|
|
|
|
&movdqa ($T2,&QWP(16*(0-9),"edx")); # r0^2
|
|
|
|
|
&movdqa (&QWP(16*1,"eax"),$T1);
|
|
|
|
|
&movdqa (&QWP(16*2,"eax"),$D2);
|
|
|
|
|
&movdqa (&QWP(16*3,"eax"),$D3);
|
|
|
|
|
&movdqa (&QWP(16*4,"eax"),$D4);
|
|
|
|
|
|
|
|
|
|
################################################################
|
|
|
|
|
# d4 = h4*r0 + h0*r4 + h1*r3 + h2*r2 + h3*r1
|
|
|
|
|
# d3 = h3*r0 + h0*r3 + h1*r2 + h2*r1 + h4*5*r4
|
|
|
|
|
# d2 = h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3
|
|
|
|
|
# d1 = h1*r0 + h0*r1 + h2*5*r4 + h3*5*r3 + h4*5*r2
|
|
|
|
|
# d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
|
|
|
|
|
|
|
|
|
|
&movdqa ($D1,$T0);
|
|
|
|
|
&pmuludq ($T0,$T2); # h0*r0
|
|
|
|
|
&movdqa ($D0,$T1);
|
|
|
|
|
&pmuludq ($T1,$T2); # h1*r0
|
|
|
|
|
&pmuludq ($D2,$T2); # h2*r0
|
|
|
|
|
&pmuludq ($D3,$T2); # h3*r0
|
|
|
|
|
&pmuludq ($D4,$T2); # h4*r0
|
|
|
|
|
|
|
|
|
|
sub pmuladd_alt {
|
|
|
|
|
my $addr = shift;
|
|
|
|
|
|
|
|
|
|
&pmuludq ($D0,&$addr(8)); # h1*s4
|
|
|
|
|
&movdqa ($T2,$D1);
|
|
|
|
|
&pmuludq ($D1,&$addr(1)); # h0*r1
|
|
|
|
|
&paddq ($D0,$T0);
|
|
|
|
|
&movdqa ($T0,$T2);
|
|
|
|
|
&pmuludq ($T2,&$addr(2)); # h0*r2
|
|
|
|
|
&paddq ($D1,$T1);
|
|
|
|
|
&movdqa ($T1,$T0);
|
|
|
|
|
&pmuludq ($T0,&$addr(3)); # h0*r3
|
|
|
|
|
&paddq ($D2,$T2);
|
|
|
|
|
&movdqa ($T2,&QWP(16*1,"eax")); # pull h1
|
|
|
|
|
&pmuludq ($T1,&$addr(4)); # h0*r4
|
|
|
|
|
&paddq ($D3,$T0);
|
|
|
|
|
|
|
|
|
|
&movdqa ($T0,$T2);
|
|
|
|
|
&pmuludq ($T2,&$addr(1)); # h1*r1
|
|
|
|
|
&paddq ($D4,$T1);
|
|
|
|
|
&movdqa ($T1,$T0);
|
|
|
|
|
&pmuludq ($T0,&$addr(2)); # h1*r2
|
|
|
|
|
&paddq ($D2,$T2);
|
|
|
|
|
&movdqa ($T2,&QWP(16*2,"eax")); # pull h2
|
|
|
|
|
&pmuludq ($T1,&$addr(3)); # h1*r3
|
|
|
|
|
&paddq ($D3,$T0);
|
|
|
|
|
&movdqa ($T0,$T2);
|
|
|
|
|
&pmuludq ($T2,&$addr(7)); # h2*s3
|
|
|
|
|
&paddq ($D4,$T1);
|
|
|
|
|
&movdqa ($T1,$T0);
|
|
|
|
|
&pmuludq ($T0,&$addr(8)); # h2*s4
|
|
|
|
|
&paddq ($D0,$T2);
|
|
|
|
|
|
|
|
|
|
&movdqa ($T2,$T1);
|
|
|
|
|
&pmuludq ($T1,&$addr(1)); # h2*r1
|
|
|
|
|
&paddq ($D1,$T0);
|
|
|
|
|
&movdqa ($T0,&QWP(16*3,"eax")); # pull h3
|
|
|
|
|
&pmuludq ($T2,&$addr(2)); # h2*r2
|
|
|
|
|
&paddq ($D3,$T1);
|
|
|
|
|
&movdqa ($T1,$T0);
|
|
|
|
|
&pmuludq ($T0,&$addr(6)); # h3*s2
|
|
|
|
|
&paddq ($D4,$T2);
|
|
|
|
|
&movdqa ($T2,$T1);
|
|
|
|
|
&pmuludq ($T1,&$addr(7)); # h3*s3
|
|
|
|
|
&paddq ($D0,$T0);
|
|
|
|
|
&movdqa ($T0,$T2);
|
|
|
|
|
&pmuludq ($T2,&$addr(8)); # h3*s4
|
|
|
|
|
&paddq ($D1,$T1);
|
|
|
|
|
|
|
|
|
|
&movdqa ($T1,&QWP(16*4,"eax")); # pull h4
|
|
|
|
|
&pmuludq ($T0,&$addr(1)); # h3*r1
|
|
|
|
|
&paddq ($D2,$T2);
|
|
|
|
|
&movdqa ($T2,$T1);
|
|
|
|
|
&pmuludq ($T1,&$addr(8)); # h4*s4
|
|
|
|
|
&paddq ($D4,$T0);
|
|
|
|
|
&movdqa ($T0,$T2);
|
|
|
|
|
&pmuludq ($T2,&$addr(5)); # h4*s1
|
|
|
|
|
&paddq ($D3,$T1);
|
|
|
|
|
&movdqa ($T1,$T0);
|
|
|
|
|
&pmuludq ($T0,&$addr(6)); # h4*s2
|
|
|
|
|
&paddq ($D0,$T2);
|
|
|
|
|
&movdqa ($MASK,&QWP(64,"ebx"));
|
|
|
|
|
&pmuludq ($T1,&$addr(7)); # h4*s3
|
|
|
|
|
&paddq ($D1,$T0);
|
|
|
|
|
&paddq ($D2,$T1);
|
|
|
|
|
}
|
|
|
|
|
&pmuladd_alt (sub { my $i=shift; &QWP(16*($i-9),"edx"); });
|
|
|
|
|
|
|
|
|
|
&load_input (-16*2,0);
|
|
|
|
|
&lea ("eax",&DWP(-16*2,"esi"));
|
|
|
|
|
&sub ("ecx",64);
|
|
|
|
|
|
|
|
|
|
&paddd ($T0,&QWP(16*(5+0),"esp")); # add hash value
|
|
|
|
|
&paddd ($T1,&QWP(16*(5+1),"esp"));
|
|
|
|
|
&paddd ($D2,&QWP(16*(5+2),"esp"));
|
|
|
|
|
&paddd ($D3,&QWP(16*(5+3),"esp"));
|
|
|
|
|
&paddd ($D4,&QWP(16*(5+4),"esp"));
|
|
|
|
|
|
|
|
|
|
&cmovb ("esi","eax");
|
|
|
|
|
&lea ("eax",&DWP(16*10,"esp"));
|
|
|
|
|
|
|
|
|
|
&movdqa ($T2,&QWP(16*0,"edx")); # r0^4
|
|
|
|
|
&movdqa (&QWP(16*1,"esp"),$D1);
|
|
|
|
|
&movdqa (&QWP(16*1,"eax"),$T1);
|
|
|
|
|
&movdqa (&QWP(16*2,"eax"),$D2);
|
|
|
|
|
&movdqa (&QWP(16*3,"eax"),$D3);
|
|
|
|
|
&movdqa (&QWP(16*4,"eax"),$D4);
|
|
|
|
|
|
|
|
|
|
################################################################
|
|
|
|
|
# d4 += h4*r0 + h0*r4 + h1*r3 + h2*r2 + h3*r1
|
|
|
|
|
# d3 += h3*r0 + h0*r3 + h1*r2 + h2*r1 + h4*5*r4
|
|
|
|
|
# d2 += h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3
|
|
|
|
|
# d1 += h1*r0 + h0*r1 + h2*5*r4 + h3*5*r3 + h4*5*r2
|
|
|
|
|
# d0 += h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1
|
|
|
|
|
|
|
|
|
|
&movdqa ($D1,$T0);
|
|
|
|
|
&pmuludq ($T0,$T2); # h0*r0
|
|
|
|
|
&paddq ($T0,$D0);
|
|
|
|
|
&movdqa ($D0,$T1);
|
|
|
|
|
&pmuludq ($T1,$T2); # h1*r0
|
|
|
|
|
&pmuludq ($D2,$T2); # h2*r0
|
|
|
|
|
&pmuludq ($D3,$T2); # h3*r0
|
|
|
|
|
&pmuludq ($D4,$T2); # h4*r0
|
|
|
|
|
|
|
|
|
|
&paddq ($T1,&QWP(16*1,"esp"));
|
|
|
|
|
&paddq ($D2,&QWP(16*2,"esp"));
|
|
|
|
|
&paddq ($D3,&QWP(16*3,"esp"));
|
|
|
|
|
&paddq ($D4,&QWP(16*4,"esp"));
|
|
|
|
|
|
|
|
|
|
&pmuladd_alt (sub { my $i=shift; &QWP(16*$i,"edx"); });
|
|
|
|
|
|
|
|
|
|
&lazy_reduction ();
|
|
|
|
|
|
|
|
|
|
&load_input (16*2,16*5);
|
|
|
|
|
|
|
|
|
|
&ja (&label("loop"));
|
|
|
|
|
|
|
|
|
|
&set_label("skip_loop");
|
|
|
|
|
################################################################
|
|
|
|
|
# multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1
|
|
|
|
|
|
|
|
|
|
&pshufd ($T2,&QWP(16*(0-9),"edx"),0x10);# r0^n
|
|
|
|
|
&add ("ecx",32);
|
|
|
|
|
&jnz (&label("long_tail"));
|
|
|
|
|
|
|
|
|
|
&paddd ($T0,$D0); # add hash value
|
|
|
|
|
&paddd ($T1,$D1);
|
|
|
|
|
&paddd ($D2,&QWP(16*7,"esp"));
|
|
|
|
|
&paddd ($D3,&QWP(16*8,"esp"));
|
|
|
|
|
&paddd ($D4,&QWP(16*9,"esp"));
|
|
|
|
|
|
|
|
|
|
&set_label("long_tail");
|
|
|
|
|
|
|
|
|
|
&movdqa (&QWP(16*0,"eax"),$T0);
|
|
|
|
|
&movdqa (&QWP(16*1,"eax"),$T1);
|
|
|
|
|
&movdqa (&QWP(16*2,"eax"),$D2);
|
|
|
|
|
&movdqa (&QWP(16*3,"eax"),$D3);
|
|
|
|
|
&movdqa (&QWP(16*4,"eax"),$D4);
|
|
|
|
|
|
|
|
|
|
################################################################
|
|
|
|
|
# d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
|
|
|
|
|
# d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
|
|
|
|
|
# d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
|
|
|
|
|
# d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
|
|
|
|
|
# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
|
|
|
|
|
|
|
|
|
|
&pmuludq ($T0,$T2); # h0*r0
|
|
|
|
|
&pmuludq ($T1,$T2); # h1*r0
|
|
|
|
|
&pmuludq ($D2,$T2); # h2*r0
|
|
|
|
|
&movdqa ($D0,$T0);
|
|
|
|
|
&pshufd ($T0,&QWP(16*(1-9),"edx"),0x10);# r1^n
|
|
|
|
|
&pmuludq ($D3,$T2); # h3*r0
|
|
|
|
|
&movdqa ($D1,$T1);
|
|
|
|
|
&pmuludq ($D4,$T2); # h4*r0
|
|
|
|
|
|
|
|
|
|
&pmuladd (sub { my ($reg,$i)=@_;
|
|
|
|
|
&pshufd ($reg,&QWP(16*($i-9),"edx"),0x10);
|
|
|
|
|
},"eax");
|
|
|
|
|
|
|
|
|
|
&jz (&label("short_tail"));
|
|
|
|
|
|
|
|
|
|
&load_input (-16*2,0);
|
|
|
|
|
|
|
|
|
|
&pshufd ($T2,&QWP(16*0,"edx"),0x10); # r0^n
|
|
|
|
|
&paddd ($T0,&QWP(16*5,"esp")); # add hash value
|
|
|
|
|
&paddd ($T1,&QWP(16*6,"esp"));
|
|
|
|
|
&paddd ($D2,&QWP(16*7,"esp"));
|
|
|
|
|
&paddd ($D3,&QWP(16*8,"esp"));
|
|
|
|
|
&paddd ($D4,&QWP(16*9,"esp"));
|
|
|
|
|
|
|
|
|
|
################################################################
|
|
|
|
|
# multiply inp[0:1] by r^4:r^3 and accumulate
|
|
|
|
|
|
|
|
|
|
&movdqa (&QWP(16*0,"esp"),$T0);
|
|
|
|
|
&pmuludq ($T0,$T2); # h0*r0
|
|
|
|
|
&movdqa (&QWP(16*1,"esp"),$T1);
|
|
|
|
|
&pmuludq ($T1,$T2); # h1*r0
|
|
|
|
|
&paddq ($D0,$T0);
|
|
|
|
|
&movdqa ($T0,$D2);
|
|
|
|
|
&pmuludq ($D2,$T2); # h2*r0
|
|
|
|
|
&paddq ($D1,$T1);
|
|
|
|
|
&movdqa ($T1,$D3);
|
|
|
|
|
&pmuludq ($D3,$T2); # h3*r0
|
|
|
|
|
&paddq ($D2,&QWP(16*2,"esp"));
|
|
|
|
|
&movdqa (&QWP(16*2,"esp"),$T0);
|
|
|
|
|
&pshufd ($T0,&QWP(16*1,"edx"),0x10); # r1^n
|
|
|
|
|
&paddq ($D3,&QWP(16*3,"esp"));
|
|
|
|
|
&movdqa (&QWP(16*3,"esp"),$T1);
|
|
|
|
|
&movdqa ($T1,$D4);
|
|
|
|
|
&pmuludq ($D4,$T2); # h4*r0
|
|
|
|
|
&paddq ($D4,&QWP(16*4,"esp"));
|
|
|
|
|
&movdqa (&QWP(16*4,"esp"),$T1);
|
|
|
|
|
|
|
|
|
|
&pmuladd (sub { my ($reg,$i)=@_;
|
|
|
|
|
&pshufd ($reg,&QWP(16*$i,"edx"),0x10);
|
|
|
|
|
});
|
|
|
|
|
|
|
|
|
|
&set_label("short_tail");
|
|
|
|
|
|
|
|
|
|
################################################################
|
|
|
|
|
# horizontal addition
|
|
|
|
|
|
2016-03-03 20:36:33 +00:00
|
|
|
&pshufd ($T1,$D4,0b01001110);
|
|
|
|
|
&pshufd ($T0,$D3,0b01001110);
|
|
|
|
|
&paddq ($D4,$T1);
|
|
|
|
|
&paddq ($D3,$T0);
|
2016-02-23 21:25:40 +00:00
|
|
|
&pshufd ($T1,$D0,0b01001110);
|
|
|
|
|
&pshufd ($T0,$D1,0b01001110);
|
2016-03-03 20:36:33 +00:00
|
|
|
&paddq ($D0,$T1);
|
|
|
|
|
&paddq ($D1,$T0);
|
2016-02-23 21:25:40 +00:00
|
|
|
&pshufd ($T1,$D2,0b01001110);
|
2016-03-03 20:36:33 +00:00
|
|
|
#&paddq ($D2,$T1);
|
|
|
|
|
|
|
|
|
|
&lazy_reduction (sub { &paddq ($D2,$T1) });
|
2016-02-23 21:25:40 +00:00
|
|
|
|
|
|
|
|
&set_label("done");
|
|
|
|
|
&movd (&DWP(-16*3+4*0,"edi"),$D0); # store hash value
|
|
|
|
|
&movd (&DWP(-16*3+4*1,"edi"),$D1);
|
|
|
|
|
&movd (&DWP(-16*3+4*2,"edi"),$D2);
|
|
|
|
|
&movd (&DWP(-16*3+4*3,"edi"),$D3);
|
|
|
|
|
&movd (&DWP(-16*3+4*4,"edi"),$D4);
|
|
|
|
|
&mov ("esp","ebp");
|
2016-03-03 20:36:33 +00:00
|
|
|
&set_label("nodata");
|
2016-02-23 21:25:40 +00:00
|
|
|
&function_end("_poly1305_blocks_sse2");
|
|
|
|
|
|
|
|
|
|
&align (32);
|
|
|
|
|
&function_begin("_poly1305_emit_sse2");
|
|
|
|
|
&mov ("ebp",&wparam(0)); # context
|
|
|
|
|
|
|
|
|
|
&cmp (&DWP(4*5,"ebp"),0); # is_base2_26?
|
|
|
|
|
&je (&label("enter_emit"));
|
|
|
|
|
|
|
|
|
|
&mov ("eax",&DWP(4*0,"ebp")); # load hash value
|
|
|
|
|
&mov ("edi",&DWP(4*1,"ebp"));
|
|
|
|
|
&mov ("ecx",&DWP(4*2,"ebp"));
|
|
|
|
|
&mov ("edx",&DWP(4*3,"ebp"));
|
|
|
|
|
&mov ("esi",&DWP(4*4,"ebp"));
|
|
|
|
|
|
|
|
|
|
&mov ("ebx","edi"); # base 2^26 -> base 2^32
|
|
|
|
|
&shl ("edi",26);
|
|
|
|
|
&shr ("ebx",6);
|
|
|
|
|
&add ("eax","edi");
|
|
|
|
|
&mov ("edi","ecx");
|
|
|
|
|
&adc ("ebx",0);
|
|
|
|
|
|
|
|
|
|
&shl ("edi",20);
|
|
|
|
|
&shr ("ecx",12);
|
|
|
|
|
&add ("ebx","edi");
|
|
|
|
|
&mov ("edi","edx");
|
|
|
|
|
&adc ("ecx",0);
|
|
|
|
|
|
|
|
|
|
&shl ("edi",14);
|
|
|
|
|
&shr ("edx",18);
|
|
|
|
|
&add ("ecx","edi");
|
|
|
|
|
&mov ("edi","esi");
|
|
|
|
|
&adc ("edx",0);
|
|
|
|
|
|
|
|
|
|
&shl ("edi",8);
|
|
|
|
|
&shr ("esi",24);
|
|
|
|
|
&add ("edx","edi");
|
|
|
|
|
&adc ("esi",0); # can be partially reduced
|
|
|
|
|
|
|
|
|
|
&mov ("edi","esi"); # final reduction
|
|
|
|
|
&and ("esi",3);
|
|
|
|
|
&shr ("edi",2);
|
|
|
|
|
&lea ("ebp",&DWP(0,"edi","edi",4)); # *5
|
|
|
|
|
&mov ("edi",&wparam(1)); # output
|
|
|
|
|
add ("eax","ebp");
|
|
|
|
|
&mov ("ebp",&wparam(2)); # key
|
|
|
|
|
adc ("ebx",0);
|
|
|
|
|
adc ("ecx",0);
|
|
|
|
|
adc ("edx",0);
|
|
|
|
|
|
|
|
|
|
&movd ($D0,"eax"); # offload original hash value
|
|
|
|
|
&add ("eax",5); # compare to modulus
|
|
|
|
|
&movd ($D1,"ebx");
|
|
|
|
|
&adc ("ebx",0);
|
|
|
|
|
&movd ($D2,"ecx");
|
|
|
|
|
&adc ("ecx",0);
|
|
|
|
|
&movd ($D3,"edx");
|
|
|
|
|
&adc ("edx",0);
|
|
|
|
|
&adc ("esi",0);
|
|
|
|
|
&shr ("esi",2); # did it carry/borrow?
|
|
|
|
|
|
|
|
|
|
&neg ("esi"); # do we choose (hash-modulus) ...
|
|
|
|
|
&and ("eax","esi");
|
|
|
|
|
&and ("ebx","esi");
|
|
|
|
|
&and ("ecx","esi");
|
|
|
|
|
&and ("edx","esi");
|
|
|
|
|
&mov (&DWP(4*0,"edi"),"eax");
|
|
|
|
|
&movd ("eax",$D0);
|
|
|
|
|
&mov (&DWP(4*1,"edi"),"ebx");
|
|
|
|
|
&movd ("ebx",$D1);
|
|
|
|
|
&mov (&DWP(4*2,"edi"),"ecx");
|
|
|
|
|
&movd ("ecx",$D2);
|
|
|
|
|
&mov (&DWP(4*3,"edi"),"edx");
|
|
|
|
|
&movd ("edx",$D3);
|
|
|
|
|
|
|
|
|
|
¬ ("esi"); # ... or original hash value?
|
|
|
|
|
&and ("eax","esi");
|
|
|
|
|
&and ("ebx","esi");
|
|
|
|
|
&or ("eax",&DWP(4*0,"edi"));
|
|
|
|
|
&and ("ecx","esi");
|
|
|
|
|
&or ("ebx",&DWP(4*1,"edi"));
|
|
|
|
|
&and ("edx","esi");
|
|
|
|
|
&or ("ecx",&DWP(4*2,"edi"));
|
|
|
|
|
&or ("edx",&DWP(4*3,"edi"));
|
|
|
|
|
|
|
|
|
|
&add ("eax",&DWP(4*0,"ebp")); # accumulate key
|
|
|
|
|
&adc ("ebx",&DWP(4*1,"ebp"));
|
|
|
|
|
&mov (&DWP(4*0,"edi"),"eax");
|
|
|
|
|
&adc ("ecx",&DWP(4*2,"ebp"));
|
|
|
|
|
&mov (&DWP(4*1,"edi"),"ebx");
|
|
|
|
|
&adc ("edx",&DWP(4*3,"ebp"));
|
|
|
|
|
&mov (&DWP(4*2,"edi"),"ecx");
|
|
|
|
|
&mov (&DWP(4*3,"edi"),"edx");
|
|
|
|
|
&function_end("_poly1305_emit_sse2");
|
|
|
|
|
|
|
|
|
|
if ($avx>1) {
|
|
|
|
|
########################################################################
|
|
|
|
|
# Note that poly1305_init_avx2 operates on %xmm, I could have used
|
|
|
|
|
# poly1305_init_sse2...
|
|
|
|
|
|
|
|
|
|
&align (32);
|
|
|
|
|
&function_begin_B("_poly1305_init_avx2");
|
|
|
|
|
&vmovdqu ($D4,&QWP(4*6,"edi")); # key base 2^32
|
|
|
|
|
&lea ("edi",&DWP(16*3,"edi")); # size optimization
|
|
|
|
|
&mov ("ebp","esp");
|
|
|
|
|
&sub ("esp",16*(9+5));
|
|
|
|
|
&and ("esp",-16);
|
|
|
|
|
|
|
|
|
|
#&vpand ($D4,$D4,&QWP(96,"ebx")); # magic mask
|
|
|
|
|
&vmovdqa ($MASK,&QWP(64,"ebx"));
|
|
|
|
|
|
|
|
|
|
&vpand ($D0,$D4,$MASK); # -> base 2^26
|
|
|
|
|
&vpsrlq ($D1,$D4,26);
|
|
|
|
|
&vpsrldq ($D3,$D4,6);
|
|
|
|
|
&vpand ($D1,$D1,$MASK);
|
|
|
|
|
&vpsrlq ($D2,$D3,4)
|
|
|
|
|
&vpsrlq ($D3,$D3,30);
|
|
|
|
|
&vpand ($D2,$D2,$MASK);
|
|
|
|
|
&vpand ($D3,$D3,$MASK);
|
|
|
|
|
&vpsrldq ($D4,$D4,13);
|
|
|
|
|
|
|
|
|
|
&lea ("edx",&DWP(16*9,"esp")); # size optimization
|
|
|
|
|
&mov ("ecx",2);
|
|
|
|
|
&set_label("square");
|
|
|
|
|
&vmovdqa (&QWP(16*0,"esp"),$D0);
|
|
|
|
|
&vmovdqa (&QWP(16*1,"esp"),$D1);
|
|
|
|
|
&vmovdqa (&QWP(16*2,"esp"),$D2);
|
|
|
|
|
&vmovdqa (&QWP(16*3,"esp"),$D3);
|
|
|
|
|
&vmovdqa (&QWP(16*4,"esp"),$D4);
|
|
|
|
|
|
|
|
|
|
&vpslld ($T1,$D1,2);
|
|
|
|
|
&vpslld ($T0,$D2,2);
|
|
|
|
|
&vpaddd ($T1,$T1,$D1); # *5
|
|
|
|
|
&vpaddd ($T0,$T0,$D2); # *5
|
|
|
|
|
&vmovdqa (&QWP(16*5,"esp"),$T1);
|
|
|
|
|
&vmovdqa (&QWP(16*6,"esp"),$T0);
|
|
|
|
|
&vpslld ($T1,$D3,2);
|
|
|
|
|
&vpslld ($T0,$D4,2);
|
|
|
|
|
&vpaddd ($T1,$T1,$D3); # *5
|
|
|
|
|
&vpaddd ($T0,$T0,$D4); # *5
|
|
|
|
|
&vmovdqa (&QWP(16*7,"esp"),$T1);
|
|
|
|
|
&vmovdqa (&QWP(16*8,"esp"),$T0);
|
|
|
|
|
|
|
|
|
|
&vpshufd ($T0,$D0,0b01000100);
|
|
|
|
|
&vmovdqa ($T1,$D1);
|
|
|
|
|
&vpshufd ($D1,$D1,0b01000100);
|
|
|
|
|
&vpshufd ($D2,$D2,0b01000100);
|
|
|
|
|
&vpshufd ($D3,$D3,0b01000100);
|
|
|
|
|
&vpshufd ($D4,$D4,0b01000100);
|
|
|
|
|
&vmovdqa (&QWP(16*0,"edx"),$T0);
|
|
|
|
|
&vmovdqa (&QWP(16*1,"edx"),$D1);
|
|
|
|
|
&vmovdqa (&QWP(16*2,"edx"),$D2);
|
|
|
|
|
&vmovdqa (&QWP(16*3,"edx"),$D3);
|
|
|
|
|
&vmovdqa (&QWP(16*4,"edx"),$D4);
|
|
|
|
|
|
|
|
|
|
################################################################
|
|
|
|
|
# d4 = h4*r0 + h3*r1 + h2*r2 + h1*r3 + h0*r4
|
|
|
|
|
# d3 = h3*r0 + h2*r1 + h1*r2 + h0*r3 + h4*5*r4
|
|
|
|
|
# d2 = h2*r0 + h1*r1 + h0*r2 + h4*5*r3 + h3*5*r4
|
|
|
|
|
# d1 = h1*r0 + h0*r1 + h4*5*r2 + h3*5*r3 + h2*5*r4
|
|
|
|
|
# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4
|
|
|
|
|
|
|
|
|
|
&vpmuludq ($D4,$D4,$D0); # h4*r0
|
|
|
|
|
&vpmuludq ($D3,$D3,$D0); # h3*r0
|
|
|
|
|
&vpmuludq ($D2,$D2,$D0); # h2*r0
|
|
|
|
|
&vpmuludq ($D1,$D1,$D0); # h1*r0
|
|
|
|
|
&vpmuludq ($D0,$T0,$D0); # h0*r0
|
|
|
|
|
|
|
|
|
|
&vpmuludq ($T0,$T1,&QWP(16*3,"edx")); # r1*h3
|
|
|
|
|
&vpaddq ($D4,$D4,$T0);
|
|
|
|
|
&vpmuludq ($T2,$T1,&QWP(16*2,"edx")); # r1*h2
|
|
|
|
|
&vpaddq ($D3,$D3,$T2);
|
|
|
|
|
&vpmuludq ($T0,$T1,&QWP(16*1,"edx")); # r1*h1
|
|
|
|
|
&vpaddq ($D2,$D2,$T0);
|
|
|
|
|
&vmovdqa ($T2,&QWP(16*5,"esp")); # s1
|
|
|
|
|
&vpmuludq ($T1,$T1,&QWP(16*0,"edx")); # r1*h0
|
|
|
|
|
&vpaddq ($D1,$D1,$T1);
|
|
|
|
|
&vmovdqa ($T0,&QWP(16*2,"esp")); # r2
|
|
|
|
|
&vpmuludq ($T2,$T2,&QWP(16*4,"edx")); # s1*h4
|
|
|
|
|
&vpaddq ($D0,$D0,$T2);
|
|
|
|
|
|
|
|
|
|
&vpmuludq ($T1,$T0,&QWP(16*2,"edx")); # r2*h2
|
|
|
|
|
&vpaddq ($D4,$D4,$T1);
|
|
|
|
|
&vpmuludq ($T2,$T0,&QWP(16*1,"edx")); # r2*h1
|
|
|
|
|
&vpaddq ($D3,$D3,$T2);
|
|
|
|
|
&vmovdqa ($T1,&QWP(16*6,"esp")); # s2
|
|
|
|
|
&vpmuludq ($T0,$T0,&QWP(16*0,"edx")); # r2*h0
|
|
|
|
|
&vpaddq ($D2,$D2,$T0);
|
|
|
|
|
&vpmuludq ($T2,$T1,&QWP(16*4,"edx")); # s2*h4
|
|
|
|
|
&vpaddq ($D1,$D1,$T2);
|
|
|
|
|
&vmovdqa ($T0,&QWP(16*3,"esp")); # r3
|
|
|
|
|
&vpmuludq ($T1,$T1,&QWP(16*3,"edx")); # s2*h3
|
|
|
|
|
&vpaddq ($D0,$D0,$T1);
|
|
|
|
|
|
|
|
|
|
&vpmuludq ($T2,$T0,&QWP(16*1,"edx")); # r3*h1
|
|
|
|
|
&vpaddq ($D4,$D4,$T2);
|
|
|
|
|
&vmovdqa ($T1,&QWP(16*7,"esp")); # s3
|
|
|
|
|
&vpmuludq ($T0,$T0,&QWP(16*0,"edx")); # r3*h0
|
|
|
|
|
&vpaddq ($D3,$D3,$T0);
|
|
|
|
|
&vpmuludq ($T2,$T1,&QWP(16*4,"edx")); # s3*h4
|
|
|
|
|
&vpaddq ($D2,$D2,$T2);
|
|
|
|
|
&vpmuludq ($T0,$T1,&QWP(16*3,"edx")); # s3*h3
|
|
|
|
|
&vpaddq ($D1,$D1,$T0);
|
|
|
|
|
&vmovdqa ($T2,&QWP(16*4,"esp")); # r4
|
|
|
|
|
&vpmuludq ($T1,$T1,&QWP(16*2,"edx")); # s3*h2
|
|
|
|
|
&vpaddq ($D0,$D0,$T1);
|
|
|
|
|
|
|
|
|
|
&vmovdqa ($T0,&QWP(16*8,"esp")); # s4
|
|
|
|
|
&vpmuludq ($T2,$T2,&QWP(16*0,"edx")); # r4*h0
|
|
|
|
|
&vpaddq ($D4,$D4,$T2);
|
|
|
|
|
&vpmuludq ($T1,$T0,&QWP(16*4,"edx")); # s4*h4
|
|
|
|
|
&vpaddq ($D3,$D3,$T1);
|
|
|
|
|
&vpmuludq ($T2,$T0,&QWP(16*1,"edx")); # s4*h1
|
|
|
|
|
&vpaddq ($D0,$D0,$T2);
|
|
|
|
|
&vpmuludq ($T1,$T0,&QWP(16*2,"edx")); # s4*h2
|
|
|
|
|
&vpaddq ($D1,$D1,$T1);
|
|
|
|
|
&vmovdqa ($MASK,&QWP(64,"ebx"));
|
|
|
|
|
&vpmuludq ($T0,$T0,&QWP(16*3,"edx")); # s4*h3
|
|
|
|
|
&vpaddq ($D2,$D2,$T0);
|
|
|
|
|
|
|
|
|
|
################################################################
|
|
|
|
|
# lazy reduction
|
|
|
|
|
&vpsrlq ($T0,$D3,26);
|
|
|
|
|
&vpand ($D3,$D3,$MASK);
|
|
|
|
|
&vpsrlq ($T1,$D0,26);
|
|
|
|
|
&vpand ($D0,$D0,$MASK);
|
|
|
|
|
&vpaddq ($D4,$D4,$T0); # h3 -> h4
|
|
|
|
|
&vpaddq ($D1,$D1,$T1); # h0 -> h1
|
|
|
|
|
&vpsrlq ($T0,$D4,26);
|
|
|
|
|
&vpand ($D4,$D4,$MASK);
|
|
|
|
|
&vpsrlq ($T1,$D1,26);
|
|
|
|
|
&vpand ($D1,$D1,$MASK);
|
|
|
|
|
&vpaddq ($D2,$D2,$T1); # h1 -> h2
|
|
|
|
|
&vpaddd ($D0,$D0,$T0);
|
|
|
|
|
&vpsllq ($T0,$T0,2);
|
|
|
|
|
&vpsrlq ($T1,$D2,26);
|
|
|
|
|
&vpand ($D2,$D2,$MASK);
|
|
|
|
|
&vpaddd ($D0,$D0,$T0); # h4 -> h0
|
|
|
|
|
&vpaddd ($D3,$D3,$T1); # h2 -> h3
|
|
|
|
|
&vpsrlq ($T1,$D3,26);
|
|
|
|
|
&vpsrlq ($T0,$D0,26);
|
|
|
|
|
&vpand ($D0,$D0,$MASK);
|
|
|
|
|
&vpand ($D3,$D3,$MASK);
|
|
|
|
|
&vpaddd ($D1,$D1,$T0); # h0 -> h1
|
|
|
|
|
&vpaddd ($D4,$D4,$T1); # h3 -> h4
|
|
|
|
|
|
|
|
|
|
&dec ("ecx");
|
|
|
|
|
&jz (&label("square_break"));
|
|
|
|
|
|
|
|
|
|
&vpunpcklqdq ($D0,$D0,&QWP(16*0,"esp")); # 0:r^1:0:r^2
|
|
|
|
|
&vpunpcklqdq ($D1,$D1,&QWP(16*1,"esp"));
|
|
|
|
|
&vpunpcklqdq ($D2,$D2,&QWP(16*2,"esp"));
|
|
|
|
|
&vpunpcklqdq ($D3,$D3,&QWP(16*3,"esp"));
|
|
|
|
|
&vpunpcklqdq ($D4,$D4,&QWP(16*4,"esp"));
|
|
|
|
|
&jmp (&label("square"));
|
|
|
|
|
|
|
|
|
|
&set_label("square_break");
|
|
|
|
|
&vpsllq ($D0,$D0,32); # -> r^3:0:r^4:0
|
|
|
|
|
&vpsllq ($D1,$D1,32);
|
|
|
|
|
&vpsllq ($D2,$D2,32);
|
|
|
|
|
&vpsllq ($D3,$D3,32);
|
|
|
|
|
&vpsllq ($D4,$D4,32);
|
|
|
|
|
&vpor ($D0,$D0,&QWP(16*0,"esp")); # r^3:r^1:r^4:r^2
|
|
|
|
|
&vpor ($D1,$D1,&QWP(16*1,"esp"));
|
|
|
|
|
&vpor ($D2,$D2,&QWP(16*2,"esp"));
|
|
|
|
|
&vpor ($D3,$D3,&QWP(16*3,"esp"));
|
|
|
|
|
&vpor ($D4,$D4,&QWP(16*4,"esp"));
|
|
|
|
|
|
|
|
|
|
&vpshufd ($D0,$D0,0b10001101); # -> r^1:r^2:r^3:r^4
|
|
|
|
|
&vpshufd ($D1,$D1,0b10001101);
|
|
|
|
|
&vpshufd ($D2,$D2,0b10001101);
|
|
|
|
|
&vpshufd ($D3,$D3,0b10001101);
|
|
|
|
|
&vpshufd ($D4,$D4,0b10001101);
|
|
|
|
|
|
|
|
|
|
&vmovdqu (&QWP(16*0,"edi"),$D0); # save the table
|
|
|
|
|
&vmovdqu (&QWP(16*1,"edi"),$D1);
|
|
|
|
|
&vmovdqu (&QWP(16*2,"edi"),$D2);
|
|
|
|
|
&vmovdqu (&QWP(16*3,"edi"),$D3);
|
|
|
|
|
&vmovdqu (&QWP(16*4,"edi"),$D4);
|
|
|
|
|
|
|
|
|
|
&vpslld ($T1,$D1,2);
|
|
|
|
|
&vpslld ($T0,$D2,2);
|
|
|
|
|
&vpaddd ($T1,$T1,$D1); # *5
|
|
|
|
|
&vpaddd ($T0,$T0,$D2); # *5
|
|
|
|
|
&vmovdqu (&QWP(16*5,"edi"),$T1);
|
|
|
|
|
&vmovdqu (&QWP(16*6,"edi"),$T0);
|
|
|
|
|
&vpslld ($T1,$D3,2);
|
|
|
|
|
&vpslld ($T0,$D4,2);
|
|
|
|
|
&vpaddd ($T1,$T1,$D3); # *5
|
|
|
|
|
&vpaddd ($T0,$T0,$D4); # *5
|
|
|
|
|
&vmovdqu (&QWP(16*7,"edi"),$T1);
|
|
|
|
|
&vmovdqu (&QWP(16*8,"edi"),$T0);
|
|
|
|
|
|
|
|
|
|
&mov ("esp","ebp");
|
|
|
|
|
&lea ("edi",&DWP(-16*3,"edi")); # size de-optimization
|
|
|
|
|
&ret ();
|
|
|
|
|
&function_end_B("_poly1305_init_avx2");
|
|
|
|
|
|
|
|
|
|
########################################################################
|
|
|
|
|
# now it's time to switch to %ymm
|
|
|
|
|
|
|
|
|
|
my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("ymm$_",(0..7));
|
|
|
|
|
my $MASK=$T2;
|
|
|
|
|
|
|
|
|
|
sub X { my $reg=shift; $reg=~s/^ymm/xmm/; $reg; }
|
|
|
|
|
|
|
|
|
|
&align (32);
|
|
|
|
|
&function_begin("_poly1305_blocks_avx2");
|
|
|
|
|
&mov ("edi",&wparam(0)); # ctx
|
|
|
|
|
&mov ("esi",&wparam(1)); # inp
|
|
|
|
|
&mov ("ecx",&wparam(2)); # len
|
|
|
|
|
|
|
|
|
|
&mov ("eax",&DWP(4*5,"edi")); # is_base2_26
|
|
|
|
|
&and ("ecx",-16);
|
|
|
|
|
&jz (&label("nodata"));
|
|
|
|
|
&cmp ("ecx",64);
|
|
|
|
|
&jae (&label("enter_avx2"));
|
|
|
|
|
&test ("eax","eax"); # is_base2_26?
|
|
|
|
|
&jz (&label("enter_blocks"));
|
|
|
|
|
|
2016-03-03 20:36:33 +00:00
|
|
|
&set_label("enter_avx2");
|
2016-02-23 21:25:40 +00:00
|
|
|
&vzeroupper ();
|
|
|
|
|
|
|
|
|
|
&call (&label("pic_point"));
|
|
|
|
|
&set_label("pic_point");
|
|
|
|
|
&blindpop("ebx");
|
|
|
|
|
&lea ("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));
|
|
|
|
|
|
|
|
|
|
&test ("eax","eax"); # is_base2_26?
|
|
|
|
|
&jnz (&label("base2_26"));
|
|
|
|
|
|
|
|
|
|
&call ("_poly1305_init_avx2");
|
|
|
|
|
|
|
|
|
|
################################################# base 2^32 -> base 2^26
|
|
|
|
|
&mov ("eax",&DWP(0,"edi"));
|
|
|
|
|
&mov ("ecx",&DWP(3,"edi"));
|
|
|
|
|
&mov ("edx",&DWP(6,"edi"));
|
|
|
|
|
&mov ("esi",&DWP(9,"edi"));
|
|
|
|
|
&mov ("ebp",&DWP(13,"edi"));
|
|
|
|
|
|
|
|
|
|
&shr ("ecx",2);
|
|
|
|
|
&and ("eax",0x3ffffff);
|
|
|
|
|
&shr ("edx",4);
|
|
|
|
|
&and ("ecx",0x3ffffff);
|
|
|
|
|
&shr ("esi",6);
|
|
|
|
|
&and ("edx",0x3ffffff);
|
|
|
|
|
|
|
|
|
|
&mov (&DWP(4*0,"edi"),"eax");
|
|
|
|
|
&mov (&DWP(4*1,"edi"),"ecx");
|
|
|
|
|
&mov (&DWP(4*2,"edi"),"edx");
|
|
|
|
|
&mov (&DWP(4*3,"edi"),"esi");
|
|
|
|
|
&mov (&DWP(4*4,"edi"),"ebp");
|
|
|
|
|
&mov (&DWP(4*5,"edi"),1); # is_base2_26
|
|
|
|
|
|
|
|
|
|
&mov ("esi",&wparam(1)); # [reload] inp
|
|
|
|
|
&mov ("ecx",&wparam(2)); # [reload] len
|
|
|
|
|
|
|
|
|
|
&set_label("base2_26");
|
|
|
|
|
&mov ("eax",&wparam(3)); # padbit
|
|
|
|
|
&mov ("ebp","esp");
|
|
|
|
|
|
|
|
|
|
&sub ("esp",32*(5+9));
|
|
|
|
|
&and ("esp",-512); # ensure that frame
|
|
|
|
|
# doesn't cross page
|
|
|
|
|
# boundary, which is
|
|
|
|
|
# essential for
|
|
|
|
|
# misaligned 32-byte
|
|
|
|
|
# loads
|
|
|
|
|
|
|
|
|
|
################################################################
|
|
|
|
|
# expand and copy pre-calculated table to stack
|
|
|
|
|
|
|
|
|
|
&vmovdqu (&X($D0),&QWP(16*(3+0),"edi"));
|
|
|
|
|
&lea ("edx",&DWP(32*5+128,"esp")); # +128 size optimization
|
|
|
|
|
&vmovdqu (&X($D1),&QWP(16*(3+1),"edi"));
|
|
|
|
|
&vmovdqu (&X($D2),&QWP(16*(3+2),"edi"));
|
|
|
|
|
&vmovdqu (&X($D3),&QWP(16*(3+3),"edi"));
|
|
|
|
|
&vmovdqu (&X($D4),&QWP(16*(3+4),"edi"));
|
|
|
|
|
&lea ("edi",&DWP(16*3,"edi")); # size optimization
|
|
|
|
|
&vpermq ($D0,$D0,0b01000000); # 00001234 -> 12343434
|
|
|
|
|
&vpermq ($D1,$D1,0b01000000);
|
|
|
|
|
&vpermq ($D2,$D2,0b01000000);
|
|
|
|
|
&vpermq ($D3,$D3,0b01000000);
|
|
|
|
|
&vpermq ($D4,$D4,0b01000000);
|
|
|
|
|
&vpshufd ($D0,$D0,0b11001000); # 12343434 -> 14243444
|
|
|
|
|
&vpshufd ($D1,$D1,0b11001000);
|
|
|
|
|
&vpshufd ($D2,$D2,0b11001000);
|
|
|
|
|
&vpshufd ($D3,$D3,0b11001000);
|
|
|
|
|
&vpshufd ($D4,$D4,0b11001000);
|
|
|
|
|
&vmovdqa (&QWP(32*0-128,"edx"),$D0);
|
|
|
|
|
&vmovdqu (&X($D0),&QWP(16*5,"edi"));
|
|
|
|
|
&vmovdqa (&QWP(32*1-128,"edx"),$D1);
|
|
|
|
|
&vmovdqu (&X($D1),&QWP(16*6,"edi"));
|
|
|
|
|
&vmovdqa (&QWP(32*2-128,"edx"),$D2);
|
|
|
|
|
&vmovdqu (&X($D2),&QWP(16*7,"edi"));
|
|
|
|
|
&vmovdqa (&QWP(32*3-128,"edx"),$D3);
|
|
|
|
|
&vmovdqu (&X($D3),&QWP(16*8,"edi"));
|
|
|
|
|
&vmovdqa (&QWP(32*4-128,"edx"),$D4);
|
|
|
|
|
&vpermq ($D0,$D0,0b01000000);
|
|
|
|
|
&vpermq ($D1,$D1,0b01000000);
|
|
|
|
|
&vpermq ($D2,$D2,0b01000000);
|
|
|
|
|
&vpermq ($D3,$D3,0b01000000);
|
|
|
|
|
&vpshufd ($D0,$D0,0b11001000);
|
|
|
|
|
&vpshufd ($D1,$D1,0b11001000);
|
|
|
|
|
&vpshufd ($D2,$D2,0b11001000);
|
|
|
|
|
&vpshufd ($D3,$D3,0b11001000);
|
|
|
|
|
&vmovdqa (&QWP(32*5-128,"edx"),$D0);
|
|
|
|
|
&vmovd (&X($D0),&DWP(-16*3+4*0,"edi"));# load hash value
|
|
|
|
|
&vmovdqa (&QWP(32*6-128,"edx"),$D1);
|
|
|
|
|
&vmovd (&X($D1),&DWP(-16*3+4*1,"edi"));
|
|
|
|
|
&vmovdqa (&QWP(32*7-128,"edx"),$D2);
|
|
|
|
|
&vmovd (&X($D2),&DWP(-16*3+4*2,"edi"));
|
|
|
|
|
&vmovdqa (&QWP(32*8-128,"edx"),$D3);
|
|
|
|
|
&vmovd (&X($D3),&DWP(-16*3+4*3,"edi"));
|
|
|
|
|
&vmovd (&X($D4),&DWP(-16*3+4*4,"edi"));
|
|
|
|
|
&vmovdqa ($MASK,&QWP(64,"ebx"));
|
|
|
|
|
&neg ("eax"); # padbit
|
|
|
|
|
|
|
|
|
|
&test ("ecx",63);
|
|
|
|
|
&jz (&label("even"));
|
|
|
|
|
|
|
|
|
|
&mov ("edx","ecx");
|
|
|
|
|
&and ("ecx",-64);
|
|
|
|
|
&and ("edx",63);
|
|
|
|
|
|
|
|
|
|
&vmovdqu (&X($T0),&QWP(16*0,"esi"));
|
|
|
|
|
&cmp ("edx",32);
|
|
|
|
|
&jb (&label("one"));
|
|
|
|
|
|
|
|
|
|
&vmovdqu (&X($T1),&QWP(16*1,"esi"));
|
|
|
|
|
&je (&label("two"));
|
|
|
|
|
|
|
|
|
|
&vinserti128 ($T0,$T0,&QWP(16*2,"esi"),1);
|
|
|
|
|
&lea ("esi",&DWP(16*3,"esi"));
|
|
|
|
|
&lea ("ebx",&DWP(8,"ebx")); # three padbits
|
|
|
|
|
&lea ("edx",&DWP(32*5+128+8,"esp")); # --:r^1:r^2:r^3 (*)
|
|
|
|
|
&jmp (&label("tail"));
|
|
|
|
|
|
|
|
|
|
&set_label("two");
|
|
|
|
|
&lea ("esi",&DWP(16*2,"esi"));
|
|
|
|
|
&lea ("ebx",&DWP(16,"ebx")); # two padbits
|
|
|
|
|
&lea ("edx",&DWP(32*5+128+16,"esp"));# --:--:r^1:r^2 (*)
|
|
|
|
|
&jmp (&label("tail"));
|
|
|
|
|
|
|
|
|
|
&set_label("one");
|
|
|
|
|
&lea ("esi",&DWP(16*1,"esi"));
|
|
|
|
|
&vpxor ($T1,$T1,$T1);
|
|
|
|
|
&lea ("ebx",&DWP(32,"ebx","eax",8)); # one or no padbits
|
|
|
|
|
&lea ("edx",&DWP(32*5+128+24,"esp"));# --:--:--:r^1 (*)
|
|
|
|
|
&jmp (&label("tail"));
|
|
|
|
|
|
|
|
|
|
# (*) spots marked with '--' are data from next table entry, but they
|
|
|
|
|
# are multiplied by 0 and therefore rendered insignificant
|
|
|
|
|
|
|
|
|
|
&set_label("even",32);
|
|
|
|
|
&vmovdqu (&X($T0),&QWP(16*0,"esi")); # load input
|
|
|
|
|
&vmovdqu (&X($T1),&QWP(16*1,"esi"));
|
|
|
|
|
&vinserti128 ($T0,$T0,&QWP(16*2,"esi"),1);
|
|
|
|
|
&vinserti128 ($T1,$T1,&QWP(16*3,"esi"),1);
|
|
|
|
|
&lea ("esi",&DWP(16*4,"esi"));
|
|
|
|
|
&sub ("ecx",64);
|
|
|
|
|
&jz (&label("tail"));
|
|
|
|
|
|
|
|
|
|
&set_label("loop");
|
|
|
|
|
################################################################
|
|
|
|
|
# ((inp[0]*r^4+r[4])*r^4+r[8])*r^4
|
|
|
|
|
# ((inp[1]*r^4+r[5])*r^4+r[9])*r^3
|
|
|
|
|
# ((inp[2]*r^4+r[6])*r^4+r[10])*r^2
|
|
|
|
|
# ((inp[3]*r^4+r[7])*r^4+r[11])*r^1
|
|
|
|
|
# \________/ \_______/
|
|
|
|
|
################################################################
|
|
|
|
|
|
|
|
|
|
sub vsplat_input {
|
|
|
|
|
&vmovdqa (&QWP(32*2,"esp"),$D2);
|
|
|
|
|
&vpsrldq ($D2,$T0,6); # splat input
|
|
|
|
|
&vmovdqa (&QWP(32*0,"esp"),$D0);
|
|
|
|
|
&vpsrldq ($D0,$T1,6);
|
|
|
|
|
&vmovdqa (&QWP(32*1,"esp"),$D1);
|
|
|
|
|
&vpunpckhqdq ($D1,$T0,$T1); # 4
|
|
|
|
|
&vpunpcklqdq ($T0,$T0,$T1); # 0:1
|
|
|
|
|
&vpunpcklqdq ($D2,$D2,$D0); # 2:3
|
|
|
|
|
|
|
|
|
|
&vpsrlq ($D0,$D2,30);
|
|
|
|
|
&vpsrlq ($D2,$D2,4);
|
|
|
|
|
&vpsrlq ($T1,$T0,26);
|
|
|
|
|
&vpsrlq ($D1,$D1,40); # 4
|
|
|
|
|
&vpand ($D2,$D2,$MASK); # 2
|
|
|
|
|
&vpand ($T0,$T0,$MASK); # 0
|
|
|
|
|
&vpand ($T1,$T1,$MASK); # 1
|
|
|
|
|
&vpand ($D0,$D0,$MASK); # 3 (*)
|
|
|
|
|
&vpor ($D1,$D1,&QWP(0,"ebx")); # padbit, yes, always
|
|
|
|
|
|
|
|
|
|
# (*) note that output is counterintuitive, inp[3:4] is
|
|
|
|
|
# returned in $D1-2, while $D3-4 are preserved;
|
|
|
|
|
}
|
|
|
|
|
&vsplat_input ();
|
|
|
|
|
|
|
|
|
|
sub vpmuladd {
|
|
|
|
|
my $addr = shift;
|
|
|
|
|
|
|
|
|
|
&vpaddq ($D2,$D2,&QWP(32*2,"esp")); # add hash value
|
|
|
|
|
&vpaddq ($T0,$T0,&QWP(32*0,"esp"));
|
|
|
|
|
&vpaddq ($T1,$T1,&QWP(32*1,"esp"));
|
|
|
|
|
&vpaddq ($D0,$D0,$D3);
|
|
|
|
|
&vpaddq ($D1,$D1,$D4);
|
|
|
|
|
|
|
|
|
|
################################################################
|
|
|
|
|
# d3 = h2*r1 + h0*r3 + h1*r2 + h3*r0 + h4*5*r4
|
|
|
|
|
# d4 = h2*r2 + h0*r4 + h1*r3 + h3*r1 + h4*r0
|
|
|
|
|
# d0 = h2*5*r3 + h0*r0 + h1*5*r4 + h3*5*r2 + h4*5*r1
|
|
|
|
|
# d1 = h2*5*r4 + h0*r1 + h1*r0 + h3*5*r3 + h4*5*r2
|
|
|
|
|
# d2 = h2*r0 + h0*r2 + h1*r1 + h3*5*r4 + h4*5*r3
|
|
|
|
|
|
|
|
|
|
&vpmuludq ($D3,$D2,&$addr(1)); # d3 = h2*r1
|
|
|
|
|
&vmovdqa (QWP(32*1,"esp"),$T1);
|
|
|
|
|
&vpmuludq ($D4,$D2,&$addr(2)); # d4 = h2*r2
|
|
|
|
|
&vmovdqa (QWP(32*3,"esp"),$D0);
|
|
|
|
|
&vpmuludq ($D0,$D2,&$addr(7)); # d0 = h2*s3
|
|
|
|
|
&vmovdqa (QWP(32*4,"esp"),$D1);
|
|
|
|
|
&vpmuludq ($D1,$D2,&$addr(8)); # d1 = h2*s4
|
|
|
|
|
&vpmuludq ($D2,$D2,&$addr(0)); # d2 = h2*r0
|
|
|
|
|
|
|
|
|
|
&vpmuludq ($T2,$T0,&$addr(3)); # h0*r3
|
|
|
|
|
&vpaddq ($D3,$D3,$T2); # d3 += h0*r3
|
|
|
|
|
&vpmuludq ($T1,$T0,&$addr(4)); # h0*r4
|
|
|
|
|
&vpaddq ($D4,$D4,$T1); # d4 + h0*r4
|
|
|
|
|
&vpmuludq ($T2,$T0,&$addr(0)); # h0*r0
|
|
|
|
|
&vpaddq ($D0,$D0,$T2); # d0 + h0*r0
|
|
|
|
|
&vmovdqa ($T2,&QWP(32*1,"esp")); # h1
|
|
|
|
|
&vpmuludq ($T1,$T0,&$addr(1)); # h0*r1
|
|
|
|
|
&vpaddq ($D1,$D1,$T1); # d1 += h0*r1
|
|
|
|
|
&vpmuludq ($T0,$T0,&$addr(2)); # h0*r2
|
|
|
|
|
&vpaddq ($D2,$D2,$T0); # d2 += h0*r2
|
|
|
|
|
|
|
|
|
|
&vpmuludq ($T1,$T2,&$addr(2)); # h1*r2
|
|
|
|
|
&vpaddq ($D3,$D3,$T1); # d3 += h1*r2
|
|
|
|
|
&vpmuludq ($T0,$T2,&$addr(3)); # h1*r3
|
|
|
|
|
&vpaddq ($D4,$D4,$T0); # d4 += h1*r3
|
|
|
|
|
&vpmuludq ($T1,$T2,&$addr(8)); # h1*s4
|
|
|
|
|
&vpaddq ($D0,$D0,$T1); # d0 += h1*s4
|
|
|
|
|
&vmovdqa ($T1,&QWP(32*3,"esp")); # h3
|
|
|
|
|
&vpmuludq ($T0,$T2,&$addr(0)); # h1*r0
|
|
|
|
|
&vpaddq ($D1,$D1,$T0); # d1 += h1*r0
|
|
|
|
|
&vpmuludq ($T2,$T2,&$addr(1)); # h1*r1
|
|
|
|
|
&vpaddq ($D2,$D2,$T2); # d2 += h1*r1
|
|
|
|
|
|
|
|
|
|
&vpmuludq ($T0,$T1,&$addr(0)); # h3*r0
|
|
|
|
|
&vpaddq ($D3,$D3,$T0); # d3 += h3*r0
|
|
|
|
|
&vpmuludq ($T2,$T1,&$addr(1)); # h3*r1
|
|
|
|
|
&vpaddq ($D4,$D4,$T2); # d4 += h3*r1
|
|
|
|
|
&vpmuludq ($T0,$T1,&$addr(6)); # h3*s2
|
|
|
|
|
&vpaddq ($D0,$D0,$T0); # d0 += h3*s2
|
|
|
|
|
&vmovdqa ($T0,&QWP(32*4,"esp")); # h4
|
|
|
|
|
&vpmuludq ($T2,$T1,&$addr(7)); # h3*s3
|
|
|
|
|
&vpaddq ($D1,$D1,$T2); # d1+= h3*s3
|
|
|
|
|
&vpmuludq ($T1,$T1,&$addr(8)); # h3*s4
|
|
|
|
|
&vpaddq ($D2,$D2,$T1); # d2 += h3*s4
|
|
|
|
|
|
|
|
|
|
&vpmuludq ($T2,$T0,&$addr(8)); # h4*s4
|
|
|
|
|
&vpaddq ($D3,$D3,$T2); # d3 += h4*s4
|
|
|
|
|
&vpmuludq ($T1,$T0,&$addr(5)); # h4*s1
|
|
|
|
|
&vpaddq ($D0,$D0,$T1); # d0 += h4*s1
|
|
|
|
|
&vpmuludq ($T2,$T0,&$addr(0)); # h4*r0
|
|
|
|
|
&vpaddq ($D4,$D4,$T2); # d4 += h4*r0
|
|
|
|
|
&vmovdqa ($MASK,&QWP(64,"ebx"));
|
|
|
|
|
&vpmuludq ($T1,$T0,&$addr(6)); # h4*s2
|
|
|
|
|
&vpaddq ($D1,$D1,$T1); # d1 += h4*s2
|
|
|
|
|
&vpmuludq ($T0,$T0,&$addr(7)); # h4*s3
|
|
|
|
|
&vpaddq ($D2,$D2,$T0); # d2 += h4*s3
|
|
|
|
|
}
|
|
|
|
|
&vpmuladd (sub { my $i=shift; &QWP(32*$i-128,"edx"); });
|
|
|
|
|
|
|
|
|
|
sub vlazy_reduction {
|
|
|
|
|
################################################################
|
|
|
|
|
# lazy reduction
|
|
|
|
|
|
|
|
|
|
&vpsrlq ($T0,$D3,26);
|
|
|
|
|
&vpand ($D3,$D3,$MASK);
|
|
|
|
|
&vpsrlq ($T1,$D0,26);
|
|
|
|
|
&vpand ($D0,$D0,$MASK);
|
|
|
|
|
&vpaddq ($D4,$D4,$T0); # h3 -> h4
|
|
|
|
|
&vpaddq ($D1,$D1,$T1); # h0 -> h1
|
|
|
|
|
&vpsrlq ($T0,$D4,26);
|
|
|
|
|
&vpand ($D4,$D4,$MASK);
|
|
|
|
|
&vpsrlq ($T1,$D1,26);
|
|
|
|
|
&vpand ($D1,$D1,$MASK);
|
|
|
|
|
&vpaddq ($D2,$D2,$T1); # h1 -> h2
|
2016-03-17 20:27:41 +00:00
|
|
|
&vpaddq ($D0,$D0,$T0);
|
2016-02-23 21:25:40 +00:00
|
|
|
&vpsllq ($T0,$T0,2);
|
|
|
|
|
&vpsrlq ($T1,$D2,26);
|
|
|
|
|
&vpand ($D2,$D2,$MASK);
|
2016-03-17 20:27:41 +00:00
|
|
|
&vpaddq ($D0,$D0,$T0); # h4 -> h0
|
|
|
|
|
&vpaddq ($D3,$D3,$T1); # h2 -> h3
|
2016-02-23 21:25:40 +00:00
|
|
|
&vpsrlq ($T1,$D3,26);
|
|
|
|
|
&vpsrlq ($T0,$D0,26);
|
|
|
|
|
&vpand ($D0,$D0,$MASK);
|
|
|
|
|
&vpand ($D3,$D3,$MASK);
|
2016-03-17 20:27:41 +00:00
|
|
|
&vpaddq ($D1,$D1,$T0); # h0 -> h1
|
|
|
|
|
&vpaddq ($D4,$D4,$T1); # h3 -> h4
|
2016-02-23 21:25:40 +00:00
|
|
|
}
|
|
|
|
|
&vlazy_reduction();
|
|
|
|
|
|
|
|
|
|
&vmovdqu (&X($T0),&QWP(16*0,"esi")); # load input
|
|
|
|
|
&vmovdqu (&X($T1),&QWP(16*1,"esi"));
|
|
|
|
|
&vinserti128 ($T0,$T0,&QWP(16*2,"esi"),1);
|
|
|
|
|
&vinserti128 ($T1,$T1,&QWP(16*3,"esi"),1);
|
|
|
|
|
&lea ("esi",&DWP(16*4,"esi"));
|
|
|
|
|
&sub ("ecx",64);
|
|
|
|
|
&jnz (&label("loop"));
|
|
|
|
|
|
|
|
|
|
&set_label("tail");
|
|
|
|
|
&vsplat_input ();
|
|
|
|
|
&and ("ebx",-64); # restore pointer
|
|
|
|
|
|
|
|
|
|
&vpmuladd (sub { my $i=shift; &QWP(4+32*$i-128,"edx"); });
|
|
|
|
|
|
|
|
|
|
################################################################
|
|
|
|
|
# horizontal addition
|
|
|
|
|
|
2016-03-03 20:36:33 +00:00
|
|
|
&vpsrldq ($T0,$D4,8);
|
|
|
|
|
&vpsrldq ($T1,$D3,8);
|
|
|
|
|
&vpaddq ($D4,$D4,$T0);
|
2016-02-23 21:25:40 +00:00
|
|
|
&vpsrldq ($T0,$D0,8);
|
2016-03-03 20:36:33 +00:00
|
|
|
&vpaddq ($D3,$D3,$T1);
|
2016-02-23 21:25:40 +00:00
|
|
|
&vpsrldq ($T1,$D1,8);
|
|
|
|
|
&vpaddq ($D0,$D0,$T0);
|
|
|
|
|
&vpsrldq ($T0,$D2,8);
|
|
|
|
|
&vpaddq ($D1,$D1,$T1);
|
2016-03-03 20:36:33 +00:00
|
|
|
&vpermq ($T1,$D4,2); # keep folding
|
2016-02-23 21:25:40 +00:00
|
|
|
&vpaddq ($D2,$D2,$T0);
|
2016-03-03 20:36:33 +00:00
|
|
|
&vpermq ($T0,$D3,2);
|
|
|
|
|
&vpaddq ($D4,$D4,$T1);
|
|
|
|
|
&vpermq ($T1,$D0,2);
|
|
|
|
|
&vpaddq ($D3,$D3,$T0);
|
2016-02-23 21:25:40 +00:00
|
|
|
&vpermq ($T0,$D1,2);
|
|
|
|
|
&vpaddq ($D0,$D0,$T1);
|
|
|
|
|
&vpermq ($T1,$D2,2);
|
|
|
|
|
&vpaddq ($D1,$D1,$T0);
|
|
|
|
|
&vpaddq ($D2,$D2,$T1);
|
2016-03-03 20:36:33 +00:00
|
|
|
|
|
|
|
|
&vlazy_reduction();
|
2016-02-23 21:25:40 +00:00
|
|
|
|
|
|
|
|
&cmp ("ecx",0);
|
|
|
|
|
&je (&label("done"));
|
|
|
|
|
|
|
|
|
|
################################################################
|
|
|
|
|
# clear all but single word
|
|
|
|
|
|
|
|
|
|
&vpshufd (&X($D0),&X($D0),0b11111100);
|
|
|
|
|
&lea ("edx",&DWP(32*5+128,"esp")); # restore pointer
|
|
|
|
|
&vpshufd (&X($D1),&X($D1),0b11111100);
|
|
|
|
|
&vpshufd (&X($D2),&X($D2),0b11111100);
|
|
|
|
|
&vpshufd (&X($D3),&X($D3),0b11111100);
|
|
|
|
|
&vpshufd (&X($D4),&X($D4),0b11111100);
|
|
|
|
|
&jmp (&label("even"));
|
|
|
|
|
|
|
|
|
|
&set_label("done",16);
|
2016-03-03 20:36:33 +00:00
|
|
|
&vmovd (&DWP(-16*3+4*0,"edi"),&X($D0));# store hash value
|
|
|
|
|
&vmovd (&DWP(-16*3+4*1,"edi"),&X($D1));
|
|
|
|
|
&vmovd (&DWP(-16*3+4*2,"edi"),&X($D2));
|
|
|
|
|
&vmovd (&DWP(-16*3+4*3,"edi"),&X($D3));
|
|
|
|
|
&vmovd (&DWP(-16*3+4*4,"edi"),&X($D4));
|
2016-02-23 21:25:40 +00:00
|
|
|
&vzeroupper ();
|
|
|
|
|
&mov ("esp","ebp");
|
2016-03-03 20:36:33 +00:00
|
|
|
&set_label("nodata");
|
2016-02-23 21:25:40 +00:00
|
|
|
&function_end("_poly1305_blocks_avx2");
|
|
|
|
|
}
|
|
|
|
|
&set_label("const_sse2",64);
|
|
|
|
|
&data_word(1<<24,0, 1<<24,0, 1<<24,0, 1<<24,0);
|
|
|
|
|
&data_word(0,0, 0,0, 0,0, 0,0);
|
|
|
|
|
&data_word(0x03ffffff,0,0x03ffffff,0, 0x03ffffff,0, 0x03ffffff,0);
|
|
|
|
|
&data_word(0x0fffffff,0x0ffffffc,0x0ffffffc,0x0ffffffc);
|
|
|
|
|
}
|
|
|
|
|
&asciz ("Poly1305 for x86, CRYPTOGAMS by <appro\@openssl.org>");
|
|
|
|
|
&align (4);
|
|
|
|
|
|
|
|
|
|
&asm_finish();
|