|
- #!/usr/bin/env perl
-
- # Copyright (c) 2014, Intel Corporation.
- #
- # Permission to use, copy, modify, and/or distribute this software for any
- # purpose with or without fee is hereby granted, provided that the above
- # copyright notice and this permission notice appear in all copies.
- #
- # THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
- # WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
- # MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
- # SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
- # WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
- # OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
- # CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE.
-
- # Developers and authors:
- # Shay Gueron (1, 2), and Vlad Krasnov (1)
- # (1) Intel Corporation, Israel Development Center
- # (2) University of Haifa
-
- # Reference:
- # S.Gueron and V.Krasnov, "Fast Prime Field Elliptic Curve Cryptography with
- # 256 Bit Primes"
-
- # Further optimization by <appro@openssl.org>:
- #
- # this/original
- # Opteron +12-49%
- # Bulldozer +14-45%
- # P4 +18-46%
- # Westmere +12-34%
- # Sandy Bridge +9-35%
- # Ivy Bridge +9-35%
- # Haswell +8-37%
- # Broadwell +18-58%
- # Atom +15-50%
- # VIA Nano +43-160%
- #
- # Ranges denote minimum and maximum improvement coefficients depending
- # on benchmark.
-
- $flavour = shift;
- $output = shift;
- if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
-
- $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
-
- $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
- ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
- ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or
- die "can't locate x86_64-xlate.pl";
-
- open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
- *STDOUT=*OUT;
-
- # TODO: enable these after testing. $avx goes to two and $addx to one.
- $avx=0;
- $addx=0;
-
- $code.=<<___;
- .text
- .extern OPENSSL_ia32cap_P
-
- # The polynomial
- .align 64
- .Lpoly:
- .quad 0xffffffffffffffff, 0x00000000ffffffff, 0x0000000000000000, 0xffffffff00000001
-
- .LOne:
- .long 1,1,1,1,1,1,1,1
- .LTwo:
- .long 2,2,2,2,2,2,2,2
- .LThree:
- .long 3,3,3,3,3,3,3,3
- .LONE_mont:
- .quad 0x0000000000000001, 0xffffffff00000000, 0xffffffffffffffff, 0x00000000fffffffe
- ___
-
- {
- ################################################################################
- # void ecp_nistz256_mul_by_2(uint64_t res[4], uint64_t a[4]);
-
- my ($a0,$a1,$a2,$a3)=map("%r$_",(8..11));
- my ($t0,$t1,$t2,$t3,$t4)=("%rax","%rdx","%rcx","%r12","%r13");
- my ($r_ptr,$a_ptr,$b_ptr)=("%rdi","%rsi","%rdx");
-
- $code.=<<___;
-
- .type ecp_nistz256_mul_by_2,\@function,2
- .align 64
- ecp_nistz256_mul_by_2:
- push %r12
- push %r13
-
- mov 8*0($a_ptr), $a0
- mov 8*1($a_ptr), $a1
- add $a0, $a0 # a0:a3+a0:a3
- mov 8*2($a_ptr), $a2
- adc $a1, $a1
- mov 8*3($a_ptr), $a3
- lea .Lpoly(%rip), $a_ptr
- mov $a0, $t0
- adc $a2, $a2
- adc $a3, $a3
- mov $a1, $t1
- sbb $t4, $t4
-
- sub 8*0($a_ptr), $a0
- mov $a2, $t2
- sbb 8*1($a_ptr), $a1
- sbb 8*2($a_ptr), $a2
- mov $a3, $t3
- sbb 8*3($a_ptr), $a3
- test $t4, $t4
-
- cmovz $t0, $a0
- cmovz $t1, $a1
- mov $a0, 8*0($r_ptr)
- cmovz $t2, $a2
- mov $a1, 8*1($r_ptr)
- cmovz $t3, $a3
- mov $a2, 8*2($r_ptr)
- mov $a3, 8*3($r_ptr)
-
- pop %r13
- pop %r12
- ret
- .size ecp_nistz256_mul_by_2,.-ecp_nistz256_mul_by_2
-
- ################################################################################
- # void ecp_nistz256_neg(uint64_t res[4], uint64_t a[4]);
- .globl ecp_nistz256_neg
- .type ecp_nistz256_neg,\@function,2
- .align 32
- ecp_nistz256_neg:
- push %r12
- push %r13
-
- xor $a0, $a0
- xor $a1, $a1
- xor $a2, $a2
- xor $a3, $a3
- xor $t4, $t4
-
- sub 8*0($a_ptr), $a0
- sbb 8*1($a_ptr), $a1
- sbb 8*2($a_ptr), $a2
- mov $a0, $t0
- sbb 8*3($a_ptr), $a3
- lea .Lpoly(%rip), $a_ptr
- mov $a1, $t1
- sbb \$0, $t4
-
- add 8*0($a_ptr), $a0
- mov $a2, $t2
- adc 8*1($a_ptr), $a1
- adc 8*2($a_ptr), $a2
- mov $a3, $t3
- adc 8*3($a_ptr), $a3
- test $t4, $t4
-
- cmovz $t0, $a0
- cmovz $t1, $a1
- mov $a0, 8*0($r_ptr)
- cmovz $t2, $a2
- mov $a1, 8*1($r_ptr)
- cmovz $t3, $a3
- mov $a2, 8*2($r_ptr)
- mov $a3, 8*3($r_ptr)
-
- pop %r13
- pop %r12
- ret
- .size ecp_nistz256_neg,.-ecp_nistz256_neg
- ___
- }
- {
- my ($r_ptr,$a_ptr,$b_org,$b_ptr)=("%rdi","%rsi","%rdx","%rbx");
- my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("%r$_",(8..15));
- my ($t0,$t1,$t2,$t3,$t4)=("%rcx","%rbp","%rbx","%rdx","%rax");
- my ($poly1,$poly3)=($acc6,$acc7);
-
- $code.=<<___;
- ################################################################################
- # void ecp_nistz256_mul_mont(
- # uint64_t res[4],
- # uint64_t a[4],
- # uint64_t b[4]);
-
- .globl ecp_nistz256_mul_mont
- .type ecp_nistz256_mul_mont,\@function,3
- .align 32
- ecp_nistz256_mul_mont:
- ___
- $code.=<<___ if ($addx);
- mov \$0x80100, %ecx
- and OPENSSL_ia32cap_P+8(%rip), %ecx
- ___
- $code.=<<___;
- .Lmul_mont:
- push %rbp
- push %rbx
- push %r12
- push %r13
- push %r14
- push %r15
- ___
- $code.=<<___ if ($addx);
- cmp \$0x80100, %ecx
- je .Lmul_montx
- ___
- $code.=<<___;
- mov $b_org, $b_ptr
- mov 8*0($b_org), %rax
- mov 8*0($a_ptr), $acc1
- mov 8*1($a_ptr), $acc2
- mov 8*2($a_ptr), $acc3
- mov 8*3($a_ptr), $acc4
-
- call __ecp_nistz256_mul_montq
- ___
- $code.=<<___ if ($addx);
- jmp .Lmul_mont_done
-
- .align 32
- .Lmul_montx:
- mov $b_org, $b_ptr
- mov 8*0($b_org), %rdx
- mov 8*0($a_ptr), $acc1
- mov 8*1($a_ptr), $acc2
- mov 8*2($a_ptr), $acc3
- mov 8*3($a_ptr), $acc4
- lea -128($a_ptr), $a_ptr # control u-op density
-
- call __ecp_nistz256_mul_montx
- ___
- $code.=<<___;
- .Lmul_mont_done:
- pop %r15
- pop %r14
- pop %r13
- pop %r12
- pop %rbx
- pop %rbp
- ret
- .size ecp_nistz256_mul_mont,.-ecp_nistz256_mul_mont
-
- .type __ecp_nistz256_mul_montq,\@abi-omnipotent
- .align 32
- __ecp_nistz256_mul_montq:
- ########################################################################
- # Multiply a by b[0]
- mov %rax, $t1
- mulq $acc1
- mov .Lpoly+8*1(%rip),$poly1
- mov %rax, $acc0
- mov $t1, %rax
- mov %rdx, $acc1
-
- mulq $acc2
- mov .Lpoly+8*3(%rip),$poly3
- add %rax, $acc1
- mov $t1, %rax
- adc \$0, %rdx
- mov %rdx, $acc2
-
- mulq $acc3
- add %rax, $acc2
- mov $t1, %rax
- adc \$0, %rdx
- mov %rdx, $acc3
-
- mulq $acc4
- add %rax, $acc3
- mov $acc0, %rax
- adc \$0, %rdx
- xor $acc5, $acc5
- mov %rdx, $acc4
-
- ########################################################################
- # First reduction step
- # Basically now we want to multiply acc[0] by p256,
- # and add the result to the acc.
- # Due to the special form of p256 we do some optimizations
- #
- # acc[0] x p256[0..1] = acc[0] x 2^96 - acc[0]
- # then we add acc[0] and get acc[0] x 2^96
-
- mov $acc0, $t1
- shl \$32, $acc0
- mulq $poly3
- shr \$32, $t1
- add $acc0, $acc1 # +=acc[0]<<96
- adc $t1, $acc2
- adc %rax, $acc3
- mov 8*1($b_ptr), %rax
- adc %rdx, $acc4
- adc \$0, $acc5
- xor $acc0, $acc0
-
- ########################################################################
- # Multiply by b[1]
- mov %rax, $t1
- mulq 8*0($a_ptr)
- add %rax, $acc1
- mov $t1, %rax
- adc \$0, %rdx
- mov %rdx, $t0
-
- mulq 8*1($a_ptr)
- add $t0, $acc2
- adc \$0, %rdx
- add %rax, $acc2
- mov $t1, %rax
- adc \$0, %rdx
- mov %rdx, $t0
-
- mulq 8*2($a_ptr)
- add $t0, $acc3
- adc \$0, %rdx
- add %rax, $acc3
- mov $t1, %rax
- adc \$0, %rdx
- mov %rdx, $t0
-
- mulq 8*3($a_ptr)
- add $t0, $acc4
- adc \$0, %rdx
- add %rax, $acc4
- mov $acc1, %rax
- adc %rdx, $acc5
- adc \$0, $acc0
-
- ########################################################################
- # Second reduction step
- mov $acc1, $t1
- shl \$32, $acc1
- mulq $poly3
- shr \$32, $t1
- add $acc1, $acc2
- adc $t1, $acc3
- adc %rax, $acc4
- mov 8*2($b_ptr), %rax
- adc %rdx, $acc5
- adc \$0, $acc0
- xor $acc1, $acc1
-
- ########################################################################
- # Multiply by b[2]
- mov %rax, $t1
- mulq 8*0($a_ptr)
- add %rax, $acc2
- mov $t1, %rax
- adc \$0, %rdx
- mov %rdx, $t0
-
- mulq 8*1($a_ptr)
- add $t0, $acc3
- adc \$0, %rdx
- add %rax, $acc3
- mov $t1, %rax
- adc \$0, %rdx
- mov %rdx, $t0
-
- mulq 8*2($a_ptr)
- add $t0, $acc4
- adc \$0, %rdx
- add %rax, $acc4
- mov $t1, %rax
- adc \$0, %rdx
- mov %rdx, $t0
-
- mulq 8*3($a_ptr)
- add $t0, $acc5
- adc \$0, %rdx
- add %rax, $acc5
- mov $acc2, %rax
- adc %rdx, $acc0
- adc \$0, $acc1
-
- ########################################################################
- # Third reduction step
- mov $acc2, $t1
- shl \$32, $acc2
- mulq $poly3
- shr \$32, $t1
- add $acc2, $acc3
- adc $t1, $acc4
- adc %rax, $acc5
- mov 8*3($b_ptr), %rax
- adc %rdx, $acc0
- adc \$0, $acc1
- xor $acc2, $acc2
-
- ########################################################################
- # Multiply by b[3]
- mov %rax, $t1
- mulq 8*0($a_ptr)
- add %rax, $acc3
- mov $t1, %rax
- adc \$0, %rdx
- mov %rdx, $t0
-
- mulq 8*1($a_ptr)
- add $t0, $acc4
- adc \$0, %rdx
- add %rax, $acc4
- mov $t1, %rax
- adc \$0, %rdx
- mov %rdx, $t0
-
- mulq 8*2($a_ptr)
- add $t0, $acc5
- adc \$0, %rdx
- add %rax, $acc5
- mov $t1, %rax
- adc \$0, %rdx
- mov %rdx, $t0
-
- mulq 8*3($a_ptr)
- add $t0, $acc0
- adc \$0, %rdx
- add %rax, $acc0
- mov $acc3, %rax
- adc %rdx, $acc1
- adc \$0, $acc2
-
- ########################################################################
- # Final reduction step
- mov $acc3, $t1
- shl \$32, $acc3
- mulq $poly3
- shr \$32, $t1
- add $acc3, $acc4
- adc $t1, $acc5
- mov $acc4, $t0
- adc %rax, $acc0
- adc %rdx, $acc1
- mov $acc5, $t1
- adc \$0, $acc2
-
- ########################################################################
- # Branch-less conditional subtraction of P
- sub \$-1, $acc4 # .Lpoly[0]
- mov $acc0, $t2
- sbb $poly1, $acc5 # .Lpoly[1]
- sbb \$0, $acc0 # .Lpoly[2]
- mov $acc1, $t3
- sbb $poly3, $acc1 # .Lpoly[3]
- sbb \$0, $acc2
-
- cmovc $t0, $acc4
- cmovc $t1, $acc5
- mov $acc4, 8*0($r_ptr)
- cmovc $t2, $acc0
- mov $acc5, 8*1($r_ptr)
- cmovc $t3, $acc1
- mov $acc0, 8*2($r_ptr)
- mov $acc1, 8*3($r_ptr)
-
- ret
- .size __ecp_nistz256_mul_montq,.-__ecp_nistz256_mul_montq
-
- ################################################################################
- # void ecp_nistz256_sqr_mont(
- # uint64_t res[4],
- # uint64_t a[4]);
-
- # we optimize the square according to S.Gueron and V.Krasnov,
- # "Speeding up Big-Number Squaring"
- .globl ecp_nistz256_sqr_mont
- .type ecp_nistz256_sqr_mont,\@function,2
- .align 32
- ecp_nistz256_sqr_mont:
- ___
- $code.=<<___ if ($addx);
- mov \$0x80100, %ecx
- and OPENSSL_ia32cap_P+8(%rip), %ecx
- ___
- $code.=<<___;
- push %rbp
- push %rbx
- push %r12
- push %r13
- push %r14
- push %r15
- ___
- $code.=<<___ if ($addx);
- cmp \$0x80100, %ecx
- je .Lsqr_montx
- ___
- $code.=<<___;
- mov 8*0($a_ptr), %rax
- mov 8*1($a_ptr), $acc6
- mov 8*2($a_ptr), $acc7
- mov 8*3($a_ptr), $acc0
-
- call __ecp_nistz256_sqr_montq
- ___
- $code.=<<___ if ($addx);
- jmp .Lsqr_mont_done
-
- .align 32
- .Lsqr_montx:
- mov 8*0($a_ptr), %rdx
- mov 8*1($a_ptr), $acc6
- mov 8*2($a_ptr), $acc7
- mov 8*3($a_ptr), $acc0
- lea -128($a_ptr), $a_ptr # control u-op density
-
- call __ecp_nistz256_sqr_montx
- ___
- $code.=<<___;
- .Lsqr_mont_done:
- pop %r15
- pop %r14
- pop %r13
- pop %r12
- pop %rbx
- pop %rbp
- ret
- .size ecp_nistz256_sqr_mont,.-ecp_nistz256_sqr_mont
-
- .type __ecp_nistz256_sqr_montq,\@abi-omnipotent
- .align 32
- __ecp_nistz256_sqr_montq:
- mov %rax, $acc5
- mulq $acc6 # a[1]*a[0]
- mov %rax, $acc1
- mov $acc7, %rax
- mov %rdx, $acc2
-
- mulq $acc5 # a[0]*a[2]
- add %rax, $acc2
- mov $acc0, %rax
- adc \$0, %rdx
- mov %rdx, $acc3
-
- mulq $acc5 # a[0]*a[3]
- add %rax, $acc3
- mov $acc7, %rax
- adc \$0, %rdx
- mov %rdx, $acc4
-
- #################################
- mulq $acc6 # a[1]*a[2]
- add %rax, $acc3
- mov $acc0, %rax
- adc \$0, %rdx
- mov %rdx, $t1
-
- mulq $acc6 # a[1]*a[3]
- add %rax, $acc4
- mov $acc0, %rax
- adc \$0, %rdx
- add $t1, $acc4
- mov %rdx, $acc5
- adc \$0, $acc5
-
- #################################
- mulq $acc7 # a[2]*a[3]
- xor $acc7, $acc7
- add %rax, $acc5
- mov 8*0($a_ptr), %rax
- mov %rdx, $acc6
- adc \$0, $acc6
-
- add $acc1, $acc1 # acc1:6<<1
- adc $acc2, $acc2
- adc $acc3, $acc3
- adc $acc4, $acc4
- adc $acc5, $acc5
- adc $acc6, $acc6
- adc \$0, $acc7
-
- mulq %rax
- mov %rax, $acc0
- mov 8*1($a_ptr), %rax
- mov %rdx, $t0
-
- mulq %rax
- add $t0, $acc1
- adc %rax, $acc2
- mov 8*2($a_ptr), %rax
- adc \$0, %rdx
- mov %rdx, $t0
-
- mulq %rax
- add $t0, $acc3
- adc %rax, $acc4
- mov 8*3($a_ptr), %rax
- adc \$0, %rdx
- mov %rdx, $t0
-
- mulq %rax
- add $t0, $acc5
- adc %rax, $acc6
- mov $acc0, %rax
- adc %rdx, $acc7
-
- mov .Lpoly+8*1(%rip), $a_ptr
- mov .Lpoly+8*3(%rip), $t1
-
- ##########################################
- # Now the reduction
- # First iteration
- mov $acc0, $t0
- shl \$32, $acc0
- mulq $t1
- shr \$32, $t0
- add $acc0, $acc1 # +=acc[0]<<96
- adc $t0, $acc2
- adc %rax, $acc3
- mov $acc1, %rax
- adc \$0, %rdx
-
- ##########################################
- # Second iteration
- mov $acc1, $t0
- shl \$32, $acc1
- mov %rdx, $acc0
- mulq $t1
- shr \$32, $t0
- add $acc1, $acc2
- adc $t0, $acc3
- adc %rax, $acc0
- mov $acc2, %rax
- adc \$0, %rdx
-
- ##########################################
- # Third iteration
- mov $acc2, $t0
- shl \$32, $acc2
- mov %rdx, $acc1
- mulq $t1
- shr \$32, $t0
- add $acc2, $acc3
- adc $t0, $acc0
- adc %rax, $acc1
- mov $acc3, %rax
- adc \$0, %rdx
-
- ###########################################
- # Last iteration
- mov $acc3, $t0
- shl \$32, $acc3
- mov %rdx, $acc2
- mulq $t1
- shr \$32, $t0
- add $acc3, $acc0
- adc $t0, $acc1
- adc %rax, $acc2
- adc \$0, %rdx
- xor $acc3, $acc3
-
- ############################################
- # Add the rest of the acc
- add $acc0, $acc4
- adc $acc1, $acc5
- mov $acc4, $acc0
- adc $acc2, $acc6
- adc %rdx, $acc7
- mov $acc5, $acc1
- adc \$0, $acc3
-
- sub \$-1, $acc4 # .Lpoly[0]
- mov $acc6, $acc2
- sbb $a_ptr, $acc5 # .Lpoly[1]
- sbb \$0, $acc6 # .Lpoly[2]
- mov $acc7, $t0
- sbb $t1, $acc7 # .Lpoly[3]
- sbb \$0, $acc3
-
- cmovc $acc0, $acc4
- cmovc $acc1, $acc5
- mov $acc4, 8*0($r_ptr)
- cmovc $acc2, $acc6
- mov $acc5, 8*1($r_ptr)
- cmovc $t0, $acc7
- mov $acc6, 8*2($r_ptr)
- mov $acc7, 8*3($r_ptr)
-
- ret
- .size __ecp_nistz256_sqr_montq,.-__ecp_nistz256_sqr_montq
- ___
-
- if ($addx) {
- $code.=<<___;
- .type __ecp_nistz256_mul_montx,\@abi-omnipotent
- .align 32
- __ecp_nistz256_mul_montx:
- ########################################################################
- # Multiply by b[0]
- mulx $acc1, $acc0, $acc1
- mulx $acc2, $t0, $acc2
- mov \$32, $poly1
- xor $acc5, $acc5 # cf=0
- mulx $acc3, $t1, $acc3
- mov .Lpoly+8*3(%rip), $poly3
- adc $t0, $acc1
- mulx $acc4, $t0, $acc4
- mov $acc0, %rdx
- adc $t1, $acc2
- shlx $poly1,$acc0,$t1
- adc $t0, $acc3
- shrx $poly1,$acc0,$t0
- adc \$0, $acc4
-
- ########################################################################
- # First reduction step
- add $t1, $acc1
- adc $t0, $acc2
-
- mulx $poly3, $t0, $t1
- mov 8*1($b_ptr), %rdx
- adc $t0, $acc3
- adc $t1, $acc4
- adc \$0, $acc5
- xor $acc0, $acc0 # $acc0=0,cf=0,of=0
-
- ########################################################################
- # Multiply by b[1]
- mulx 8*0+128($a_ptr), $t0, $t1
- adcx $t0, $acc1
- adox $t1, $acc2
-
- mulx 8*1+128($a_ptr), $t0, $t1
- adcx $t0, $acc2
- adox $t1, $acc3
-
- mulx 8*2+128($a_ptr), $t0, $t1
- adcx $t0, $acc3
- adox $t1, $acc4
-
- mulx 8*3+128($a_ptr), $t0, $t1
- mov $acc1, %rdx
- adcx $t0, $acc4
- shlx $poly1, $acc1, $t0
- adox $t1, $acc5
- shrx $poly1, $acc1, $t1
-
- adcx $acc0, $acc5
- adox $acc0, $acc0
- adc \$0, $acc0
-
- ########################################################################
- # Second reduction step
- add $t0, $acc2
- adc $t1, $acc3
-
- mulx $poly3, $t0, $t1
- mov 8*2($b_ptr), %rdx
- adc $t0, $acc4
- adc $t1, $acc5
- adc \$0, $acc0
- xor $acc1 ,$acc1 # $acc1=0,cf=0,of=0
-
- ########################################################################
- # Multiply by b[2]
- mulx 8*0+128($a_ptr), $t0, $t1
- adcx $t0, $acc2
- adox $t1, $acc3
-
- mulx 8*1+128($a_ptr), $t0, $t1
- adcx $t0, $acc3
- adox $t1, $acc4
-
- mulx 8*2+128($a_ptr), $t0, $t1
- adcx $t0, $acc4
- adox $t1, $acc5
-
- mulx 8*3+128($a_ptr), $t0, $t1
- mov $acc2, %rdx
- adcx $t0, $acc5
- shlx $poly1, $acc2, $t0
- adox $t1, $acc0
- shrx $poly1, $acc2, $t1
-
- adcx $acc1, $acc0
- adox $acc1, $acc1
- adc \$0, $acc1
-
- ########################################################################
- # Third reduction step
- add $t0, $acc3
- adc $t1, $acc4
-
- mulx $poly3, $t0, $t1
- mov 8*3($b_ptr), %rdx
- adc $t0, $acc5
- adc $t1, $acc0
- adc \$0, $acc1
- xor $acc2, $acc2 # $acc2=0,cf=0,of=0
-
- ########################################################################
- # Multiply by b[3]
- mulx 8*0+128($a_ptr), $t0, $t1
- adcx $t0, $acc3
- adox $t1, $acc4
-
- mulx 8*1+128($a_ptr), $t0, $t1
- adcx $t0, $acc4
- adox $t1, $acc5
-
- mulx 8*2+128($a_ptr), $t0, $t1
- adcx $t0, $acc5
- adox $t1, $acc0
-
- mulx 8*3+128($a_ptr), $t0, $t1
- mov $acc3, %rdx
- adcx $t0, $acc0
- shlx $poly1, $acc3, $t0
- adox $t1, $acc1
- shrx $poly1, $acc3, $t1
-
- adcx $acc2, $acc1
- adox $acc2, $acc2
- adc \$0, $acc2
-
- ########################################################################
- # Fourth reduction step
- add $t0, $acc4
- adc $t1, $acc5
-
- mulx $poly3, $t0, $t1
- mov $acc4, $t2
- mov .Lpoly+8*1(%rip), $poly1
- adc $t0, $acc0
- mov $acc5, $t3
- adc $t1, $acc1
- adc \$0, $acc2
-
- ########################################################################
- # Branch-less conditional subtraction of P
- xor %eax, %eax
- mov $acc0, $t0
- sbb \$-1, $acc4 # .Lpoly[0]
- sbb $poly1, $acc5 # .Lpoly[1]
- sbb \$0, $acc0 # .Lpoly[2]
- mov $acc1, $t1
- sbb $poly3, $acc1 # .Lpoly[3]
- sbb \$0, $acc2
-
- cmovc $t2, $acc4
- cmovc $t3, $acc5
- mov $acc4, 8*0($r_ptr)
- cmovc $t0, $acc0
- mov $acc5, 8*1($r_ptr)
- cmovc $t1, $acc1
- mov $acc0, 8*2($r_ptr)
- mov $acc1, 8*3($r_ptr)
-
- ret
- .size __ecp_nistz256_mul_montx,.-__ecp_nistz256_mul_montx
-
- .type __ecp_nistz256_sqr_montx,\@abi-omnipotent
- .align 32
- __ecp_nistz256_sqr_montx:
- mulx $acc6, $acc1, $acc2 # a[0]*a[1]
- mulx $acc7, $t0, $acc3 # a[0]*a[2]
- xor %eax, %eax
- adc $t0, $acc2
- mulx $acc0, $t1, $acc4 # a[0]*a[3]
- mov $acc6, %rdx
- adc $t1, $acc3
- adc \$0, $acc4
- xor $acc5, $acc5 # $acc5=0,cf=0,of=0
-
- #################################
- mulx $acc7, $t0, $t1 # a[1]*a[2]
- adcx $t0, $acc3
- adox $t1, $acc4
-
- mulx $acc0, $t0, $t1 # a[1]*a[3]
- mov $acc7, %rdx
- adcx $t0, $acc4
- adox $t1, $acc5
- adc \$0, $acc5
-
- #################################
- mulx $acc0, $t0, $acc6 # a[2]*a[3]
- mov 8*0+128($a_ptr), %rdx
- xor $acc7, $acc7 # $acc7=0,cf=0,of=0
- adcx $acc1, $acc1 # acc1:6<<1
- adox $t0, $acc5
- adcx $acc2, $acc2
- adox $acc7, $acc6 # of=0
-
- mulx %rdx, $acc0, $t1
- mov 8*1+128($a_ptr), %rdx
- adcx $acc3, $acc3
- adox $t1, $acc1
- adcx $acc4, $acc4
- mulx %rdx, $t0, $t4
- mov 8*2+128($a_ptr), %rdx
- adcx $acc5, $acc5
- adox $t0, $acc2
- adcx $acc6, $acc6
- .byte 0x67
- mulx %rdx, $t0, $t1
- mov 8*3+128($a_ptr), %rdx
- adox $t4, $acc3
- adcx $acc7, $acc7
- adox $t0, $acc4
- mov \$32, $a_ptr
- adox $t1, $acc5
- .byte 0x67,0x67
- mulx %rdx, $t0, $t4
- mov $acc0, %rdx
- adox $t0, $acc6
- shlx $a_ptr, $acc0, $t0
- adox $t4, $acc7
- shrx $a_ptr, $acc0, $t4
- mov .Lpoly+8*3(%rip), $t1
-
- # reduction step 1
- add $t0, $acc1
- adc $t4, $acc2
-
- mulx $t1, $t0, $acc0
- mov $acc1, %rdx
- adc $t0, $acc3
- shlx $a_ptr, $acc1, $t0
- adc \$0, $acc0
- shrx $a_ptr, $acc1, $t4
-
- # reduction step 2
- add $t0, $acc2
- adc $t4, $acc3
-
- mulx $t1, $t0, $acc1
- mov $acc2, %rdx
- adc $t0, $acc0
- shlx $a_ptr, $acc2, $t0
- adc \$0, $acc1
- shrx $a_ptr, $acc2, $t4
-
- # reduction step 3
- add $t0, $acc3
- adc $t4, $acc0
-
- mulx $t1, $t0, $acc2
- mov $acc3, %rdx
- adc $t0, $acc1
- shlx $a_ptr, $acc3, $t0
- adc \$0, $acc2
- shrx $a_ptr, $acc3, $t4
-
- # reduction step 4
- add $t0, $acc0
- adc $t4, $acc1
-
- mulx $t1, $t0, $acc3
- adc $t0, $acc2
- adc \$0, $acc3
-
- xor $t3, $t3 # cf=0
- adc $acc0, $acc4 # accumulate upper half
- mov .Lpoly+8*1(%rip), $a_ptr
- adc $acc1, $acc5
- mov $acc4, $acc0
- adc $acc2, $acc6
- adc $acc3, $acc7
- mov $acc5, $acc1
- adc \$0, $t3
-
- xor %eax, %eax # cf=0
- sbb \$-1, $acc4 # .Lpoly[0]
- mov $acc6, $acc2
- sbb $a_ptr, $acc5 # .Lpoly[1]
- sbb \$0, $acc6 # .Lpoly[2]
- mov $acc7, $acc3
- sbb $t1, $acc7 # .Lpoly[3]
- sbb \$0, $t3
-
- cmovc $acc0, $acc4
- cmovc $acc1, $acc5
- mov $acc4, 8*0($r_ptr)
- cmovc $acc2, $acc6
- mov $acc5, 8*1($r_ptr)
- cmovc $acc3, $acc7
- mov $acc6, 8*2($r_ptr)
- mov $acc7, 8*3($r_ptr)
-
- ret
- .size __ecp_nistz256_sqr_montx,.-__ecp_nistz256_sqr_montx
- ___
- }
- }
- {
- my ($r_ptr,$in_ptr)=("%rdi","%rsi");
- my ($acc0,$acc1,$acc2,$acc3)=map("%r$_",(8..11));
- my ($t0,$t1,$t2)=("%rcx","%r12","%r13");
-
- $code.=<<___;
- ################################################################################
- # void ecp_nistz256_from_mont(
- # uint64_t res[4],
- # uint64_t in[4]);
- # This one performs Montgomery multiplication by 1, so we only need the reduction
-
- .globl ecp_nistz256_from_mont
- .type ecp_nistz256_from_mont,\@function,2
- .align 32
- ecp_nistz256_from_mont:
- push %r12
- push %r13
-
- mov 8*0($in_ptr), %rax
- mov .Lpoly+8*3(%rip), $t2
- mov 8*1($in_ptr), $acc1
- mov 8*2($in_ptr), $acc2
- mov 8*3($in_ptr), $acc3
- mov %rax, $acc0
- mov .Lpoly+8*1(%rip), $t1
-
- #########################################
- # First iteration
- mov %rax, $t0
- shl \$32, $acc0
- mulq $t2
- shr \$32, $t0
- add $acc0, $acc1
- adc $t0, $acc2
- adc %rax, $acc3
- mov $acc1, %rax
- adc \$0, %rdx
-
- #########################################
- # Second iteration
- mov $acc1, $t0
- shl \$32, $acc1
- mov %rdx, $acc0
- mulq $t2
- shr \$32, $t0
- add $acc1, $acc2
- adc $t0, $acc3
- adc %rax, $acc0
- mov $acc2, %rax
- adc \$0, %rdx
-
- ##########################################
- # Third iteration
- mov $acc2, $t0
- shl \$32, $acc2
- mov %rdx, $acc1
- mulq $t2
- shr \$32, $t0
- add $acc2, $acc3
- adc $t0, $acc0
- adc %rax, $acc1
- mov $acc3, %rax
- adc \$0, %rdx
-
- ###########################################
- # Last iteration
- mov $acc3, $t0
- shl \$32, $acc3
- mov %rdx, $acc2
- mulq $t2
- shr \$32, $t0
- add $acc3, $acc0
- adc $t0, $acc1
- mov $acc0, $t0
- adc %rax, $acc2
- mov $acc1, $in_ptr
- adc \$0, %rdx
-
- sub \$-1, $acc0
- mov $acc2, %rax
- sbb $t1, $acc1
- sbb \$0, $acc2
- mov %rdx, $acc3
- sbb $t2, %rdx
- sbb $t2, $t2
-
- cmovnz $t0, $acc0
- cmovnz $in_ptr, $acc1
- mov $acc0, 8*0($r_ptr)
- cmovnz %rax, $acc2
- mov $acc1, 8*1($r_ptr)
- cmovz %rdx, $acc3
- mov $acc2, 8*2($r_ptr)
- mov $acc3, 8*3($r_ptr)
-
- pop %r13
- pop %r12
- ret
- .size ecp_nistz256_from_mont,.-ecp_nistz256_from_mont
- ___
- }
- {
- my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
- my ($ONE,$INDEX,$Ra,$Rb,$Rc,$Rd,$Re,$Rf)=map("%xmm$_",(0..7));
- my ($M0,$T0a,$T0b,$T0c,$T0d,$T0e,$T0f,$TMP0)=map("%xmm$_",(8..15));
- my ($M1,$T2a,$T2b,$TMP2,$M2,$T2a,$T2b,$TMP2)=map("%xmm$_",(8..15));
-
- $code.=<<___;
- ################################################################################
- # void ecp_nistz256_select_w5(uint64_t *val, uint64_t *in_t, int index);
- .globl ecp_nistz256_select_w5
- .type ecp_nistz256_select_w5,\@abi-omnipotent
- .align 32
- ecp_nistz256_select_w5:
- ___
- $code.=<<___ if ($avx>1);
- mov OPENSSL_ia32cap_P+8(%rip), %eax
- test \$`1<<5`, %eax
- jnz .Lavx2_select_w5
- ___
- $code.=<<___ if ($win64);
- lea -0x88(%rsp), %rax
- .LSEH_begin_ecp_nistz256_select_w5:
- .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
- .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6, -0x20(%rax)
- .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7, -0x10(%rax)
- .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8, 0(%rax)
- .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9, 0x10(%rax)
- .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10, 0x20(%rax)
- .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11, 0x30(%rax)
- .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12, 0x40(%rax)
- .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13, 0x50(%rax)
- .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14, 0x60(%rax)
- .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15, 0x70(%rax)
- ___
- $code.=<<___;
- movdqa .LOne(%rip), $ONE
- movd $index, $INDEX
-
- pxor $Ra, $Ra
- pxor $Rb, $Rb
- pxor $Rc, $Rc
- pxor $Rd, $Rd
- pxor $Re, $Re
- pxor $Rf, $Rf
-
- movdqa $ONE, $M0
- pshufd \$0, $INDEX, $INDEX
-
- mov \$16, %rax
- .Lselect_loop_sse_w5:
-
- movdqa $M0, $TMP0
- paddd $ONE, $M0
- pcmpeqd $INDEX, $TMP0
-
- movdqa 16*0($in_t), $T0a
- movdqa 16*1($in_t), $T0b
- movdqa 16*2($in_t), $T0c
- movdqa 16*3($in_t), $T0d
- movdqa 16*4($in_t), $T0e
- movdqa 16*5($in_t), $T0f
- lea 16*6($in_t), $in_t
-
- pand $TMP0, $T0a
- pand $TMP0, $T0b
- por $T0a, $Ra
- pand $TMP0, $T0c
- por $T0b, $Rb
- pand $TMP0, $T0d
- por $T0c, $Rc
- pand $TMP0, $T0e
- por $T0d, $Rd
- pand $TMP0, $T0f
- por $T0e, $Re
- por $T0f, $Rf
-
- dec %rax
- jnz .Lselect_loop_sse_w5
-
- movdqu $Ra, 16*0($val)
- movdqu $Rb, 16*1($val)
- movdqu $Rc, 16*2($val)
- movdqu $Rd, 16*3($val)
- movdqu $Re, 16*4($val)
- movdqu $Rf, 16*5($val)
- ___
- $code.=<<___ if ($win64);
- movaps (%rsp), %xmm6
- movaps 0x10(%rsp), %xmm7
- movaps 0x20(%rsp), %xmm8
- movaps 0x30(%rsp), %xmm9
- movaps 0x40(%rsp), %xmm10
- movaps 0x50(%rsp), %xmm11
- movaps 0x60(%rsp), %xmm12
- movaps 0x70(%rsp), %xmm13
- movaps 0x80(%rsp), %xmm14
- movaps 0x90(%rsp), %xmm15
- lea 0xa8(%rsp), %rsp
- .LSEH_end_ecp_nistz256_select_w5:
- ___
- $code.=<<___;
- ret
- .size ecp_nistz256_select_w5,.-ecp_nistz256_select_w5
-
- ################################################################################
- # void ecp_nistz256_select_w7(uint64_t *val, uint64_t *in_t, int index);
- .globl ecp_nistz256_select_w7
- .type ecp_nistz256_select_w7,\@abi-omnipotent
- .align 32
- ecp_nistz256_select_w7:
- ___
- $code.=<<___ if ($avx>1);
- mov OPENSSL_ia32cap_P+8(%rip), %eax
- test \$`1<<5`, %eax
- jnz .Lavx2_select_w7
- ___
- $code.=<<___ if ($win64);
- lea -0x88(%rsp), %rax
- .LSEH_begin_ecp_nistz256_select_w7:
- .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
- .byte 0x0f,0x29,0x70,0xe0 #movaps %xmm6, -0x20(%rax)
- .byte 0x0f,0x29,0x78,0xf0 #movaps %xmm7, -0x10(%rax)
- .byte 0x44,0x0f,0x29,0x00 #movaps %xmm8, 0(%rax)
- .byte 0x44,0x0f,0x29,0x48,0x10 #movaps %xmm9, 0x10(%rax)
- .byte 0x44,0x0f,0x29,0x50,0x20 #movaps %xmm10, 0x20(%rax)
- .byte 0x44,0x0f,0x29,0x58,0x30 #movaps %xmm11, 0x30(%rax)
- .byte 0x44,0x0f,0x29,0x60,0x40 #movaps %xmm12, 0x40(%rax)
- .byte 0x44,0x0f,0x29,0x68,0x50 #movaps %xmm13, 0x50(%rax)
- .byte 0x44,0x0f,0x29,0x70,0x60 #movaps %xmm14, 0x60(%rax)
- .byte 0x44,0x0f,0x29,0x78,0x70 #movaps %xmm15, 0x70(%rax)
- ___
- $code.=<<___;
- movdqa .LOne(%rip), $M0
- movd $index, $INDEX
-
- pxor $Ra, $Ra
- pxor $Rb, $Rb
- pxor $Rc, $Rc
- pxor $Rd, $Rd
-
- movdqa $M0, $ONE
- pshufd \$0, $INDEX, $INDEX
- mov \$64, %rax
-
- .Lselect_loop_sse_w7:
- movdqa $M0, $TMP0
- paddd $ONE, $M0
- movdqa 16*0($in_t), $T0a
- movdqa 16*1($in_t), $T0b
- pcmpeqd $INDEX, $TMP0
- movdqa 16*2($in_t), $T0c
- movdqa 16*3($in_t), $T0d
- lea 16*4($in_t), $in_t
-
- pand $TMP0, $T0a
- pand $TMP0, $T0b
- por $T0a, $Ra
- pand $TMP0, $T0c
- por $T0b, $Rb
- pand $TMP0, $T0d
- por $T0c, $Rc
- prefetcht0 255($in_t)
- por $T0d, $Rd
-
- dec %rax
- jnz .Lselect_loop_sse_w7
-
- movdqu $Ra, 16*0($val)
- movdqu $Rb, 16*1($val)
- movdqu $Rc, 16*2($val)
- movdqu $Rd, 16*3($val)
- ___
- $code.=<<___ if ($win64);
- movaps (%rsp), %xmm6
- movaps 0x10(%rsp), %xmm7
- movaps 0x20(%rsp), %xmm8
- movaps 0x30(%rsp), %xmm9
- movaps 0x40(%rsp), %xmm10
- movaps 0x50(%rsp), %xmm11
- movaps 0x60(%rsp), %xmm12
- movaps 0x70(%rsp), %xmm13
- movaps 0x80(%rsp), %xmm14
- movaps 0x90(%rsp), %xmm15
- lea 0xa8(%rsp), %rsp
- .LSEH_end_ecp_nistz256_select_w7:
- ___
- $code.=<<___;
- ret
- .size ecp_nistz256_select_w7,.-ecp_nistz256_select_w7
- ___
- }
- if ($avx>1) {
- my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
- my ($TWO,$INDEX,$Ra,$Rb,$Rc)=map("%ymm$_",(0..4));
- my ($M0,$T0a,$T0b,$T0c,$TMP0)=map("%ymm$_",(5..9));
- my ($M1,$T1a,$T1b,$T1c,$TMP1)=map("%ymm$_",(10..14));
-
- $code.=<<___;
- ################################################################################
- # void ecp_nistz256_avx2_select_w5(uint64_t *val, uint64_t *in_t, int index);
- .type ecp_nistz256_avx2_select_w5,\@abi-omnipotent
- .align 32
- ecp_nistz256_avx2_select_w5:
- .Lavx2_select_w5:
- vzeroupper
- ___
- $code.=<<___ if ($win64);
- lea -0x88(%rsp), %rax
- .LSEH_begin_ecp_nistz256_avx2_select_w5:
- .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
- .byte 0xc5,0xf8,0x29,0x70,0xe0 #vmovaps %xmm6, -0x20(%rax)
- .byte 0xc5,0xf8,0x29,0x78,0xf0 #vmovaps %xmm7, -0x10(%rax)
- .byte 0xc5,0x78,0x29,0x40,0x00 #vmovaps %xmm8, 8(%rax)
- .byte 0xc5,0x78,0x29,0x48,0x10 #vmovaps %xmm9, 0x10(%rax)
- .byte 0xc5,0x78,0x29,0x50,0x20 #vmovaps %xmm10, 0x20(%rax)
- .byte 0xc5,0x78,0x29,0x58,0x30 #vmovaps %xmm11, 0x30(%rax)
- .byte 0xc5,0x78,0x29,0x60,0x40 #vmovaps %xmm12, 0x40(%rax)
- .byte 0xc5,0x78,0x29,0x68,0x50 #vmovaps %xmm13, 0x50(%rax)
- .byte 0xc5,0x78,0x29,0x70,0x60 #vmovaps %xmm14, 0x60(%rax)
- .byte 0xc5,0x78,0x29,0x78,0x70 #vmovaps %xmm15, 0x70(%rax)
- ___
- $code.=<<___;
- vmovdqa .LTwo(%rip), $TWO
-
- vpxor $Ra, $Ra, $Ra
- vpxor $Rb, $Rb, $Rb
- vpxor $Rc, $Rc, $Rc
-
- vmovdqa .LOne(%rip), $M0
- vmovdqa .LTwo(%rip), $M1
-
- vmovd $index, %xmm1
- vpermd $INDEX, $Ra, $INDEX
-
- mov \$8, %rax
- .Lselect_loop_avx2_w5:
-
- vmovdqa 32*0($in_t), $T0a
- vmovdqa 32*1($in_t), $T0b
- vmovdqa 32*2($in_t), $T0c
-
- vmovdqa 32*3($in_t), $T1a
- vmovdqa 32*4($in_t), $T1b
- vmovdqa 32*5($in_t), $T1c
-
- vpcmpeqd $INDEX, $M0, $TMP0
- vpcmpeqd $INDEX, $M1, $TMP1
-
- vpaddd $TWO, $M0, $M0
- vpaddd $TWO, $M1, $M1
- lea 32*6($in_t), $in_t
-
- vpand $TMP0, $T0a, $T0a
- vpand $TMP0, $T0b, $T0b
- vpand $TMP0, $T0c, $T0c
- vpand $TMP1, $T1a, $T1a
- vpand $TMP1, $T1b, $T1b
- vpand $TMP1, $T1c, $T1c
-
- vpxor $T0a, $Ra, $Ra
- vpxor $T0b, $Rb, $Rb
- vpxor $T0c, $Rc, $Rc
- vpxor $T1a, $Ra, $Ra
- vpxor $T1b, $Rb, $Rb
- vpxor $T1c, $Rc, $Rc
-
- dec %rax
- jnz .Lselect_loop_avx2_w5
-
- vmovdqu $Ra, 32*0($val)
- vmovdqu $Rb, 32*1($val)
- vmovdqu $Rc, 32*2($val)
- vzeroupper
- ___
- $code.=<<___ if ($win64);
- movaps (%rsp), %xmm6
- movaps 0x10(%rsp), %xmm7
- movaps 0x20(%rsp), %xmm8
- movaps 0x30(%rsp), %xmm9
- movaps 0x40(%rsp), %xmm10
- movaps 0x50(%rsp), %xmm11
- movaps 0x60(%rsp), %xmm12
- movaps 0x70(%rsp), %xmm13
- movaps 0x80(%rsp), %xmm14
- movaps 0x90(%rsp), %xmm15
- lea 0xa8(%rsp), %rsp
- .LSEH_end_ecp_nistz256_avx2_select_w5:
- ___
- $code.=<<___;
- ret
- .size ecp_nistz256_avx2_select_w5,.-ecp_nistz256_avx2_select_w5
- ___
- }
- if ($avx>1) {
- my ($val,$in_t,$index)=$win64?("%rcx","%rdx","%r8d"):("%rdi","%rsi","%edx");
- my ($THREE,$INDEX,$Ra,$Rb)=map("%ymm$_",(0..3));
- my ($M0,$T0a,$T0b,$TMP0)=map("%ymm$_",(4..7));
- my ($M1,$T1a,$T1b,$TMP1)=map("%ymm$_",(8..11));
- my ($M2,$T2a,$T2b,$TMP2)=map("%ymm$_",(12..15));
-
- $code.=<<___;
-
- ################################################################################
- # void ecp_nistz256_avx2_select_w7(uint64_t *val, uint64_t *in_t, int index);
- .globl ecp_nistz256_avx2_select_w7
- .type ecp_nistz256_avx2_select_w7,\@abi-omnipotent
- .align 32
- ecp_nistz256_avx2_select_w7:
- .Lavx2_select_w7:
- vzeroupper
- ___
- $code.=<<___ if ($win64);
- lea -0x88(%rsp), %rax
- .LSEH_begin_ecp_nistz256_avx2_select_w7:
- .byte 0x48,0x8d,0x60,0xe0 #lea -0x20(%rax), %rsp
- .byte 0xc5,0xf8,0x29,0x70,0xe0 #vmovaps %xmm6, -0x20(%rax)
- .byte 0xc5,0xf8,0x29,0x78,0xf0 #vmovaps %xmm7, -0x10(%rax)
- .byte 0xc5,0x78,0x29,0x40,0x00 #vmovaps %xmm8, 8(%rax)
- .byte 0xc5,0x78,0x29,0x48,0x10 #vmovaps %xmm9, 0x10(%rax)
- .byte 0xc5,0x78,0x29,0x50,0x20 #vmovaps %xmm10, 0x20(%rax)
- .byte 0xc5,0x78,0x29,0x58,0x30 #vmovaps %xmm11, 0x30(%rax)
- .byte 0xc5,0x78,0x29,0x60,0x40 #vmovaps %xmm12, 0x40(%rax)
- .byte 0xc5,0x78,0x29,0x68,0x50 #vmovaps %xmm13, 0x50(%rax)
- .byte 0xc5,0x78,0x29,0x70,0x60 #vmovaps %xmm14, 0x60(%rax)
- .byte 0xc5,0x78,0x29,0x78,0x70 #vmovaps %xmm15, 0x70(%rax)
- ___
- $code.=<<___;
- vmovdqa .LThree(%rip), $THREE
-
- vpxor $Ra, $Ra, $Ra
- vpxor $Rb, $Rb, $Rb
-
- vmovdqa .LOne(%rip), $M0
- vmovdqa .LTwo(%rip), $M1
- vmovdqa .LThree(%rip), $M2
-
- vmovd $index, %xmm1
- vpermd $INDEX, $Ra, $INDEX
- # Skip index = 0, because it is implicitly the point at infinity
-
- mov \$21, %rax
- .Lselect_loop_avx2_w7:
-
- vmovdqa 32*0($in_t), $T0a
- vmovdqa 32*1($in_t), $T0b
-
- vmovdqa 32*2($in_t), $T1a
- vmovdqa 32*3($in_t), $T1b
-
- vmovdqa 32*4($in_t), $T2a
- vmovdqa 32*5($in_t), $T2b
-
- vpcmpeqd $INDEX, $M0, $TMP0
- vpcmpeqd $INDEX, $M1, $TMP1
- vpcmpeqd $INDEX, $M2, $TMP2
-
- vpaddd $THREE, $M0, $M0
- vpaddd $THREE, $M1, $M1
- vpaddd $THREE, $M2, $M2
- lea 32*6($in_t), $in_t
-
- vpand $TMP0, $T0a, $T0a
- vpand $TMP0, $T0b, $T0b
- vpand $TMP1, $T1a, $T1a
- vpand $TMP1, $T1b, $T1b
- vpand $TMP2, $T2a, $T2a
- vpand $TMP2, $T2b, $T2b
-
- vpxor $T0a, $Ra, $Ra
- vpxor $T0b, $Rb, $Rb
- vpxor $T1a, $Ra, $Ra
- vpxor $T1b, $Rb, $Rb
- vpxor $T2a, $Ra, $Ra
- vpxor $T2b, $Rb, $Rb
-
- dec %rax
- jnz .Lselect_loop_avx2_w7
-
-
- vmovdqa 32*0($in_t), $T0a
- vmovdqa 32*1($in_t), $T0b
-
- vpcmpeqd $INDEX, $M0, $TMP0
-
- vpand $TMP0, $T0a, $T0a
- vpand $TMP0, $T0b, $T0b
-
- vpxor $T0a, $Ra, $Ra
- vpxor $T0b, $Rb, $Rb
-
- vmovdqu $Ra, 32*0($val)
- vmovdqu $Rb, 32*1($val)
- vzeroupper
- ___
- $code.=<<___ if ($win64);
- movaps (%rsp), %xmm6
- movaps 0x10(%rsp), %xmm7
- movaps 0x20(%rsp), %xmm8
- movaps 0x30(%rsp), %xmm9
- movaps 0x40(%rsp), %xmm10
- movaps 0x50(%rsp), %xmm11
- movaps 0x60(%rsp), %xmm12
- movaps 0x70(%rsp), %xmm13
- movaps 0x80(%rsp), %xmm14
- movaps 0x90(%rsp), %xmm15
- lea 0xa8(%rsp), %rsp
- .LSEH_end_ecp_nistz256_avx2_select_w7:
- ___
- $code.=<<___;
- ret
- .size ecp_nistz256_avx2_select_w7,.-ecp_nistz256_avx2_select_w7
- ___
- } else {
- $code.=<<___;
- .globl ecp_nistz256_avx2_select_w7
- .type ecp_nistz256_avx2_select_w7,\@function,3
- .align 32
- ecp_nistz256_avx2_select_w7:
- .byte 0x0f,0x0b # ud2
- ret
- .size ecp_nistz256_avx2_select_w7,.-ecp_nistz256_avx2_select_w7
- ___
- }
- {{{
- ########################################################################
- # This block implements higher level point_double, point_add and
- # point_add_affine. The key to performance in this case is to allow
- # out-of-order execution logic to overlap computations from next step
- # with tail processing from current step. By using tailored calling
- # sequence we minimize inter-step overhead to give processor better
- # shot at overlapping operations...
- #
- # You will notice that input data is copied to stack. Trouble is that
- # there are no registers to spare for holding original pointers and
- # reloading them, pointers, would create undesired dependencies on
- # effective addresses calculation paths. In other words it's too done
- # to favour out-of-order execution logic.
- # <appro@openssl.org>
-
- my ($r_ptr,$a_ptr,$b_org,$b_ptr)=("%rdi","%rsi","%rdx","%rbx");
- my ($acc0,$acc1,$acc2,$acc3,$acc4,$acc5,$acc6,$acc7)=map("%r$_",(8..15));
- my ($t0,$t1,$t2,$t3,$t4)=("%rax","%rbp","%rcx",$acc4,$acc4);
- my ($poly1,$poly3)=($acc6,$acc7);
-
- sub load_for_mul () {
- my ($a,$b,$src0) = @_;
- my $bias = $src0 eq "%rax" ? 0 : -128;
-
- " mov $b, $src0
- lea $b, $b_ptr
- mov 8*0+$a, $acc1
- mov 8*1+$a, $acc2
- lea $bias+$a, $a_ptr
- mov 8*2+$a, $acc3
- mov 8*3+$a, $acc4"
- }
-
- sub load_for_sqr () {
- my ($a,$src0) = @_;
- my $bias = $src0 eq "%rax" ? 0 : -128;
-
- " mov 8*0+$a, $src0
- mov 8*1+$a, $acc6
- lea $bias+$a, $a_ptr
- mov 8*2+$a, $acc7
- mov 8*3+$a, $acc0"
- }
-
- {
- ########################################################################
- # operate in 4-5-0-1 "name space" that matches multiplication output
- #
- my ($a0,$a1,$a2,$a3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
-
- $code.=<<___;
- .type __ecp_nistz256_add_toq,\@abi-omnipotent
- .align 32
- __ecp_nistz256_add_toq:
- add 8*0($b_ptr), $a0
- adc 8*1($b_ptr), $a1
- mov $a0, $t0
- adc 8*2($b_ptr), $a2
- adc 8*3($b_ptr), $a3
- mov $a1, $t1
- sbb $t4, $t4
-
- sub \$-1, $a0
- mov $a2, $t2
- sbb $poly1, $a1
- sbb \$0, $a2
- mov $a3, $t3
- sbb $poly3, $a3
- test $t4, $t4
-
- cmovz $t0, $a0
- cmovz $t1, $a1
- mov $a0, 8*0($r_ptr)
- cmovz $t2, $a2
- mov $a1, 8*1($r_ptr)
- cmovz $t3, $a3
- mov $a2, 8*2($r_ptr)
- mov $a3, 8*3($r_ptr)
-
- ret
- .size __ecp_nistz256_add_toq,.-__ecp_nistz256_add_toq
-
- .type __ecp_nistz256_sub_fromq,\@abi-omnipotent
- .align 32
- __ecp_nistz256_sub_fromq:
- sub 8*0($b_ptr), $a0
- sbb 8*1($b_ptr), $a1
- mov $a0, $t0
- sbb 8*2($b_ptr), $a2
- sbb 8*3($b_ptr), $a3
- mov $a1, $t1
- sbb $t4, $t4
-
- add \$-1, $a0
- mov $a2, $t2
- adc $poly1, $a1
- adc \$0, $a2
- mov $a3, $t3
- adc $poly3, $a3
- test $t4, $t4
-
- cmovz $t0, $a0
- cmovz $t1, $a1
- mov $a0, 8*0($r_ptr)
- cmovz $t2, $a2
- mov $a1, 8*1($r_ptr)
- cmovz $t3, $a3
- mov $a2, 8*2($r_ptr)
- mov $a3, 8*3($r_ptr)
-
- ret
- .size __ecp_nistz256_sub_fromq,.-__ecp_nistz256_sub_fromq
-
- .type __ecp_nistz256_subq,\@abi-omnipotent
- .align 32
- __ecp_nistz256_subq:
- sub $a0, $t0
- sbb $a1, $t1
- mov $t0, $a0
- sbb $a2, $t2
- sbb $a3, $t3
- mov $t1, $a1
- sbb $t4, $t4
-
- add \$-1, $t0
- mov $t2, $a2
- adc $poly1, $t1
- adc \$0, $t2
- mov $t3, $a3
- adc $poly3, $t3
- test $t4, $t4
-
- cmovnz $t0, $a0
- cmovnz $t1, $a1
- cmovnz $t2, $a2
- cmovnz $t3, $a3
-
- ret
- .size __ecp_nistz256_subq,.-__ecp_nistz256_subq
-
- .type __ecp_nistz256_mul_by_2q,\@abi-omnipotent
- .align 32
- __ecp_nistz256_mul_by_2q:
- add $a0, $a0 # a0:a3+a0:a3
- adc $a1, $a1
- mov $a0, $t0
- adc $a2, $a2
- adc $a3, $a3
- mov $a1, $t1
- sbb $t4, $t4
-
- sub \$-1, $a0
- mov $a2, $t2
- sbb $poly1, $a1
- sbb \$0, $a2
- mov $a3, $t3
- sbb $poly3, $a3
- test $t4, $t4
-
- cmovz $t0, $a0
- cmovz $t1, $a1
- mov $a0, 8*0($r_ptr)
- cmovz $t2, $a2
- mov $a1, 8*1($r_ptr)
- cmovz $t3, $a3
- mov $a2, 8*2($r_ptr)
- mov $a3, 8*3($r_ptr)
-
- ret
- .size __ecp_nistz256_mul_by_2q,.-__ecp_nistz256_mul_by_2q
- ___
- }
- sub gen_double () {
- my $x = shift;
- my ($src0,$sfx,$bias);
- my ($S,$M,$Zsqr,$in_x,$tmp0)=map(32*$_,(0..4));
-
- if ($x ne "x") {
- $src0 = "%rax";
- $sfx = "";
- $bias = 0;
-
- $code.=<<___;
- .globl ecp_nistz256_point_double
- .type ecp_nistz256_point_double,\@function,2
- .align 32
- ecp_nistz256_point_double:
- ___
- $code.=<<___ if ($addx);
- mov \$0x80100, %ecx
- and OPENSSL_ia32cap_P+8(%rip), %ecx
- cmp \$0x80100, %ecx
- je .Lpoint_doublex
- ___
- } else {
- $src0 = "%rdx";
- $sfx = "x";
- $bias = 128;
-
- $code.=<<___;
- .type ecp_nistz256_point_doublex,\@function,2
- .align 32
- ecp_nistz256_point_doublex:
- .Lpoint_doublex:
- ___
- }
- $code.=<<___;
- push %rbp
- push %rbx
- push %r12
- push %r13
- push %r14
- push %r15
- sub \$32*5+8, %rsp
-
- .Lpoint_double_shortcut$x:
- movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr.x
- mov $a_ptr, $b_ptr # backup copy
- movdqu 0x10($a_ptr), %xmm1
- mov 0x20+8*0($a_ptr), $acc4 # load in_y in "5-4-0-1" order
- mov 0x20+8*1($a_ptr), $acc5
- mov 0x20+8*2($a_ptr), $acc0
- mov 0x20+8*3($a_ptr), $acc1
- mov .Lpoly+8*1(%rip), $poly1
- mov .Lpoly+8*3(%rip), $poly3
- movdqa %xmm0, $in_x(%rsp)
- movdqa %xmm1, $in_x+0x10(%rsp)
- lea 0x20($r_ptr), $acc2
- lea 0x40($r_ptr), $acc3
- movq $r_ptr, %xmm0
- movq $acc2, %xmm1
- movq $acc3, %xmm2
-
- lea $S(%rsp), $r_ptr
- call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(S, in_y);
-
- mov 0x40+8*0($a_ptr), $src0
- mov 0x40+8*1($a_ptr), $acc6
- mov 0x40+8*2($a_ptr), $acc7
- mov 0x40+8*3($a_ptr), $acc0
- lea 0x40-$bias($a_ptr), $a_ptr
- lea $Zsqr(%rsp), $r_ptr
- call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Zsqr, in_z);
-
- `&load_for_sqr("$S(%rsp)", "$src0")`
- lea $S(%rsp), $r_ptr
- call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(S, S);
-
- mov 0x20($b_ptr), $src0 # $b_ptr is still valid
- mov 0x40+8*0($b_ptr), $acc1
- mov 0x40+8*1($b_ptr), $acc2
- mov 0x40+8*2($b_ptr), $acc3
- mov 0x40+8*3($b_ptr), $acc4
- lea 0x40-$bias($b_ptr), $a_ptr
- lea 0x20($b_ptr), $b_ptr
- movq %xmm2, $r_ptr
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, in_z, in_y);
- call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(res_z, res_z);
-
- mov $in_x+8*0(%rsp), $acc4 # "5-4-0-1" order
- mov $in_x+8*1(%rsp), $acc5
- lea $Zsqr(%rsp), $b_ptr
- mov $in_x+8*2(%rsp), $acc0
- mov $in_x+8*3(%rsp), $acc1
- lea $M(%rsp), $r_ptr
- call __ecp_nistz256_add_to$x # p256_add(M, in_x, Zsqr);
-
- mov $in_x+8*0(%rsp), $acc4 # "5-4-0-1" order
- mov $in_x+8*1(%rsp), $acc5
- lea $Zsqr(%rsp), $b_ptr
- mov $in_x+8*2(%rsp), $acc0
- mov $in_x+8*3(%rsp), $acc1
- lea $Zsqr(%rsp), $r_ptr
- call __ecp_nistz256_sub_from$x # p256_sub(Zsqr, in_x, Zsqr);
-
- `&load_for_sqr("$S(%rsp)", "$src0")`
- movq %xmm1, $r_ptr
- call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(res_y, S);
- ___
- {
- ######## ecp_nistz256_div_by_2(res_y, res_y); ##########################
- # operate in 4-5-6-7 "name space" that matches squaring output
- #
- my ($poly1,$poly3)=($a_ptr,$t1);
- my ($a0,$a1,$a2,$a3,$t3,$t4,$t1)=($acc4,$acc5,$acc6,$acc7,$acc0,$acc1,$acc2);
-
- $code.=<<___;
- xor $t4, $t4
- mov $a0, $t0
- add \$-1, $a0
- mov $a1, $t1
- adc $poly1, $a1
- mov $a2, $t2
- adc \$0, $a2
- mov $a3, $t3
- adc $poly3, $a3
- adc \$0, $t4
- xor $a_ptr, $a_ptr # borrow $a_ptr
- test \$1, $t0
-
- cmovz $t0, $a0
- cmovz $t1, $a1
- cmovz $t2, $a2
- cmovz $t3, $a3
- cmovz $a_ptr, $t4
-
- mov $a1, $t0 # a0:a3>>1
- shr \$1, $a0
- shl \$63, $t0
- mov $a2, $t1
- shr \$1, $a1
- or $t0, $a0
- shl \$63, $t1
- mov $a3, $t2
- shr \$1, $a2
- or $t1, $a1
- shl \$63, $t2
- mov $a0, 8*0($r_ptr)
- shr \$1, $a3
- mov $a1, 8*1($r_ptr)
- shl \$63, $t4
- or $t2, $a2
- or $t4, $a3
- mov $a2, 8*2($r_ptr)
- mov $a3, 8*3($r_ptr)
- ___
- }
- $code.=<<___;
- `&load_for_mul("$M(%rsp)", "$Zsqr(%rsp)", "$src0")`
- lea $M(%rsp), $r_ptr
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(M, M, Zsqr);
-
- lea $tmp0(%rsp), $r_ptr
- call __ecp_nistz256_mul_by_2$x
-
- lea $M(%rsp), $b_ptr
- lea $M(%rsp), $r_ptr
- call __ecp_nistz256_add_to$x # p256_mul_by_3(M, M);
-
- `&load_for_mul("$S(%rsp)", "$in_x(%rsp)", "$src0")`
- lea $S(%rsp), $r_ptr
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(S, S, in_x);
-
- lea $tmp0(%rsp), $r_ptr
- call __ecp_nistz256_mul_by_2$x # p256_mul_by_2(tmp0, S);
-
- `&load_for_sqr("$M(%rsp)", "$src0")`
- movq %xmm0, $r_ptr
- call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(res_x, M);
-
- lea $tmp0(%rsp), $b_ptr
- mov $acc6, $acc0 # harmonize sqr output and sub input
- mov $acc7, $acc1
- mov $a_ptr, $poly1
- mov $t1, $poly3
- call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, tmp0);
-
- mov $S+8*0(%rsp), $t0
- mov $S+8*1(%rsp), $t1
- mov $S+8*2(%rsp), $t2
- mov $S+8*3(%rsp), $acc2 # "4-5-0-1" order
- lea $S(%rsp), $r_ptr
- call __ecp_nistz256_sub$x # p256_sub(S, S, res_x);
-
- mov $M(%rsp), $src0
- lea $M(%rsp), $b_ptr
- mov $acc4, $acc6 # harmonize sub output and mul input
- xor %ecx, %ecx
- mov $acc4, $S+8*0(%rsp) # have to save:-(
- mov $acc5, $acc2
- mov $acc5, $S+8*1(%rsp)
- cmovz $acc0, $acc3
- mov $acc0, $S+8*2(%rsp)
- lea $S-$bias(%rsp), $a_ptr
- cmovz $acc1, $acc4
- mov $acc1, $S+8*3(%rsp)
- mov $acc6, $acc1
- lea $S(%rsp), $r_ptr
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(S, S, M);
-
- movq %xmm1, $b_ptr
- movq %xmm1, $r_ptr
- call __ecp_nistz256_sub_from$x # p256_sub(res_y, S, res_y);
-
- add \$32*5+8, %rsp
- pop %r15
- pop %r14
- pop %r13
- pop %r12
- pop %rbx
- pop %rbp
- ret
- .size ecp_nistz256_point_double$sfx,.-ecp_nistz256_point_double$sfx
- ___
- }
- &gen_double("q");
-
- sub gen_add () {
- my $x = shift;
- my ($src0,$sfx,$bias);
- my ($H,$Hsqr,$R,$Rsqr,$Hcub,
- $U1,$U2,$S1,$S2,
- $res_x,$res_y,$res_z,
- $in1_x,$in1_y,$in1_z,
- $in2_x,$in2_y,$in2_z)=map(32*$_,(0..17));
- my ($Z1sqr, $Z2sqr) = ($Hsqr, $Rsqr);
-
- if ($x ne "x") {
- $src0 = "%rax";
- $sfx = "";
- $bias = 0;
-
- $code.=<<___;
- .globl ecp_nistz256_point_add
- .type ecp_nistz256_point_add,\@function,3
- .align 32
- ecp_nistz256_point_add:
- ___
- $code.=<<___ if ($addx);
- mov \$0x80100, %ecx
- and OPENSSL_ia32cap_P+8(%rip), %ecx
- cmp \$0x80100, %ecx
- je .Lpoint_addx
- ___
- } else {
- $src0 = "%rdx";
- $sfx = "x";
- $bias = 128;
-
- $code.=<<___;
- .type ecp_nistz256_point_addx,\@function,3
- .align 32
- ecp_nistz256_point_addx:
- .Lpoint_addx:
- ___
- }
- $code.=<<___;
- push %rbp
- push %rbx
- push %r12
- push %r13
- push %r14
- push %r15
- sub \$32*18+8, %rsp
-
- movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr
- movdqu 0x10($a_ptr), %xmm1
- movdqu 0x20($a_ptr), %xmm2
- movdqu 0x30($a_ptr), %xmm3
- movdqu 0x40($a_ptr), %xmm4
- movdqu 0x50($a_ptr), %xmm5
- mov $a_ptr, $b_ptr # reassign
- mov $b_org, $a_ptr # reassign
- movdqa %xmm0, $in1_x(%rsp)
- movdqa %xmm1, $in1_x+0x10(%rsp)
- por %xmm0, %xmm1
- movdqa %xmm2, $in1_y(%rsp)
- movdqa %xmm3, $in1_y+0x10(%rsp)
- por %xmm2, %xmm3
- movdqa %xmm4, $in1_z(%rsp)
- movdqa %xmm5, $in1_z+0x10(%rsp)
- por %xmm1, %xmm3
-
- movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$b_ptr
- pshufd \$0xb1, %xmm3, %xmm5
- movdqu 0x10($a_ptr), %xmm1
- movdqu 0x20($a_ptr), %xmm2
- por %xmm3, %xmm5
- movdqu 0x30($a_ptr), %xmm3
- mov 0x40+8*0($a_ptr), $src0 # load original in2_z
- mov 0x40+8*1($a_ptr), $acc6
- mov 0x40+8*2($a_ptr), $acc7
- mov 0x40+8*3($a_ptr), $acc0
- movdqa %xmm0, $in2_x(%rsp)
- pshufd \$0x1e, %xmm5, %xmm4
- movdqa %xmm1, $in2_x+0x10(%rsp)
- por %xmm0, %xmm1
- movq $r_ptr, %xmm0 # save $r_ptr
- movdqa %xmm2, $in2_y(%rsp)
- movdqa %xmm3, $in2_y+0x10(%rsp)
- por %xmm2, %xmm3
- por %xmm4, %xmm5
- pxor %xmm4, %xmm4
- por %xmm1, %xmm3
-
- lea 0x40-$bias($a_ptr), $a_ptr # $a_ptr is still valid
- mov $src0, $in2_z+8*0(%rsp) # make in2_z copy
- mov $acc6, $in2_z+8*1(%rsp)
- mov $acc7, $in2_z+8*2(%rsp)
- mov $acc0, $in2_z+8*3(%rsp)
- lea $Z2sqr(%rsp), $r_ptr # Z2^2
- call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z2sqr, in2_z);
-
- pcmpeqd %xmm4, %xmm5
- pshufd \$0xb1, %xmm3, %xmm4
- por %xmm3, %xmm4
- pshufd \$0, %xmm5, %xmm5 # in1infty
- pshufd \$0x1e, %xmm4, %xmm3
- por %xmm3, %xmm4
- pxor %xmm3, %xmm3
- pcmpeqd %xmm3, %xmm4
- pshufd \$0, %xmm4, %xmm4 # in2infty
- mov 0x40+8*0($b_ptr), $src0 # load original in1_z
- mov 0x40+8*1($b_ptr), $acc6
- mov 0x40+8*2($b_ptr), $acc7
- mov 0x40+8*3($b_ptr), $acc0
- movq $b_ptr, %xmm1
-
- lea 0x40-$bias($b_ptr), $a_ptr
- lea $Z1sqr(%rsp), $r_ptr # Z1^2
- call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z1sqr, in1_z);
-
- `&load_for_mul("$Z2sqr(%rsp)", "$in2_z(%rsp)", "$src0")`
- lea $S1(%rsp), $r_ptr # S1 = Z2^3
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(S1, Z2sqr, in2_z);
-
- `&load_for_mul("$Z1sqr(%rsp)", "$in1_z(%rsp)", "$src0")`
- lea $S2(%rsp), $r_ptr # S2 = Z1^3
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Z1sqr, in1_z);
-
- `&load_for_mul("$S1(%rsp)", "$in1_y(%rsp)", "$src0")`
- lea $S1(%rsp), $r_ptr # S1 = Y1*Z2^3
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(S1, S1, in1_y);
-
- `&load_for_mul("$S2(%rsp)", "$in2_y(%rsp)", "$src0")`
- lea $S2(%rsp), $r_ptr # S2 = Y2*Z1^3
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S2, in2_y);
-
- lea $S1(%rsp), $b_ptr
- lea $R(%rsp), $r_ptr # R = S2 - S1
- call __ecp_nistz256_sub_from$x # p256_sub(R, S2, S1);
-
- or $acc5, $acc4 # see if result is zero
- movdqa %xmm4, %xmm2
- or $acc0, $acc4
- or $acc1, $acc4
- por %xmm5, %xmm2 # in1infty || in2infty
- movq $acc4, %xmm3
-
- `&load_for_mul("$Z2sqr(%rsp)", "$in1_x(%rsp)", "$src0")`
- lea $U1(%rsp), $r_ptr # U1 = X1*Z2^2
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(U1, in1_x, Z2sqr);
-
- `&load_for_mul("$Z1sqr(%rsp)", "$in2_x(%rsp)", "$src0")`
- lea $U2(%rsp), $r_ptr # U2 = X2*Z1^2
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, in2_x, Z1sqr);
-
- lea $U1(%rsp), $b_ptr
- lea $H(%rsp), $r_ptr # H = U2 - U1
- call __ecp_nistz256_sub_from$x # p256_sub(H, U2, U1);
-
- or $acc5, $acc4 # see if result is zero
- or $acc0, $acc4
- or $acc1, $acc4
-
- .byte 0x3e # predict taken
- jnz .Ladd_proceed$x # is_equal(U1,U2)?
- movq %xmm2, $acc0
- movq %xmm3, $acc1
- test $acc0, $acc0
- jnz .Ladd_proceed$x # (in1infty || in2infty)?
- test $acc1, $acc1
- jz .Ladd_double$x # is_equal(S1,S2)?
-
- movq %xmm0, $r_ptr # restore $r_ptr
- pxor %xmm0, %xmm0
- movdqu %xmm0, 0x00($r_ptr)
- movdqu %xmm0, 0x10($r_ptr)
- movdqu %xmm0, 0x20($r_ptr)
- movdqu %xmm0, 0x30($r_ptr)
- movdqu %xmm0, 0x40($r_ptr)
- movdqu %xmm0, 0x50($r_ptr)
- jmp .Ladd_done$x
-
- .align 32
- .Ladd_double$x:
- movq %xmm1, $a_ptr # restore $a_ptr
- movq %xmm0, $r_ptr # restore $r_ptr
- add \$`32*(18-5)`, %rsp # difference in frame sizes
- jmp .Lpoint_double_shortcut$x
-
- .align 32
- .Ladd_proceed$x:
- `&load_for_sqr("$R(%rsp)", "$src0")`
- lea $Rsqr(%rsp), $r_ptr # R^2
- call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Rsqr, R);
-
- `&load_for_mul("$H(%rsp)", "$in1_z(%rsp)", "$src0")`
- lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, H, in1_z);
-
- `&load_for_sqr("$H(%rsp)", "$src0")`
- lea $Hsqr(%rsp), $r_ptr # H^2
- call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Hsqr, H);
-
- `&load_for_mul("$res_z(%rsp)", "$in2_z(%rsp)", "$src0")`
- lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, res_z, in2_z);
-
- `&load_for_mul("$Hsqr(%rsp)", "$H(%rsp)", "$src0")`
- lea $Hcub(%rsp), $r_ptr # H^3
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(Hcub, Hsqr, H);
-
- `&load_for_mul("$Hsqr(%rsp)", "$U1(%rsp)", "$src0")`
- lea $U2(%rsp), $r_ptr # U1*H^2
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, U1, Hsqr);
- ___
- {
- #######################################################################
- # operate in 4-5-0-1 "name space" that matches multiplication output
- #
- my ($acc0,$acc1,$acc2,$acc3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
- my ($poly1, $poly3)=($acc6,$acc7);
-
- $code.=<<___;
- #lea $U2(%rsp), $a_ptr
- #lea $Hsqr(%rsp), $r_ptr # 2*U1*H^2
- #call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2);
-
- add $acc0, $acc0 # a0:a3+a0:a3
- lea $Rsqr(%rsp), $a_ptr
- adc $acc1, $acc1
- mov $acc0, $t0
- adc $acc2, $acc2
- adc $acc3, $acc3
- mov $acc1, $t1
- sbb $t4, $t4
-
- sub \$-1, $acc0
- mov $acc2, $t2
- sbb $poly1, $acc1
- sbb \$0, $acc2
- mov $acc3, $t3
- sbb $poly3, $acc3
- test $t4, $t4
-
- cmovz $t0, $acc0
- mov 8*0($a_ptr), $t0
- cmovz $t1, $acc1
- mov 8*1($a_ptr), $t1
- cmovz $t2, $acc2
- mov 8*2($a_ptr), $t2
- cmovz $t3, $acc3
- mov 8*3($a_ptr), $t3
-
- call __ecp_nistz256_sub$x # p256_sub(res_x, Rsqr, Hsqr);
-
- lea $Hcub(%rsp), $b_ptr
- lea $res_x(%rsp), $r_ptr
- call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, Hcub);
-
- mov $U2+8*0(%rsp), $t0
- mov $U2+8*1(%rsp), $t1
- mov $U2+8*2(%rsp), $t2
- mov $U2+8*3(%rsp), $t3
- lea $res_y(%rsp), $r_ptr
-
- call __ecp_nistz256_sub$x # p256_sub(res_y, U2, res_x);
-
- mov $acc0, 8*0($r_ptr) # save the result, as
- mov $acc1, 8*1($r_ptr) # __ecp_nistz256_sub doesn't
- mov $acc2, 8*2($r_ptr)
- mov $acc3, 8*3($r_ptr)
- ___
- }
- $code.=<<___;
- `&load_for_mul("$S1(%rsp)", "$Hcub(%rsp)", "$src0")`
- lea $S2(%rsp), $r_ptr
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S1, Hcub);
-
- `&load_for_mul("$R(%rsp)", "$res_y(%rsp)", "$src0")`
- lea $res_y(%rsp), $r_ptr
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_y, R, res_y);
-
- lea $S2(%rsp), $b_ptr
- lea $res_y(%rsp), $r_ptr
- call __ecp_nistz256_sub_from$x # p256_sub(res_y, res_y, S2);
-
- movq %xmm0, $r_ptr # restore $r_ptr
-
- movdqa %xmm5, %xmm0 # copy_conditional(res_z, in2_z, in1infty);
- movdqa %xmm5, %xmm1
- pandn $res_z(%rsp), %xmm0
- movdqa %xmm5, %xmm2
- pandn $res_z+0x10(%rsp), %xmm1
- movdqa %xmm5, %xmm3
- pand $in2_z(%rsp), %xmm2
- pand $in2_z+0x10(%rsp), %xmm3
- por %xmm0, %xmm2
- por %xmm1, %xmm3
-
- movdqa %xmm4, %xmm0 # copy_conditional(res_z, in1_z, in2infty);
- movdqa %xmm4, %xmm1
- pandn %xmm2, %xmm0
- movdqa %xmm4, %xmm2
- pandn %xmm3, %xmm1
- movdqa %xmm4, %xmm3
- pand $in1_z(%rsp), %xmm2
- pand $in1_z+0x10(%rsp), %xmm3
- por %xmm0, %xmm2
- por %xmm1, %xmm3
- movdqu %xmm2, 0x40($r_ptr)
- movdqu %xmm3, 0x50($r_ptr)
-
- movdqa %xmm5, %xmm0 # copy_conditional(res_x, in2_x, in1infty);
- movdqa %xmm5, %xmm1
- pandn $res_x(%rsp), %xmm0
- movdqa %xmm5, %xmm2
- pandn $res_x+0x10(%rsp), %xmm1
- movdqa %xmm5, %xmm3
- pand $in2_x(%rsp), %xmm2
- pand $in2_x+0x10(%rsp), %xmm3
- por %xmm0, %xmm2
- por %xmm1, %xmm3
-
- movdqa %xmm4, %xmm0 # copy_conditional(res_x, in1_x, in2infty);
- movdqa %xmm4, %xmm1
- pandn %xmm2, %xmm0
- movdqa %xmm4, %xmm2
- pandn %xmm3, %xmm1
- movdqa %xmm4, %xmm3
- pand $in1_x(%rsp), %xmm2
- pand $in1_x+0x10(%rsp), %xmm3
- por %xmm0, %xmm2
- por %xmm1, %xmm3
- movdqu %xmm2, 0x00($r_ptr)
- movdqu %xmm3, 0x10($r_ptr)
-
- movdqa %xmm5, %xmm0 # copy_conditional(res_y, in2_y, in1infty);
- movdqa %xmm5, %xmm1
- pandn $res_y(%rsp), %xmm0
- movdqa %xmm5, %xmm2
- pandn $res_y+0x10(%rsp), %xmm1
- movdqa %xmm5, %xmm3
- pand $in2_y(%rsp), %xmm2
- pand $in2_y+0x10(%rsp), %xmm3
- por %xmm0, %xmm2
- por %xmm1, %xmm3
-
- movdqa %xmm4, %xmm0 # copy_conditional(res_y, in1_y, in2infty);
- movdqa %xmm4, %xmm1
- pandn %xmm2, %xmm0
- movdqa %xmm4, %xmm2
- pandn %xmm3, %xmm1
- movdqa %xmm4, %xmm3
- pand $in1_y(%rsp), %xmm2
- pand $in1_y+0x10(%rsp), %xmm3
- por %xmm0, %xmm2
- por %xmm1, %xmm3
- movdqu %xmm2, 0x20($r_ptr)
- movdqu %xmm3, 0x30($r_ptr)
-
- .Ladd_done$x:
- add \$32*18+8, %rsp
- pop %r15
- pop %r14
- pop %r13
- pop %r12
- pop %rbx
- pop %rbp
- ret
- .size ecp_nistz256_point_add$sfx,.-ecp_nistz256_point_add$sfx
- ___
- }
- &gen_add("q");
-
- sub gen_add_affine () {
- my $x = shift;
- my ($src0,$sfx,$bias);
- my ($U2,$S2,$H,$R,$Hsqr,$Hcub,$Rsqr,
- $res_x,$res_y,$res_z,
- $in1_x,$in1_y,$in1_z,
- $in2_x,$in2_y)=map(32*$_,(0..14));
- my $Z1sqr = $S2;
-
- if ($x ne "x") {
- $src0 = "%rax";
- $sfx = "";
- $bias = 0;
-
- $code.=<<___;
- .globl ecp_nistz256_point_add_affine
- .type ecp_nistz256_point_add_affine,\@function,3
- .align 32
- ecp_nistz256_point_add_affine:
- ___
- $code.=<<___ if ($addx);
- mov \$0x80100, %ecx
- and OPENSSL_ia32cap_P+8(%rip), %ecx
- cmp \$0x80100, %ecx
- je .Lpoint_add_affinex
- ___
- } else {
- $src0 = "%rdx";
- $sfx = "x";
- $bias = 128;
-
- $code.=<<___;
- .type ecp_nistz256_point_add_affinex,\@function,3
- .align 32
- ecp_nistz256_point_add_affinex:
- .Lpoint_add_affinex:
- ___
- }
- $code.=<<___;
- push %rbp
- push %rbx
- push %r12
- push %r13
- push %r14
- push %r15
- sub \$32*15+8, %rsp
-
- movdqu 0x00($a_ptr), %xmm0 # copy *(P256_POINT *)$a_ptr
- mov $b_org, $b_ptr # reassign
- movdqu 0x10($a_ptr), %xmm1
- movdqu 0x20($a_ptr), %xmm2
- movdqu 0x30($a_ptr), %xmm3
- movdqu 0x40($a_ptr), %xmm4
- movdqu 0x50($a_ptr), %xmm5
- mov 0x40+8*0($a_ptr), $src0 # load original in1_z
- mov 0x40+8*1($a_ptr), $acc6
- mov 0x40+8*2($a_ptr), $acc7
- mov 0x40+8*3($a_ptr), $acc0
- movdqa %xmm0, $in1_x(%rsp)
- movdqa %xmm1, $in1_x+0x10(%rsp)
- por %xmm0, %xmm1
- movdqa %xmm2, $in1_y(%rsp)
- movdqa %xmm3, $in1_y+0x10(%rsp)
- por %xmm2, %xmm3
- movdqa %xmm4, $in1_z(%rsp)
- movdqa %xmm5, $in1_z+0x10(%rsp)
- por %xmm1, %xmm3
-
- movdqu 0x00($b_ptr), %xmm0 # copy *(P256_POINT_AFFINE *)$b_ptr
- pshufd \$0xb1, %xmm3, %xmm5
- movdqu 0x10($b_ptr), %xmm1
- movdqu 0x20($b_ptr), %xmm2
- por %xmm3, %xmm5
- movdqu 0x30($b_ptr), %xmm3
- movdqa %xmm0, $in2_x(%rsp)
- pshufd \$0x1e, %xmm5, %xmm4
- movdqa %xmm1, $in2_x+0x10(%rsp)
- por %xmm0, %xmm1
- movq $r_ptr, %xmm0 # save $r_ptr
- movdqa %xmm2, $in2_y(%rsp)
- movdqa %xmm3, $in2_y+0x10(%rsp)
- por %xmm2, %xmm3
- por %xmm4, %xmm5
- pxor %xmm4, %xmm4
- por %xmm1, %xmm3
-
- lea 0x40-$bias($a_ptr), $a_ptr # $a_ptr is still valid
- lea $Z1sqr(%rsp), $r_ptr # Z1^2
- call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Z1sqr, in1_z);
-
- pcmpeqd %xmm4, %xmm5
- pshufd \$0xb1, %xmm3, %xmm4
- mov 0x00($b_ptr), $src0 # $b_ptr is still valid
- #lea 0x00($b_ptr), $b_ptr
- mov $acc4, $acc1 # harmonize sqr output and mul input
- por %xmm3, %xmm4
- pshufd \$0, %xmm5, %xmm5 # in1infty
- pshufd \$0x1e, %xmm4, %xmm3
- mov $acc5, $acc2
- por %xmm3, %xmm4
- pxor %xmm3, %xmm3
- mov $acc6, $acc3
- pcmpeqd %xmm3, %xmm4
- pshufd \$0, %xmm4, %xmm4 # in2infty
-
- lea $Z1sqr-$bias(%rsp), $a_ptr
- mov $acc7, $acc4
- lea $U2(%rsp), $r_ptr # U2 = X2*Z1^2
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, Z1sqr, in2_x);
-
- lea $in1_x(%rsp), $b_ptr
- lea $H(%rsp), $r_ptr # H = U2 - U1
- call __ecp_nistz256_sub_from$x # p256_sub(H, U2, in1_x);
-
- `&load_for_mul("$Z1sqr(%rsp)", "$in1_z(%rsp)", "$src0")`
- lea $S2(%rsp), $r_ptr # S2 = Z1^3
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Z1sqr, in1_z);
-
- `&load_for_mul("$H(%rsp)", "$in1_z(%rsp)", "$src0")`
- lea $res_z(%rsp), $r_ptr # Z3 = H*Z1*Z2
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(res_z, H, in1_z);
-
- `&load_for_mul("$S2(%rsp)", "$in2_y(%rsp)", "$src0")`
- lea $S2(%rsp), $r_ptr # S2 = Y2*Z1^3
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, S2, in2_y);
-
- lea $in1_y(%rsp), $b_ptr
- lea $R(%rsp), $r_ptr # R = S2 - S1
- call __ecp_nistz256_sub_from$x # p256_sub(R, S2, in1_y);
-
- `&load_for_sqr("$H(%rsp)", "$src0")`
- lea $Hsqr(%rsp), $r_ptr # H^2
- call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Hsqr, H);
-
- `&load_for_sqr("$R(%rsp)", "$src0")`
- lea $Rsqr(%rsp), $r_ptr # R^2
- call __ecp_nistz256_sqr_mont$x # p256_sqr_mont(Rsqr, R);
-
- `&load_for_mul("$H(%rsp)", "$Hsqr(%rsp)", "$src0")`
- lea $Hcub(%rsp), $r_ptr # H^3
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(Hcub, Hsqr, H);
-
- `&load_for_mul("$Hsqr(%rsp)", "$in1_x(%rsp)", "$src0")`
- lea $U2(%rsp), $r_ptr # U1*H^2
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(U2, in1_x, Hsqr);
- ___
- {
- #######################################################################
- # operate in 4-5-0-1 "name space" that matches multiplication output
- #
- my ($acc0,$acc1,$acc2,$acc3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
- my ($poly1, $poly3)=($acc6,$acc7);
-
- $code.=<<___;
- #lea $U2(%rsp), $a_ptr
- #lea $Hsqr(%rsp), $r_ptr # 2*U1*H^2
- #call __ecp_nistz256_mul_by_2 # ecp_nistz256_mul_by_2(Hsqr, U2);
-
- add $acc0, $acc0 # a0:a3+a0:a3
- lea $Rsqr(%rsp), $a_ptr
- adc $acc1, $acc1
- mov $acc0, $t0
- adc $acc2, $acc2
- adc $acc3, $acc3
- mov $acc1, $t1
- sbb $t4, $t4
-
- sub \$-1, $acc0
- mov $acc2, $t2
- sbb $poly1, $acc1
- sbb \$0, $acc2
- mov $acc3, $t3
- sbb $poly3, $acc3
- test $t4, $t4
-
- cmovz $t0, $acc0
- mov 8*0($a_ptr), $t0
- cmovz $t1, $acc1
- mov 8*1($a_ptr), $t1
- cmovz $t2, $acc2
- mov 8*2($a_ptr), $t2
- cmovz $t3, $acc3
- mov 8*3($a_ptr), $t3
-
- call __ecp_nistz256_sub$x # p256_sub(res_x, Rsqr, Hsqr);
-
- lea $Hcub(%rsp), $b_ptr
- lea $res_x(%rsp), $r_ptr
- call __ecp_nistz256_sub_from$x # p256_sub(res_x, res_x, Hcub);
-
- mov $U2+8*0(%rsp), $t0
- mov $U2+8*1(%rsp), $t1
- mov $U2+8*2(%rsp), $t2
- mov $U2+8*3(%rsp), $t3
- lea $H(%rsp), $r_ptr
-
- call __ecp_nistz256_sub$x # p256_sub(H, U2, res_x);
-
- mov $acc0, 8*0($r_ptr) # save the result, as
- mov $acc1, 8*1($r_ptr) # __ecp_nistz256_sub doesn't
- mov $acc2, 8*2($r_ptr)
- mov $acc3, 8*3($r_ptr)
- ___
- }
- $code.=<<___;
- `&load_for_mul("$Hcub(%rsp)", "$in1_y(%rsp)", "$src0")`
- lea $S2(%rsp), $r_ptr
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(S2, Hcub, in1_y);
-
- `&load_for_mul("$H(%rsp)", "$R(%rsp)", "$src0")`
- lea $H(%rsp), $r_ptr
- call __ecp_nistz256_mul_mont$x # p256_mul_mont(H, H, R);
-
- lea $S2(%rsp), $b_ptr
- lea $res_y(%rsp), $r_ptr
- call __ecp_nistz256_sub_from$x # p256_sub(res_y, H, S2);
-
- movq %xmm0, $r_ptr # restore $r_ptr
-
- movdqa %xmm5, %xmm0 # copy_conditional(res_z, ONE, in1infty);
- movdqa %xmm5, %xmm1
- pandn $res_z(%rsp), %xmm0
- movdqa %xmm5, %xmm2
- pandn $res_z+0x10(%rsp), %xmm1
- movdqa %xmm5, %xmm3
- pand .LONE_mont(%rip), %xmm2
- pand .LONE_mont+0x10(%rip), %xmm3
- por %xmm0, %xmm2
- por %xmm1, %xmm3
-
- movdqa %xmm4, %xmm0 # copy_conditional(res_z, in1_z, in2infty);
- movdqa %xmm4, %xmm1
- pandn %xmm2, %xmm0
- movdqa %xmm4, %xmm2
- pandn %xmm3, %xmm1
- movdqa %xmm4, %xmm3
- pand $in1_z(%rsp), %xmm2
- pand $in1_z+0x10(%rsp), %xmm3
- por %xmm0, %xmm2
- por %xmm1, %xmm3
- movdqu %xmm2, 0x40($r_ptr)
- movdqu %xmm3, 0x50($r_ptr)
-
- movdqa %xmm5, %xmm0 # copy_conditional(res_x, in2_x, in1infty);
- movdqa %xmm5, %xmm1
- pandn $res_x(%rsp), %xmm0
- movdqa %xmm5, %xmm2
- pandn $res_x+0x10(%rsp), %xmm1
- movdqa %xmm5, %xmm3
- pand $in2_x(%rsp), %xmm2
- pand $in2_x+0x10(%rsp), %xmm3
- por %xmm0, %xmm2
- por %xmm1, %xmm3
-
- movdqa %xmm4, %xmm0 # copy_conditional(res_x, in1_x, in2infty);
- movdqa %xmm4, %xmm1
- pandn %xmm2, %xmm0
- movdqa %xmm4, %xmm2
- pandn %xmm3, %xmm1
- movdqa %xmm4, %xmm3
- pand $in1_x(%rsp), %xmm2
- pand $in1_x+0x10(%rsp), %xmm3
- por %xmm0, %xmm2
- por %xmm1, %xmm3
- movdqu %xmm2, 0x00($r_ptr)
- movdqu %xmm3, 0x10($r_ptr)
-
- movdqa %xmm5, %xmm0 # copy_conditional(res_y, in2_y, in1infty);
- movdqa %xmm5, %xmm1
- pandn $res_y(%rsp), %xmm0
- movdqa %xmm5, %xmm2
- pandn $res_y+0x10(%rsp), %xmm1
- movdqa %xmm5, %xmm3
- pand $in2_y(%rsp), %xmm2
- pand $in2_y+0x10(%rsp), %xmm3
- por %xmm0, %xmm2
- por %xmm1, %xmm3
-
- movdqa %xmm4, %xmm0 # copy_conditional(res_y, in1_y, in2infty);
- movdqa %xmm4, %xmm1
- pandn %xmm2, %xmm0
- movdqa %xmm4, %xmm2
- pandn %xmm3, %xmm1
- movdqa %xmm4, %xmm3
- pand $in1_y(%rsp), %xmm2
- pand $in1_y+0x10(%rsp), %xmm3
- por %xmm0, %xmm2
- por %xmm1, %xmm3
- movdqu %xmm2, 0x20($r_ptr)
- movdqu %xmm3, 0x30($r_ptr)
-
- add \$32*15+8, %rsp
- pop %r15
- pop %r14
- pop %r13
- pop %r12
- pop %rbx
- pop %rbp
- ret
- .size ecp_nistz256_point_add_affine$sfx,.-ecp_nistz256_point_add_affine$sfx
- ___
- }
- &gen_add_affine("q");
-
- ########################################################################
- # AD*X magic
- #
- if ($addx) { {
- ########################################################################
- # operate in 4-5-0-1 "name space" that matches multiplication output
- #
- my ($a0,$a1,$a2,$a3,$t3,$t4)=($acc4,$acc5,$acc0,$acc1,$acc2,$acc3);
-
- $code.=<<___;
- .type __ecp_nistz256_add_tox,\@abi-omnipotent
- .align 32
- __ecp_nistz256_add_tox:
- xor $t4, $t4
- adc 8*0($b_ptr), $a0
- adc 8*1($b_ptr), $a1
- mov $a0, $t0
- adc 8*2($b_ptr), $a2
- adc 8*3($b_ptr), $a3
- mov $a1, $t1
- adc \$0, $t4
-
- xor $t3, $t3
- sbb \$-1, $a0
- mov $a2, $t2
- sbb $poly1, $a1
- sbb \$0, $a2
- mov $a3, $t3
- sbb $poly3, $a3
-
- bt \$0, $t4
- cmovnc $t0, $a0
- cmovnc $t1, $a1
- mov $a0, 8*0($r_ptr)
- cmovnc $t2, $a2
- mov $a1, 8*1($r_ptr)
- cmovnc $t3, $a3
- mov $a2, 8*2($r_ptr)
- mov $a3, 8*3($r_ptr)
-
- ret
- .size __ecp_nistz256_add_tox,.-__ecp_nistz256_add_tox
-
- .type __ecp_nistz256_sub_fromx,\@abi-omnipotent
- .align 32
- __ecp_nistz256_sub_fromx:
- xor $t4, $t4
- sbb 8*0($b_ptr), $a0
- sbb 8*1($b_ptr), $a1
- mov $a0, $t0
- sbb 8*2($b_ptr), $a2
- sbb 8*3($b_ptr), $a3
- mov $a1, $t1
- sbb \$0, $t4
-
- xor $t3, $t3
- adc \$-1, $a0
- mov $a2, $t2
- adc $poly1, $a1
- adc \$0, $a2
- mov $a3, $t3
- adc $poly3, $a3
-
- bt \$0, $t4
- cmovnc $t0, $a0
- cmovnc $t1, $a1
- mov $a0, 8*0($r_ptr)
- cmovnc $t2, $a2
- mov $a1, 8*1($r_ptr)
- cmovnc $t3, $a3
- mov $a2, 8*2($r_ptr)
- mov $a3, 8*3($r_ptr)
-
- ret
- .size __ecp_nistz256_sub_fromx,.-__ecp_nistz256_sub_fromx
-
- .type __ecp_nistz256_subx,\@abi-omnipotent
- .align 32
- __ecp_nistz256_subx:
- xor $t4, $t4
- sbb $a0, $t0
- sbb $a1, $t1
- mov $t0, $a0
- sbb $a2, $t2
- sbb $a3, $t3
- mov $t1, $a1
- sbb \$0, $t4
-
- xor $a3 ,$a3
- adc \$-1, $t0
- mov $t2, $a2
- adc $poly1, $t1
- adc \$0, $t2
- mov $t3, $a3
- adc $poly3, $t3
-
- bt \$0, $t4
- cmovc $t0, $a0
- cmovc $t1, $a1
- cmovc $t2, $a2
- cmovc $t3, $a3
-
- ret
- .size __ecp_nistz256_subx,.-__ecp_nistz256_subx
-
- .type __ecp_nistz256_mul_by_2x,\@abi-omnipotent
- .align 32
- __ecp_nistz256_mul_by_2x:
- xor $t4, $t4
- adc $a0, $a0 # a0:a3+a0:a3
- adc $a1, $a1
- mov $a0, $t0
- adc $a2, $a2
- adc $a3, $a3
- mov $a1, $t1
- adc \$0, $t4
-
- xor $t3, $t3
- sbb \$-1, $a0
- mov $a2, $t2
- sbb $poly1, $a1
- sbb \$0, $a2
- mov $a3, $t3
- sbb $poly3, $a3
-
- bt \$0, $t4
- cmovnc $t0, $a0
- cmovnc $t1, $a1
- mov $a0, 8*0($r_ptr)
- cmovnc $t2, $a2
- mov $a1, 8*1($r_ptr)
- cmovnc $t3, $a3
- mov $a2, 8*2($r_ptr)
- mov $a3, 8*3($r_ptr)
-
- ret
- .size __ecp_nistz256_mul_by_2x,.-__ecp_nistz256_mul_by_2x
- ___
- }
- &gen_double("x");
- &gen_add("x");
- &gen_add_affine("x");
- }
- }}}
-
- $code =~ s/\`([^\`]*)\`/eval $1/gem;
- print $code;
- close STDOUT;
|