9edbc7ff9f
This reverts commit e907ed4c4b. CPUID
checks have been added so hopefully this time sticks.
Change-Id: I5e0e5b87427c1230132681f936b3c70bac8263b8
Reviewed-on: https://boringssl-review.googlesource.com/c/32924
Commit-Queue: David Benjamin <davidben@google.com>
Reviewed-by: David Benjamin <davidben@google.com>
CQ-Verified: CQ bot account: commit-bot@chromium.org <commit-bot@chromium.org>
406 lines
9.0 KiB
Raku
406 lines
9.0 KiB
Raku
# Copyright (c) 2018, Amazon Inc.
|
|
#
|
|
# Permission to use, copy, modify, and/or distribute this software for any
|
|
# purpose with or without fee is hereby granted, provided that the above
|
|
# copyright notice and this permission notice appear in all copies.
|
|
#
|
|
# THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
|
# WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
|
# MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
|
|
# SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
|
# WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
|
|
# OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
|
|
# CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
|
|
#
|
|
# Written by Nir Drucker, and Shay Gueron
|
|
# AWS Cryptographic Algorithms Group
|
|
# (ndrucker@amazon.com, gueron@amazon.com)
|
|
# based on BN_mod_inverse_odd
|
|
|
|
$flavour = shift;
|
|
$output = shift;
|
|
if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }
|
|
|
|
$win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);
|
|
|
|
$0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;
|
|
( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or
|
|
( $xlate="${dir}../../../perlasm/x86_64-xlate.pl" and -f $xlate) or
|
|
die "can't locate x86_64-xlate.pl";
|
|
|
|
open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";
|
|
*STDOUT=*OUT;
|
|
|
|
#############################################################################
|
|
# extern int beeu_mod_inverse_vartime(BN_ULONG out[P256_LIMBS],
|
|
# BN_ULONG a[P256_LIMBS],
|
|
# BN_ULONG n[P256_LIMBS]);
|
|
#
|
|
# (Binary Extended Euclidean Algorithm.
|
|
# See https://en.wikipedia.org/wiki/Binary_GCD_algorithm)
|
|
#
|
|
# Assumption 1: n is odd for the BEEU
|
|
# Assumption 2: 1 < a < n < 2^256
|
|
|
|
$out = "%rdi";
|
|
$a = "%rsi";
|
|
$n = "%rdx";
|
|
|
|
# X/Y will hold the inverse parameter
|
|
# Assumption: X,Y<2^(256)
|
|
$x0 = "%r8";
|
|
$x1 = "%r9";
|
|
$x2 = "%r10";
|
|
$x3 = "%r11";
|
|
# borrow from out (out is needed only at the end)
|
|
$x4 = "%rdi";
|
|
$y0 = "%r12";
|
|
$y1 = "%r13";
|
|
$y2 = "%r14";
|
|
$y3 = "%r15";
|
|
$y4 = "%rbp";
|
|
$shift = "%rcx";
|
|
$t0 = "%rax";
|
|
$t1 = "%rbx";
|
|
$t2 = "%rsi";
|
|
# borrow
|
|
$t3 = "%rcx";
|
|
|
|
$T0 = "%xmm0";
|
|
$T1 = "%xmm1";
|
|
|
|
# Offsets on the stack
|
|
$out_rsp = 0;
|
|
$shift_rsp = $out_rsp+0x8;
|
|
$a_rsp0 = $shift_rsp+0x8;
|
|
$a_rsp1 = $a_rsp0+0x8;
|
|
$a_rsp2 = $a_rsp1+0x8;
|
|
$a_rsp3 = $a_rsp2+0x8;
|
|
$b_rsp0 = $a_rsp3+0x8;
|
|
$b_rsp1 = $b_rsp0+0x8;
|
|
$b_rsp2 = $b_rsp1+0x8;
|
|
$b_rsp3 = $b_rsp2+0x8;
|
|
|
|
# Borrow when a_rsp/b_rsp are no longer needed.
|
|
$y_rsp0 = $a_rsp0;
|
|
$y_rsp1 = $y_rsp0+0x8;
|
|
$y_rsp2 = $y_rsp1+0x8;
|
|
$y_rsp3 = $y_rsp2+0x8;
|
|
$y_rsp4 = $y_rsp3+0x8;
|
|
$last_rsp_offset = $b_rsp3+0x8;
|
|
|
|
sub TEST_B_ZERO {
|
|
return <<___;
|
|
xorq $t1, $t1
|
|
or $b_rsp0(%rsp), $t1
|
|
or $b_rsp1(%rsp), $t1
|
|
or $b_rsp2(%rsp), $t1
|
|
or $b_rsp3(%rsp), $t1
|
|
jz .Lbeeu_loop_end
|
|
___
|
|
}
|
|
|
|
$g_next_label = 0;
|
|
|
|
sub SHIFT1 {
|
|
my ($var0, $var1, $var2, $var3, $var4) = @_;
|
|
my $label = ".Lshift1_${g_next_label}";
|
|
$g_next_label++;
|
|
|
|
return <<___;
|
|
# Ensure X is even and divide by two.
|
|
movq \$1, $t1
|
|
andq $var0, $t1
|
|
jz $label
|
|
add 0*8($n), $var0
|
|
adc 1*8($n), $var1
|
|
adc 2*8($n), $var2
|
|
adc 3*8($n), $var3
|
|
adc \$0, $var4
|
|
|
|
$label:
|
|
shrdq \$1, $var1, $var0
|
|
shrdq \$1, $var2, $var1
|
|
shrdq \$1, $var3, $var2
|
|
shrdq \$1, $var4, $var3
|
|
shrq \$1, $var4
|
|
___
|
|
}
|
|
|
|
sub SHIFT256 {
|
|
my ($var) = @_;
|
|
return <<___;
|
|
# Copy shifted values.
|
|
# Remember not to override t3=rcx
|
|
movq 1*8+$var(%rsp), $t0
|
|
movq 2*8+$var(%rsp), $t1
|
|
movq 3*8+$var(%rsp), $t2
|
|
|
|
shrdq %cl, $t0, 0*8+$var(%rsp)
|
|
shrdq %cl, $t1, 1*8+$var(%rsp)
|
|
shrdq %cl, $t2, 2*8+$var(%rsp)
|
|
|
|
shrq %cl, $t2
|
|
mov $t2, 3*8+$var(%rsp)
|
|
___
|
|
}
|
|
|
|
$code.=<<___;
|
|
.text
|
|
|
|
.type beeu_mod_inverse_vartime,\@function
|
|
.hidden beeu_mod_inverse_vartime
|
|
.globl beeu_mod_inverse_vartime
|
|
.align 32
|
|
beeu_mod_inverse_vartime:
|
|
.cfi_startproc
|
|
push %rbp
|
|
.cfi_push rbp
|
|
movq %rsp, %rbp
|
|
.cfi_def_cfa_register rbp
|
|
|
|
push %r12
|
|
.cfi_push r12
|
|
push %r13
|
|
.cfi_push r13
|
|
push %r14
|
|
.cfi_push r14
|
|
push %r15
|
|
.cfi_push r15
|
|
push %rbx
|
|
.cfi_push rbx
|
|
push %rsi
|
|
.cfi_push rsi
|
|
|
|
sub \$$last_rsp_offset, %rsp
|
|
movq $out, $out_rsp(%rsp)
|
|
|
|
# X=1, Y=0
|
|
movq \$1, $x0
|
|
xorq $x1, $x1
|
|
xorq $x2, $x2
|
|
xorq $x3, $x3
|
|
xorq $x4, $x4
|
|
|
|
xorq $y0, $y0
|
|
xorq $y1, $y1
|
|
xorq $y2, $y2
|
|
xorq $y3, $y3
|
|
xorq $y4, $y4
|
|
|
|
# Copy a/n into B/A on the stack.
|
|
vmovdqu 0*8($a), $T0
|
|
vmovdqu 2*8($a), $T1
|
|
vmovdqu $T0, $b_rsp0(%rsp)
|
|
vmovdqu $T1, $b_rsp2(%rsp)
|
|
|
|
vmovdqu 0*8($n), $T0
|
|
vmovdqu 2*8($n), $T1
|
|
vmovdqu $T0, $a_rsp0(%rsp)
|
|
vmovdqu $T1, $a_rsp2(%rsp)
|
|
|
|
.Lbeeu_loop:
|
|
${\TEST_B_ZERO}
|
|
|
|
# 0 < B < |n|,
|
|
# 0 < A <= |n|,
|
|
# (1) X*a == B (mod |n|),
|
|
# (2) (-1)*Y*a == A (mod |n|)
|
|
|
|
# Now divide B by the maximum possible power of two in the
|
|
# integers, and divide X by the same value mod |n|. When we're
|
|
# done, (1) still holds.
|
|
movq \$1, $shift
|
|
|
|
# Note that B > 0
|
|
.Lbeeu_shift_loop_XB:
|
|
movq $shift, $t1
|
|
andq $b_rsp0(%rsp), $t1
|
|
jnz .Lbeeu_shift_loop_end_XB
|
|
|
|
${\SHIFT1($x0, $x1, $x2, $x3, $x4)}
|
|
shl \$1, $shift
|
|
|
|
# Test wraparound of the shift parameter. The probability to have 32 zeroes
|
|
# in a row is small Therefore having the value below equal \$0x8000000 or
|
|
# \$0x8000 does not affect the performance. We choose 0x8000000 because it
|
|
# is the maximal immediate value possible.
|
|
cmp \$0x8000000, $shift
|
|
jne .Lbeeu_shift_loop_XB
|
|
|
|
.Lbeeu_shift_loop_end_XB:
|
|
bsf $shift, $shift
|
|
test $shift, $shift
|
|
jz .Lbeeu_no_shift_XB
|
|
|
|
${\SHIFT256($b_rsp0)}
|
|
|
|
.Lbeeu_no_shift_XB:
|
|
# Same for A and Y. Afterwards, (2) still holds.
|
|
movq \$1, $shift
|
|
|
|
# Note that A > 0
|
|
.Lbeeu_shift_loop_YA:
|
|
movq $shift, $t1
|
|
andq $a_rsp0(%rsp), $t1
|
|
jnz .Lbeeu_shift_loop_end_YA
|
|
|
|
${\SHIFT1($y0, $y1, $y2, $y3, $y4)}
|
|
shl \$1, $shift
|
|
|
|
# Test wraparound of the shift parameter. The probability to have 32 zeroes
|
|
# in a row is small therefore having the value below equal \$0x8000000 or
|
|
# \$0x8000 Does not affect the performance. We choose 0x8000000 because it
|
|
# is the maximal immediate value possible.
|
|
cmp \$0x8000000, $shift
|
|
jne .Lbeeu_shift_loop_YA
|
|
|
|
.Lbeeu_shift_loop_end_YA:
|
|
bsf $shift, $shift
|
|
test $shift, $shift
|
|
jz .Lbeeu_no_shift_YA
|
|
|
|
${\SHIFT256($a_rsp0)}
|
|
|
|
.Lbeeu_no_shift_YA:
|
|
# T = B-A (A,B < 2^256)
|
|
mov $b_rsp0(%rsp), $t0
|
|
mov $b_rsp1(%rsp), $t1
|
|
mov $b_rsp2(%rsp), $t2
|
|
mov $b_rsp3(%rsp), $t3
|
|
sub $a_rsp0(%rsp), $t0
|
|
sbb $a_rsp1(%rsp), $t1
|
|
sbb $a_rsp2(%rsp), $t2
|
|
sbb $a_rsp3(%rsp), $t3 # borrow from shift
|
|
jnc .Lbeeu_B_bigger_than_A
|
|
|
|
# A = A - B
|
|
mov $a_rsp0(%rsp), $t0
|
|
mov $a_rsp1(%rsp), $t1
|
|
mov $a_rsp2(%rsp), $t2
|
|
mov $a_rsp3(%rsp), $t3
|
|
sub $b_rsp0(%rsp), $t0
|
|
sbb $b_rsp1(%rsp), $t1
|
|
sbb $b_rsp2(%rsp), $t2
|
|
sbb $b_rsp3(%rsp), $t3
|
|
mov $t0, $a_rsp0(%rsp)
|
|
mov $t1, $a_rsp1(%rsp)
|
|
mov $t2, $a_rsp2(%rsp)
|
|
mov $t3, $a_rsp3(%rsp)
|
|
|
|
# Y = Y + X
|
|
add $x0, $y0
|
|
adc $x1, $y1
|
|
adc $x2, $y2
|
|
adc $x3, $y3
|
|
adc $x4, $y4
|
|
jmp .Lbeeu_loop
|
|
|
|
.Lbeeu_B_bigger_than_A:
|
|
# B = T = B - A
|
|
mov $t0, $b_rsp0(%rsp)
|
|
mov $t1, $b_rsp1(%rsp)
|
|
mov $t2, $b_rsp2(%rsp)
|
|
mov $t3, $b_rsp3(%rsp)
|
|
|
|
# X = Y + X
|
|
add $y0, $x0
|
|
adc $y1, $x1
|
|
adc $y2, $x2
|
|
adc $y3, $x3
|
|
adc $y4, $x4
|
|
|
|
jmp .Lbeeu_loop
|
|
|
|
.Lbeeu_loop_end:
|
|
# The Euclid's algorithm loop ends when A == beeu(a,n);
|
|
# Therefore (-1)*Y*a == A (mod |n|), Y>0
|
|
|
|
# Verify that A = 1 ==> (-1)*Y*a = A = 1 (mod |n|)
|
|
mov $a_rsp0(%rsp), $t1
|
|
sub \$1, $t1
|
|
or $a_rsp1(%rsp), $t1
|
|
or $a_rsp2(%rsp), $t1
|
|
or $a_rsp3(%rsp), $t1
|
|
# If not, fail.
|
|
jnz .Lbeeu_err
|
|
|
|
# From this point on, we no longer need X
|
|
# Therefore we use it as a temporary storage.
|
|
# X = n
|
|
movq 0*8($n), $x0
|
|
movq 1*8($n), $x1
|
|
movq 2*8($n), $x2
|
|
movq 3*8($n), $x3
|
|
xorq $x4, $x4
|
|
|
|
.Lbeeu_reduction_loop:
|
|
movq $y0, $y_rsp0(%rsp)
|
|
movq $y1, $y_rsp1(%rsp)
|
|
movq $y2, $y_rsp2(%rsp)
|
|
movq $y3, $y_rsp3(%rsp)
|
|
movq $y4, $y_rsp4(%rsp)
|
|
|
|
# If Y>n ==> Y=Y-n
|
|
sub $x0, $y0
|
|
sbb $x1, $y1
|
|
sbb $x2, $y2
|
|
sbb $x3, $y3
|
|
sbb \$0, $y4
|
|
|
|
# Choose old Y or new Y
|
|
cmovc $y_rsp0(%rsp), $y0
|
|
cmovc $y_rsp1(%rsp), $y1
|
|
cmovc $y_rsp2(%rsp), $y2
|
|
cmovc $y_rsp3(%rsp), $y3
|
|
jnc .Lbeeu_reduction_loop
|
|
|
|
# X = n - Y (n, Y < 2^256), (Cancel the (-1))
|
|
sub $y0, $x0
|
|
sbb $y1, $x1
|
|
sbb $y2, $x2
|
|
sbb $y3, $x3
|
|
|
|
.Lbeeu_save:
|
|
# Save the inverse(<2^256) to out.
|
|
mov $out_rsp(%rsp), $out
|
|
|
|
movq $x0, 0*8($out)
|
|
movq $x1, 1*8($out)
|
|
movq $x2, 2*8($out)
|
|
movq $x3, 3*8($out)
|
|
|
|
# Return 1.
|
|
movq \$1, %rax
|
|
jmp .Lbeeu_finish
|
|
|
|
.Lbeeu_err:
|
|
# Return 0.
|
|
xorq %rax, %rax
|
|
|
|
.Lbeeu_finish:
|
|
add \$$last_rsp_offset, %rsp
|
|
pop %rsi
|
|
.cfi_pop rsi
|
|
pop %rbx
|
|
.cfi_pop rbx
|
|
pop %r15
|
|
.cfi_pop r15
|
|
pop %r14
|
|
.cfi_pop r14
|
|
pop %r13
|
|
.cfi_pop r13
|
|
pop %r12
|
|
.cfi_pop r12
|
|
pop %rbp
|
|
.cfi_pop rbp
|
|
.cfi_def_cfa rsp, 8
|
|
.cfi_endproc
|
|
ret
|
|
|
|
.size beeu_mod_inverse_vartime, .-beeu_mod_inverse_vartime
|
|
___
|
|
|
|
print $code;
|
|
close STDOUT;
|