kris
/
boringssl
1
0
Форк 0
Код Проблеми 0 Запити на злиття 0 Релізи 0 Вікі Активність
Ви не можете вибрати більше 25 тем Теми мають розпочинатися з літери або цифри, можуть містити дефіси (-) і не повинні перевищувати 35 символів.
528 Коміти
12 Гілки
38 MiB
 
 
 
 
 
 
Дерево: f4b4952719
Гілки Теги
${ item.name }
Створити гілку ${ searchTerm }
з 'f4b4952719'
${ noResults }
boringssl/crypto/bn/asm/rsaz-avx2.pl

1895 рядки
51 KiB
Неформатований Звинувачення Історія 

  1. #!/usr/bin/env perl

  2. 

  3. ##############################################################################

  4. #                                                                            #

  5. #  Copyright (c) 2012, Intel Corporation                                     #

  6. #                                                                            #

  7. #  All rights reserved.                                                      #

  8. #                                                                            #

  9. #  Redistribution and use in source and binary forms, with or without        #

  10. #  modification, are permitted provided that the following conditions are    #

  11. #  met:                                                                      #

  12. #                                                                            #

  13. #  *  Redistributions of source code must retain the above copyright         #

  14. #     notice, this list of conditions and the following disclaimer.          #

  15. #                                                                            #

  16. #  *  Redistributions in binary form must reproduce the above copyright      #

  17. #     notice, this list of conditions and the following disclaimer in the    #

  18. #     documentation and/or other materials provided with the                 #

  19. #     distribution.                                                          #

  20. #                                                                            #

  21. #  *  Neither the name of the Intel Corporation nor the names of its         #

  22. #     contributors may be used to endorse or promote products derived from   #

  23. #     this software without specific prior written permission.               #

  24. #                                                                            #

  25. #                                                                            #

  26. #  THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY          #

  27. #  EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE         #

  28. #  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR        #

  29. #  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR            #

  30. #  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,     #

  31. #  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,       #

  32. #  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR        #

  33. #  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF    #

  34. #  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING      #

  35. #  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS        #

  36. #  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.              #

  37. #                                                                            #

  38. ##############################################################################

  39. # Developers and authors:                                                    #

  40. # Shay Gueron (1, 2), and Vlad Krasnov (1)                                   #

  41. # (1) Intel Corporation, Israel Development Center, Haifa, Israel            #

  42. # (2) University of Haifa, Israel                                            #

  43. ##############################################################################

  44. # Reference:                                                                 #

  45. # [1] S. Gueron, V. Krasnov: "Software Implementation of Modular             #

  46. #     Exponentiation,  Using Advanced Vector Instructions Architectures",    #

  47. #     F. Ozbudak and F. Rodriguez-Henriquez (Eds.): WAIFI 2012, LNCS 7369,   #

  48. #     pp. 119?135, 2012. Springer-Verlag Berlin Heidelberg 2012              #

  49. # [2] S. Gueron: "Efficient Software Implementations of Modular              #

  50. #     Exponentiation", Journal of Cryptographic Engineering 2:31-43 (2012).  #

  51. # [3] S. Gueron, V. Krasnov: "Speeding up Big-numbers Squaring",IEEE         #

  52. #     Proceedings of 9th International Conference on Information Technology: #

  53. #     New Generations (ITNG 2012), pp.821-823 (2012)                         #

  54. # [4] S. Gueron, V. Krasnov: "[PATCH] Efficient and side channel analysis    #

  55. #     resistant 1024-bit modular exponentiation, for optimizing RSA2048      #

  56. #     on AVX2 capable x86_64 platforms",                                     #

  57. #     http://rt.openssl.org/Ticket/Display.html?id=2850&user=guest&pass=guest#

  58. ##############################################################################

  59. #

  60. # +13% improvement over original submission by <appro@openssl.org>

  61. #

  62. # rsa2048 sign/sec	OpenSSL 1.0.1	scalar(*)	this

  63. # 2.3GHz Haswell	621		765/+23%	1113/+79%

  64. #

  65. # (*)	if system doesn't support AVX2, for reference purposes;

  66. 

  67. $flavour = shift;

  68. $output  = shift;

  69. if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }

  70. 

  71. $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);

  72. 

  73. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;

  74. ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or

  75. ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or

  76. die "can't locate x86_64-xlate.pl";

  77. 

  78. if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`

  79. 		=~ /GNU assembler version ([2-9]\.[0-9]+)/) {

  80. 	$avx = ($1>=2.19) + ($1>=2.22);

  81. 	$addx = ($1>=2.23);

  82. }

  83. 

  84. if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&

  85. 	    `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {

  86. 	$avx = ($1>=2.09) + ($1>=2.10);

  87. 	$addx = ($1>=2.10);

  88. }

  89. 

  90. if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&

  91. 	    `ml64 2>&1` =~ /Version ([0-9]+)\./) {

  92. 	$avx = ($1>=10) + ($1>=11);

  93. 	$addx = ($1>=11);

  94. }

  95. 

  96. if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9])\.([0-9]+)/) {

  97. 	my $ver = $2 + $3/100.0;	# 3.1->3.01, 3.10->3.10

  98. 	$avx = ($ver>=3.0) + ($ver>=3.01);

  99. 	$addx = ($ver>=3.03);

  100. }

  101. 

  102. open OUT,"| $^X $xlate $flavour $output";

  103. *STDOUT = *OUT;

  104. 

  105. if ($avx>1) {{{

  106. { # void AMS_WW(

  107. my $rp="%rdi";	# BN_ULONG *rp,

  108. my $ap="%rsi";	# const BN_ULONG *ap,

  109. my $np="%rdx";	# const BN_ULONG *np,

  110. my $n0="%ecx";	# const BN_ULONG n0,

  111. my $rep="%r8d";	# int repeat);

  112. 

  113. # The registers that hold the accumulated redundant result

  114. # The AMM works on 1024 bit operands, and redundant word size is 29

  115. # Therefore: ceil(1024/29)/4 = 9

  116. my $ACC0="%ymm0";

  117. my $ACC1="%ymm1";

  118. my $ACC2="%ymm2";

  119. my $ACC3="%ymm3";

  120. my $ACC4="%ymm4";

  121. my $ACC5="%ymm5";

  122. my $ACC6="%ymm6";

  123. my $ACC7="%ymm7";

  124. my $ACC8="%ymm8";

  125. my $ACC9="%ymm9";

  126. # Registers that hold the broadcasted words of bp, currently used

  127. my $B1="%ymm10";

  128. my $B2="%ymm11";

  129. # Registers that hold the broadcasted words of Y, currently used

  130. my $Y1="%ymm12";

  131. my $Y2="%ymm13";

  132. # Helper registers

  133. my $TEMP1="%ymm14";

  134. my $AND_MASK="%ymm15";

  135. # alu registers that hold the first words of the ACC

  136. my $r0="%r9";

  137. my $r1="%r10";

  138. my $r2="%r11";

  139. my $r3="%r12";

  140. 

  141. my $i="%r14d";			# loop counter

  142. my $tmp = "%r15";

  143. 

  144. my $FrameSize=32*18+32*8;	# place for A^2 and 2*A

  145. 

  146. my $aap=$r0;

  147. my $tp0="%rbx";

  148. my $tp1=$r3;

  149. my $tpa=$tmp;

  150. 

  151. $np="%r13";			# reassigned argument

  152. 

  153. $code.=<<___;

  154. .text

  155. 

  156. .globl	rsaz_1024_sqr_avx2

  157. .type	rsaz_1024_sqr_avx2,\@function,5

  158. .align	64

  159. rsaz_1024_sqr_avx2:		# 702 cycles, 14% faster than rsaz_1024_mul_avx2

  160. 	lea	(%rsp), %rax

  161. 	push	%rbx

  162. 	push	%rbp

  163. 	push	%r12

  164. 	push	%r13

  165. 	push	%r14

  166. 	push	%r15

  167. 	vzeroupper

  168. ___

  169. $code.=<<___ if ($win64);

  170. 	lea	-0xa8(%rsp),%rsp

  171. 	vmovaps	%xmm6,-0xd8(%rax)

  172. 	vmovaps	%xmm7,-0xc8(%rax)

  173. 	vmovaps	%xmm8,-0xb8(%rax)

  174. 	vmovaps	%xmm9,-0xa8(%rax)

  175. 	vmovaps	%xmm10,-0x98(%rax)

  176. 	vmovaps	%xmm11,-0x88(%rax)

  177. 	vmovaps	%xmm12,-0x78(%rax)

  178. 	vmovaps	%xmm13,-0x68(%rax)

  179. 	vmovaps	%xmm14,-0x58(%rax)

  180. 	vmovaps	%xmm15,-0x48(%rax)

  181. .Lsqr_1024_body:

  182. ___

  183. $code.=<<___;

  184. 	mov	%rax,%rbp

  185. 	mov	%rdx, $np			# reassigned argument

  186. 	sub	\$$FrameSize, %rsp

  187. 	mov	$np, $tmp

  188. 	sub	\$-128, $rp			# size optimization

  189. 	sub	\$-128, $ap

  190. 	sub	\$-128, $np

  191. 

  192. 	and	\$4095, $tmp			# see if $np crosses page

  193. 	add	\$32*10, $tmp

  194. 	shr	\$12, $tmp

  195. 	vpxor	$ACC9,$ACC9,$ACC9

  196. 	jz	.Lsqr_1024_no_n_copy

  197. 

  198. 	# unaligned 256-bit load that crosses page boundary can

  199. 	# cause >2x performance degradation here, so if $np does

  200. 	# cross page boundary, copy it to stack and make sure stack

  201. 	# frame doesn't...

  202. 	sub		\$32*10,%rsp

  203. 	vmovdqu		32*0-128($np), $ACC0

  204. 	and		\$-2048, %rsp

  205. 	vmovdqu		32*1-128($np), $ACC1

  206. 	vmovdqu		32*2-128($np), $ACC2

  207. 	vmovdqu		32*3-128($np), $ACC3

  208. 	vmovdqu		32*4-128($np), $ACC4

  209. 	vmovdqu		32*5-128($np), $ACC5

  210. 	vmovdqu		32*6-128($np), $ACC6

  211. 	vmovdqu		32*7-128($np), $ACC7

  212. 	vmovdqu		32*8-128($np), $ACC8

  213. 	lea		$FrameSize+128(%rsp),$np

  214. 	vmovdqu		$ACC0, 32*0-128($np)

  215. 	vmovdqu		$ACC1, 32*1-128($np)

  216. 	vmovdqu		$ACC2, 32*2-128($np)

  217. 	vmovdqu		$ACC3, 32*3-128($np)

  218. 	vmovdqu		$ACC4, 32*4-128($np)

  219. 	vmovdqu		$ACC5, 32*5-128($np)

  220. 	vmovdqu		$ACC6, 32*6-128($np)

  221. 	vmovdqu		$ACC7, 32*7-128($np)

  222. 	vmovdqu		$ACC8, 32*8-128($np)

  223. 	vmovdqu		$ACC9, 32*9-128($np)	# $ACC9 is zero

  224. 

  225. .Lsqr_1024_no_n_copy:

  226. 	and		\$-1024, %rsp

  227. 

  228. 	vmovdqu		32*1-128($ap), $ACC1

  229. 	vmovdqu		32*2-128($ap), $ACC2

  230. 	vmovdqu		32*3-128($ap), $ACC3

  231. 	vmovdqu		32*4-128($ap), $ACC4

  232. 	vmovdqu		32*5-128($ap), $ACC5

  233. 	vmovdqu		32*6-128($ap), $ACC6

  234. 	vmovdqu		32*7-128($ap), $ACC7

  235. 	vmovdqu		32*8-128($ap), $ACC8

  236. 

  237. 	lea	192(%rsp), $tp0			# 64+128=192

  238. 	vpbroadcastq	.Land_mask(%rip), $AND_MASK

  239. 	jmp	.LOOP_GRANDE_SQR_1024

  240. 

  241. .align	32

  242. .LOOP_GRANDE_SQR_1024:

  243. 	lea	32*18+128(%rsp), $aap		# size optimization

  244. 	lea	448(%rsp), $tp1			# 64+128+256=448

  245. 

  246. 	# the squaring is performed as described in Variant B of

  247. 	# "Speeding up Big-Number Squaring", so start by calculating

  248. 	# the A*2=A+A vector

  249. 	vpaddq		$ACC1, $ACC1, $ACC1

  250. 	 vpbroadcastq	32*0-128($ap), $B1

  251. 	vpaddq		$ACC2, $ACC2, $ACC2

  252. 	vmovdqa		$ACC1, 32*0-128($aap)

  253. 	vpaddq		$ACC3, $ACC3, $ACC3

  254. 	vmovdqa		$ACC2, 32*1-128($aap)

  255. 	vpaddq		$ACC4, $ACC4, $ACC4

  256. 	vmovdqa		$ACC3, 32*2-128($aap)

  257. 	vpaddq		$ACC5, $ACC5, $ACC5

  258. 	vmovdqa		$ACC4, 32*3-128($aap)

  259. 	vpaddq		$ACC6, $ACC6, $ACC6

  260. 	vmovdqa		$ACC5, 32*4-128($aap)

  261. 	vpaddq		$ACC7, $ACC7, $ACC7

  262. 	vmovdqa		$ACC6, 32*5-128($aap)

  263. 	vpaddq		$ACC8, $ACC8, $ACC8

  264. 	vmovdqa		$ACC7, 32*6-128($aap)

  265. 	vpxor		$ACC9, $ACC9, $ACC9

  266. 	vmovdqa		$ACC8, 32*7-128($aap)

  267. 

  268. 	vpmuludq	32*0-128($ap), $B1, $ACC0

  269. 	 vpbroadcastq	32*1-128($ap), $B2

  270. 	 vmovdqu	$ACC9, 32*9-192($tp0)	# zero upper half

  271. 	vpmuludq	$B1, $ACC1, $ACC1

  272. 	 vmovdqu	$ACC9, 32*10-448($tp1)

  273. 	vpmuludq	$B1, $ACC2, $ACC2

  274. 	 vmovdqu	$ACC9, 32*11-448($tp1)

  275. 	vpmuludq	$B1, $ACC3, $ACC3

  276. 	 vmovdqu	$ACC9, 32*12-448($tp1)

  277. 	vpmuludq	$B1, $ACC4, $ACC4

  278. 	 vmovdqu	$ACC9, 32*13-448($tp1)

  279. 	vpmuludq	$B1, $ACC5, $ACC5

  280. 	 vmovdqu	$ACC9, 32*14-448($tp1)

  281. 	vpmuludq	$B1, $ACC6, $ACC6

  282. 	 vmovdqu	$ACC9, 32*15-448($tp1)

  283. 	vpmuludq	$B1, $ACC7, $ACC7

  284. 	 vmovdqu	$ACC9, 32*16-448($tp1)

  285. 	vpmuludq	$B1, $ACC8, $ACC8

  286. 	 vpbroadcastq	32*2-128($ap), $B1

  287. 	 vmovdqu	$ACC9, 32*17-448($tp1)

  288. 

  289. 	mov	$ap, $tpa

  290. 	mov 	\$4, $i

  291. 	jmp	.Lsqr_entry_1024

  292. ___

  293. $TEMP0=$Y1;

  294. $TEMP2=$Y2;

  295. $code.=<<___;

  296. .align	32

  297. .LOOP_SQR_1024:

  298. 	 vpbroadcastq	32*1-128($tpa), $B2

  299. 	vpmuludq	32*0-128($ap), $B1, $ACC0

  300. 	vpaddq		32*0-192($tp0), $ACC0, $ACC0

  301. 	vpmuludq	32*0-128($aap), $B1, $ACC1

  302. 	vpaddq		32*1-192($tp0), $ACC1, $ACC1

  303. 	vpmuludq	32*1-128($aap), $B1, $ACC2

  304. 	vpaddq		32*2-192($tp0), $ACC2, $ACC2

  305. 	vpmuludq	32*2-128($aap), $B1, $ACC3

  306. 	vpaddq		32*3-192($tp0), $ACC3, $ACC3

  307. 	vpmuludq	32*3-128($aap), $B1, $ACC4

  308. 	vpaddq		32*4-192($tp0), $ACC4, $ACC4

  309. 	vpmuludq	32*4-128($aap), $B1, $ACC5

  310. 	vpaddq		32*5-192($tp0), $ACC5, $ACC5

  311. 	vpmuludq	32*5-128($aap), $B1, $ACC6

  312. 	vpaddq		32*6-192($tp0), $ACC6, $ACC6

  313. 	vpmuludq	32*6-128($aap), $B1, $ACC7

  314. 	vpaddq		32*7-192($tp0), $ACC7, $ACC7

  315. 	vpmuludq	32*7-128($aap), $B1, $ACC8

  316. 	 vpbroadcastq	32*2-128($tpa), $B1

  317. 	vpaddq		32*8-192($tp0), $ACC8, $ACC8

  318. .Lsqr_entry_1024:

  319. 	vmovdqu		$ACC0, 32*0-192($tp0)

  320. 	vmovdqu		$ACC1, 32*1-192($tp0)

  321. 

  322. 	vpmuludq	32*1-128($ap), $B2, $TEMP0

  323. 	vpaddq		$TEMP0, $ACC2, $ACC2

  324. 	vpmuludq	32*1-128($aap), $B2, $TEMP1

  325. 	vpaddq		$TEMP1, $ACC3, $ACC3

  326. 	vpmuludq	32*2-128($aap), $B2, $TEMP2

  327. 	vpaddq		$TEMP2, $ACC4, $ACC4

  328. 	vpmuludq	32*3-128($aap), $B2, $TEMP0

  329. 	vpaddq		$TEMP0, $ACC5, $ACC5

  330. 	vpmuludq	32*4-128($aap), $B2, $TEMP1

  331. 	vpaddq		$TEMP1, $ACC6, $ACC6

  332. 	vpmuludq	32*5-128($aap), $B2, $TEMP2

  333. 	vpaddq		$TEMP2, $ACC7, $ACC7

  334. 	vpmuludq	32*6-128($aap), $B2, $TEMP0

  335. 	vpaddq		$TEMP0, $ACC8, $ACC8

  336. 	vpmuludq	32*7-128($aap), $B2, $ACC0

  337. 	 vpbroadcastq	32*3-128($tpa), $B2

  338. 	vpaddq		32*9-192($tp0), $ACC0, $ACC0

  339. 

  340. 	vmovdqu		$ACC2, 32*2-192($tp0)

  341. 	vmovdqu		$ACC3, 32*3-192($tp0)

  342. 

  343. 	vpmuludq	32*2-128($ap), $B1, $TEMP2

  344. 	vpaddq		$TEMP2, $ACC4, $ACC4

  345. 	vpmuludq	32*2-128($aap), $B1, $TEMP0

  346. 	vpaddq		$TEMP0, $ACC5, $ACC5

  347. 	vpmuludq	32*3-128($aap), $B1, $TEMP1

  348. 	vpaddq		$TEMP1, $ACC6, $ACC6

  349. 	vpmuludq	32*4-128($aap), $B1, $TEMP2

  350. 	vpaddq		$TEMP2, $ACC7, $ACC7

  351. 	vpmuludq	32*5-128($aap), $B1, $TEMP0

  352. 	vpaddq		$TEMP0, $ACC8, $ACC8

  353. 	vpmuludq	32*6-128($aap), $B1, $TEMP1

  354. 	vpaddq		$TEMP1, $ACC0, $ACC0

  355. 	vpmuludq	32*7-128($aap), $B1, $ACC1

  356. 	 vpbroadcastq	32*4-128($tpa), $B1

  357. 	vpaddq		32*10-448($tp1), $ACC1, $ACC1

  358. 

  359. 	vmovdqu		$ACC4, 32*4-192($tp0)

  360. 	vmovdqu		$ACC5, 32*5-192($tp0)

  361. 

  362. 	vpmuludq	32*3-128($ap), $B2, $TEMP0

  363. 	vpaddq		$TEMP0, $ACC6, $ACC6

  364. 	vpmuludq	32*3-128($aap), $B2, $TEMP1

  365. 	vpaddq		$TEMP1, $ACC7, $ACC7

  366. 	vpmuludq	32*4-128($aap), $B2, $TEMP2

  367. 	vpaddq		$TEMP2, $ACC8, $ACC8

  368. 	vpmuludq	32*5-128($aap), $B2, $TEMP0

  369. 	vpaddq		$TEMP0, $ACC0, $ACC0

  370. 	vpmuludq	32*6-128($aap), $B2, $TEMP1

  371. 	vpaddq		$TEMP1, $ACC1, $ACC1

  372. 	vpmuludq	32*7-128($aap), $B2, $ACC2

  373. 	 vpbroadcastq	32*5-128($tpa), $B2

  374. 	vpaddq		32*11-448($tp1), $ACC2, $ACC2	

  375. 

  376. 	vmovdqu		$ACC6, 32*6-192($tp0)

  377. 	vmovdqu		$ACC7, 32*7-192($tp0)

  378. 

  379. 	vpmuludq	32*4-128($ap), $B1, $TEMP0

  380. 	vpaddq		$TEMP0, $ACC8, $ACC8

  381. 	vpmuludq	32*4-128($aap), $B1, $TEMP1

  382. 	vpaddq		$TEMP1, $ACC0, $ACC0

  383. 	vpmuludq	32*5-128($aap), $B1, $TEMP2

  384. 	vpaddq		$TEMP2, $ACC1, $ACC1

  385. 	vpmuludq	32*6-128($aap), $B1, $TEMP0

  386. 	vpaddq		$TEMP0, $ACC2, $ACC2

  387. 	vpmuludq	32*7-128($aap), $B1, $ACC3

  388. 	 vpbroadcastq	32*6-128($tpa), $B1

  389. 	vpaddq		32*12-448($tp1), $ACC3, $ACC3

  390. 

  391. 	vmovdqu		$ACC8, 32*8-192($tp0)

  392. 	vmovdqu		$ACC0, 32*9-192($tp0)

  393. 	lea		8($tp0), $tp0

  394. 

  395. 	vpmuludq	32*5-128($ap), $B2, $TEMP2

  396. 	vpaddq		$TEMP2, $ACC1, $ACC1

  397. 	vpmuludq	32*5-128($aap), $B2, $TEMP0

  398. 	vpaddq		$TEMP0, $ACC2, $ACC2

  399. 	vpmuludq	32*6-128($aap), $B2, $TEMP1

  400. 	vpaddq		$TEMP1, $ACC3, $ACC3

  401. 	vpmuludq	32*7-128($aap), $B2, $ACC4

  402. 	 vpbroadcastq	32*7-128($tpa), $B2

  403. 	vpaddq		32*13-448($tp1), $ACC4, $ACC4

  404. 

  405. 	vmovdqu		$ACC1, 32*10-448($tp1)

  406. 	vmovdqu		$ACC2, 32*11-448($tp1)

  407. 

  408. 	vpmuludq	32*6-128($ap), $B1, $TEMP0

  409. 	vpaddq		$TEMP0, $ACC3, $ACC3

  410. 	vpmuludq	32*6-128($aap), $B1, $TEMP1

  411. 	 vpbroadcastq	32*8-128($tpa), $ACC0		# borrow $ACC0 for $B1

  412. 	vpaddq		$TEMP1, $ACC4, $ACC4

  413. 	vpmuludq	32*7-128($aap), $B1, $ACC5

  414. 	 vpbroadcastq	32*0+8-128($tpa), $B1		# for next iteration

  415. 	vpaddq		32*14-448($tp1), $ACC5, $ACC5

  416. 

  417. 	vmovdqu		$ACC3, 32*12-448($tp1)

  418. 	vmovdqu		$ACC4, 32*13-448($tp1)

  419. 	lea		8($tpa), $tpa

  420. 

  421. 	vpmuludq	32*7-128($ap), $B2, $TEMP0

  422. 	vpaddq		$TEMP0, $ACC5, $ACC5

  423. 	vpmuludq	32*7-128($aap), $B2, $ACC6

  424. 	vpaddq		32*15-448($tp1), $ACC6, $ACC6

  425. 

  426. 	vpmuludq	32*8-128($ap), $ACC0, $ACC7

  427. 	vmovdqu		$ACC5, 32*14-448($tp1)

  428. 	vpaddq		32*16-448($tp1), $ACC7, $ACC7

  429. 	vmovdqu		$ACC6, 32*15-448($tp1)

  430. 	vmovdqu		$ACC7, 32*16-448($tp1)

  431. 	lea		8($tp1), $tp1

  432. 

  433. 	dec	$i        

  434. 	jnz	.LOOP_SQR_1024

  435. ___

  436. $ZERO = $ACC9;

  437. $TEMP0 = $B1;

  438. $TEMP2 = $B2;

  439. $TEMP3 = $Y1;

  440. $TEMP4 = $Y2;

  441. $code.=<<___;

  442. 	#we need to fix indexes 32-39 to avoid overflow

  443. 	vmovdqu		32*8(%rsp), $ACC8		# 32*8-192($tp0),

  444. 	vmovdqu		32*9(%rsp), $ACC1		# 32*9-192($tp0)

  445. 	vmovdqu		32*10(%rsp), $ACC2		# 32*10-192($tp0)

  446. 	lea		192(%rsp), $tp0			# 64+128=192

  447. 

  448. 	vpsrlq		\$29, $ACC8, $TEMP1

  449. 	vpand		$AND_MASK, $ACC8, $ACC8

  450. 	vpsrlq		\$29, $ACC1, $TEMP2

  451. 	vpand		$AND_MASK, $ACC1, $ACC1

  452. 

  453. 	vpermq		\$0x93, $TEMP1, $TEMP1

  454. 	vpxor		$ZERO, $ZERO, $ZERO

  455. 	vpermq		\$0x93, $TEMP2, $TEMP2

  456. 

  457. 	vpblendd	\$3, $ZERO, $TEMP1, $TEMP0

  458. 	vpblendd	\$3, $TEMP1, $TEMP2, $TEMP1

  459. 	vpaddq		$TEMP0, $ACC8, $ACC8

  460. 	vpblendd	\$3, $TEMP2, $ZERO, $TEMP2

  461. 	vpaddq		$TEMP1, $ACC1, $ACC1

  462. 	vpaddq		$TEMP2, $ACC2, $ACC2

  463. 	vmovdqu		$ACC1, 32*9-192($tp0)

  464. 	vmovdqu		$ACC2, 32*10-192($tp0)

  465. 

  466. 	mov	(%rsp), %rax

  467. 	mov	8(%rsp), $r1

  468. 	mov	16(%rsp), $r2

  469. 	mov	24(%rsp), $r3

  470. 	vmovdqu	32*1(%rsp), $ACC1

  471. 	vmovdqu	32*2-192($tp0), $ACC2

  472. 	vmovdqu	32*3-192($tp0), $ACC3

  473. 	vmovdqu	32*4-192($tp0), $ACC4

  474. 	vmovdqu	32*5-192($tp0), $ACC5

  475. 	vmovdqu	32*6-192($tp0), $ACC6

  476. 	vmovdqu	32*7-192($tp0), $ACC7

  477. 

  478. 	mov	%rax, $r0

  479. 	imull	$n0, %eax

  480. 	and	\$0x1fffffff, %eax

  481. 	vmovd	%eax, $Y1

  482. 

  483. 	mov	%rax, %rdx

  484. 	imulq	-128($np), %rax

  485. 	 vpbroadcastq	$Y1, $Y1

  486. 	add	%rax, $r0

  487. 	mov	%rdx, %rax

  488. 	imulq	8-128($np), %rax

  489. 	shr	\$29, $r0

  490. 	add	%rax, $r1

  491. 	mov	%rdx, %rax

  492. 	imulq	16-128($np), %rax

  493. 	add	$r0, $r1

  494. 	add	%rax, $r2

  495. 	imulq	24-128($np), %rdx

  496. 	add	%rdx, $r3

  497. 

  498. 	mov	$r1, %rax

  499. 	imull	$n0, %eax

  500. 	and	\$0x1fffffff, %eax

  501. 

  502. 	mov \$9, $i

  503. 	jmp .LOOP_REDUCE_1024

  504. 

  505. .align	32

  506. .LOOP_REDUCE_1024:

  507. 	vmovd	%eax, $Y2

  508. 	vpbroadcastq	$Y2, $Y2

  509. 

  510. 	vpmuludq	32*1-128($np), $Y1, $TEMP0

  511. 	 mov	%rax, %rdx

  512. 	 imulq	-128($np), %rax

  513. 	vpaddq		$TEMP0, $ACC1, $ACC1

  514. 	 add	%rax, $r1

  515. 	vpmuludq	32*2-128($np), $Y1, $TEMP1

  516. 	 mov	%rdx, %rax

  517. 	 imulq	8-128($np), %rax

  518. 	vpaddq		$TEMP1, $ACC2, $ACC2

  519. 	vpmuludq	32*3-128($np), $Y1, $TEMP2

  520. 	 .byte	0x67

  521. 	 add	%rax, $r2

  522. 	 .byte	0x67

  523. 	 mov	%rdx, %rax

  524. 	 imulq	16-128($np), %rax

  525. 	 shr	\$29, $r1

  526. 	vpaddq		$TEMP2, $ACC3, $ACC3

  527. 	vpmuludq	32*4-128($np), $Y1, $TEMP0

  528. 	 add	%rax, $r3

  529. 	 add	$r1, $r2

  530. 	vpaddq		$TEMP0, $ACC4, $ACC4

  531. 	vpmuludq	32*5-128($np), $Y1, $TEMP1

  532. 	 mov	$r2, %rax

  533. 	 imull	$n0, %eax

  534. 	vpaddq		$TEMP1, $ACC5, $ACC5

  535. 	vpmuludq	32*6-128($np), $Y1, $TEMP2

  536. 	 and	\$0x1fffffff, %eax

  537. 	vpaddq		$TEMP2, $ACC6, $ACC6

  538. 	vpmuludq	32*7-128($np), $Y1, $TEMP0

  539. 	vpaddq		$TEMP0, $ACC7, $ACC7

  540. 	vpmuludq	32*8-128($np), $Y1, $TEMP1

  541. 	 vmovd	%eax, $Y1

  542. 	 #vmovdqu	32*1-8-128($np), $TEMP2		# moved below

  543. 	vpaddq		$TEMP1, $ACC8, $ACC8

  544. 	 #vmovdqu	32*2-8-128($np), $TEMP0		# moved below

  545. 	 vpbroadcastq	$Y1, $Y1

  546. 

  547. 	vpmuludq	32*1-8-128($np), $Y2, $TEMP2	# see above

  548. 	vmovdqu		32*3-8-128($np), $TEMP1

  549. 	 mov	%rax, %rdx

  550. 	 imulq	-128($np), %rax

  551. 	vpaddq		$TEMP2, $ACC1, $ACC1

  552. 	vpmuludq	32*2-8-128($np), $Y2, $TEMP0	# see above

  553. 	vmovdqu		32*4-8-128($np), $TEMP2

  554. 	 add	%rax, $r2

  555. 	 mov	%rdx, %rax

  556. 	 imulq	8-128($np), %rax

  557. 	vpaddq		$TEMP0, $ACC2, $ACC2

  558. 	 add	$r3, %rax

  559. 	 shr	\$29, $r2

  560. 	vpmuludq	$Y2, $TEMP1, $TEMP1

  561. 	vmovdqu		32*5-8-128($np), $TEMP0

  562. 	 add	$r2, %rax

  563. 	vpaddq		$TEMP1, $ACC3, $ACC3

  564. 	vpmuludq	$Y2, $TEMP2, $TEMP2

  565. 	vmovdqu		32*6-8-128($np), $TEMP1

  566. 	 .byte	0x67

  567. 	 mov	%rax, $r3

  568. 	 imull	$n0, %eax

  569. 	vpaddq		$TEMP2, $ACC4, $ACC4

  570. 	vpmuludq	$Y2, $TEMP0, $TEMP0

  571. 	.byte	0xc4,0x41,0x7e,0x6f,0x9d,0x58,0x00,0x00,0x00	# vmovdqu		32*7-8-128($np), $TEMP2

  572. 	 and	\$0x1fffffff, %eax

  573. 	vpaddq		$TEMP0, $ACC5, $ACC5

  574. 	vpmuludq	$Y2, $TEMP1, $TEMP1

  575. 	vmovdqu		32*8-8-128($np), $TEMP0

  576. 	vpaddq		$TEMP1, $ACC6, $ACC6

  577. 	vpmuludq	$Y2, $TEMP2, $TEMP2

  578. 	vmovdqu		32*9-8-128($np), $ACC9

  579. 	 vmovd	%eax, $ACC0			# borrow ACC0 for Y2

  580. 	 imulq	-128($np), %rax

  581. 	vpaddq		$TEMP2, $ACC7, $ACC7

  582. 	vpmuludq	$Y2, $TEMP0, $TEMP0

  583. 	 vmovdqu	32*1-16-128($np), $TEMP1

  584. 	 vpbroadcastq	$ACC0, $ACC0

  585. 	vpaddq		$TEMP0, $ACC8, $ACC8

  586. 	vpmuludq	$Y2, $ACC9, $ACC9

  587. 	 vmovdqu	32*2-16-128($np), $TEMP2

  588. 	 add	%rax, $r3

  589. 

  590. ___

  591. ($ACC0,$Y2)=($Y2,$ACC0);

  592. $code.=<<___;

  593. 	 vmovdqu	32*1-24-128($np), $ACC0

  594. 	vpmuludq	$Y1, $TEMP1, $TEMP1

  595. 	vmovdqu		32*3-16-128($np), $TEMP0

  596. 	vpaddq		$TEMP1, $ACC1, $ACC1

  597. 	 vpmuludq	$Y2, $ACC0, $ACC0

  598. 	vpmuludq	$Y1, $TEMP2, $TEMP2

  599. 	.byte	0xc4,0x41,0x7e,0x6f,0xb5,0xf0,0xff,0xff,0xff	# vmovdqu		32*4-16-128($np), $TEMP1

  600. 	 vpaddq		$ACC1, $ACC0, $ACC0

  601. 	vpaddq		$TEMP2, $ACC2, $ACC2

  602. 	vpmuludq	$Y1, $TEMP0, $TEMP0

  603. 	vmovdqu		32*5-16-128($np), $TEMP2

  604. 	 .byte	0x67

  605. 	 vmovq		$ACC0, %rax

  606. 	 vmovdqu	$ACC0, (%rsp)		# transfer $r0-$r3

  607. 	vpaddq		$TEMP0, $ACC3, $ACC3

  608. 	vpmuludq	$Y1, $TEMP1, $TEMP1

  609. 	vmovdqu		32*6-16-128($np), $TEMP0

  610. 	vpaddq		$TEMP1, $ACC4, $ACC4

  611. 	vpmuludq	$Y1, $TEMP2, $TEMP2

  612. 	vmovdqu		32*7-16-128($np), $TEMP1

  613. 	vpaddq		$TEMP2, $ACC5, $ACC5

  614. 	vpmuludq	$Y1, $TEMP0, $TEMP0

  615. 	vmovdqu		32*8-16-128($np), $TEMP2

  616. 	vpaddq		$TEMP0, $ACC6, $ACC6

  617. 	vpmuludq	$Y1, $TEMP1, $TEMP1

  618. 	 shr	\$29, $r3

  619. 	vmovdqu		32*9-16-128($np), $TEMP0

  620. 	 add	$r3, %rax

  621. 	vpaddq		$TEMP1, $ACC7, $ACC7

  622. 	vpmuludq	$Y1, $TEMP2, $TEMP2

  623. 	 #vmovdqu	32*2-24-128($np), $TEMP1	# moved below

  624. 	 mov	%rax, $r0

  625. 	 imull	$n0, %eax

  626. 	vpaddq		$TEMP2, $ACC8, $ACC8

  627. 	vpmuludq	$Y1, $TEMP0, $TEMP0

  628. 	 and	\$0x1fffffff, %eax

  629. 	 vmovd	%eax, $Y1

  630. 	 vmovdqu	32*3-24-128($np), $TEMP2

  631. 	.byte	0x67

  632. 	vpaddq		$TEMP0, $ACC9, $ACC9

  633. 	 vpbroadcastq	$Y1, $Y1

  634. 

  635. 	vpmuludq	32*2-24-128($np), $Y2, $TEMP1	# see above

  636. 	vmovdqu		32*4-24-128($np), $TEMP0

  637. 	 mov	%rax, %rdx

  638. 	 imulq	-128($np), %rax

  639. 	 mov	8(%rsp), $r1

  640. 	vpaddq		$TEMP1, $ACC2, $ACC1

  641. 	vpmuludq	$Y2, $TEMP2, $TEMP2

  642. 	vmovdqu		32*5-24-128($np), $TEMP1

  643. 	 add	%rax, $r0

  644. 	 mov	%rdx, %rax

  645. 	 imulq	8-128($np), %rax

  646. 	 .byte	0x67

  647. 	 shr	\$29, $r0

  648. 	 mov	16(%rsp), $r2

  649. 	vpaddq		$TEMP2, $ACC3, $ACC2

  650. 	vpmuludq	$Y2, $TEMP0, $TEMP0

  651. 	vmovdqu		32*6-24-128($np), $TEMP2

  652. 	 add	%rax, $r1

  653. 	 mov	%rdx, %rax

  654. 	 imulq	16-128($np), %rax

  655. 	vpaddq		$TEMP0, $ACC4, $ACC3

  656. 	vpmuludq	$Y2, $TEMP1, $TEMP1

  657. 	vmovdqu		32*7-24-128($np), $TEMP0

  658. 	 imulq	24-128($np), %rdx		# future $r3

  659. 	 add	%rax, $r2

  660. 	 lea	($r0,$r1), %rax

  661. 	vpaddq		$TEMP1, $ACC5, $ACC4

  662. 	vpmuludq	$Y2, $TEMP2, $TEMP2

  663. 	vmovdqu		32*8-24-128($np), $TEMP1

  664. 	 mov	%rax, $r1

  665. 	 imull	$n0, %eax

  666. 	vpmuludq	$Y2, $TEMP0, $TEMP0

  667. 	vpaddq		$TEMP2, $ACC6, $ACC5

  668. 	vmovdqu		32*9-24-128($np), $TEMP2

  669. 	 and	\$0x1fffffff, %eax

  670. 	vpaddq		$TEMP0, $ACC7, $ACC6

  671. 	vpmuludq	$Y2, $TEMP1, $TEMP1

  672. 	 add	24(%rsp), %rdx

  673. 	vpaddq		$TEMP1, $ACC8, $ACC7

  674. 	vpmuludq	$Y2, $TEMP2, $TEMP2

  675. 	vpaddq		$TEMP2, $ACC9, $ACC8

  676. 	 vmovq	$r3, $ACC9

  677. 	 mov	%rdx, $r3

  678. 

  679. 	dec	$i

  680. 	jnz	.LOOP_REDUCE_1024

  681. ___

  682. ($ACC0,$Y2)=($Y2,$ACC0);

  683. $code.=<<___;

  684. 	lea	448(%rsp), $tp1			# size optimization

  685. 	vpaddq	$ACC9, $Y2, $ACC0

  686. 	vpxor	$ZERO, $ZERO, $ZERO

  687. 

  688. 	vpaddq		32*9-192($tp0), $ACC0, $ACC0

  689. 	vpaddq		32*10-448($tp1), $ACC1, $ACC1

  690. 	vpaddq		32*11-448($tp1), $ACC2, $ACC2

  691. 	vpaddq		32*12-448($tp1), $ACC3, $ACC3

  692. 	vpaddq		32*13-448($tp1), $ACC4, $ACC4

  693. 	vpaddq		32*14-448($tp1), $ACC5, $ACC5

  694. 	vpaddq		32*15-448($tp1), $ACC6, $ACC6

  695. 	vpaddq		32*16-448($tp1), $ACC7, $ACC7

  696. 	vpaddq		32*17-448($tp1), $ACC8, $ACC8

  697. 

  698. 	vpsrlq		\$29, $ACC0, $TEMP1

  699. 	vpand		$AND_MASK, $ACC0, $ACC0

  700. 	vpsrlq		\$29, $ACC1, $TEMP2

  701. 	vpand		$AND_MASK, $ACC1, $ACC1

  702. 	vpsrlq		\$29, $ACC2, $TEMP3

  703. 	vpermq		\$0x93, $TEMP1, $TEMP1

  704. 	vpand		$AND_MASK, $ACC2, $ACC2

  705. 	vpsrlq		\$29, $ACC3, $TEMP4

  706. 	vpermq		\$0x93, $TEMP2, $TEMP2

  707. 	vpand		$AND_MASK, $ACC3, $ACC3

  708. 	vpermq		\$0x93, $TEMP3, $TEMP3

  709. 

  710. 	vpblendd	\$3, $ZERO, $TEMP1, $TEMP0

  711. 	vpermq		\$0x93, $TEMP4, $TEMP4

  712. 	vpblendd	\$3, $TEMP1, $TEMP2, $TEMP1

  713. 	vpaddq		$TEMP0, $ACC0, $ACC0

  714. 	vpblendd	\$3, $TEMP2, $TEMP3, $TEMP2

  715. 	vpaddq		$TEMP1, $ACC1, $ACC1

  716. 	vpblendd	\$3, $TEMP3, $TEMP4, $TEMP3

  717. 	vpaddq		$TEMP2, $ACC2, $ACC2

  718. 	vpblendd	\$3, $TEMP4, $ZERO, $TEMP4

  719. 	vpaddq		$TEMP3, $ACC3, $ACC3

  720. 	vpaddq		$TEMP4, $ACC4, $ACC4

  721. 

  722. 	vpsrlq		\$29, $ACC0, $TEMP1

  723. 	vpand		$AND_MASK, $ACC0, $ACC0

  724. 	vpsrlq		\$29, $ACC1, $TEMP2

  725. 	vpand		$AND_MASK, $ACC1, $ACC1

  726. 	vpsrlq		\$29, $ACC2, $TEMP3

  727. 	vpermq		\$0x93, $TEMP1, $TEMP1

  728. 	vpand		$AND_MASK, $ACC2, $ACC2

  729. 	vpsrlq		\$29, $ACC3, $TEMP4

  730. 	vpermq		\$0x93, $TEMP2, $TEMP2

  731. 	vpand		$AND_MASK, $ACC3, $ACC3

  732. 	vpermq		\$0x93, $TEMP3, $TEMP3

  733. 

  734. 	vpblendd	\$3, $ZERO, $TEMP1, $TEMP0

  735. 	vpermq		\$0x93, $TEMP4, $TEMP4

  736. 	vpblendd	\$3, $TEMP1, $TEMP2, $TEMP1

  737. 	vpaddq		$TEMP0, $ACC0, $ACC0

  738. 	vpblendd	\$3, $TEMP2, $TEMP3, $TEMP2

  739. 	vpaddq		$TEMP1, $ACC1, $ACC1

  740. 	vmovdqu		$ACC0, 32*0-128($rp)

  741. 	vpblendd	\$3, $TEMP3, $TEMP4, $TEMP3

  742. 	vpaddq		$TEMP2, $ACC2, $ACC2

  743. 	vmovdqu		$ACC1, 32*1-128($rp)

  744. 	vpblendd	\$3, $TEMP4, $ZERO, $TEMP4

  745. 	vpaddq		$TEMP3, $ACC3, $ACC3

  746. 	vmovdqu		$ACC2, 32*2-128($rp)

  747. 	vpaddq		$TEMP4, $ACC4, $ACC4

  748. 	vmovdqu		$ACC3, 32*3-128($rp)

  749. ___

  750. $TEMP5=$ACC0;

  751. $code.=<<___;

  752. 	vpsrlq		\$29, $ACC4, $TEMP1

  753. 	vpand		$AND_MASK, $ACC4, $ACC4

  754. 	vpsrlq		\$29, $ACC5, $TEMP2

  755. 	vpand		$AND_MASK, $ACC5, $ACC5

  756. 	vpsrlq		\$29, $ACC6, $TEMP3

  757. 	vpermq		\$0x93, $TEMP1, $TEMP1

  758. 	vpand		$AND_MASK, $ACC6, $ACC6

  759. 	vpsrlq		\$29, $ACC7, $TEMP4

  760. 	vpermq		\$0x93, $TEMP2, $TEMP2

  761. 	vpand		$AND_MASK, $ACC7, $ACC7

  762. 	vpsrlq		\$29, $ACC8, $TEMP5

  763. 	vpermq		\$0x93, $TEMP3, $TEMP3

  764. 	vpand		$AND_MASK, $ACC8, $ACC8

  765. 	vpermq		\$0x93, $TEMP4, $TEMP4

  766. 

  767. 	vpblendd	\$3, $ZERO, $TEMP1, $TEMP0

  768. 	vpermq		\$0x93, $TEMP5, $TEMP5

  769. 	vpblendd	\$3, $TEMP1, $TEMP2, $TEMP1

  770. 	vpaddq		$TEMP0, $ACC4, $ACC4

  771. 	vpblendd	\$3, $TEMP2, $TEMP3, $TEMP2

  772. 	vpaddq		$TEMP1, $ACC5, $ACC5

  773. 	vpblendd	\$3, $TEMP3, $TEMP4, $TEMP3

  774. 	vpaddq		$TEMP2, $ACC6, $ACC6

  775. 	vpblendd	\$3, $TEMP4, $TEMP5, $TEMP4

  776. 	vpaddq		$TEMP3, $ACC7, $ACC7

  777. 	vpaddq		$TEMP4, $ACC8, $ACC8

  778.      

  779. 	vpsrlq		\$29, $ACC4, $TEMP1

  780. 	vpand		$AND_MASK, $ACC4, $ACC4

  781. 	vpsrlq		\$29, $ACC5, $TEMP2

  782. 	vpand		$AND_MASK, $ACC5, $ACC5

  783. 	vpsrlq		\$29, $ACC6, $TEMP3

  784. 	vpermq		\$0x93, $TEMP1, $TEMP1

  785. 	vpand		$AND_MASK, $ACC6, $ACC6

  786. 	vpsrlq		\$29, $ACC7, $TEMP4

  787. 	vpermq		\$0x93, $TEMP2, $TEMP2

  788. 	vpand		$AND_MASK, $ACC7, $ACC7

  789. 	vpsrlq		\$29, $ACC8, $TEMP5

  790. 	vpermq		\$0x93, $TEMP3, $TEMP3

  791. 	vpand		$AND_MASK, $ACC8, $ACC8

  792. 	vpermq		\$0x93, $TEMP4, $TEMP4

  793. 

  794. 	vpblendd	\$3, $ZERO, $TEMP1, $TEMP0

  795. 	vpermq		\$0x93, $TEMP5, $TEMP5

  796. 	vpblendd	\$3, $TEMP1, $TEMP2, $TEMP1

  797. 	vpaddq		$TEMP0, $ACC4, $ACC4

  798. 	vpblendd	\$3, $TEMP2, $TEMP3, $TEMP2

  799. 	vpaddq		$TEMP1, $ACC5, $ACC5

  800. 	vmovdqu		$ACC4, 32*4-128($rp)

  801. 	vpblendd	\$3, $TEMP3, $TEMP4, $TEMP3

  802. 	vpaddq		$TEMP2, $ACC6, $ACC6

  803. 	vmovdqu		$ACC5, 32*5-128($rp)

  804. 	vpblendd	\$3, $TEMP4, $TEMP5, $TEMP4

  805. 	vpaddq		$TEMP3, $ACC7, $ACC7

  806. 	vmovdqu		$ACC6, 32*6-128($rp)

  807. 	vpaddq		$TEMP4, $ACC8, $ACC8

  808. 	vmovdqu		$ACC7, 32*7-128($rp)

  809. 	vmovdqu		$ACC8, 32*8-128($rp)

  810. 

  811. 	mov	$rp, $ap

  812. 	dec	$rep

  813. 	jne	.LOOP_GRANDE_SQR_1024

  814. 

  815. 	vzeroall

  816. 	mov	%rbp, %rax

  817. ___

  818. $code.=<<___ if ($win64);

  819. 	movaps	-0xd8(%rax),%xmm6

  820. 	movaps	-0xc8(%rax),%xmm7

  821. 	movaps	-0xb8(%rax),%xmm8

  822. 	movaps	-0xa8(%rax),%xmm9

  823. 	movaps	-0x98(%rax),%xmm10

  824. 	movaps	-0x88(%rax),%xmm11

  825. 	movaps	-0x78(%rax),%xmm12

  826. 	movaps	-0x68(%rax),%xmm13

  827. 	movaps	-0x58(%rax),%xmm14

  828. 	movaps	-0x48(%rax),%xmm15

  829. ___

  830. $code.=<<___;

  831. 	mov	-48(%rax),%r15

  832. 	mov	-40(%rax),%r14

  833. 	mov	-32(%rax),%r13

  834. 	mov	-24(%rax),%r12

  835. 	mov	-16(%rax),%rbp

  836. 	mov	-8(%rax),%rbx

  837. 	lea	(%rax),%rsp		# restore %rsp

  838. .Lsqr_1024_epilogue:

  839. 	ret

  840. .size	rsaz_1024_sqr_avx2,.-rsaz_1024_sqr_avx2

  841. ___

  842. }

  843. 

  844. { # void AMM_WW(

  845. my $rp="%rdi";	# BN_ULONG *rp,

  846. my $ap="%rsi";	# const BN_ULONG *ap,

  847. my $bp="%rdx";	# const BN_ULONG *bp,

  848. my $np="%rcx";	# const BN_ULONG *np,

  849. my $n0="%r8d";	# unsigned int n0);

  850. 

  851. # The registers that hold the accumulated redundant result

  852. # The AMM works on 1024 bit operands, and redundant word size is 29

  853. # Therefore: ceil(1024/29)/4 = 9

  854. my $ACC0="%ymm0";

  855. my $ACC1="%ymm1";

  856. my $ACC2="%ymm2";

  857. my $ACC3="%ymm3";

  858. my $ACC4="%ymm4";

  859. my $ACC5="%ymm5";

  860. my $ACC6="%ymm6";

  861. my $ACC7="%ymm7";

  862. my $ACC8="%ymm8";

  863. my $ACC9="%ymm9";

  864. 

  865. # Registers that hold the broadcasted words of multiplier, currently used

  866. my $Bi="%ymm10";

  867. my $Yi="%ymm11";

  868. 

  869. # Helper registers

  870. my $TEMP0=$ACC0;

  871. my $TEMP1="%ymm12";

  872. my $TEMP2="%ymm13";

  873. my $ZERO="%ymm14";

  874. my $AND_MASK="%ymm15";

  875. 

  876. # alu registers that hold the first words of the ACC

  877. my $r0="%r9";

  878. my $r1="%r10";

  879. my $r2="%r11";

  880. my $r3="%r12";

  881. 

  882. my $i="%r14d";

  883. my $tmp="%r15";

  884. 

  885. $bp="%r13";	# reassigned argument

  886. 

  887. $code.=<<___;

  888. .globl	rsaz_1024_mul_avx2

  889. .type	rsaz_1024_mul_avx2,\@function,5

  890. .align	64

  891. rsaz_1024_mul_avx2:

  892. 	lea	(%rsp), %rax

  893. 	push	%rbx

  894. 	push	%rbp

  895. 	push	%r12

  896. 	push	%r13

  897. 	push	%r14

  898. 	push	%r15

  899. ___

  900. $code.=<<___ if ($win64);

  901. 	vzeroupper

  902. 	lea	-0xa8(%rsp),%rsp

  903. 	vmovaps	%xmm6,-0xd8(%rax)

  904. 	vmovaps	%xmm7,-0xc8(%rax)

  905. 	vmovaps	%xmm8,-0xb8(%rax)

  906. 	vmovaps	%xmm9,-0xa8(%rax)

  907. 	vmovaps	%xmm10,-0x98(%rax)

  908. 	vmovaps	%xmm11,-0x88(%rax)

  909. 	vmovaps	%xmm12,-0x78(%rax)

  910. 	vmovaps	%xmm13,-0x68(%rax)

  911. 	vmovaps	%xmm14,-0x58(%rax)

  912. 	vmovaps	%xmm15,-0x48(%rax)

  913. .Lmul_1024_body:

  914. ___

  915. $code.=<<___;

  916. 	mov	%rax,%rbp

  917. 	vzeroall

  918. 	mov	%rdx, $bp	# reassigned argument

  919. 	sub	\$64,%rsp

  920. 

  921. 	# unaligned 256-bit load that crosses page boundary can

  922. 	# cause severe performance degradation here, so if $ap does

  923. 	# cross page boundary, swap it with $bp [meaning that caller

  924. 	# is advised to lay down $ap and $bp next to each other, so

  925. 	# that only one can cross page boundary].

  926. 	.byte	0x67,0x67

  927. 	mov	$ap, $tmp

  928. 	and	\$4095, $tmp

  929. 	add	\$32*10, $tmp

  930. 	shr	\$12, $tmp

  931. 	mov	$ap, $tmp

  932. 	cmovnz	$bp, $ap

  933. 	cmovnz	$tmp, $bp

  934. 

  935. 	mov	$np, $tmp

  936. 	sub	\$-128,$ap	# size optimization

  937. 	sub	\$-128,$np

  938. 	sub	\$-128,$rp

  939. 

  940. 	and	\$4095, $tmp	# see if $np crosses page

  941. 	add	\$32*10, $tmp

  942. 	.byte	0x67,0x67

  943. 	shr	\$12, $tmp

  944. 	jz	.Lmul_1024_no_n_copy

  945. 

  946. 	# unaligned 256-bit load that crosses page boundary can

  947. 	# cause severe performance degradation here, so if $np does

  948. 	# cross page boundary, copy it to stack and make sure stack

  949. 	# frame doesn't...

  950. 	sub		\$32*10,%rsp

  951. 	vmovdqu		32*0-128($np), $ACC0

  952. 	and		\$-512, %rsp

  953. 	vmovdqu		32*1-128($np), $ACC1

  954. 	vmovdqu		32*2-128($np), $ACC2

  955. 	vmovdqu		32*3-128($np), $ACC3

  956. 	vmovdqu		32*4-128($np), $ACC4

  957. 	vmovdqu		32*5-128($np), $ACC5

  958. 	vmovdqu		32*6-128($np), $ACC6

  959. 	vmovdqu		32*7-128($np), $ACC7

  960. 	vmovdqu		32*8-128($np), $ACC8

  961. 	lea		64+128(%rsp),$np

  962. 	vmovdqu		$ACC0, 32*0-128($np)

  963. 	vpxor		$ACC0, $ACC0, $ACC0

  964. 	vmovdqu		$ACC1, 32*1-128($np)

  965. 	vpxor		$ACC1, $ACC1, $ACC1

  966. 	vmovdqu		$ACC2, 32*2-128($np)

  967. 	vpxor		$ACC2, $ACC2, $ACC2

  968. 	vmovdqu		$ACC3, 32*3-128($np)

  969. 	vpxor		$ACC3, $ACC3, $ACC3

  970. 	vmovdqu		$ACC4, 32*4-128($np)

  971. 	vpxor		$ACC4, $ACC4, $ACC4

  972. 	vmovdqu		$ACC5, 32*5-128($np)

  973. 	vpxor		$ACC5, $ACC5, $ACC5

  974. 	vmovdqu		$ACC6, 32*6-128($np)

  975. 	vpxor		$ACC6, $ACC6, $ACC6

  976. 	vmovdqu		$ACC7, 32*7-128($np)

  977. 	vpxor		$ACC7, $ACC7, $ACC7

  978. 	vmovdqu		$ACC8, 32*8-128($np)

  979. 	vmovdqa		$ACC0, $ACC8

  980. 	vmovdqu		$ACC9, 32*9-128($np)	# $ACC9 is zero after vzeroall

  981. .Lmul_1024_no_n_copy:

  982. 	and	\$-64,%rsp

  983. 

  984. 	mov	($bp), %rbx

  985. 	vpbroadcastq ($bp), $Bi

  986. 	vmovdqu	$ACC0, (%rsp)			# clear top of stack

  987. 	xor	$r0, $r0

  988. 	.byte	0x67

  989. 	xor	$r1, $r1

  990. 	xor	$r2, $r2

  991. 	xor	$r3, $r3

  992. 

  993. 	vmovdqu	.Land_mask(%rip), $AND_MASK

  994. 	mov	\$9, $i

  995. 	vmovdqu	$ACC9, 32*9-128($rp)		# $ACC9 is zero after vzeroall

  996. 	jmp	.Loop_mul_1024

  997. 

  998. .align	32

  999. .Loop_mul_1024:

  1000. 	 vpsrlq		\$29, $ACC3, $ACC9		# correct $ACC3(*)

  1001. 	mov	%rbx, %rax

  1002. 	imulq	-128($ap), %rax

  1003. 	add	$r0, %rax

  1004. 	mov	%rbx, $r1

  1005. 	imulq	8-128($ap), $r1

  1006. 	add	8(%rsp), $r1

  1007. 

  1008. 	mov	%rax, $r0

  1009. 	imull	$n0, %eax

  1010. 	and	\$0x1fffffff, %eax

  1011. 

  1012. 	 mov	%rbx, $r2

  1013. 	 imulq	16-128($ap), $r2

  1014. 	 add	16(%rsp), $r2

  1015. 

  1016. 	 mov	%rbx, $r3

  1017. 	 imulq	24-128($ap), $r3

  1018. 	 add	24(%rsp), $r3

  1019. 	vpmuludq	32*1-128($ap),$Bi,$TEMP0

  1020. 	 vmovd		%eax, $Yi

  1021. 	vpaddq		$TEMP0,$ACC1,$ACC1

  1022. 	vpmuludq	32*2-128($ap),$Bi,$TEMP1

  1023. 	 vpbroadcastq	$Yi, $Yi

  1024. 	vpaddq		$TEMP1,$ACC2,$ACC2

  1025. 	vpmuludq	32*3-128($ap),$Bi,$TEMP2

  1026. 	 vpand		$AND_MASK, $ACC3, $ACC3		# correct $ACC3

  1027. 	vpaddq		$TEMP2,$ACC3,$ACC3

  1028. 	vpmuludq	32*4-128($ap),$Bi,$TEMP0

  1029. 	vpaddq		$TEMP0,$ACC4,$ACC4

  1030. 	vpmuludq	32*5-128($ap),$Bi,$TEMP1

  1031. 	vpaddq		$TEMP1,$ACC5,$ACC5

  1032. 	vpmuludq	32*6-128($ap),$Bi,$TEMP2

  1033. 	vpaddq		$TEMP2,$ACC6,$ACC6

  1034. 	vpmuludq	32*7-128($ap),$Bi,$TEMP0

  1035. 	 vpermq		\$0x93, $ACC9, $ACC9		# correct $ACC3

  1036. 	vpaddq		$TEMP0,$ACC7,$ACC7

  1037. 	vpmuludq	32*8-128($ap),$Bi,$TEMP1

  1038. 	 vpbroadcastq	8($bp), $Bi

  1039. 	vpaddq		$TEMP1,$ACC8,$ACC8

  1040. 

  1041. 	mov	%rax,%rdx

  1042. 	imulq	-128($np),%rax

  1043. 	add	%rax,$r0

  1044. 	mov	%rdx,%rax

  1045. 	imulq	8-128($np),%rax

  1046. 	add	%rax,$r1

  1047. 	mov	%rdx,%rax

  1048. 	imulq	16-128($np),%rax

  1049. 	add	%rax,$r2

  1050. 	shr	\$29, $r0

  1051. 	imulq	24-128($np),%rdx

  1052. 	add	%rdx,$r3

  1053. 	add	$r0, $r1

  1054. 

  1055. 	vpmuludq	32*1-128($np),$Yi,$TEMP2

  1056. 	 vmovq		$Bi, %rbx

  1057. 	vpaddq		$TEMP2,$ACC1,$ACC1

  1058. 	vpmuludq	32*2-128($np),$Yi,$TEMP0

  1059. 	vpaddq		$TEMP0,$ACC2,$ACC2

  1060. 	vpmuludq	32*3-128($np),$Yi,$TEMP1

  1061. 	vpaddq		$TEMP1,$ACC3,$ACC3

  1062. 	vpmuludq	32*4-128($np),$Yi,$TEMP2

  1063. 	vpaddq		$TEMP2,$ACC4,$ACC4

  1064. 	vpmuludq	32*5-128($np),$Yi,$TEMP0

  1065. 	vpaddq		$TEMP0,$ACC5,$ACC5

  1066. 	vpmuludq	32*6-128($np),$Yi,$TEMP1

  1067. 	vpaddq		$TEMP1,$ACC6,$ACC6

  1068. 	vpmuludq	32*7-128($np),$Yi,$TEMP2

  1069. 	 vpblendd	\$3, $ZERO, $ACC9, $ACC9	# correct $ACC3

  1070. 	vpaddq		$TEMP2,$ACC7,$ACC7

  1071. 	vpmuludq	32*8-128($np),$Yi,$TEMP0

  1072. 	 vpaddq		$ACC9, $ACC3, $ACC3		# correct $ACC3

  1073. 	vpaddq		$TEMP0,$ACC8,$ACC8

  1074. 

  1075. 	mov	%rbx, %rax

  1076. 	imulq	-128($ap),%rax

  1077. 	add	%rax,$r1

  1078. 	 vmovdqu	-8+32*1-128($ap),$TEMP1

  1079. 	mov	%rbx, %rax

  1080. 	imulq	8-128($ap),%rax

  1081. 	add	%rax,$r2

  1082. 	 vmovdqu	-8+32*2-128($ap),$TEMP2

  1083. 

  1084. 	mov	$r1, %rax

  1085. 	imull	$n0, %eax

  1086. 	and	\$0x1fffffff, %eax

  1087. 

  1088. 	 imulq	16-128($ap),%rbx

  1089. 	 add	%rbx,$r3

  1090. 	vpmuludq	$Bi,$TEMP1,$TEMP1

  1091. 	 vmovd		%eax, $Yi

  1092. 	vmovdqu		-8+32*3-128($ap),$TEMP0

  1093. 	vpaddq		$TEMP1,$ACC1,$ACC1

  1094. 	vpmuludq	$Bi,$TEMP2,$TEMP2

  1095. 	 vpbroadcastq	$Yi, $Yi

  1096. 	vmovdqu		-8+32*4-128($ap),$TEMP1

  1097. 	vpaddq		$TEMP2,$ACC2,$ACC2

  1098. 	vpmuludq	$Bi,$TEMP0,$TEMP0

  1099. 	vmovdqu		-8+32*5-128($ap),$TEMP2

  1100. 	vpaddq		$TEMP0,$ACC3,$ACC3

  1101. 	vpmuludq	$Bi,$TEMP1,$TEMP1

  1102. 	vmovdqu		-8+32*6-128($ap),$TEMP0

  1103. 	vpaddq		$TEMP1,$ACC4,$ACC4

  1104. 	vpmuludq	$Bi,$TEMP2,$TEMP2

  1105. 	vmovdqu		-8+32*7-128($ap),$TEMP1

  1106. 	vpaddq		$TEMP2,$ACC5,$ACC5

  1107. 	vpmuludq	$Bi,$TEMP0,$TEMP0

  1108. 	vmovdqu		-8+32*8-128($ap),$TEMP2

  1109. 	vpaddq		$TEMP0,$ACC6,$ACC6

  1110. 	vpmuludq	$Bi,$TEMP1,$TEMP1

  1111. 	vmovdqu		-8+32*9-128($ap),$ACC9

  1112. 	vpaddq		$TEMP1,$ACC7,$ACC7

  1113. 	vpmuludq	$Bi,$TEMP2,$TEMP2

  1114. 	vpaddq		$TEMP2,$ACC8,$ACC8

  1115. 	vpmuludq	$Bi,$ACC9,$ACC9

  1116. 	 vpbroadcastq	16($bp), $Bi

  1117. 

  1118. 	mov	%rax,%rdx

  1119. 	imulq	-128($np),%rax

  1120. 	add	%rax,$r1

  1121. 	 vmovdqu	-8+32*1-128($np),$TEMP0

  1122. 	mov	%rdx,%rax

  1123. 	imulq	8-128($np),%rax

  1124. 	add	%rax,$r2

  1125. 	 vmovdqu	-8+32*2-128($np),$TEMP1

  1126. 	shr	\$29, $r1

  1127. 	imulq	16-128($np),%rdx

  1128. 	add	%rdx,$r3

  1129. 	add	$r1, $r2

  1130. 

  1131. 	vpmuludq	$Yi,$TEMP0,$TEMP0

  1132. 	 vmovq		$Bi, %rbx

  1133. 	vmovdqu		-8+32*3-128($np),$TEMP2

  1134. 	vpaddq		$TEMP0,$ACC1,$ACC1

  1135. 	vpmuludq	$Yi,$TEMP1,$TEMP1

  1136. 	vmovdqu		-8+32*4-128($np),$TEMP0

  1137. 	vpaddq		$TEMP1,$ACC2,$ACC2

  1138. 	vpmuludq	$Yi,$TEMP2,$TEMP2

  1139. 	vmovdqu		-8+32*5-128($np),$TEMP1

  1140. 	vpaddq		$TEMP2,$ACC3,$ACC3

  1141. 	vpmuludq	$Yi,$TEMP0,$TEMP0

  1142. 	vmovdqu		-8+32*6-128($np),$TEMP2

  1143. 	vpaddq		$TEMP0,$ACC4,$ACC4

  1144. 	vpmuludq	$Yi,$TEMP1,$TEMP1

  1145. 	vmovdqu		-8+32*7-128($np),$TEMP0

  1146. 	vpaddq		$TEMP1,$ACC5,$ACC5

  1147. 	vpmuludq	$Yi,$TEMP2,$TEMP2

  1148. 	vmovdqu		-8+32*8-128($np),$TEMP1

  1149. 	vpaddq		$TEMP2,$ACC6,$ACC6

  1150. 	vpmuludq	$Yi,$TEMP0,$TEMP0

  1151. 	vmovdqu		-8+32*9-128($np),$TEMP2

  1152. 	vpaddq		$TEMP0,$ACC7,$ACC7

  1153. 	vpmuludq	$Yi,$TEMP1,$TEMP1

  1154. 	vpaddq		$TEMP1,$ACC8,$ACC8

  1155. 	vpmuludq	$Yi,$TEMP2,$TEMP2

  1156. 	vpaddq		$TEMP2,$ACC9,$ACC9

  1157. 

  1158. 	 vmovdqu	-16+32*1-128($ap),$TEMP0

  1159. 	mov	%rbx,%rax

  1160. 	imulq	-128($ap),%rax

  1161. 	add	$r2,%rax

  1162. 

  1163. 	 vmovdqu	-16+32*2-128($ap),$TEMP1

  1164. 	mov	%rax,$r2

  1165. 	imull	$n0, %eax

  1166. 	and	\$0x1fffffff, %eax

  1167. 

  1168. 	 imulq	8-128($ap),%rbx

  1169. 	 add	%rbx,$r3

  1170. 	vpmuludq	$Bi,$TEMP0,$TEMP0

  1171. 	 vmovd		%eax, $Yi

  1172. 	vmovdqu		-16+32*3-128($ap),$TEMP2

  1173. 	vpaddq		$TEMP0,$ACC1,$ACC1

  1174. 	vpmuludq	$Bi,$TEMP1,$TEMP1

  1175. 	 vpbroadcastq	$Yi, $Yi

  1176. 	vmovdqu		-16+32*4-128($ap),$TEMP0

  1177. 	vpaddq		$TEMP1,$ACC2,$ACC2

  1178. 	vpmuludq	$Bi,$TEMP2,$TEMP2

  1179. 	vmovdqu		-16+32*5-128($ap),$TEMP1

  1180. 	vpaddq		$TEMP2,$ACC3,$ACC3

  1181. 	vpmuludq	$Bi,$TEMP0,$TEMP0

  1182. 	vmovdqu		-16+32*6-128($ap),$TEMP2

  1183. 	vpaddq		$TEMP0,$ACC4,$ACC4

  1184. 	vpmuludq	$Bi,$TEMP1,$TEMP1

  1185. 	vmovdqu		-16+32*7-128($ap),$TEMP0

  1186. 	vpaddq		$TEMP1,$ACC5,$ACC5

  1187. 	vpmuludq	$Bi,$TEMP2,$TEMP2

  1188. 	vmovdqu		-16+32*8-128($ap),$TEMP1

  1189. 	vpaddq		$TEMP2,$ACC6,$ACC6

  1190. 	vpmuludq	$Bi,$TEMP0,$TEMP0

  1191. 	vmovdqu		-16+32*9-128($ap),$TEMP2

  1192. 	vpaddq		$TEMP0,$ACC7,$ACC7

  1193. 	vpmuludq	$Bi,$TEMP1,$TEMP1

  1194. 	vpaddq		$TEMP1,$ACC8,$ACC8

  1195. 	vpmuludq	$Bi,$TEMP2,$TEMP2

  1196. 	 vpbroadcastq	24($bp), $Bi

  1197. 	vpaddq		$TEMP2,$ACC9,$ACC9

  1198. 

  1199. 	 vmovdqu	-16+32*1-128($np),$TEMP0

  1200. 	mov	%rax,%rdx

  1201. 	imulq	-128($np),%rax

  1202. 	add	%rax,$r2

  1203. 	 vmovdqu	-16+32*2-128($np),$TEMP1

  1204. 	imulq	8-128($np),%rdx

  1205. 	add	%rdx,$r3

  1206. 	shr	\$29, $r2

  1207. 

  1208. 	vpmuludq	$Yi,$TEMP0,$TEMP0

  1209. 	 vmovq		$Bi, %rbx

  1210. 	vmovdqu		-16+32*3-128($np),$TEMP2

  1211. 	vpaddq		$TEMP0,$ACC1,$ACC1

  1212. 	vpmuludq	$Yi,$TEMP1,$TEMP1

  1213. 	vmovdqu		-16+32*4-128($np),$TEMP0

  1214. 	vpaddq		$TEMP1,$ACC2,$ACC2

  1215. 	vpmuludq	$Yi,$TEMP2,$TEMP2

  1216. 	vmovdqu		-16+32*5-128($np),$TEMP1

  1217. 	vpaddq		$TEMP2,$ACC3,$ACC3

  1218. 	vpmuludq	$Yi,$TEMP0,$TEMP0

  1219. 	vmovdqu		-16+32*6-128($np),$TEMP2

  1220. 	vpaddq		$TEMP0,$ACC4,$ACC4

  1221. 	vpmuludq	$Yi,$TEMP1,$TEMP1

  1222. 	vmovdqu		-16+32*7-128($np),$TEMP0

  1223. 	vpaddq		$TEMP1,$ACC5,$ACC5

  1224. 	vpmuludq	$Yi,$TEMP2,$TEMP2

  1225. 	vmovdqu		-16+32*8-128($np),$TEMP1

  1226. 	vpaddq		$TEMP2,$ACC6,$ACC6

  1227. 	vpmuludq	$Yi,$TEMP0,$TEMP0

  1228. 	vmovdqu		-16+32*9-128($np),$TEMP2

  1229. 	vpaddq		$TEMP0,$ACC7,$ACC7

  1230. 	vpmuludq	$Yi,$TEMP1,$TEMP1

  1231. 	 vmovdqu	-24+32*1-128($ap),$TEMP0

  1232. 	vpaddq		$TEMP1,$ACC8,$ACC8

  1233. 	vpmuludq	$Yi,$TEMP2,$TEMP2

  1234. 	 vmovdqu	-24+32*2-128($ap),$TEMP1

  1235. 	vpaddq		$TEMP2,$ACC9,$ACC9

  1236. 

  1237. 	add	$r2, $r3

  1238. 	imulq	-128($ap),%rbx

  1239. 	add	%rbx,$r3

  1240. 

  1241. 	mov	$r3, %rax

  1242. 	imull	$n0, %eax

  1243. 	and	\$0x1fffffff, %eax

  1244. 

  1245. 	vpmuludq	$Bi,$TEMP0,$TEMP0

  1246. 	 vmovd		%eax, $Yi

  1247. 	vmovdqu		-24+32*3-128($ap),$TEMP2

  1248. 	vpaddq		$TEMP0,$ACC1,$ACC1

  1249. 	vpmuludq	$Bi,$TEMP1,$TEMP1

  1250. 	 vpbroadcastq	$Yi, $Yi

  1251. 	vmovdqu		-24+32*4-128($ap),$TEMP0

  1252. 	vpaddq		$TEMP1,$ACC2,$ACC2

  1253. 	vpmuludq	$Bi,$TEMP2,$TEMP2

  1254. 	vmovdqu		-24+32*5-128($ap),$TEMP1

  1255. 	vpaddq		$TEMP2,$ACC3,$ACC3

  1256. 	vpmuludq	$Bi,$TEMP0,$TEMP0

  1257. 	vmovdqu		-24+32*6-128($ap),$TEMP2

  1258. 	vpaddq		$TEMP0,$ACC4,$ACC4

  1259. 	vpmuludq	$Bi,$TEMP1,$TEMP1

  1260. 	vmovdqu		-24+32*7-128($ap),$TEMP0

  1261. 	vpaddq		$TEMP1,$ACC5,$ACC5

  1262. 	vpmuludq	$Bi,$TEMP2,$TEMP2

  1263. 	vmovdqu		-24+32*8-128($ap),$TEMP1

  1264. 	vpaddq		$TEMP2,$ACC6,$ACC6

  1265. 	vpmuludq	$Bi,$TEMP0,$TEMP0

  1266. 	vmovdqu		-24+32*9-128($ap),$TEMP2

  1267. 	vpaddq		$TEMP0,$ACC7,$ACC7

  1268. 	vpmuludq	$Bi,$TEMP1,$TEMP1

  1269. 	vpaddq		$TEMP1,$ACC8,$ACC8

  1270. 	vpmuludq	$Bi,$TEMP2,$TEMP2

  1271. 	 vpbroadcastq	32($bp), $Bi

  1272. 	vpaddq		$TEMP2,$ACC9,$ACC9

  1273. 	 add		\$32, $bp			# $bp++

  1274. 

  1275. 	vmovdqu		-24+32*1-128($np),$TEMP0

  1276. 	imulq	-128($np),%rax

  1277. 	add	%rax,$r3

  1278. 	shr	\$29, $r3

  1279. 

  1280. 	vmovdqu		-24+32*2-128($np),$TEMP1

  1281. 	vpmuludq	$Yi,$TEMP0,$TEMP0

  1282. 	 vmovq		$Bi, %rbx

  1283. 	vmovdqu		-24+32*3-128($np),$TEMP2

  1284. 	vpaddq		$TEMP0,$ACC1,$ACC0		# $ACC0==$TEMP0

  1285. 	vpmuludq	$Yi,$TEMP1,$TEMP1

  1286. 	 vmovdqu	$ACC0, (%rsp)			# transfer $r0-$r3

  1287. 	vpaddq		$TEMP1,$ACC2,$ACC1

  1288. 	vmovdqu		-24+32*4-128($np),$TEMP0

  1289. 	vpmuludq	$Yi,$TEMP2,$TEMP2

  1290. 	vmovdqu		-24+32*5-128($np),$TEMP1

  1291. 	vpaddq		$TEMP2,$ACC3,$ACC2

  1292. 	vpmuludq	$Yi,$TEMP0,$TEMP0

  1293. 	vmovdqu		-24+32*6-128($np),$TEMP2

  1294. 	vpaddq		$TEMP0,$ACC4,$ACC3

  1295. 	vpmuludq	$Yi,$TEMP1,$TEMP1

  1296. 	vmovdqu		-24+32*7-128($np),$TEMP0

  1297. 	vpaddq		$TEMP1,$ACC5,$ACC4

  1298. 	vpmuludq	$Yi,$TEMP2,$TEMP2

  1299. 	vmovdqu		-24+32*8-128($np),$TEMP1

  1300. 	vpaddq		$TEMP2,$ACC6,$ACC5

  1301. 	vpmuludq	$Yi,$TEMP0,$TEMP0

  1302. 	vmovdqu		-24+32*9-128($np),$TEMP2

  1303. 	 mov	$r3, $r0

  1304. 	vpaddq		$TEMP0,$ACC7,$ACC6

  1305. 	vpmuludq	$Yi,$TEMP1,$TEMP1

  1306. 	 add	(%rsp), $r0

  1307. 	vpaddq		$TEMP1,$ACC8,$ACC7

  1308. 	vpmuludq	$Yi,$TEMP2,$TEMP2

  1309. 	 vmovq	$r3, $TEMP1

  1310. 	vpaddq		$TEMP2,$ACC9,$ACC8

  1311. 

  1312. 	dec	$i

  1313. 	jnz	.Loop_mul_1024

  1314. ___

  1315. 

  1316. # (*)	Original implementation was correcting ACC1-ACC3 for overflow

  1317. #	after 7 loop runs, or after 28 iterations, or 56 additions.

  1318. #	But as we underutilize resources, it's possible to correct in

  1319. #	each iteration with marginal performance loss. But then, as

  1320. #	we do it in each iteration, we can correct less digits, and

  1321. #	avoid performance penalties completely. Also note that we

  1322. #	correct only three digits out of four. This works because

  1323. #	most significant digit is subjected to less additions.

  1324. 

  1325. $TEMP0 = $ACC9;

  1326. $TEMP3 = $Bi;

  1327. $TEMP4 = $Yi;

  1328. $code.=<<___;

  1329. 	vpermq		\$0, $AND_MASK, $AND_MASK

  1330. 	vpaddq		(%rsp), $TEMP1, $ACC0

  1331. 

  1332. 	vpsrlq		\$29, $ACC0, $TEMP1

  1333. 	vpand		$AND_MASK, $ACC0, $ACC0

  1334. 	vpsrlq		\$29, $ACC1, $TEMP2

  1335. 	vpand		$AND_MASK, $ACC1, $ACC1

  1336. 	vpsrlq		\$29, $ACC2, $TEMP3

  1337. 	vpermq		\$0x93, $TEMP1, $TEMP1

  1338. 	vpand		$AND_MASK, $ACC2, $ACC2

  1339. 	vpsrlq		\$29, $ACC3, $TEMP4

  1340. 	vpermq		\$0x93, $TEMP2, $TEMP2

  1341. 	vpand		$AND_MASK, $ACC3, $ACC3

  1342. 

  1343. 	vpblendd	\$3, $ZERO, $TEMP1, $TEMP0

  1344. 	vpermq		\$0x93, $TEMP3, $TEMP3

  1345. 	vpblendd	\$3, $TEMP1, $TEMP2, $TEMP1

  1346. 	vpermq		\$0x93, $TEMP4, $TEMP4

  1347. 	vpaddq		$TEMP0, $ACC0, $ACC0

  1348. 	vpblendd	\$3, $TEMP2, $TEMP3, $TEMP2

  1349. 	vpaddq		$TEMP1, $ACC1, $ACC1

  1350. 	vpblendd	\$3, $TEMP3, $TEMP4, $TEMP3

  1351. 	vpaddq		$TEMP2, $ACC2, $ACC2

  1352. 	vpblendd	\$3, $TEMP4, $ZERO, $TEMP4

  1353. 	vpaddq		$TEMP3, $ACC3, $ACC3

  1354. 	vpaddq		$TEMP4, $ACC4, $ACC4

  1355. 

  1356. 	vpsrlq		\$29, $ACC0, $TEMP1

  1357. 	vpand		$AND_MASK, $ACC0, $ACC0

  1358. 	vpsrlq		\$29, $ACC1, $TEMP2

  1359. 	vpand		$AND_MASK, $ACC1, $ACC1

  1360. 	vpsrlq		\$29, $ACC2, $TEMP3

  1361. 	vpermq		\$0x93, $TEMP1, $TEMP1

  1362. 	vpand		$AND_MASK, $ACC2, $ACC2

  1363. 	vpsrlq		\$29, $ACC3, $TEMP4

  1364. 	vpermq		\$0x93, $TEMP2, $TEMP2

  1365. 	vpand		$AND_MASK, $ACC3, $ACC3

  1366. 	vpermq		\$0x93, $TEMP3, $TEMP3

  1367. 

  1368. 	vpblendd	\$3, $ZERO, $TEMP1, $TEMP0

  1369. 	vpermq		\$0x93, $TEMP4, $TEMP4

  1370. 	vpblendd	\$3, $TEMP1, $TEMP2, $TEMP1

  1371. 	vpaddq		$TEMP0, $ACC0, $ACC0

  1372. 	vpblendd	\$3, $TEMP2, $TEMP3, $TEMP2

  1373. 	vpaddq		$TEMP1, $ACC1, $ACC1

  1374. 	vpblendd	\$3, $TEMP3, $TEMP4, $TEMP3

  1375. 	vpaddq		$TEMP2, $ACC2, $ACC2

  1376. 	vpblendd	\$3, $TEMP4, $ZERO, $TEMP4

  1377. 	vpaddq		$TEMP3, $ACC3, $ACC3

  1378. 	vpaddq		$TEMP4, $ACC4, $ACC4

  1379. 

  1380. 	vmovdqu		$ACC0, 0-128($rp)

  1381. 	vmovdqu		$ACC1, 32-128($rp)

  1382. 	vmovdqu		$ACC2, 64-128($rp)

  1383. 	vmovdqu		$ACC3, 96-128($rp)

  1384. ___

  1385. 

  1386. $TEMP5=$ACC0;

  1387. $code.=<<___;

  1388. 	vpsrlq		\$29, $ACC4, $TEMP1

  1389. 	vpand		$AND_MASK, $ACC4, $ACC4

  1390. 	vpsrlq		\$29, $ACC5, $TEMP2

  1391. 	vpand		$AND_MASK, $ACC5, $ACC5

  1392. 	vpsrlq		\$29, $ACC6, $TEMP3

  1393. 	vpermq		\$0x93, $TEMP1, $TEMP1

  1394. 	vpand		$AND_MASK, $ACC6, $ACC6

  1395. 	vpsrlq		\$29, $ACC7, $TEMP4

  1396. 	vpermq		\$0x93, $TEMP2, $TEMP2

  1397. 	vpand		$AND_MASK, $ACC7, $ACC7

  1398. 	vpsrlq		\$29, $ACC8, $TEMP5

  1399. 	vpermq		\$0x93, $TEMP3, $TEMP3

  1400. 	vpand		$AND_MASK, $ACC8, $ACC8

  1401. 	vpermq		\$0x93, $TEMP4, $TEMP4

  1402. 

  1403. 	vpblendd	\$3, $ZERO, $TEMP1, $TEMP0

  1404. 	vpermq		\$0x93, $TEMP5, $TEMP5

  1405. 	vpblendd	\$3, $TEMP1, $TEMP2, $TEMP1

  1406. 	vpaddq		$TEMP0, $ACC4, $ACC4

  1407. 	vpblendd	\$3, $TEMP2, $TEMP3, $TEMP2

  1408. 	vpaddq		$TEMP1, $ACC5, $ACC5

  1409. 	vpblendd	\$3, $TEMP3, $TEMP4, $TEMP3

  1410. 	vpaddq		$TEMP2, $ACC6, $ACC6

  1411. 	vpblendd	\$3, $TEMP4, $TEMP5, $TEMP4

  1412. 	vpaddq		$TEMP3, $ACC7, $ACC7

  1413. 	vpaddq		$TEMP4, $ACC8, $ACC8

  1414. 

  1415. 	vpsrlq		\$29, $ACC4, $TEMP1

  1416. 	vpand		$AND_MASK, $ACC4, $ACC4

  1417. 	vpsrlq		\$29, $ACC5, $TEMP2

  1418. 	vpand		$AND_MASK, $ACC5, $ACC5

  1419. 	vpsrlq		\$29, $ACC6, $TEMP3

  1420. 	vpermq		\$0x93, $TEMP1, $TEMP1

  1421. 	vpand		$AND_MASK, $ACC6, $ACC6

  1422. 	vpsrlq		\$29, $ACC7, $TEMP4

  1423. 	vpermq		\$0x93, $TEMP2, $TEMP2

  1424. 	vpand		$AND_MASK, $ACC7, $ACC7

  1425. 	vpsrlq		\$29, $ACC8, $TEMP5

  1426. 	vpermq		\$0x93, $TEMP3, $TEMP3

  1427. 	vpand		$AND_MASK, $ACC8, $ACC8

  1428. 	vpermq		\$0x93, $TEMP4, $TEMP4

  1429. 

  1430. 	vpblendd	\$3, $ZERO, $TEMP1, $TEMP0

  1431. 	vpermq		\$0x93, $TEMP5, $TEMP5

  1432. 	vpblendd	\$3, $TEMP1, $TEMP2, $TEMP1

  1433. 	vpaddq		$TEMP0, $ACC4, $ACC4

  1434. 	vpblendd	\$3, $TEMP2, $TEMP3, $TEMP2

  1435. 	vpaddq		$TEMP1, $ACC5, $ACC5

  1436. 	vpblendd	\$3, $TEMP3, $TEMP4, $TEMP3

  1437. 	vpaddq		$TEMP2, $ACC6, $ACC6

  1438. 	vpblendd	\$3, $TEMP4, $TEMP5, $TEMP4

  1439. 	vpaddq		$TEMP3, $ACC7, $ACC7

  1440. 	vpaddq		$TEMP4, $ACC8, $ACC8

  1441. 

  1442. 	vmovdqu		$ACC4, 128-128($rp)

  1443. 	vmovdqu		$ACC5, 160-128($rp)    

  1444. 	vmovdqu		$ACC6, 192-128($rp)

  1445. 	vmovdqu		$ACC7, 224-128($rp)

  1446. 	vmovdqu		$ACC8, 256-128($rp)

  1447. 	vzeroupper

  1448. 

  1449. 	mov	%rbp, %rax

  1450. ___

  1451. $code.=<<___ if ($win64);

  1452. 	movaps	-0xd8(%rax),%xmm6

  1453. 	movaps	-0xc8(%rax),%xmm7

  1454. 	movaps	-0xb8(%rax),%xmm8

  1455. 	movaps	-0xa8(%rax),%xmm9

  1456. 	movaps	-0x98(%rax),%xmm10

  1457. 	movaps	-0x88(%rax),%xmm11

  1458. 	movaps	-0x78(%rax),%xmm12

  1459. 	movaps	-0x68(%rax),%xmm13

  1460. 	movaps	-0x58(%rax),%xmm14

  1461. 	movaps	-0x48(%rax),%xmm15

  1462. ___

  1463. $code.=<<___;

  1464. 	mov	-48(%rax),%r15

  1465. 	mov	-40(%rax),%r14

  1466. 	mov	-32(%rax),%r13

  1467. 	mov	-24(%rax),%r12

  1468. 	mov	-16(%rax),%rbp

  1469. 	mov	-8(%rax),%rbx

  1470. 	lea	(%rax),%rsp		# restore %rsp

  1471. .Lmul_1024_epilogue:

  1472. 	ret

  1473. .size	rsaz_1024_mul_avx2,.-rsaz_1024_mul_avx2

  1474. ___

  1475. }

  1476. {

  1477. my ($out,$inp) = $win64 ? ("%rcx","%rdx") : ("%rdi","%rsi");

  1478. my @T = map("%r$_",(8..11));

  1479. 

  1480. $code.=<<___;

  1481. .globl	rsaz_1024_red2norm_avx2

  1482. .type	rsaz_1024_red2norm_avx2,\@abi-omnipotent

  1483. .align	32

  1484. rsaz_1024_red2norm_avx2:

  1485. 	sub	\$-128,$inp	# size optimization

  1486. 	xor	%rax,%rax

  1487. ___

  1488. 

  1489. for ($j=0,$i=0; $i<16; $i++) {

  1490.     my $k=0;

  1491.     while (29*$j<64*($i+1)) {	# load data till boundary

  1492. 	$code.="	mov	`8*$j-128`($inp), @T[0]\n";

  1493. 	$j++; $k++; push(@T,shift(@T));

  1494.     }

  1495.     $l=$k;

  1496.     while ($k>1) {		# shift loaded data but last value

  1497. 	$code.="	shl	\$`29*($j-$k)`,@T[-$k]\n";

  1498. 	$k--;

  1499.     }

  1500.     $code.=<<___;		# shift last value

  1501. 	mov	@T[-1], @T[0]

  1502. 	shl	\$`29*($j-1)`, @T[-1]

  1503. 	shr	\$`-29*($j-1)`, @T[0]

  1504. ___

  1505.     while ($l) {		# accumulate all values

  1506. 	$code.="	add	@T[-$l], %rax\n";

  1507. 	$l--;

  1508.     }

  1509. 	$code.=<<___;

  1510. 	adc	\$0, @T[0]	# consume eventual carry

  1511. 	mov	%rax, 8*$i($out)

  1512. 	mov	@T[0], %rax

  1513. ___

  1514.     push(@T,shift(@T));

  1515. }

  1516. $code.=<<___;

  1517. 	ret

  1518. .size	rsaz_1024_red2norm_avx2,.-rsaz_1024_red2norm_avx2

  1519. 

  1520. .globl	rsaz_1024_norm2red_avx2

  1521. .type	rsaz_1024_norm2red_avx2,\@abi-omnipotent

  1522. .align	32

  1523. rsaz_1024_norm2red_avx2:

  1524. 	sub	\$-128,$out	# size optimization

  1525. 	mov	($inp),@T[0]

  1526. 	mov	\$0x1fffffff,%eax

  1527. ___

  1528. for ($j=0,$i=0; $i<16; $i++) {

  1529.     $code.="	mov	`8*($i+1)`($inp),@T[1]\n"	if ($i<15);

  1530.     $code.="	xor	@T[1],@T[1]\n"			if ($i==15);

  1531.     my $k=1;

  1532.     while (29*($j+1)<64*($i+1)) {

  1533.     	$code.=<<___;

  1534. 	mov	@T[0],@T[-$k]

  1535. 	shr	\$`29*$j`,@T[-$k]

  1536. 	and	%rax,@T[-$k]				# &0x1fffffff

  1537. 	mov	@T[-$k],`8*$j-128`($out)

  1538. ___

  1539. 	$j++; $k++;

  1540.     }

  1541.     $code.=<<___;

  1542. 	shrd	\$`29*$j`,@T[1],@T[0]

  1543. 	and	%rax,@T[0]

  1544. 	mov	@T[0],`8*$j-128`($out)

  1545. ___

  1546.     $j++;

  1547.     push(@T,shift(@T));

  1548. }

  1549. $code.=<<___;

  1550. 	mov	@T[0],`8*$j-128`($out)			# zero

  1551. 	mov	@T[0],`8*($j+1)-128`($out)

  1552. 	mov	@T[0],`8*($j+2)-128`($out)

  1553. 	mov	@T[0],`8*($j+3)-128`($out)

  1554. 	ret

  1555. .size	rsaz_1024_norm2red_avx2,.-rsaz_1024_norm2red_avx2

  1556. ___

  1557. }

  1558. {

  1559. my ($out,$inp,$power) = $win64 ? ("%rcx","%rdx","%r8d") : ("%rdi","%rsi","%edx");

  1560. 

  1561. $code.=<<___;

  1562. .globl	rsaz_1024_scatter5_avx2

  1563. .type	rsaz_1024_scatter5_avx2,\@abi-omnipotent

  1564. .align	32

  1565. rsaz_1024_scatter5_avx2:

  1566. 	vzeroupper

  1567. 	vmovdqu	.Lscatter_permd(%rip),%ymm5

  1568. 	shl	\$4,$power

  1569. 	lea	($out,$power),$out

  1570. 	mov	\$9,%eax

  1571. 	jmp	.Loop_scatter_1024

  1572. 

  1573. .align	32

  1574. .Loop_scatter_1024:

  1575. 	vmovdqu		($inp),%ymm0

  1576. 	lea		32($inp),$inp

  1577. 	vpermd		%ymm0,%ymm5,%ymm0

  1578. 	vmovdqu		%xmm0,($out)

  1579. 	lea		16*32($out),$out

  1580. 	dec	%eax

  1581. 	jnz	.Loop_scatter_1024

  1582. 

  1583. 	vzeroupper

  1584. 	ret

  1585. .size	rsaz_1024_scatter5_avx2,.-rsaz_1024_scatter5_avx2

  1586. 

  1587. .globl	rsaz_1024_gather5_avx2

  1588. .type	rsaz_1024_gather5_avx2,\@abi-omnipotent

  1589. .align	32

  1590. rsaz_1024_gather5_avx2:

  1591. ___

  1592. $code.=<<___ if ($win64);

  1593. 	lea	-0x88(%rsp),%rax

  1594. 	vzeroupper

  1595. .LSEH_begin_rsaz_1024_gather5:

  1596. 	# I can't trust assembler to use specific encoding:-(

  1597. 	.byte	0x48,0x8d,0x60,0xe0		#lea	-0x20(%rax),%rsp

  1598. 	.byte	0xc5,0xf8,0x29,0x70,0xe0	#vmovaps %xmm6,-0x20(%rax)

  1599. 	.byte	0xc5,0xf8,0x29,0x78,0xf0	#vmovaps %xmm7,-0x10(%rax)

  1600. 	.byte	0xc5,0x78,0x29,0x40,0x00	#vmovaps %xmm8,0(%rax)

  1601. 	.byte	0xc5,0x78,0x29,0x48,0x10	#vmovaps %xmm9,0x10(%rax)

  1602. 	.byte	0xc5,0x78,0x29,0x50,0x20	#vmovaps %xmm10,0x20(%rax)

  1603. 	.byte	0xc5,0x78,0x29,0x58,0x30	#vmovaps %xmm11,0x30(%rax)

  1604. 	.byte	0xc5,0x78,0x29,0x60,0x40	#vmovaps %xmm12,0x40(%rax)

  1605. 	.byte	0xc5,0x78,0x29,0x68,0x50	#vmovaps %xmm13,0x50(%rax)

  1606. 	.byte	0xc5,0x78,0x29,0x70,0x60	#vmovaps %xmm14,0x60(%rax)

  1607. 	.byte	0xc5,0x78,0x29,0x78,0x70	#vmovaps %xmm15,0x70(%rax)

  1608. ___

  1609. $code.=<<___;

  1610. 	lea	.Lgather_table(%rip),%r11

  1611. 	mov	$power,%eax

  1612. 	and	\$3,$power

  1613. 	shr	\$2,%eax			# cache line number

  1614. 	shl	\$4,$power			# offset within cache line

  1615. 

  1616. 	vmovdqu		-32(%r11),%ymm7		# .Lgather_permd

  1617. 	vpbroadcastb	8(%r11,%rax), %xmm8

  1618. 	vpbroadcastb	7(%r11,%rax), %xmm9

  1619. 	vpbroadcastb	6(%r11,%rax), %xmm10

  1620. 	vpbroadcastb	5(%r11,%rax), %xmm11

  1621. 	vpbroadcastb	4(%r11,%rax), %xmm12

  1622. 	vpbroadcastb	3(%r11,%rax), %xmm13

  1623. 	vpbroadcastb	2(%r11,%rax), %xmm14

  1624. 	vpbroadcastb	1(%r11,%rax), %xmm15

  1625. 

  1626. 	lea	64($inp,$power),$inp

  1627. 	mov	\$64,%r11			# size optimization

  1628. 	mov	\$9,%eax

  1629. 	jmp	.Loop_gather_1024

  1630. 

  1631. .align	32

  1632. .Loop_gather_1024:

  1633. 	vpand		-64($inp),		%xmm8,%xmm0

  1634. 	vpand		($inp),			%xmm9,%xmm1

  1635. 	vpand		64($inp),		%xmm10,%xmm2

  1636. 	vpand		($inp,%r11,2),		%xmm11,%xmm3

  1637. 	 vpor					%xmm0,%xmm1,%xmm1

  1638. 	vpand		64($inp,%r11,2),	%xmm12,%xmm4

  1639. 	 vpor					%xmm2,%xmm3,%xmm3

  1640. 	vpand		($inp,%r11,4),		%xmm13,%xmm5

  1641. 	 vpor					%xmm1,%xmm3,%xmm3

  1642. 	vpand		64($inp,%r11,4),	%xmm14,%xmm6

  1643. 	 vpor					%xmm4,%xmm5,%xmm5

  1644. 	vpand		-128($inp,%r11,8),	%xmm15,%xmm2

  1645. 	lea		($inp,%r11,8),$inp

  1646. 	 vpor					%xmm3,%xmm5,%xmm5

  1647. 	 vpor					%xmm2,%xmm6,%xmm6

  1648. 	 vpor					%xmm5,%xmm6,%xmm6

  1649. 	vpermd		%ymm6,%ymm7,%ymm6

  1650. 	vmovdqu		%ymm6,($out)

  1651. 	lea		32($out),$out

  1652. 	dec	%eax

  1653. 	jnz	.Loop_gather_1024

  1654. 

  1655. 	vpxor	%ymm0,%ymm0,%ymm0

  1656. 	vmovdqu	%ymm0,($out)

  1657. 	vzeroupper

  1658. ___

  1659. $code.=<<___ if ($win64);

  1660. 	movaps	(%rsp),%xmm6

  1661. 	movaps	0x10(%rsp),%xmm7

  1662. 	movaps	0x20(%rsp),%xmm8

  1663. 	movaps	0x30(%rsp),%xmm9

  1664. 	movaps	0x40(%rsp),%xmm10

  1665. 	movaps	0x50(%rsp),%xmm11

  1666. 	movaps	0x60(%rsp),%xmm12

  1667. 	movaps	0x70(%rsp),%xmm13

  1668. 	movaps	0x80(%rsp),%xmm14

  1669. 	movaps	0x90(%rsp),%xmm15

  1670. 	lea	0xa8(%rsp),%rsp

  1671. .LSEH_end_rsaz_1024_gather5:

  1672. ___

  1673. $code.=<<___;

  1674. 	ret

  1675. .size	rsaz_1024_gather5_avx2,.-rsaz_1024_gather5_avx2

  1676. ___

  1677. }

  1678. 

  1679. $code.=<<___;

  1680. .extern	OPENSSL_ia32cap_P

  1681. .globl	rsaz_avx2_eligible

  1682. .type	rsaz_avx2_eligible,\@abi-omnipotent

  1683. .align	32

  1684. rsaz_avx2_eligible:

  1685. 	mov	OPENSSL_ia32cap_P+8(%rip),%eax

  1686. ___

  1687. $code.=<<___	if ($addx);

  1688. 	mov	\$`1<<8|1<<19`,%ecx

  1689. 	mov	\$0,%edx

  1690. 	and	%eax,%ecx

  1691. 	cmp	\$`1<<8|1<<19`,%ecx	# check for BMI2+AD*X

  1692. 	cmove	%edx,%eax

  1693. ___

  1694. $code.=<<___;

  1695. 	and	\$`1<<5`,%eax

  1696. 	shr	\$5,%eax

  1697. 	ret

  1698. .size	rsaz_avx2_eligible,.-rsaz_avx2_eligible

  1699. 

  1700. .align	64

  1701. .Land_mask:

  1702. 	.quad	0x1fffffff,0x1fffffff,0x1fffffff,-1

  1703. .Lscatter_permd:

  1704. 	.long	0,2,4,6,7,7,7,7

  1705. .Lgather_permd:

  1706. 	.long	0,7,1,7,2,7,3,7

  1707. .Lgather_table:

  1708. 	.byte	0,0,0,0,0,0,0,0, 0xff,0,0,0,0,0,0,0

  1709. .align	64

  1710. ___

  1711. 

  1712. if ($win64) {

  1713. $rec="%rcx";

  1714. $frame="%rdx";

  1715. $context="%r8";

  1716. $disp="%r9";

  1717. 

  1718. $code.=<<___

  1719. .extern	__imp_RtlVirtualUnwind

  1720. .type	rsaz_se_handler,\@abi-omnipotent

  1721. .align	16

  1722. rsaz_se_handler:

  1723. 	push	%rsi

  1724. 	push	%rdi

  1725. 	push	%rbx

  1726. 	push	%rbp

  1727. 	push	%r12

  1728. 	push	%r13

  1729. 	push	%r14

  1730. 	push	%r15

  1731. 	pushfq

  1732. 	sub	\$64,%rsp

  1733. 

  1734. 	mov	120($context),%rax	# pull context->Rax

  1735. 	mov	248($context),%rbx	# pull context->Rip

  1736. 

  1737. 	mov	8($disp),%rsi		# disp->ImageBase

  1738. 	mov	56($disp),%r11		# disp->HandlerData

  1739. 

  1740. 	mov	0(%r11),%r10d		# HandlerData[0]

  1741. 	lea	(%rsi,%r10),%r10	# prologue label

  1742. 	cmp	%r10,%rbx		# context->Rip<prologue label

  1743. 	jb	.Lcommon_seh_tail

  1744. 

  1745. 	mov	152($context),%rax	# pull context->Rsp

  1746. 

  1747. 	mov	4(%r11),%r10d		# HandlerData[1]

  1748. 	lea	(%rsi,%r10),%r10	# epilogue label

  1749. 	cmp	%r10,%rbx		# context->Rip>=epilogue label

  1750. 	jae	.Lcommon_seh_tail

  1751. 

  1752. 	mov	160($context),%rax	# pull context->Rbp

  1753. 

  1754. 	mov	-48(%rax),%r15

  1755. 	mov	-40(%rax),%r14

  1756. 	mov	-32(%rax),%r13

  1757. 	mov	-24(%rax),%r12

  1758. 	mov	-16(%rax),%rbp

  1759. 	mov	-8(%rax),%rbx

  1760. 	mov	%r15,240($context)

  1761. 	mov	%r14,232($context)

  1762. 	mov	%r13,224($context)

  1763. 	mov	%r12,216($context)

  1764. 	mov	%rbp,160($context)

  1765. 	mov	%rbx,144($context)

  1766. 

  1767. 	lea	-0xd8(%rax),%rsi	# %xmm save area

  1768. 	lea	512($context),%rdi	# & context.Xmm6

  1769. 	mov	\$20,%ecx		# 10*sizeof(%xmm0)/sizeof(%rax)

  1770. 	.long	0xa548f3fc		# cld; rep movsq

  1771. 

  1772. .Lcommon_seh_tail:

  1773. 	mov	8(%rax),%rdi

  1774. 	mov	16(%rax),%rsi

  1775. 	mov	%rax,152($context)	# restore context->Rsp

  1776. 	mov	%rsi,168($context)	# restore context->Rsi

  1777. 	mov	%rdi,176($context)	# restore context->Rdi

  1778. 

  1779. 	mov	40($disp),%rdi		# disp->ContextRecord

  1780. 	mov	$context,%rsi		# context

  1781. 	mov	\$154,%ecx		# sizeof(CONTEXT)

  1782. 	.long	0xa548f3fc		# cld; rep movsq

  1783. 

  1784. 	mov	$disp,%rsi

  1785. 	xor	%rcx,%rcx		# arg1, UNW_FLAG_NHANDLER

  1786. 	mov	8(%rsi),%rdx		# arg2, disp->ImageBase

  1787. 	mov	0(%rsi),%r8		# arg3, disp->ControlPc

  1788. 	mov	16(%rsi),%r9		# arg4, disp->FunctionEntry

  1789. 	mov	40(%rsi),%r10		# disp->ContextRecord

  1790. 	lea	56(%rsi),%r11		# &disp->HandlerData

  1791. 	lea	24(%rsi),%r12		# &disp->EstablisherFrame

  1792. 	mov	%r10,32(%rsp)		# arg5

  1793. 	mov	%r11,40(%rsp)		# arg6

  1794. 	mov	%r12,48(%rsp)		# arg7

  1795. 	mov	%rcx,56(%rsp)		# arg8, (NULL)

  1796. 	call	*__imp_RtlVirtualUnwind(%rip)

  1797. 

  1798. 	mov	\$1,%eax		# ExceptionContinueSearch

  1799. 	add	\$64,%rsp

  1800. 	popfq

  1801. 	pop	%r15

  1802. 	pop	%r14

  1803. 	pop	%r13

  1804. 	pop	%r12

  1805. 	pop	%rbp

  1806. 	pop	%rbx

  1807. 	pop	%rdi

  1808. 	pop	%rsi

  1809. 	ret

  1810. .size	rsaz_se_handler,.-rsaz_se_handler

  1811. 

  1812. .section	.pdata

  1813. .align	4

  1814. 	.rva	.LSEH_begin_rsaz_1024_sqr_avx2

  1815. 	.rva	.LSEH_end_rsaz_1024_sqr_avx2

  1816. 	.rva	.LSEH_info_rsaz_1024_sqr_avx2

  1817. 

  1818. 	.rva	.LSEH_begin_rsaz_1024_mul_avx2

  1819. 	.rva	.LSEH_end_rsaz_1024_mul_avx2

  1820. 	.rva	.LSEH_info_rsaz_1024_mul_avx2

  1821. 

  1822. 	.rva	.LSEH_begin_rsaz_1024_gather5

  1823. 	.rva	.LSEH_end_rsaz_1024_gather5

  1824. 	.rva	.LSEH_info_rsaz_1024_gather5

  1825. .section	.xdata

  1826. .align	8

  1827. .LSEH_info_rsaz_1024_sqr_avx2:

  1828. 	.byte	9,0,0,0

  1829. 	.rva	rsaz_se_handler

  1830. 	.rva	.Lsqr_1024_body,.Lsqr_1024_epilogue

  1831. .LSEH_info_rsaz_1024_mul_avx2:

  1832. 	.byte	9,0,0,0

  1833. 	.rva	rsaz_se_handler

  1834. 	.rva	.Lmul_1024_body,.Lmul_1024_epilogue

  1835. .LSEH_info_rsaz_1024_gather5:

  1836. 	.byte	0x01,0x33,0x16,0x00

  1837. 	.byte	0x36,0xf8,0x09,0x00	#vmovaps 0x90(rsp),xmm15

  1838. 	.byte	0x31,0xe8,0x08,0x00	#vmovaps 0x80(rsp),xmm14

  1839. 	.byte	0x2c,0xd8,0x07,0x00	#vmovaps 0x70(rsp),xmm13

  1840. 	.byte	0x27,0xc8,0x06,0x00	#vmovaps 0x60(rsp),xmm12

  1841. 	.byte	0x22,0xb8,0x05,0x00	#vmovaps 0x50(rsp),xmm11

  1842. 	.byte	0x1d,0xa8,0x04,0x00	#vmovaps 0x40(rsp),xmm10

  1843. 	.byte	0x18,0x98,0x03,0x00	#vmovaps 0x30(rsp),xmm9

  1844. 	.byte	0x13,0x88,0x02,0x00	#vmovaps 0x20(rsp),xmm8

  1845. 	.byte	0x0e,0x78,0x01,0x00	#vmovaps 0x10(rsp),xmm7

  1846. 	.byte	0x09,0x68,0x00,0x00	#vmovaps 0x00(rsp),xmm6

  1847. 	.byte	0x04,0x01,0x15,0x00	#sub	rsp,0xa8

  1848. ___

  1849. }

  1850. 

  1851. foreach (split("\n",$code)) {

  1852. 	s/\`([^\`]*)\`/eval($1)/ge;

  1853. 

  1854. 	s/\b(sh[rl]d?\s+\$)(-?[0-9]+)/$1.$2%64/ge		or

  1855. 

  1856. 	s/\b(vmov[dq])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go		or

  1857. 	s/\b(vmovdqu)\b(.+)%x%ymm([0-9]+)/$1$2%xmm$3/go		or

  1858. 	s/\b(vpinsr[qd])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go	or

  1859. 	s/\b(vpextr[qd])\b(.+)%ymm([0-9]+)/$1$2%xmm$3/go	or

  1860. 	s/\b(vpbroadcast[qd]\s+)%ymm([0-9]+)/$1%xmm$2/go;

  1861. 	print $_,"\n";

  1862. }

  1863. 

  1864. }}} else {{{

  1865. print <<___;	# assembler is too old

  1866. .text

  1867. 

  1868. .globl	rsaz_avx2_eligible

  1869. .type	rsaz_avx2_eligible,\@abi-omnipotent

  1870. rsaz_avx2_eligible:

  1871. 	xor	%eax,%eax

  1872. 	ret

  1873. .size	rsaz_avx2_eligible,.-rsaz_avx2_eligible

  1874. 

  1875. .globl	rsaz_1024_sqr_avx2

  1876. .globl	rsaz_1024_mul_avx2

  1877. .globl	rsaz_1024_norm2red_avx2

  1878. .globl	rsaz_1024_red2norm_avx2

  1879. .globl	rsaz_1024_scatter5_avx2

  1880. .globl	rsaz_1024_gather5_avx2

  1881. .type	rsaz_1024_sqr_avx2,\@abi-omnipotent

  1882. rsaz_1024_sqr_avx2:

  1883. rsaz_1024_mul_avx2:

  1884. rsaz_1024_norm2red_avx2:

  1885. rsaz_1024_red2norm_avx2:

  1886. rsaz_1024_scatter5_avx2:

  1887. rsaz_1024_gather5_avx2:

  1888. 	.byte	0x0f,0x0b	# ud2

  1889. 	ret

  1890. .size	rsaz_1024_sqr_avx2,.-rsaz_1024_sqr_avx2

  1891. ___

  1892. }}}

  1893. 

  1894. close STDOUT;