kris
/
boringssl
1
0
Forkuj 0
Kod Zgłoszenia 0 Oczekujące zmiany 0 Wydania 0 Wiki Aktywność
Nie możesz wybrać więcej, niż 25 tematów Tematy muszą się zaczynać od litery lub cyfry, mogą zawierać myślniki ('-') i mogą mieć do 35 znaków.
2891 Commity
12 Gałęzie
38 MiB
 
 
 
 
 
 
Drzewo: 71dd6660e8
Gałęzie Tagi
${ item.name }
Utwórz gałąź ${ searchTerm }
z '71dd6660e8'
${ noResults }
boringssl/crypto/bn/asm/rsaz-x86_64.pl

2339 wiersze
46 KiB
Czysty Wina Historia 

  1. #!/usr/bin/env perl

  2. 

  3. ##############################################################################

  4. #                                                                            #

  5. #  Copyright (c) 2012, Intel Corporation                                     #

  6. #                                                                            #

  7. #  All rights reserved.                                                      #

  8. #                                                                            #

  9. #  Redistribution and use in source and binary forms, with or without        #

  10. #  modification, are permitted provided that the following conditions are    #

  11. #  met:                                                                      #

  12. #                                                                            #

  13. #  *  Redistributions of source code must retain the above copyright         #

  14. #     notice, this list of conditions and the following disclaimer.          #

  15. #                                                                            #

  16. #  *  Redistributions in binary form must reproduce the above copyright      #

  17. #     notice, this list of conditions and the following disclaimer in the    #

  18. #     documentation and/or other materials provided with the                 #

  19. #     distribution.                                                          #

  20. #                                                                            #

  21. #  *  Neither the name of the Intel Corporation nor the names of its         #

  22. #     contributors may be used to endorse or promote products derived from   #

  23. #     this software without specific prior written permission.               #

  24. #                                                                            #

  25. #                                                                            #

  26. #  THIS SOFTWARE IS PROVIDED BY INTEL CORPORATION ""AS IS"" AND ANY          #

  27. #  EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE         #

  28. #  IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR        #

  29. #  PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL INTEL CORPORATION OR            #

  30. #  CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,     #

  31. #  EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,       #

  32. #  PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR        #

  33. #  PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF    #

  34. #  LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING      #

  35. #  NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS        #

  36. #  SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.              #

  37. #                                                                            #

  38. ##############################################################################

  39. # Developers and authors:                                                    #

  40. # Shay Gueron (1, 2), and Vlad Krasnov (1)                                   #

  41. # (1) Intel Architecture Group, Microprocessor and Chipset Development,      #

  42. #     Israel Development Center, Haifa, Israel                               #

  43. # (2) University of Haifa                                                    #

  44. ##############################################################################

  45. # Reference:                                                                 #

  46. # [1] S. Gueron, "Efficient Software Implementations of Modular              #

  47. #     Exponentiation", http://eprint.iacr.org/2011/239                       #

  48. # [2] S. Gueron, V. Krasnov. "Speeding up Big-Numbers Squaring".             #

  49. #     IEEE Proceedings of 9th International Conference on Information        #

  50. #     Technology: New Generations (ITNG 2012), 821-823 (2012).               #

  51. # [3] S. Gueron, Efficient Software Implementations of Modular Exponentiation#

  52. #     Journal of Cryptographic Engineering 2:31-43 (2012).                   #

  53. # [4] S. Gueron, V. Krasnov: "[PATCH] Efficient and side channel analysis    #

  54. #     resistant 512-bit and 1024-bit modular exponentiation for optimizing   #

  55. #     RSA1024 and RSA2048 on x86_64 platforms",                              #

  56. #     http://rt.openssl.org/Ticket/Display.html?id=2582&user=guest&pass=guest#

  57. ##############################################################################

  58. 

  59. # While original submission covers 512- and 1024-bit exponentiation,

  60. # this module is limited to 512-bit version only (and as such

  61. # accelerates RSA1024 sign). This is because improvement for longer

  62. # keys is not high enough to justify the effort, highest measured

  63. # was ~5% on Westmere. [This is relative to OpenSSL 1.0.2, upcoming

  64. # for the moment of this writing!] Nor does this module implement

  65. # "monolithic" complete exponentiation jumbo-subroutine, but adheres

  66. # to more modular mixture of C and assembly. And it's optimized even

  67. # for processors other than Intel Core family (see table below for

  68. # improvement coefficients).

  69. # 						<appro@openssl.org>

  70. #

  71. # RSA1024 sign/sec	this/original	|this/rsax(*)	this/fips(*)

  72. #			----------------+---------------------------

  73. # Opteron		+13%		|+5%		+20%

  74. # Bulldozer		-0%		|-1%		+10%

  75. # P4			+11%		|+7%		+8%

  76. # Westmere		+5%		|+14%		+17%

  77. # Sandy Bridge		+2%		|+12%		+29%

  78. # Ivy Bridge		+1%		|+11%		+35%

  79. # Haswell(**)		-0%		|+12%		+39%

  80. # Atom			+13%		|+11%		+4%

  81. # VIA Nano		+70%		|+9%		+25%

  82. #

  83. # (*)	rsax engine and fips numbers are presented for reference

  84. #	purposes;

  85. # (**)	MULX was attempted, but found to give only marginal improvement;

  86. 

  87. $flavour = shift;

  88. $output  = shift;

  89. if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }

  90. 

  91. $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);

  92. 

  93. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;

  94. ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or

  95. ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or

  96. die "can't locate x86_64-xlate.pl";

  97. 

  98. open OUT,"| \"$^X\" \"$xlate\" $flavour \"$output\"";

  99. *STDOUT=*OUT;

  100. 

  101. # In upstream, this is controlled by shelling out to the compiler to check

  102. # versions, but BoringSSL is intended to be used with pre-generated perlasm

  103. # output, so this isn't useful anyway.

  104. #

  105. # TODO(davidben): Enable this after testing. $addx goes up to 1.

  106. $addx = 0;

  107. 

  108. ($out, $inp, $mod) = ("%rdi", "%rsi", "%rbp");	# common internal API

  109. {

  110. my ($out,$inp,$mod,$n0,$times) = ("%rdi","%rsi","%rdx","%rcx","%r8d");

  111. 

  112. $code.=<<___;

  113. .text

  114. 

  115. .extern	OPENSSL_ia32cap_P

  116. 

  117. .globl	rsaz_512_sqr

  118. .type	rsaz_512_sqr,\@function,5

  119. .align	32

  120. rsaz_512_sqr:				# 25-29% faster than rsaz_512_mul

  121. 	push	%rbx

  122. 	push	%rbp

  123. 	push	%r12

  124. 	push	%r13

  125. 	push	%r14

  126. 	push	%r15

  127. 

  128. 	subq	\$128+24, %rsp

  129. .Lsqr_body:

  130. 	movq	$mod, %rbp		# common argument

  131. 	movq	($inp), %rdx

  132. 	movq	8($inp), %rax

  133. 	movq	$n0, 128(%rsp)

  134. ___

  135. $code.=<<___ if ($addx);

  136. 	movl	\$0x80100,%r11d

  137. 	andl	OPENSSL_ia32cap_P+8(%rip),%r11d

  138. 	cmpl	\$0x80100,%r11d		# check for MULX and ADO/CX

  139. 	je	.Loop_sqrx

  140. ___

  141. $code.=<<___;

  142. 	jmp	.Loop_sqr

  143. 

  144. .align	32

  145. .Loop_sqr:

  146. 	movl	$times,128+8(%rsp)

  147. #first iteration

  148. 	movq	%rdx, %rbx

  149. 	mulq	%rdx

  150. 	movq	%rax, %r8

  151. 	movq	16($inp), %rax

  152. 	movq	%rdx, %r9

  153. 

  154. 	mulq	%rbx

  155. 	addq	%rax, %r9

  156. 	movq	24($inp), %rax

  157. 	movq	%rdx, %r10

  158. 	adcq	\$0, %r10

  159. 

  160. 	mulq	%rbx

  161. 	addq	%rax, %r10

  162. 	movq	32($inp), %rax

  163. 	movq	%rdx, %r11

  164. 	adcq	\$0, %r11

  165. 

  166. 	mulq	%rbx

  167. 	addq	%rax, %r11

  168. 	movq	40($inp), %rax

  169. 	movq	%rdx, %r12

  170. 	adcq	\$0, %r12

  171. 

  172. 	mulq	%rbx

  173. 	addq	%rax, %r12

  174. 	movq	48($inp), %rax

  175. 	movq	%rdx, %r13

  176. 	adcq	\$0, %r13

  177. 

  178. 	mulq	%rbx

  179. 	addq	%rax, %r13

  180. 	movq	56($inp), %rax

  181. 	movq	%rdx, %r14

  182. 	adcq	\$0, %r14

  183. 

  184. 	mulq	%rbx

  185. 	addq	%rax, %r14

  186. 	movq	%rbx, %rax

  187. 	movq	%rdx, %r15

  188. 	adcq	\$0, %r15

  189. 

  190. 	addq	%r8, %r8		#shlq	\$1, %r8

  191. 	movq	%r9, %rcx

  192. 	adcq	%r9, %r9		#shld	\$1, %r8, %r9

  193. 

  194. 	mulq	%rax

  195. 	movq	%rax, (%rsp)

  196. 	addq	%rdx, %r8

  197. 	adcq	\$0, %r9

  198. 

  199. 	movq	%r8, 8(%rsp)

  200. 	shrq	\$63, %rcx

  201. 

  202. #second iteration

  203. 	movq	8($inp), %r8

  204. 	movq	16($inp), %rax

  205. 	mulq	%r8

  206. 	addq	%rax, %r10

  207. 	movq	24($inp), %rax

  208. 	movq	%rdx, %rbx

  209. 	adcq	\$0, %rbx

  210. 

  211. 	mulq	%r8

  212. 	addq	%rax, %r11

  213. 	movq	32($inp), %rax

  214. 	adcq	\$0, %rdx

  215. 	addq	%rbx, %r11

  216. 	movq	%rdx, %rbx

  217. 	adcq	\$0, %rbx

  218. 

  219. 	mulq	%r8

  220. 	addq	%rax, %r12

  221. 	movq	40($inp), %rax

  222. 	adcq	\$0, %rdx

  223. 	addq	%rbx, %r12

  224. 	movq	%rdx, %rbx

  225. 	adcq	\$0, %rbx

  226. 

  227. 	mulq	%r8

  228. 	addq	%rax, %r13

  229. 	movq	48($inp), %rax

  230. 	adcq	\$0, %rdx

  231. 	addq	%rbx, %r13

  232. 	movq	%rdx, %rbx

  233. 	adcq	\$0, %rbx

  234. 

  235. 	mulq	%r8

  236. 	addq	%rax, %r14

  237. 	movq	56($inp), %rax

  238. 	adcq	\$0, %rdx

  239. 	addq	%rbx, %r14

  240. 	movq	%rdx, %rbx

  241. 	adcq	\$0, %rbx

  242. 

  243. 	mulq	%r8

  244. 	addq	%rax, %r15

  245. 	movq	%r8, %rax

  246. 	adcq	\$0, %rdx

  247. 	addq	%rbx, %r15

  248. 	movq	%rdx, %r8

  249. 	movq	%r10, %rdx

  250. 	adcq	\$0, %r8

  251. 

  252. 	add	%rdx, %rdx

  253. 	lea	(%rcx,%r10,2), %r10	#shld	\$1, %rcx, %r10

  254. 	movq	%r11, %rbx

  255. 	adcq	%r11, %r11		#shld	\$1, %r10, %r11

  256. 

  257. 	mulq	%rax

  258. 	addq	%rax, %r9

  259. 	adcq	%rdx, %r10

  260. 	adcq	\$0, %r11

  261. 

  262. 	movq	%r9, 16(%rsp)

  263. 	movq	%r10, 24(%rsp)

  264. 	shrq	\$63, %rbx

  265. 	

  266. #third iteration

  267. 	movq	16($inp), %r9	

  268. 	movq	24($inp), %rax

  269. 	mulq	%r9

  270. 	addq	%rax, %r12

  271. 	movq	32($inp), %rax

  272. 	movq	%rdx, %rcx

  273. 	adcq	\$0, %rcx

  274. 

  275. 	mulq	%r9

  276. 	addq	%rax, %r13

  277. 	movq	40($inp), %rax

  278. 	adcq	\$0, %rdx

  279. 	addq	%rcx, %r13

  280. 	movq	%rdx, %rcx

  281. 	adcq	\$0, %rcx

  282. 

  283. 	mulq	%r9

  284. 	addq	%rax, %r14

  285. 	movq	48($inp), %rax

  286. 	adcq	\$0, %rdx

  287. 	addq	%rcx, %r14

  288. 	movq	%rdx, %rcx

  289. 	adcq	\$0, %rcx

  290. 

  291. 	mulq	%r9

  292. 	 movq	%r12, %r10

  293. 	 lea	(%rbx,%r12,2), %r12	#shld	\$1, %rbx, %r12

  294. 	addq	%rax, %r15

  295. 	movq	56($inp), %rax

  296. 	adcq	\$0, %rdx

  297. 	addq	%rcx, %r15

  298. 	movq	%rdx, %rcx

  299. 	adcq	\$0, %rcx

  300. 

  301. 	mulq	%r9

  302. 	 shrq	\$63, %r10

  303. 	addq	%rax, %r8

  304. 	movq	%r9, %rax

  305. 	adcq	\$0, %rdx

  306. 	addq	%rcx, %r8

  307. 	movq	%rdx, %r9

  308. 	adcq	\$0, %r9

  309. 

  310. 	movq	%r13, %rcx

  311. 	leaq	(%r10,%r13,2), %r13	#shld	\$1, %r12, %r13

  312. 

  313. 	mulq	%rax

  314. 	addq	%rax, %r11

  315. 	adcq	%rdx, %r12

  316. 	adcq	\$0, %r13

  317. 

  318. 	movq	%r11, 32(%rsp)

  319. 	movq	%r12, 40(%rsp)

  320. 	shrq	\$63, %rcx

  321. 

  322. #fourth iteration

  323. 	movq	24($inp), %r10

  324. 	movq	32($inp), %rax

  325. 	mulq	%r10

  326. 	addq	%rax, %r14

  327. 	movq	40($inp), %rax

  328. 	movq	%rdx, %rbx

  329. 	adcq	\$0, %rbx

  330. 

  331. 	mulq	%r10

  332. 	addq	%rax, %r15

  333. 	movq	48($inp), %rax

  334. 	adcq	\$0, %rdx

  335. 	addq	%rbx, %r15

  336. 	movq	%rdx, %rbx

  337. 	adcq	\$0, %rbx

  338. 

  339. 	mulq	%r10

  340. 	 movq	%r14, %r12

  341. 	 leaq	(%rcx,%r14,2), %r14	#shld	\$1, %rcx, %r14

  342. 	addq	%rax, %r8

  343. 	movq	56($inp), %rax

  344. 	adcq	\$0, %rdx

  345. 	addq	%rbx, %r8

  346. 	movq	%rdx, %rbx

  347. 	adcq	\$0, %rbx

  348. 

  349. 	mulq	%r10

  350. 	 shrq	\$63, %r12

  351. 	addq	%rax, %r9

  352. 	movq	%r10, %rax

  353. 	adcq	\$0, %rdx

  354. 	addq	%rbx, %r9

  355. 	movq	%rdx, %r10

  356. 	adcq	\$0, %r10

  357. 

  358. 	movq	%r15, %rbx

  359. 	leaq	(%r12,%r15,2),%r15	#shld	\$1, %r14, %r15

  360. 

  361. 	mulq	%rax

  362. 	addq	%rax, %r13

  363. 	adcq	%rdx, %r14

  364. 	adcq	\$0, %r15

  365. 

  366. 	movq	%r13, 48(%rsp)

  367. 	movq	%r14, 56(%rsp)

  368. 	shrq	\$63, %rbx

  369. 

  370. #fifth iteration

  371. 	movq	32($inp), %r11

  372. 	movq	40($inp), %rax

  373. 	mulq	%r11

  374. 	addq	%rax, %r8

  375. 	movq	48($inp), %rax

  376. 	movq	%rdx, %rcx

  377. 	adcq	\$0, %rcx

  378. 

  379. 	mulq	%r11

  380. 	addq	%rax, %r9

  381. 	movq	56($inp), %rax

  382. 	adcq	\$0, %rdx

  383. 	 movq	%r8, %r12

  384. 	 leaq	(%rbx,%r8,2), %r8	#shld	\$1, %rbx, %r8

  385. 	addq	%rcx, %r9

  386. 	movq	%rdx, %rcx

  387. 	adcq	\$0, %rcx

  388. 

  389. 	mulq	%r11

  390. 	 shrq	\$63, %r12

  391. 	addq	%rax, %r10

  392. 	movq	%r11, %rax

  393. 	adcq	\$0, %rdx

  394. 	addq	%rcx, %r10

  395. 	movq	%rdx, %r11

  396. 	adcq	\$0, %r11

  397. 

  398. 	movq	%r9, %rcx

  399. 	leaq	(%r12,%r9,2), %r9	#shld	\$1, %r8, %r9

  400. 

  401. 	mulq	%rax

  402. 	addq	%rax, %r15

  403. 	adcq	%rdx, %r8

  404. 	adcq	\$0, %r9

  405. 

  406. 	movq	%r15, 64(%rsp)

  407. 	movq	%r8, 72(%rsp)

  408. 	shrq	\$63, %rcx

  409. 

  410. #sixth iteration

  411. 	movq	40($inp), %r12

  412. 	movq	48($inp), %rax

  413. 	mulq	%r12

  414. 	addq	%rax, %r10

  415. 	movq	56($inp), %rax

  416. 	movq	%rdx, %rbx

  417. 	adcq	\$0, %rbx

  418. 

  419. 	mulq	%r12

  420. 	addq	%rax, %r11

  421. 	movq	%r12, %rax

  422. 	 movq	%r10, %r15

  423. 	 leaq	(%rcx,%r10,2), %r10	#shld	\$1, %rcx, %r10

  424. 	adcq	\$0, %rdx

  425. 	 shrq	\$63, %r15

  426. 	addq	%rbx, %r11

  427. 	movq	%rdx, %r12

  428. 	adcq	\$0, %r12

  429. 

  430. 	movq	%r11, %rbx

  431. 	leaq	(%r15,%r11,2), %r11	#shld	\$1, %r10, %r11

  432. 

  433. 	mulq	%rax

  434. 	addq	%rax, %r9

  435. 	adcq	%rdx, %r10

  436. 	adcq	\$0, %r11

  437. 

  438. 	movq	%r9, 80(%rsp)

  439. 	movq	%r10, 88(%rsp)

  440. 

  441. #seventh iteration

  442. 	movq	48($inp), %r13

  443. 	movq	56($inp), %rax

  444. 	mulq	%r13

  445. 	addq	%rax, %r12

  446. 	movq	%r13, %rax

  447. 	movq	%rdx, %r13

  448. 	adcq	\$0, %r13

  449. 

  450. 	xorq	%r14, %r14

  451. 	shlq	\$1, %rbx

  452. 	adcq	%r12, %r12		#shld	\$1, %rbx, %r12

  453. 	adcq	%r13, %r13		#shld	\$1, %r12, %r13

  454. 	adcq	%r14, %r14		#shld	\$1, %r13, %r14

  455. 

  456. 	mulq	%rax

  457. 	addq	%rax, %r11

  458. 	adcq	%rdx, %r12

  459. 	adcq	\$0, %r13

  460. 

  461. 	movq	%r11, 96(%rsp)

  462. 	movq	%r12, 104(%rsp)

  463. 

  464. #eighth iteration

  465. 	movq	56($inp), %rax

  466. 	mulq	%rax

  467. 	addq	%rax, %r13

  468. 	adcq	\$0, %rdx

  469. 

  470. 	addq	%rdx, %r14

  471. 

  472. 	movq	%r13, 112(%rsp)

  473. 	movq	%r14, 120(%rsp)

  474. 

  475. 	movq	(%rsp), %r8

  476. 	movq	8(%rsp), %r9

  477. 	movq	16(%rsp), %r10

  478. 	movq	24(%rsp), %r11

  479. 	movq	32(%rsp), %r12

  480. 	movq	40(%rsp), %r13

  481. 	movq	48(%rsp), %r14

  482. 	movq	56(%rsp), %r15

  483. 

  484. 	call	__rsaz_512_reduce

  485. 

  486. 	addq	64(%rsp), %r8

  487. 	adcq	72(%rsp), %r9

  488. 	adcq	80(%rsp), %r10

  489. 	adcq	88(%rsp), %r11

  490. 	adcq	96(%rsp), %r12

  491. 	adcq	104(%rsp), %r13

  492. 	adcq	112(%rsp), %r14

  493. 	adcq	120(%rsp), %r15

  494. 	sbbq	%rcx, %rcx

  495. 

  496. 	call	__rsaz_512_subtract

  497. 

  498. 	movq	%r8, %rdx

  499. 	movq	%r9, %rax

  500. 	movl	128+8(%rsp), $times

  501. 	movq	$out, $inp

  502. 

  503. 	decl	$times

  504. 	jnz	.Loop_sqr

  505. ___

  506. if ($addx) {

  507. $code.=<<___;

  508. 	jmp	.Lsqr_tail

  509. 

  510. .align	32

  511. .Loop_sqrx:

  512. 	movl	$times,128+8(%rsp)

  513. 	movq	$out, %xmm0		# off-load

  514. 	movq	%rbp, %xmm1		# off-load

  515. #first iteration	

  516. 	mulx	%rax, %r8, %r9

  517. 

  518. 	mulx	16($inp), %rcx, %r10

  519. 	xor	%rbp, %rbp		# cf=0, of=0

  520. 

  521. 	mulx	24($inp), %rax, %r11

  522. 	adcx	%rcx, %r9

  523. 

  524. 	mulx	32($inp), %rcx, %r12

  525. 	adcx	%rax, %r10

  526. 

  527. 	mulx	40($inp), %rax, %r13

  528. 	adcx	%rcx, %r11

  529. 

  530. 	.byte	0xc4,0x62,0xf3,0xf6,0xb6,0x30,0x00,0x00,0x00	# mulx	48($inp), %rcx, %r14

  531. 	adcx	%rax, %r12

  532. 	adcx	%rcx, %r13

  533. 

  534. 	.byte	0xc4,0x62,0xfb,0xf6,0xbe,0x38,0x00,0x00,0x00	# mulx	56($inp), %rax, %r15

  535. 	adcx	%rax, %r14

  536. 	adcx	%rbp, %r15		# %rbp is 0

  537. 

  538. 	mov	%r9, %rcx

  539. 	shld	\$1, %r8, %r9

  540. 	shl	\$1, %r8

  541. 

  542. 	xor	%ebp, %ebp

  543. 	mulx	%rdx, %rax, %rdx

  544. 	adcx	%rdx, %r8

  545. 	 mov	8($inp), %rdx

  546. 	adcx	%rbp, %r9

  547. 

  548. 	mov	%rax, (%rsp)

  549. 	mov	%r8, 8(%rsp)

  550. 

  551. #second iteration	

  552. 	mulx	16($inp), %rax, %rbx

  553. 	adox	%rax, %r10

  554. 	adcx	%rbx, %r11

  555. 

  556. 	.byte	0xc4,0x62,0xc3,0xf6,0x86,0x18,0x00,0x00,0x00	# mulx	24($inp), $out, %r8

  557. 	adox	$out, %r11

  558. 	adcx	%r8, %r12

  559. 

  560. 	mulx	32($inp), %rax, %rbx

  561. 	adox	%rax, %r12

  562. 	adcx	%rbx, %r13

  563. 

  564. 	mulx	40($inp), $out, %r8

  565. 	adox	$out, %r13

  566. 	adcx	%r8, %r14

  567. 

  568. 	.byte	0xc4,0xe2,0xfb,0xf6,0x9e,0x30,0x00,0x00,0x00	# mulx	48($inp), %rax, %rbx

  569. 	adox	%rax, %r14

  570. 	adcx	%rbx, %r15

  571. 

  572. 	.byte	0xc4,0x62,0xc3,0xf6,0x86,0x38,0x00,0x00,0x00	# mulx	56($inp), $out, %r8

  573. 	adox	$out, %r15

  574. 	adcx	%rbp, %r8

  575. 	adox	%rbp, %r8

  576. 

  577. 	mov	%r11, %rbx

  578. 	shld	\$1, %r10, %r11

  579. 	shld	\$1, %rcx, %r10

  580. 

  581. 	xor	%ebp,%ebp

  582. 	mulx	%rdx, %rax, %rcx

  583. 	 mov	16($inp), %rdx

  584. 	adcx	%rax, %r9

  585. 	adcx	%rcx, %r10

  586. 	adcx	%rbp, %r11

  587. 

  588. 	mov	%r9, 16(%rsp)

  589. 	.byte	0x4c,0x89,0x94,0x24,0x18,0x00,0x00,0x00		# mov	%r10, 24(%rsp)

  590. 	

  591. #third iteration	

  592. 	.byte	0xc4,0x62,0xc3,0xf6,0x8e,0x18,0x00,0x00,0x00	# mulx	24($inp), $out, %r9

  593. 	adox	$out, %r12

  594. 	adcx	%r9, %r13

  595. 

  596. 	mulx	32($inp), %rax, %rcx

  597. 	adox	%rax, %r13

  598. 	adcx	%rcx, %r14

  599. 

  600. 	mulx	40($inp), $out, %r9

  601. 	adox	$out, %r14

  602. 	adcx	%r9, %r15

  603. 

  604. 	.byte	0xc4,0xe2,0xfb,0xf6,0x8e,0x30,0x00,0x00,0x00	# mulx	48($inp), %rax, %rcx

  605. 	adox	%rax, %r15

  606. 	adcx	%rcx, %r8

  607. 

  608. 	.byte	0xc4,0x62,0xc3,0xf6,0x8e,0x38,0x00,0x00,0x00	# mulx	56($inp), $out, %r9

  609. 	adox	$out, %r8

  610. 	adcx	%rbp, %r9

  611. 	adox	%rbp, %r9

  612. 

  613. 	mov	%r13, %rcx

  614. 	shld	\$1, %r12, %r13

  615. 	shld	\$1, %rbx, %r12

  616. 

  617. 	xor	%ebp, %ebp

  618. 	mulx	%rdx, %rax, %rdx

  619. 	adcx	%rax, %r11

  620. 	adcx	%rdx, %r12

  621. 	 mov	24($inp), %rdx

  622. 	adcx	%rbp, %r13

  623. 

  624. 	mov	%r11, 32(%rsp)

  625. 	.byte	0x4c,0x89,0xa4,0x24,0x28,0x00,0x00,0x00		# mov	%r12, 40(%rsp)

  626. 	

  627. #fourth iteration	

  628. 	.byte	0xc4,0xe2,0xfb,0xf6,0x9e,0x20,0x00,0x00,0x00	# mulx	32($inp), %rax, %rbx

  629. 	adox	%rax, %r14

  630. 	adcx	%rbx, %r15

  631. 

  632. 	mulx	40($inp), $out, %r10

  633. 	adox	$out, %r15

  634. 	adcx	%r10, %r8

  635. 

  636. 	mulx	48($inp), %rax, %rbx

  637. 	adox	%rax, %r8

  638. 	adcx	%rbx, %r9

  639. 

  640. 	mulx	56($inp), $out, %r10

  641. 	adox	$out, %r9

  642. 	adcx	%rbp, %r10

  643. 	adox	%rbp, %r10

  644. 

  645. 	.byte	0x66

  646. 	mov	%r15, %rbx

  647. 	shld	\$1, %r14, %r15

  648. 	shld	\$1, %rcx, %r14

  649. 

  650. 	xor	%ebp, %ebp

  651. 	mulx	%rdx, %rax, %rdx

  652. 	adcx	%rax, %r13

  653. 	adcx	%rdx, %r14

  654. 	 mov	32($inp), %rdx

  655. 	adcx	%rbp, %r15

  656. 

  657. 	mov	%r13, 48(%rsp)

  658. 	mov	%r14, 56(%rsp)

  659. 	

  660. #fifth iteration	

  661. 	.byte	0xc4,0x62,0xc3,0xf6,0x9e,0x28,0x00,0x00,0x00	# mulx	40($inp), $out, %r11

  662. 	adox	$out, %r8

  663. 	adcx	%r11, %r9

  664. 

  665. 	mulx	48($inp), %rax, %rcx

  666. 	adox	%rax, %r9

  667. 	adcx	%rcx, %r10

  668. 

  669. 	mulx	56($inp), $out, %r11

  670. 	adox	$out, %r10

  671. 	adcx	%rbp, %r11

  672. 	adox	%rbp, %r11

  673. 

  674. 	mov	%r9, %rcx

  675. 	shld	\$1, %r8, %r9

  676. 	shld	\$1, %rbx, %r8

  677. 

  678. 	xor	%ebp, %ebp

  679. 	mulx	%rdx, %rax, %rdx

  680. 	adcx	%rax, %r15

  681. 	adcx	%rdx, %r8

  682. 	 mov	40($inp), %rdx

  683. 	adcx	%rbp, %r9

  684. 

  685. 	mov	%r15, 64(%rsp)

  686. 	mov	%r8, 72(%rsp)

  687. 	

  688. #sixth iteration	

  689. 	.byte	0xc4,0xe2,0xfb,0xf6,0x9e,0x30,0x00,0x00,0x00	# mulx	48($inp), %rax, %rbx

  690. 	adox	%rax, %r10

  691. 	adcx	%rbx, %r11

  692. 

  693. 	.byte	0xc4,0x62,0xc3,0xf6,0xa6,0x38,0x00,0x00,0x00	# mulx	56($inp), $out, %r12

  694. 	adox	$out, %r11

  695. 	adcx	%rbp, %r12

  696. 	adox	%rbp, %r12

  697. 

  698. 	mov	%r11, %rbx

  699. 	shld	\$1, %r10, %r11

  700. 	shld	\$1, %rcx, %r10

  701. 

  702. 	xor	%ebp, %ebp

  703. 	mulx	%rdx, %rax, %rdx

  704. 	adcx	%rax, %r9

  705. 	adcx	%rdx, %r10

  706. 	 mov	48($inp), %rdx

  707. 	adcx	%rbp, %r11

  708. 

  709. 	mov	%r9, 80(%rsp)

  710. 	mov	%r10, 88(%rsp)

  711. 

  712. #seventh iteration

  713. 	.byte	0xc4,0x62,0xfb,0xf6,0xae,0x38,0x00,0x00,0x00	# mulx	56($inp), %rax, %r13

  714. 	adox	%rax, %r12

  715. 	adox	%rbp, %r13

  716. 

  717. 	xor	%r14, %r14

  718. 	shld	\$1, %r13, %r14

  719. 	shld	\$1, %r12, %r13

  720. 	shld	\$1, %rbx, %r12

  721. 

  722. 	xor	%ebp, %ebp

  723. 	mulx	%rdx, %rax, %rdx

  724. 	adcx	%rax, %r11

  725. 	adcx	%rdx, %r12

  726. 	 mov	56($inp), %rdx

  727. 	adcx	%rbp, %r13

  728. 

  729. 	.byte	0x4c,0x89,0x9c,0x24,0x60,0x00,0x00,0x00		# mov	%r11, 96(%rsp)

  730. 	.byte	0x4c,0x89,0xa4,0x24,0x68,0x00,0x00,0x00		# mov	%r12, 104(%rsp)

  731. 

  732. #eighth iteration

  733. 	mulx	%rdx, %rax, %rdx

  734. 	adox	%rax, %r13

  735. 	adox	%rbp, %rdx

  736. 

  737. 	.byte	0x66

  738. 	add	%rdx, %r14

  739. 

  740. 	movq	%r13, 112(%rsp)

  741. 	movq	%r14, 120(%rsp)

  742. 	movq	%xmm0, $out

  743. 	movq	%xmm1, %rbp

  744. 

  745. 	movq	128(%rsp), %rdx		# pull $n0

  746. 	movq	(%rsp), %r8

  747. 	movq	8(%rsp), %r9

  748. 	movq	16(%rsp), %r10

  749. 	movq	24(%rsp), %r11

  750. 	movq	32(%rsp), %r12

  751. 	movq	40(%rsp), %r13

  752. 	movq	48(%rsp), %r14

  753. 	movq	56(%rsp), %r15

  754. 

  755. 	call	__rsaz_512_reducex

  756. 

  757. 	addq	64(%rsp), %r8

  758. 	adcq	72(%rsp), %r9

  759. 	adcq	80(%rsp), %r10

  760. 	adcq	88(%rsp), %r11

  761. 	adcq	96(%rsp), %r12

  762. 	adcq	104(%rsp), %r13

  763. 	adcq	112(%rsp), %r14

  764. 	adcq	120(%rsp), %r15

  765. 	sbbq	%rcx, %rcx

  766. 

  767. 	call	__rsaz_512_subtract

  768. 

  769. 	movq	%r8, %rdx

  770. 	movq	%r9, %rax

  771. 	movl	128+8(%rsp), $times

  772. 	movq	$out, $inp

  773. 

  774. 	decl	$times

  775. 	jnz	.Loop_sqrx

  776. 

  777. .Lsqr_tail:

  778. ___

  779. }

  780. $code.=<<___;

  781. 

  782. 	leaq	128+24+48(%rsp), %rax

  783. 	movq	-48(%rax), %r15

  784. 	movq	-40(%rax), %r14

  785. 	movq	-32(%rax), %r13

  786. 	movq	-24(%rax), %r12

  787. 	movq	-16(%rax), %rbp

  788. 	movq	-8(%rax), %rbx

  789. 	leaq	(%rax), %rsp

  790. .Lsqr_epilogue:

  791. 	ret

  792. .size	rsaz_512_sqr,.-rsaz_512_sqr

  793. ___

  794. }

  795. {

  796. my ($out,$ap,$bp,$mod,$n0) = ("%rdi","%rsi","%rdx","%rcx","%r8");

  797. $code.=<<___;

  798. .globl	rsaz_512_mul

  799. .type	rsaz_512_mul,\@function,5

  800. .align	32

  801. rsaz_512_mul:

  802. 	push	%rbx

  803. 	push	%rbp

  804. 	push	%r12

  805. 	push	%r13

  806. 	push	%r14

  807. 	push	%r15

  808. 

  809. 	subq	\$128+24, %rsp

  810. .Lmul_body:

  811. 	movq	$out, %xmm0		# off-load arguments

  812. 	movq	$mod, %xmm1

  813. 	movq	$n0, 128(%rsp)

  814. ___

  815. $code.=<<___ if ($addx);

  816. 	movl	\$0x80100,%r11d

  817. 	andl	OPENSSL_ia32cap_P+8(%rip),%r11d

  818. 	cmpl	\$0x80100,%r11d		# check for MULX and ADO/CX

  819. 	je	.Lmulx

  820. ___

  821. $code.=<<___;

  822. 	movq	($bp), %rbx		# pass b[0]

  823. 	movq	$bp, %rbp		# pass argument

  824. 	call	__rsaz_512_mul

  825. 

  826. 	movq	%xmm0, $out

  827. 	movq	%xmm1, %rbp

  828. 

  829. 	movq	(%rsp), %r8

  830. 	movq	8(%rsp), %r9

  831. 	movq	16(%rsp), %r10

  832. 	movq	24(%rsp), %r11

  833. 	movq	32(%rsp), %r12

  834. 	movq	40(%rsp), %r13

  835. 	movq	48(%rsp), %r14

  836. 	movq	56(%rsp), %r15

  837. 

  838. 	call	__rsaz_512_reduce

  839. ___

  840. $code.=<<___ if ($addx);

  841. 	jmp	.Lmul_tail

  842. 

  843. .align	32

  844. .Lmulx:

  845. 	movq	$bp, %rbp		# pass argument

  846. 	movq	($bp), %rdx		# pass b[0]

  847. 	call	__rsaz_512_mulx

  848. 

  849. 	movq	%xmm0, $out

  850. 	movq	%xmm1, %rbp

  851. 

  852. 	movq	128(%rsp), %rdx		# pull $n0

  853. 	movq	(%rsp), %r8

  854. 	movq	8(%rsp), %r9

  855. 	movq	16(%rsp), %r10

  856. 	movq	24(%rsp), %r11

  857. 	movq	32(%rsp), %r12

  858. 	movq	40(%rsp), %r13

  859. 	movq	48(%rsp), %r14

  860. 	movq	56(%rsp), %r15

  861. 

  862. 	call	__rsaz_512_reducex

  863. .Lmul_tail:

  864. ___

  865. $code.=<<___;

  866. 	addq	64(%rsp), %r8

  867. 	adcq	72(%rsp), %r9

  868. 	adcq	80(%rsp), %r10

  869. 	adcq	88(%rsp), %r11

  870. 	adcq	96(%rsp), %r12

  871. 	adcq	104(%rsp), %r13

  872. 	adcq	112(%rsp), %r14

  873. 	adcq	120(%rsp), %r15

  874. 	sbbq	%rcx, %rcx

  875. 

  876. 	call	__rsaz_512_subtract

  877. 

  878. 	leaq	128+24+48(%rsp), %rax

  879. 	movq	-48(%rax), %r15

  880. 	movq	-40(%rax), %r14

  881. 	movq	-32(%rax), %r13

  882. 	movq	-24(%rax), %r12

  883. 	movq	-16(%rax), %rbp

  884. 	movq	-8(%rax), %rbx

  885. 	leaq	(%rax), %rsp

  886. .Lmul_epilogue:

  887. 	ret

  888. .size	rsaz_512_mul,.-rsaz_512_mul

  889. ___

  890. }

  891. {

  892. my ($out,$ap,$bp,$mod,$n0,$pwr) = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9d");

  893. $code.=<<___;

  894. .globl	rsaz_512_mul_gather4

  895. .type	rsaz_512_mul_gather4,\@function,6

  896. .align	32

  897. rsaz_512_mul_gather4:

  898. 	push	%rbx

  899. 	push	%rbp

  900. 	push	%r12

  901. 	push	%r13

  902. 	push	%r14

  903. 	push	%r15

  904. 

  905. 	subq	\$`128+24+($win64?0xb0:0)`, %rsp

  906. ___

  907. $code.=<<___	if ($win64);

  908. 	movaps	%xmm6,0xa0(%rsp)

  909. 	movaps	%xmm7,0xb0(%rsp)

  910. 	movaps	%xmm8,0xc0(%rsp)

  911. 	movaps	%xmm9,0xd0(%rsp)

  912. 	movaps	%xmm10,0xe0(%rsp)

  913. 	movaps	%xmm11,0xf0(%rsp)

  914. 	movaps	%xmm12,0x100(%rsp)

  915. 	movaps	%xmm13,0x110(%rsp)

  916. 	movaps	%xmm14,0x120(%rsp)

  917. 	movaps	%xmm15,0x130(%rsp)

  918. ___

  919. $code.=<<___;

  920. .Lmul_gather4_body:

  921. 	movd	$pwr,%xmm8

  922. 	movdqa	.Linc+16(%rip),%xmm1	# 00000002000000020000000200000002

  923. 	movdqa	.Linc(%rip),%xmm0	# 00000001000000010000000000000000

  924. 

  925. 	pshufd	\$0,%xmm8,%xmm8		# broadcast $power

  926. 	movdqa	%xmm1,%xmm7

  927. 	movdqa	%xmm1,%xmm2

  928. ___

  929. ########################################################################

  930. # calculate mask by comparing 0..15 to $power

  931. #

  932. for($i=0;$i<4;$i++) {

  933. $code.=<<___;

  934. 	paddd	%xmm`$i`,%xmm`$i+1`

  935. 	pcmpeqd	%xmm8,%xmm`$i`

  936. 	movdqa	%xmm7,%xmm`$i+3`

  937. ___

  938. }

  939. for(;$i<7;$i++) {

  940. $code.=<<___;

  941. 	paddd	%xmm`$i`,%xmm`$i+1`

  942. 	pcmpeqd	%xmm8,%xmm`$i`

  943. ___

  944. }

  945. $code.=<<___;

  946. 	pcmpeqd	%xmm8,%xmm7

  947. 

  948. 	movdqa	16*0($bp),%xmm8

  949. 	movdqa	16*1($bp),%xmm9

  950. 	movdqa	16*2($bp),%xmm10

  951. 	movdqa	16*3($bp),%xmm11

  952. 	pand	%xmm0,%xmm8

  953. 	movdqa	16*4($bp),%xmm12

  954. 	pand	%xmm1,%xmm9

  955. 	movdqa	16*5($bp),%xmm13

  956. 	pand	%xmm2,%xmm10

  957. 	movdqa	16*6($bp),%xmm14

  958. 	pand	%xmm3,%xmm11

  959. 	movdqa	16*7($bp),%xmm15

  960. 	leaq	128($bp), %rbp

  961. 	pand	%xmm4,%xmm12

  962. 	pand	%xmm5,%xmm13

  963. 	pand	%xmm6,%xmm14

  964. 	pand	%xmm7,%xmm15

  965. 	por	%xmm10,%xmm8

  966. 	por	%xmm11,%xmm9

  967. 	por	%xmm12,%xmm8

  968. 	por	%xmm13,%xmm9

  969. 	por	%xmm14,%xmm8

  970. 	por	%xmm15,%xmm9

  971. 

  972. 	por	%xmm9,%xmm8

  973. 	pshufd	\$0x4e,%xmm8,%xmm9

  974. 	por	%xmm9,%xmm8

  975. ___

  976. $code.=<<___ if ($addx);

  977. 	movl	\$0x80100,%r11d

  978. 	andl	OPENSSL_ia32cap_P+8(%rip),%r11d

  979. 	cmpl	\$0x80100,%r11d		# check for MULX and ADO/CX

  980. 	je	.Lmulx_gather

  981. ___

  982. $code.=<<___;

  983. 	movq	%xmm8,%rbx

  984. 

  985. 	movq	$n0, 128(%rsp)		# off-load arguments

  986. 	movq	$out, 128+8(%rsp)

  987. 	movq	$mod, 128+16(%rsp)

  988. 

  989. 	movq	($ap), %rax

  990. 	 movq	8($ap), %rcx

  991. 	mulq	%rbx			# 0 iteration

  992. 	movq	%rax, (%rsp)

  993. 	movq	%rcx, %rax

  994. 	movq	%rdx, %r8

  995. 

  996. 	mulq	%rbx

  997. 	addq	%rax, %r8

  998. 	movq	16($ap), %rax

  999. 	movq	%rdx, %r9

  1000. 	adcq	\$0, %r9

  1001. 

  1002. 	mulq	%rbx

  1003. 	addq	%rax, %r9

  1004. 	movq	24($ap), %rax

  1005. 	movq	%rdx, %r10

  1006. 	adcq	\$0, %r10

  1007. 

  1008. 	mulq	%rbx

  1009. 	addq	%rax, %r10

  1010. 	movq	32($ap), %rax

  1011. 	movq	%rdx, %r11

  1012. 	adcq	\$0, %r11

  1013. 

  1014. 	mulq	%rbx

  1015. 	addq	%rax, %r11

  1016. 	movq	40($ap), %rax

  1017. 	movq	%rdx, %r12

  1018. 	adcq	\$0, %r12

  1019. 

  1020. 	mulq	%rbx

  1021. 	addq	%rax, %r12

  1022. 	movq	48($ap), %rax

  1023. 	movq	%rdx, %r13

  1024. 	adcq	\$0, %r13

  1025. 

  1026. 	mulq	%rbx

  1027. 	addq	%rax, %r13

  1028. 	movq	56($ap), %rax

  1029. 	movq	%rdx, %r14

  1030. 	adcq	\$0, %r14

  1031. 	

  1032. 	mulq	%rbx

  1033. 	addq	%rax, %r14

  1034. 	 movq	($ap), %rax

  1035. 	movq	%rdx, %r15

  1036. 	adcq	\$0, %r15

  1037. 

  1038. 	leaq	8(%rsp), %rdi

  1039. 	movl	\$7, %ecx

  1040. 	jmp	.Loop_mul_gather

  1041. 

  1042. .align	32

  1043. .Loop_mul_gather:

  1044. 	movdqa	16*0(%rbp),%xmm8

  1045. 	movdqa	16*1(%rbp),%xmm9

  1046. 	movdqa	16*2(%rbp),%xmm10

  1047. 	movdqa	16*3(%rbp),%xmm11

  1048. 	pand	%xmm0,%xmm8

  1049. 	movdqa	16*4(%rbp),%xmm12

  1050. 	pand	%xmm1,%xmm9

  1051. 	movdqa	16*5(%rbp),%xmm13

  1052. 	pand	%xmm2,%xmm10

  1053. 	movdqa	16*6(%rbp),%xmm14

  1054. 	pand	%xmm3,%xmm11

  1055. 	movdqa	16*7(%rbp),%xmm15

  1056. 	leaq	128(%rbp), %rbp

  1057. 	pand	%xmm4,%xmm12

  1058. 	pand	%xmm5,%xmm13

  1059. 	pand	%xmm6,%xmm14

  1060. 	pand	%xmm7,%xmm15

  1061. 	por	%xmm10,%xmm8

  1062. 	por	%xmm11,%xmm9

  1063. 	por	%xmm12,%xmm8

  1064. 	por	%xmm13,%xmm9

  1065. 	por	%xmm14,%xmm8

  1066. 	por	%xmm15,%xmm9

  1067. 

  1068. 	por	%xmm9,%xmm8

  1069. 	pshufd	\$0x4e,%xmm8,%xmm9

  1070. 	por	%xmm9,%xmm8

  1071. 	movq	%xmm8,%rbx

  1072. 

  1073. 	mulq	%rbx

  1074. 	addq	%rax, %r8

  1075. 	movq	8($ap), %rax

  1076. 	movq	%r8, (%rdi)

  1077. 	movq	%rdx, %r8

  1078. 	adcq	\$0, %r8

  1079. 

  1080. 	mulq	%rbx

  1081. 	addq	%rax, %r9

  1082. 	movq	16($ap), %rax

  1083. 	adcq	\$0, %rdx

  1084. 	addq	%r9, %r8

  1085. 	movq	%rdx, %r9

  1086. 	adcq	\$0, %r9

  1087. 

  1088. 	mulq	%rbx

  1089. 	addq	%rax, %r10

  1090. 	movq	24($ap), %rax

  1091. 	adcq	\$0, %rdx

  1092. 	addq	%r10, %r9

  1093. 	movq	%rdx, %r10

  1094. 	adcq	\$0, %r10

  1095. 

  1096. 	mulq	%rbx

  1097. 	addq	%rax, %r11

  1098. 	movq	32($ap), %rax

  1099. 	adcq	\$0, %rdx

  1100. 	addq	%r11, %r10

  1101. 	movq	%rdx, %r11

  1102. 	adcq	\$0, %r11

  1103. 

  1104. 	mulq	%rbx

  1105. 	addq	%rax, %r12

  1106. 	movq	40($ap), %rax

  1107. 	adcq	\$0, %rdx

  1108. 	addq	%r12, %r11

  1109. 	movq	%rdx, %r12

  1110. 	adcq	\$0, %r12

  1111. 

  1112. 	mulq	%rbx

  1113. 	addq	%rax, %r13

  1114. 	movq	48($ap), %rax

  1115. 	adcq	\$0, %rdx

  1116. 	addq	%r13, %r12

  1117. 	movq	%rdx, %r13

  1118. 	adcq	\$0, %r13

  1119. 

  1120. 	mulq	%rbx

  1121. 	addq	%rax, %r14

  1122. 	movq	56($ap), %rax

  1123. 	adcq	\$0, %rdx

  1124. 	addq	%r14, %r13

  1125. 	movq	%rdx, %r14

  1126. 	adcq	\$0, %r14

  1127. 

  1128. 	mulq	%rbx

  1129. 	addq	%rax, %r15

  1130. 	 movq	($ap), %rax

  1131. 	adcq	\$0, %rdx

  1132. 	addq	%r15, %r14

  1133. 	movq	%rdx, %r15	

  1134. 	adcq	\$0, %r15

  1135. 

  1136. 	leaq	8(%rdi), %rdi

  1137. 

  1138. 	decl	%ecx

  1139. 	jnz	.Loop_mul_gather

  1140. 

  1141. 	movq	%r8, (%rdi)

  1142. 	movq	%r9, 8(%rdi)

  1143. 	movq	%r10, 16(%rdi)

  1144. 	movq	%r11, 24(%rdi)

  1145. 	movq	%r12, 32(%rdi)

  1146. 	movq	%r13, 40(%rdi)

  1147. 	movq	%r14, 48(%rdi)

  1148. 	movq	%r15, 56(%rdi)

  1149. 

  1150. 	movq	128+8(%rsp), $out

  1151. 	movq	128+16(%rsp), %rbp

  1152. 

  1153. 	movq	(%rsp), %r8

  1154. 	movq	8(%rsp), %r9

  1155. 	movq	16(%rsp), %r10

  1156. 	movq	24(%rsp), %r11

  1157. 	movq	32(%rsp), %r12

  1158. 	movq	40(%rsp), %r13

  1159. 	movq	48(%rsp), %r14

  1160. 	movq	56(%rsp), %r15

  1161. 

  1162. 	call	__rsaz_512_reduce

  1163. ___

  1164. $code.=<<___ if ($addx);

  1165. 	jmp	.Lmul_gather_tail

  1166. 

  1167. .align	32

  1168. .Lmulx_gather:

  1169. 	movq	%xmm8,%rdx

  1170. 

  1171. 	mov	$n0, 128(%rsp)		# off-load arguments

  1172. 	mov	$out, 128+8(%rsp)

  1173. 	mov	$mod, 128+16(%rsp)

  1174. 

  1175. 	mulx	($ap), %rbx, %r8	# 0 iteration

  1176. 	mov	%rbx, (%rsp)

  1177. 	xor	%edi, %edi		# cf=0, of=0

  1178. 

  1179. 	mulx	8($ap), %rax, %r9

  1180. 

  1181. 	mulx	16($ap), %rbx, %r10

  1182. 	adcx	%rax, %r8

  1183. 

  1184. 	mulx	24($ap), %rax, %r11

  1185. 	adcx	%rbx, %r9

  1186. 

  1187. 	mulx	32($ap), %rbx, %r12

  1188. 	adcx	%rax, %r10

  1189. 

  1190. 	mulx	40($ap), %rax, %r13

  1191. 	adcx	%rbx, %r11

  1192. 

  1193. 	mulx	48($ap), %rbx, %r14

  1194. 	adcx	%rax, %r12

  1195. 	

  1196. 	mulx	56($ap), %rax, %r15

  1197. 	adcx	%rbx, %r13

  1198. 	adcx	%rax, %r14

  1199. 	.byte	0x67

  1200. 	mov	%r8, %rbx

  1201. 	adcx	%rdi, %r15		# %rdi is 0

  1202. 

  1203. 	mov	\$-7, %rcx

  1204. 	jmp	.Loop_mulx_gather

  1205. 

  1206. .align	32

  1207. .Loop_mulx_gather:

  1208. 	movdqa	16*0(%rbp),%xmm8

  1209. 	movdqa	16*1(%rbp),%xmm9

  1210. 	movdqa	16*2(%rbp),%xmm10

  1211. 	movdqa	16*3(%rbp),%xmm11

  1212. 	pand	%xmm0,%xmm8

  1213. 	movdqa	16*4(%rbp),%xmm12

  1214. 	pand	%xmm1,%xmm9

  1215. 	movdqa	16*5(%rbp),%xmm13

  1216. 	pand	%xmm2,%xmm10

  1217. 	movdqa	16*6(%rbp),%xmm14

  1218. 	pand	%xmm3,%xmm11

  1219. 	movdqa	16*7(%rbp),%xmm15

  1220. 	leaq	128(%rbp), %rbp

  1221. 	pand	%xmm4,%xmm12

  1222. 	pand	%xmm5,%xmm13

  1223. 	pand	%xmm6,%xmm14

  1224. 	pand	%xmm7,%xmm15

  1225. 	por	%xmm10,%xmm8

  1226. 	por	%xmm11,%xmm9

  1227. 	por	%xmm12,%xmm8

  1228. 	por	%xmm13,%xmm9

  1229. 	por	%xmm14,%xmm8

  1230. 	por	%xmm15,%xmm9

  1231. 

  1232. 	por	%xmm9,%xmm8

  1233. 	pshufd	\$0x4e,%xmm8,%xmm9

  1234. 	por	%xmm9,%xmm8

  1235. 	movq	%xmm8,%rdx

  1236. 

  1237. 	.byte	0xc4,0x62,0xfb,0xf6,0x86,0x00,0x00,0x00,0x00	# mulx	($ap), %rax, %r8

  1238. 	adcx	%rax, %rbx

  1239. 	adox	%r9, %r8

  1240. 

  1241. 	mulx	8($ap), %rax, %r9

  1242. 	adcx	%rax, %r8

  1243. 	adox	%r10, %r9

  1244. 

  1245. 	mulx	16($ap), %rax, %r10

  1246. 	adcx	%rax, %r9

  1247. 	adox	%r11, %r10

  1248. 

  1249. 	.byte	0xc4,0x62,0xfb,0xf6,0x9e,0x18,0x00,0x00,0x00	# mulx	24($ap), %rax, %r11

  1250. 	adcx	%rax, %r10

  1251. 	adox	%r12, %r11

  1252. 

  1253. 	mulx	32($ap), %rax, %r12

  1254. 	adcx	%rax, %r11

  1255. 	adox	%r13, %r12

  1256. 

  1257. 	mulx	40($ap), %rax, %r13

  1258. 	adcx	%rax, %r12

  1259. 	adox	%r14, %r13

  1260. 

  1261. 	.byte	0xc4,0x62,0xfb,0xf6,0xb6,0x30,0x00,0x00,0x00	# mulx	48($ap), %rax, %r14

  1262. 	adcx	%rax, %r13

  1263. 	.byte	0x67

  1264. 	adox	%r15, %r14

  1265. 

  1266. 	mulx	56($ap), %rax, %r15

  1267. 	 mov	%rbx, 64(%rsp,%rcx,8)

  1268. 	adcx	%rax, %r14

  1269. 	adox	%rdi, %r15

  1270. 	mov	%r8, %rbx

  1271. 	adcx	%rdi, %r15		# cf=0

  1272. 

  1273. 	inc	%rcx			# of=0

  1274. 	jnz	.Loop_mulx_gather

  1275. 

  1276. 	mov	%r8, 64(%rsp)

  1277. 	mov	%r9, 64+8(%rsp)

  1278. 	mov	%r10, 64+16(%rsp)

  1279. 	mov	%r11, 64+24(%rsp)

  1280. 	mov	%r12, 64+32(%rsp)

  1281. 	mov	%r13, 64+40(%rsp)

  1282. 	mov	%r14, 64+48(%rsp)

  1283. 	mov	%r15, 64+56(%rsp)

  1284. 

  1285. 	mov	128(%rsp), %rdx		# pull arguments

  1286. 	mov	128+8(%rsp), $out

  1287. 	mov	128+16(%rsp), %rbp

  1288. 

  1289. 	mov	(%rsp), %r8

  1290. 	mov	8(%rsp), %r9

  1291. 	mov	16(%rsp), %r10

  1292. 	mov	24(%rsp), %r11

  1293. 	mov	32(%rsp), %r12

  1294. 	mov	40(%rsp), %r13

  1295. 	mov	48(%rsp), %r14

  1296. 	mov	56(%rsp), %r15

  1297. 

  1298. 	call	__rsaz_512_reducex

  1299. 

  1300. .Lmul_gather_tail:

  1301. ___

  1302. $code.=<<___;

  1303. 	addq	64(%rsp), %r8

  1304. 	adcq	72(%rsp), %r9

  1305. 	adcq	80(%rsp), %r10

  1306. 	adcq	88(%rsp), %r11

  1307. 	adcq	96(%rsp), %r12

  1308. 	adcq	104(%rsp), %r13

  1309. 	adcq	112(%rsp), %r14

  1310. 	adcq	120(%rsp), %r15

  1311. 	sbbq	%rcx, %rcx

  1312. 

  1313. 	call	__rsaz_512_subtract

  1314. 

  1315. 	leaq	128+24+48(%rsp), %rax

  1316. ___

  1317. $code.=<<___	if ($win64);

  1318. 	movaps	0xa0-0xc8(%rax),%xmm6

  1319. 	movaps	0xb0-0xc8(%rax),%xmm7

  1320. 	movaps	0xc0-0xc8(%rax),%xmm8

  1321. 	movaps	0xd0-0xc8(%rax),%xmm9

  1322. 	movaps	0xe0-0xc8(%rax),%xmm10

  1323. 	movaps	0xf0-0xc8(%rax),%xmm11

  1324. 	movaps	0x100-0xc8(%rax),%xmm12

  1325. 	movaps	0x110-0xc8(%rax),%xmm13

  1326. 	movaps	0x120-0xc8(%rax),%xmm14

  1327. 	movaps	0x130-0xc8(%rax),%xmm15

  1328. 	lea	0xb0(%rax),%rax

  1329. ___

  1330. $code.=<<___;

  1331. 	movq	-48(%rax), %r15

  1332. 	movq	-40(%rax), %r14

  1333. 	movq	-32(%rax), %r13

  1334. 	movq	-24(%rax), %r12

  1335. 	movq	-16(%rax), %rbp

  1336. 	movq	-8(%rax), %rbx

  1337. 	leaq	(%rax), %rsp

  1338. .Lmul_gather4_epilogue:

  1339. 	ret

  1340. .size	rsaz_512_mul_gather4,.-rsaz_512_mul_gather4

  1341. ___

  1342. }

  1343. {

  1344. my ($out,$ap,$mod,$n0,$tbl,$pwr) = ("%rdi","%rsi","%rdx","%rcx","%r8","%r9d");

  1345. $code.=<<___;

  1346. .globl	rsaz_512_mul_scatter4

  1347. .type	rsaz_512_mul_scatter4,\@function,6

  1348. .align	32

  1349. rsaz_512_mul_scatter4:

  1350. 	push	%rbx

  1351. 	push	%rbp

  1352. 	push	%r12

  1353. 	push	%r13

  1354. 	push	%r14

  1355. 	push	%r15

  1356. 

  1357. 	mov	$pwr, $pwr

  1358. 	subq	\$128+24, %rsp

  1359. .Lmul_scatter4_body:

  1360. 	leaq	($tbl,$pwr,8), $tbl

  1361. 	movq	$out, %xmm0		# off-load arguments

  1362. 	movq	$mod, %xmm1

  1363. 	movq	$tbl, %xmm2

  1364. 	movq	$n0, 128(%rsp)

  1365. 

  1366. 	movq	$out, %rbp

  1367. ___

  1368. $code.=<<___ if ($addx);

  1369. 	movl	\$0x80100,%r11d

  1370. 	andl	OPENSSL_ia32cap_P+8(%rip),%r11d

  1371. 	cmpl	\$0x80100,%r11d		# check for MULX and ADO/CX

  1372. 	je	.Lmulx_scatter

  1373. ___

  1374. $code.=<<___;

  1375. 	movq	($out),%rbx		# pass b[0]

  1376. 	call	__rsaz_512_mul

  1377. 

  1378. 	movq	%xmm0, $out

  1379. 	movq	%xmm1, %rbp

  1380. 

  1381. 	movq	(%rsp), %r8

  1382. 	movq	8(%rsp), %r9

  1383. 	movq	16(%rsp), %r10

  1384. 	movq	24(%rsp), %r11

  1385. 	movq	32(%rsp), %r12

  1386. 	movq	40(%rsp), %r13

  1387. 	movq	48(%rsp), %r14

  1388. 	movq	56(%rsp), %r15

  1389. 

  1390. 	call	__rsaz_512_reduce

  1391. ___

  1392. $code.=<<___ if ($addx);

  1393. 	jmp	.Lmul_scatter_tail

  1394. 	

  1395. .align	32

  1396. .Lmulx_scatter:

  1397. 	movq	($out), %rdx		# pass b[0]

  1398. 	call	__rsaz_512_mulx

  1399. 

  1400. 	movq	%xmm0, $out

  1401. 	movq	%xmm1, %rbp

  1402. 

  1403. 	movq	128(%rsp), %rdx		# pull $n0

  1404. 	movq	(%rsp), %r8

  1405. 	movq	8(%rsp), %r9

  1406. 	movq	16(%rsp), %r10

  1407. 	movq	24(%rsp), %r11

  1408. 	movq	32(%rsp), %r12

  1409. 	movq	40(%rsp), %r13

  1410. 	movq	48(%rsp), %r14

  1411. 	movq	56(%rsp), %r15

  1412. 

  1413. 	call	__rsaz_512_reducex

  1414. 

  1415. .Lmul_scatter_tail:

  1416. ___

  1417. $code.=<<___;

  1418. 	addq	64(%rsp), %r8

  1419. 	adcq	72(%rsp), %r9

  1420. 	adcq	80(%rsp), %r10

  1421. 	adcq	88(%rsp), %r11

  1422. 	adcq	96(%rsp), %r12

  1423. 	adcq	104(%rsp), %r13

  1424. 	adcq	112(%rsp), %r14

  1425. 	adcq	120(%rsp), %r15

  1426. 	movq	%xmm2, $inp

  1427. 	sbbq	%rcx, %rcx

  1428. 

  1429. 	call	__rsaz_512_subtract

  1430. 

  1431. 	movq	%r8, 128*0($inp)	# scatter

  1432. 	movq	%r9, 128*1($inp)

  1433. 	movq	%r10, 128*2($inp)

  1434. 	movq	%r11, 128*3($inp)

  1435. 	movq	%r12, 128*4($inp)

  1436. 	movq	%r13, 128*5($inp)

  1437. 	movq	%r14, 128*6($inp)

  1438. 	movq	%r15, 128*7($inp)

  1439. 

  1440. 	leaq	128+24+48(%rsp), %rax

  1441. 	movq	-48(%rax), %r15

  1442. 	movq	-40(%rax), %r14

  1443. 	movq	-32(%rax), %r13

  1444. 	movq	-24(%rax), %r12

  1445. 	movq	-16(%rax), %rbp

  1446. 	movq	-8(%rax), %rbx

  1447. 	leaq	(%rax), %rsp

  1448. .Lmul_scatter4_epilogue:

  1449. 	ret

  1450. .size	rsaz_512_mul_scatter4,.-rsaz_512_mul_scatter4

  1451. ___

  1452. }

  1453. {

  1454. my ($out,$inp,$mod,$n0) = ("%rdi","%rsi","%rdx","%rcx");

  1455. $code.=<<___;

  1456. .globl	rsaz_512_mul_by_one

  1457. .type	rsaz_512_mul_by_one,\@function,4

  1458. .align	32

  1459. rsaz_512_mul_by_one:

  1460. 	push	%rbx

  1461. 	push	%rbp

  1462. 	push	%r12

  1463. 	push	%r13

  1464. 	push	%r14

  1465. 	push	%r15

  1466. 

  1467. 	subq	\$128+24, %rsp

  1468. .Lmul_by_one_body:

  1469. ___

  1470. $code.=<<___ if ($addx);

  1471. 	movl	OPENSSL_ia32cap_P+8(%rip),%eax

  1472. ___

  1473. $code.=<<___;

  1474. 	movq	$mod, %rbp	# reassign argument

  1475. 	movq	$n0, 128(%rsp)

  1476. 

  1477. 	movq	($inp), %r8

  1478. 	pxor	%xmm0, %xmm0

  1479. 	movq	8($inp), %r9

  1480. 	movq	16($inp), %r10

  1481. 	movq	24($inp), %r11

  1482. 	movq	32($inp), %r12

  1483. 	movq	40($inp), %r13

  1484. 	movq	48($inp), %r14

  1485. 	movq	56($inp), %r15

  1486. 

  1487. 	movdqa	%xmm0, (%rsp)

  1488. 	movdqa	%xmm0, 16(%rsp)

  1489. 	movdqa	%xmm0, 32(%rsp)

  1490. 	movdqa	%xmm0, 48(%rsp)

  1491. 	movdqa	%xmm0, 64(%rsp)

  1492. 	movdqa	%xmm0, 80(%rsp)

  1493. 	movdqa	%xmm0, 96(%rsp)

  1494. ___

  1495. $code.=<<___ if ($addx);

  1496. 	andl	\$0x80100,%eax

  1497. 	cmpl	\$0x80100,%eax		# check for MULX and ADO/CX

  1498. 	je	.Lby_one_callx

  1499. ___

  1500. $code.=<<___;

  1501. 	call	__rsaz_512_reduce

  1502. ___

  1503. $code.=<<___ if ($addx);

  1504. 	jmp	.Lby_one_tail

  1505. .align	32

  1506. .Lby_one_callx:

  1507. 	movq	128(%rsp), %rdx		# pull $n0

  1508. 	call	__rsaz_512_reducex

  1509. .Lby_one_tail:

  1510. ___

  1511. $code.=<<___;

  1512. 	movq	%r8, ($out)

  1513. 	movq	%r9, 8($out)

  1514. 	movq	%r10, 16($out)

  1515. 	movq	%r11, 24($out)

  1516. 	movq	%r12, 32($out)

  1517. 	movq	%r13, 40($out)

  1518. 	movq	%r14, 48($out)

  1519. 	movq	%r15, 56($out)

  1520. 

  1521. 	leaq	128+24+48(%rsp), %rax

  1522. 	movq	-48(%rax), %r15

  1523. 	movq	-40(%rax), %r14

  1524. 	movq	-32(%rax), %r13

  1525. 	movq	-24(%rax), %r12

  1526. 	movq	-16(%rax), %rbp

  1527. 	movq	-8(%rax), %rbx

  1528. 	leaq	(%rax), %rsp

  1529. .Lmul_by_one_epilogue:

  1530. 	ret

  1531. .size	rsaz_512_mul_by_one,.-rsaz_512_mul_by_one

  1532. ___

  1533. }

  1534. {	# __rsaz_512_reduce

  1535. 	#

  1536. 	# input:	%r8-%r15, %rbp - mod, 128(%rsp) - n0

  1537. 	# output:	%r8-%r15

  1538. 	# clobbers:	everything except %rbp and %rdi

  1539. $code.=<<___;

  1540. .type	__rsaz_512_reduce,\@abi-omnipotent

  1541. .align	32

  1542. __rsaz_512_reduce:

  1543. 	movq	%r8, %rbx

  1544. 	imulq	128+8(%rsp), %rbx

  1545. 	movq	0(%rbp), %rax

  1546. 	movl	\$8, %ecx

  1547. 	jmp	.Lreduction_loop

  1548. 

  1549. .align	32

  1550. .Lreduction_loop:

  1551. 	mulq	%rbx

  1552. 	movq	8(%rbp), %rax

  1553. 	negq	%r8

  1554. 	movq	%rdx, %r8

  1555. 	adcq	\$0, %r8

  1556. 

  1557. 	mulq	%rbx

  1558. 	addq	%rax, %r9

  1559. 	movq	16(%rbp), %rax

  1560. 	adcq	\$0, %rdx

  1561. 	addq	%r9, %r8

  1562. 	movq	%rdx, %r9

  1563. 	adcq	\$0, %r9

  1564. 

  1565. 	mulq	%rbx

  1566. 	addq	%rax, %r10

  1567. 	movq	24(%rbp), %rax

  1568. 	adcq	\$0, %rdx

  1569. 	addq	%r10, %r9

  1570. 	movq	%rdx, %r10

  1571. 	adcq	\$0, %r10

  1572. 

  1573. 	mulq	%rbx

  1574. 	addq	%rax, %r11

  1575. 	movq	32(%rbp), %rax

  1576. 	adcq	\$0, %rdx

  1577. 	addq	%r11, %r10

  1578. 	 movq	128+8(%rsp), %rsi

  1579. 	#movq	%rdx, %r11

  1580. 	#adcq	\$0, %r11

  1581. 	adcq	\$0, %rdx

  1582. 	movq	%rdx, %r11

  1583. 

  1584. 	mulq	%rbx

  1585. 	addq	%rax, %r12

  1586. 	movq	40(%rbp), %rax

  1587. 	adcq	\$0, %rdx

  1588. 	 imulq	%r8, %rsi

  1589. 	addq	%r12, %r11

  1590. 	movq	%rdx, %r12

  1591. 	adcq	\$0, %r12

  1592. 

  1593. 	mulq	%rbx

  1594. 	addq	%rax, %r13

  1595. 	movq	48(%rbp), %rax

  1596. 	adcq	\$0, %rdx

  1597. 	addq	%r13, %r12

  1598. 	movq	%rdx, %r13

  1599. 	adcq	\$0, %r13

  1600. 

  1601. 	mulq	%rbx

  1602. 	addq	%rax, %r14

  1603. 	movq	56(%rbp), %rax

  1604. 	adcq	\$0, %rdx

  1605. 	addq	%r14, %r13

  1606. 	movq	%rdx, %r14

  1607. 	adcq	\$0, %r14

  1608. 

  1609. 	mulq	%rbx

  1610. 	 movq	%rsi, %rbx

  1611. 	addq	%rax, %r15

  1612. 	 movq	0(%rbp), %rax

  1613. 	adcq	\$0, %rdx

  1614. 	addq	%r15, %r14

  1615. 	movq	%rdx, %r15

  1616. 	adcq	\$0, %r15

  1617. 

  1618. 	decl	%ecx

  1619. 	jne	.Lreduction_loop

  1620. 

  1621. 	ret

  1622. .size	__rsaz_512_reduce,.-__rsaz_512_reduce

  1623. ___

  1624. }

  1625. if ($addx) {

  1626. 	# __rsaz_512_reducex

  1627. 	#

  1628. 	# input:	%r8-%r15, %rbp - mod, 128(%rsp) - n0

  1629. 	# output:	%r8-%r15

  1630. 	# clobbers:	everything except %rbp and %rdi

  1631. $code.=<<___;

  1632. .type	__rsaz_512_reducex,\@abi-omnipotent

  1633. .align	32

  1634. __rsaz_512_reducex:

  1635. 	#movq	128+8(%rsp), %rdx		# pull $n0

  1636. 	imulq	%r8, %rdx

  1637. 	xorq	%rsi, %rsi			# cf=0,of=0

  1638. 	movl	\$8, %ecx

  1639. 	jmp	.Lreduction_loopx

  1640. 

  1641. .align	32

  1642. .Lreduction_loopx:

  1643. 	mov	%r8, %rbx

  1644. 	mulx	0(%rbp), %rax, %r8

  1645. 	adcx	%rbx, %rax

  1646. 	adox	%r9, %r8

  1647. 

  1648. 	mulx	8(%rbp), %rax, %r9

  1649. 	adcx	%rax, %r8

  1650. 	adox	%r10, %r9

  1651. 

  1652. 	mulx	16(%rbp), %rbx, %r10

  1653. 	adcx	%rbx, %r9

  1654. 	adox	%r11, %r10

  1655. 

  1656. 	mulx	24(%rbp), %rbx, %r11

  1657. 	adcx	%rbx, %r10

  1658. 	adox	%r12, %r11

  1659. 

  1660. 	.byte	0xc4,0x62,0xe3,0xf6,0xa5,0x20,0x00,0x00,0x00	# mulx	32(%rbp), %rbx, %r12

  1661. 	 mov	%rdx, %rax

  1662. 	 mov	%r8, %rdx

  1663. 	adcx	%rbx, %r11

  1664. 	adox	%r13, %r12

  1665. 

  1666. 	 mulx	128+8(%rsp), %rbx, %rdx

  1667. 	 mov	%rax, %rdx

  1668. 

  1669. 	mulx	40(%rbp), %rax, %r13

  1670. 	adcx	%rax, %r12

  1671. 	adox	%r14, %r13

  1672. 

  1673. 	.byte	0xc4,0x62,0xfb,0xf6,0xb5,0x30,0x00,0x00,0x00	# mulx	48(%rbp), %rax, %r14

  1674. 	adcx	%rax, %r13

  1675. 	adox	%r15, %r14

  1676. 

  1677. 	mulx	56(%rbp), %rax, %r15

  1678. 	 mov	%rbx, %rdx

  1679. 	adcx	%rax, %r14

  1680. 	adox	%rsi, %r15			# %rsi is 0

  1681. 	adcx	%rsi, %r15			# cf=0

  1682. 

  1683. 	decl	%ecx				# of=0

  1684. 	jne	.Lreduction_loopx

  1685. 

  1686. 	ret

  1687. .size	__rsaz_512_reducex,.-__rsaz_512_reducex

  1688. ___

  1689. }

  1690. {	# __rsaz_512_subtract

  1691. 	# input: %r8-%r15, %rdi - $out, %rbp - $mod, %rcx - mask

  1692. 	# output:

  1693. 	# clobbers: everything but %rdi, %rsi and %rbp

  1694. $code.=<<___;

  1695. .type	__rsaz_512_subtract,\@abi-omnipotent

  1696. .align	32

  1697. __rsaz_512_subtract:

  1698. 	movq	%r8, ($out)

  1699. 	movq	%r9, 8($out)

  1700. 	movq	%r10, 16($out)

  1701. 	movq	%r11, 24($out)

  1702. 	movq	%r12, 32($out)

  1703. 	movq	%r13, 40($out)

  1704. 	movq	%r14, 48($out)

  1705. 	movq	%r15, 56($out)

  1706. 

  1707. 	movq	0($mod), %r8

  1708. 	movq	8($mod), %r9

  1709. 	negq	%r8

  1710. 	notq	%r9

  1711. 	andq	%rcx, %r8

  1712. 	movq	16($mod), %r10

  1713. 	andq	%rcx, %r9

  1714. 	notq	%r10

  1715. 	movq	24($mod), %r11

  1716. 	andq	%rcx, %r10

  1717. 	notq	%r11

  1718. 	movq	32($mod), %r12

  1719. 	andq	%rcx, %r11

  1720. 	notq	%r12

  1721. 	movq	40($mod), %r13

  1722. 	andq	%rcx, %r12

  1723. 	notq	%r13

  1724. 	movq	48($mod), %r14

  1725. 	andq	%rcx, %r13

  1726. 	notq	%r14

  1727. 	movq	56($mod), %r15

  1728. 	andq	%rcx, %r14

  1729. 	notq	%r15

  1730. 	andq	%rcx, %r15

  1731. 

  1732. 	addq	($out), %r8

  1733. 	adcq	8($out), %r9

  1734. 	adcq	16($out), %r10

  1735. 	adcq	24($out), %r11

  1736. 	adcq	32($out), %r12

  1737. 	adcq	40($out), %r13

  1738. 	adcq	48($out), %r14

  1739. 	adcq	56($out), %r15

  1740. 

  1741. 	movq	%r8, ($out)

  1742. 	movq	%r9, 8($out)

  1743. 	movq	%r10, 16($out)

  1744. 	movq	%r11, 24($out)

  1745. 	movq	%r12, 32($out)

  1746. 	movq	%r13, 40($out)

  1747. 	movq	%r14, 48($out)

  1748. 	movq	%r15, 56($out)

  1749. 

  1750. 	ret

  1751. .size	__rsaz_512_subtract,.-__rsaz_512_subtract

  1752. ___

  1753. }

  1754. {	# __rsaz_512_mul

  1755. 	#

  1756. 	# input: %rsi - ap, %rbp - bp

  1757. 	# ouput:

  1758. 	# clobbers: everything

  1759. my ($ap,$bp) = ("%rsi","%rbp");

  1760. $code.=<<___;

  1761. .type	__rsaz_512_mul,\@abi-omnipotent

  1762. .align	32

  1763. __rsaz_512_mul:

  1764. 	leaq	8(%rsp), %rdi

  1765. 

  1766. 	movq	($ap), %rax

  1767. 	mulq	%rbx

  1768. 	movq	%rax, (%rdi)

  1769. 	movq	8($ap), %rax

  1770. 	movq	%rdx, %r8

  1771. 

  1772. 	mulq	%rbx

  1773. 	addq	%rax, %r8

  1774. 	movq	16($ap), %rax

  1775. 	movq	%rdx, %r9

  1776. 	adcq	\$0, %r9

  1777. 

  1778. 	mulq	%rbx

  1779. 	addq	%rax, %r9

  1780. 	movq	24($ap), %rax

  1781. 	movq	%rdx, %r10

  1782. 	adcq	\$0, %r10

  1783. 

  1784. 	mulq	%rbx

  1785. 	addq	%rax, %r10

  1786. 	movq	32($ap), %rax

  1787. 	movq	%rdx, %r11

  1788. 	adcq	\$0, %r11

  1789. 

  1790. 	mulq	%rbx

  1791. 	addq	%rax, %r11

  1792. 	movq	40($ap), %rax

  1793. 	movq	%rdx, %r12

  1794. 	adcq	\$0, %r12

  1795. 

  1796. 	mulq	%rbx

  1797. 	addq	%rax, %r12

  1798. 	movq	48($ap), %rax

  1799. 	movq	%rdx, %r13

  1800. 	adcq	\$0, %r13

  1801. 

  1802. 	mulq	%rbx

  1803. 	addq	%rax, %r13

  1804. 	movq	56($ap), %rax

  1805. 	movq	%rdx, %r14

  1806. 	adcq	\$0, %r14

  1807. 	

  1808. 	mulq	%rbx

  1809. 	addq	%rax, %r14

  1810. 	 movq	($ap), %rax

  1811. 	movq	%rdx, %r15

  1812. 	adcq	\$0, %r15

  1813. 

  1814. 	leaq	8($bp), $bp

  1815. 	leaq	8(%rdi), %rdi

  1816. 

  1817. 	movl	\$7, %ecx

  1818. 	jmp	.Loop_mul

  1819. 

  1820. .align	32

  1821. .Loop_mul:

  1822. 	movq	($bp), %rbx

  1823. 	mulq	%rbx

  1824. 	addq	%rax, %r8

  1825. 	movq	8($ap), %rax

  1826. 	movq	%r8, (%rdi)

  1827. 	movq	%rdx, %r8

  1828. 	adcq	\$0, %r8

  1829. 

  1830. 	mulq	%rbx

  1831. 	addq	%rax, %r9

  1832. 	movq	16($ap), %rax

  1833. 	adcq	\$0, %rdx

  1834. 	addq	%r9, %r8

  1835. 	movq	%rdx, %r9

  1836. 	adcq	\$0, %r9

  1837. 

  1838. 	mulq	%rbx

  1839. 	addq	%rax, %r10

  1840. 	movq	24($ap), %rax

  1841. 	adcq	\$0, %rdx

  1842. 	addq	%r10, %r9

  1843. 	movq	%rdx, %r10

  1844. 	adcq	\$0, %r10

  1845. 

  1846. 	mulq	%rbx

  1847. 	addq	%rax, %r11

  1848. 	movq	32($ap), %rax

  1849. 	adcq	\$0, %rdx

  1850. 	addq	%r11, %r10

  1851. 	movq	%rdx, %r11

  1852. 	adcq	\$0, %r11

  1853. 

  1854. 	mulq	%rbx

  1855. 	addq	%rax, %r12

  1856. 	movq	40($ap), %rax

  1857. 	adcq	\$0, %rdx

  1858. 	addq	%r12, %r11

  1859. 	movq	%rdx, %r12

  1860. 	adcq	\$0, %r12

  1861. 

  1862. 	mulq	%rbx

  1863. 	addq	%rax, %r13

  1864. 	movq	48($ap), %rax

  1865. 	adcq	\$0, %rdx

  1866. 	addq	%r13, %r12

  1867. 	movq	%rdx, %r13

  1868. 	adcq	\$0, %r13

  1869. 

  1870. 	mulq	%rbx

  1871. 	addq	%rax, %r14

  1872. 	movq	56($ap), %rax

  1873. 	adcq	\$0, %rdx

  1874. 	addq	%r14, %r13

  1875. 	movq	%rdx, %r14

  1876. 	 leaq	8($bp), $bp

  1877. 	adcq	\$0, %r14

  1878. 

  1879. 	mulq	%rbx

  1880. 	addq	%rax, %r15

  1881. 	 movq	($ap), %rax

  1882. 	adcq	\$0, %rdx

  1883. 	addq	%r15, %r14

  1884. 	movq	%rdx, %r15	

  1885. 	adcq	\$0, %r15

  1886. 

  1887. 	leaq	8(%rdi), %rdi

  1888. 

  1889. 	decl	%ecx

  1890. 	jnz	.Loop_mul

  1891. 

  1892. 	movq	%r8, (%rdi)

  1893. 	movq	%r9, 8(%rdi)

  1894. 	movq	%r10, 16(%rdi)

  1895. 	movq	%r11, 24(%rdi)

  1896. 	movq	%r12, 32(%rdi)

  1897. 	movq	%r13, 40(%rdi)

  1898. 	movq	%r14, 48(%rdi)

  1899. 	movq	%r15, 56(%rdi)

  1900. 

  1901. 	ret

  1902. .size	__rsaz_512_mul,.-__rsaz_512_mul

  1903. ___

  1904. }

  1905. if ($addx) {

  1906. 	# __rsaz_512_mulx

  1907. 	#

  1908. 	# input: %rsi - ap, %rbp - bp

  1909. 	# ouput:

  1910. 	# clobbers: everything

  1911. my ($ap,$bp,$zero) = ("%rsi","%rbp","%rdi");

  1912. $code.=<<___;

  1913. .type	__rsaz_512_mulx,\@abi-omnipotent

  1914. .align	32

  1915. __rsaz_512_mulx:

  1916. 	mulx	($ap), %rbx, %r8	# initial %rdx preloaded by caller

  1917. 	mov	\$-6, %rcx

  1918. 

  1919. 	mulx	8($ap), %rax, %r9

  1920. 	movq	%rbx, 8(%rsp)

  1921. 

  1922. 	mulx	16($ap), %rbx, %r10

  1923. 	adc	%rax, %r8

  1924. 

  1925. 	mulx	24($ap), %rax, %r11

  1926. 	adc	%rbx, %r9

  1927. 

  1928. 	mulx	32($ap), %rbx, %r12

  1929. 	adc	%rax, %r10

  1930. 

  1931. 	mulx	40($ap), %rax, %r13

  1932. 	adc	%rbx, %r11

  1933. 

  1934. 	mulx	48($ap), %rbx, %r14

  1935. 	adc	%rax, %r12

  1936. 

  1937. 	mulx	56($ap), %rax, %r15

  1938. 	 mov	8($bp), %rdx

  1939. 	adc	%rbx, %r13

  1940. 	adc	%rax, %r14

  1941. 	adc	\$0, %r15

  1942. 

  1943. 	xor	$zero, $zero		# cf=0,of=0

  1944. 	jmp	.Loop_mulx

  1945. 

  1946. .align	32

  1947. .Loop_mulx:

  1948. 	movq	%r8, %rbx

  1949. 	mulx	($ap), %rax, %r8

  1950. 	adcx	%rax, %rbx

  1951. 	adox	%r9, %r8

  1952. 

  1953. 	mulx	8($ap), %rax, %r9

  1954. 	adcx	%rax, %r8

  1955. 	adox	%r10, %r9

  1956. 

  1957. 	mulx	16($ap), %rax, %r10

  1958. 	adcx	%rax, %r9

  1959. 	adox	%r11, %r10

  1960. 

  1961. 	mulx	24($ap), %rax, %r11

  1962. 	adcx	%rax, %r10

  1963. 	adox	%r12, %r11

  1964. 

  1965. 	.byte	0x3e,0xc4,0x62,0xfb,0xf6,0xa6,0x20,0x00,0x00,0x00	# mulx	32($ap), %rax, %r12

  1966. 	adcx	%rax, %r11

  1967. 	adox	%r13, %r12

  1968. 

  1969. 	mulx	40($ap), %rax, %r13

  1970. 	adcx	%rax, %r12

  1971. 	adox	%r14, %r13

  1972. 

  1973. 	mulx	48($ap), %rax, %r14

  1974. 	adcx	%rax, %r13

  1975. 	adox	%r15, %r14

  1976. 

  1977. 	mulx	56($ap), %rax, %r15

  1978. 	 movq	64($bp,%rcx,8), %rdx

  1979. 	 movq	%rbx, 8+64-8(%rsp,%rcx,8)

  1980. 	adcx	%rax, %r14

  1981. 	adox	$zero, %r15

  1982. 	adcx	$zero, %r15		# cf=0

  1983. 

  1984. 	inc	%rcx			# of=0

  1985. 	jnz	.Loop_mulx

  1986. 

  1987. 	movq	%r8, %rbx

  1988. 	mulx	($ap), %rax, %r8

  1989. 	adcx	%rax, %rbx

  1990. 	adox	%r9, %r8

  1991. 

  1992. 	.byte	0xc4,0x62,0xfb,0xf6,0x8e,0x08,0x00,0x00,0x00	# mulx	8($ap), %rax, %r9

  1993. 	adcx	%rax, %r8

  1994. 	adox	%r10, %r9

  1995. 

  1996. 	.byte	0xc4,0x62,0xfb,0xf6,0x96,0x10,0x00,0x00,0x00	# mulx	16($ap), %rax, %r10

  1997. 	adcx	%rax, %r9

  1998. 	adox	%r11, %r10

  1999. 

  2000. 	mulx	24($ap), %rax, %r11

  2001. 	adcx	%rax, %r10

  2002. 	adox	%r12, %r11

  2003. 

  2004. 	mulx	32($ap), %rax, %r12

  2005. 	adcx	%rax, %r11

  2006. 	adox	%r13, %r12

  2007. 

  2008. 	mulx	40($ap), %rax, %r13

  2009. 	adcx	%rax, %r12

  2010. 	adox	%r14, %r13

  2011. 

  2012. 	.byte	0xc4,0x62,0xfb,0xf6,0xb6,0x30,0x00,0x00,0x00	# mulx	48($ap), %rax, %r14

  2013. 	adcx	%rax, %r13

  2014. 	adox	%r15, %r14

  2015. 

  2016. 	.byte	0xc4,0x62,0xfb,0xf6,0xbe,0x38,0x00,0x00,0x00	# mulx	56($ap), %rax, %r15

  2017. 	adcx	%rax, %r14

  2018. 	adox	$zero, %r15

  2019. 	adcx	$zero, %r15

  2020. 

  2021. 	mov	%rbx, 8+64-8(%rsp)

  2022. 	mov	%r8, 8+64(%rsp)

  2023. 	mov	%r9, 8+64+8(%rsp)

  2024. 	mov	%r10, 8+64+16(%rsp)

  2025. 	mov	%r11, 8+64+24(%rsp)

  2026. 	mov	%r12, 8+64+32(%rsp)

  2027. 	mov	%r13, 8+64+40(%rsp)

  2028. 	mov	%r14, 8+64+48(%rsp)

  2029. 	mov	%r15, 8+64+56(%rsp)

  2030. 

  2031. 	ret

  2032. .size	__rsaz_512_mulx,.-__rsaz_512_mulx

  2033. ___

  2034. }

  2035. {

  2036. my ($out,$inp,$power)= $win64 ? ("%rcx","%rdx","%r8d") : ("%rdi","%rsi","%edx");

  2037. $code.=<<___;

  2038. .globl	rsaz_512_scatter4

  2039. .type	rsaz_512_scatter4,\@abi-omnipotent

  2040. .align	16

  2041. rsaz_512_scatter4:

  2042. 	leaq	($out,$power,8), $out

  2043. 	movl	\$8, %r9d

  2044. 	jmp	.Loop_scatter

  2045. .align	16

  2046. .Loop_scatter:

  2047. 	movq	($inp), %rax

  2048. 	leaq	8($inp), $inp

  2049. 	movq	%rax, ($out)

  2050. 	leaq	128($out), $out

  2051. 	decl	%r9d

  2052. 	jnz	.Loop_scatter

  2053. 	ret

  2054. .size	rsaz_512_scatter4,.-rsaz_512_scatter4

  2055. 

  2056. .globl	rsaz_512_gather4

  2057. .type	rsaz_512_gather4,\@abi-omnipotent

  2058. .align	16

  2059. rsaz_512_gather4:

  2060. ___

  2061. $code.=<<___	if ($win64);

  2062. .LSEH_begin_rsaz_512_gather4:

  2063. 	.byte	0x48,0x81,0xec,0xa8,0x00,0x00,0x00	# sub    $0xa8,%rsp

  2064. 	.byte	0x0f,0x29,0x34,0x24			# movaps %xmm6,(%rsp)

  2065. 	.byte	0x0f,0x29,0x7c,0x24,0x10		# movaps %xmm7,0x10(%rsp)

  2066. 	.byte	0x44,0x0f,0x29,0x44,0x24,0x20		# movaps %xmm8,0x20(%rsp)

  2067. 	.byte	0x44,0x0f,0x29,0x4c,0x24,0x30		# movaps %xmm9,0x30(%rsp)

  2068. 	.byte	0x44,0x0f,0x29,0x54,0x24,0x40		# movaps %xmm10,0x40(%rsp)

  2069. 	.byte	0x44,0x0f,0x29,0x5c,0x24,0x50		# movaps %xmm11,0x50(%rsp)

  2070. 	.byte	0x44,0x0f,0x29,0x64,0x24,0x60		# movaps %xmm12,0x60(%rsp)

  2071. 	.byte	0x44,0x0f,0x29,0x6c,0x24,0x70		# movaps %xmm13,0x70(%rsp)

  2072. 	.byte	0x44,0x0f,0x29,0xb4,0x24,0x80,0,0,0	# movaps %xmm14,0x80(%rsp)

  2073. 	.byte	0x44,0x0f,0x29,0xbc,0x24,0x90,0,0,0	# movaps %xmm15,0x90(%rsp)

  2074. ___

  2075. $code.=<<___;

  2076. 	movd	$power,%xmm8

  2077. 	movdqa	.Linc+16(%rip),%xmm1	# 00000002000000020000000200000002

  2078. 	movdqa	.Linc(%rip),%xmm0	# 00000001000000010000000000000000

  2079. 

  2080. 	pshufd	\$0,%xmm8,%xmm8		# broadcast $power

  2081. 	movdqa	%xmm1,%xmm7

  2082. 	movdqa	%xmm1,%xmm2

  2083. ___

  2084. ########################################################################

  2085. # calculate mask by comparing 0..15 to $power

  2086. #

  2087. for($i=0;$i<4;$i++) {

  2088. $code.=<<___;

  2089. 	paddd	%xmm`$i`,%xmm`$i+1`

  2090. 	pcmpeqd	%xmm8,%xmm`$i`

  2091. 	movdqa	%xmm7,%xmm`$i+3`

  2092. ___

  2093. }

  2094. for(;$i<7;$i++) {

  2095. $code.=<<___;

  2096. 	paddd	%xmm`$i`,%xmm`$i+1`

  2097. 	pcmpeqd	%xmm8,%xmm`$i`

  2098. ___

  2099. }

  2100. $code.=<<___;

  2101. 	pcmpeqd	%xmm8,%xmm7

  2102. 	movl	\$8, %r9d

  2103. 	jmp	.Loop_gather

  2104. .align	16

  2105. .Loop_gather:

  2106. 	movdqa	16*0($inp),%xmm8

  2107. 	movdqa	16*1($inp),%xmm9

  2108. 	movdqa	16*2($inp),%xmm10

  2109. 	movdqa	16*3($inp),%xmm11

  2110. 	pand	%xmm0,%xmm8

  2111. 	movdqa	16*4($inp),%xmm12

  2112. 	pand	%xmm1,%xmm9

  2113. 	movdqa	16*5($inp),%xmm13

  2114. 	pand	%xmm2,%xmm10

  2115. 	movdqa	16*6($inp),%xmm14

  2116. 	pand	%xmm3,%xmm11

  2117. 	movdqa	16*7($inp),%xmm15

  2118. 	leaq	128($inp), $inp

  2119. 	pand	%xmm4,%xmm12

  2120. 	pand	%xmm5,%xmm13

  2121. 	pand	%xmm6,%xmm14

  2122. 	pand	%xmm7,%xmm15

  2123. 	por	%xmm10,%xmm8

  2124. 	por	%xmm11,%xmm9

  2125. 	por	%xmm12,%xmm8

  2126. 	por	%xmm13,%xmm9

  2127. 	por	%xmm14,%xmm8

  2128. 	por	%xmm15,%xmm9

  2129. 

  2130. 	por	%xmm9,%xmm8

  2131. 	pshufd	\$0x4e,%xmm8,%xmm9

  2132. 	por	%xmm9,%xmm8

  2133. 	movq	%xmm8,($out)

  2134. 	leaq	8($out), $out

  2135. 	decl	%r9d

  2136. 	jnz	.Loop_gather

  2137. ___

  2138. $code.=<<___	if ($win64);

  2139. 	movaps	0x00(%rsp),%xmm6

  2140. 	movaps	0x10(%rsp),%xmm7

  2141. 	movaps	0x20(%rsp),%xmm8

  2142. 	movaps	0x30(%rsp),%xmm9

  2143. 	movaps	0x40(%rsp),%xmm10

  2144. 	movaps	0x50(%rsp),%xmm11

  2145. 	movaps	0x60(%rsp),%xmm12

  2146. 	movaps	0x70(%rsp),%xmm13

  2147. 	movaps	0x80(%rsp),%xmm14

  2148. 	movaps	0x90(%rsp),%xmm15

  2149. 	add	\$0xa8,%rsp

  2150. ___

  2151. $code.=<<___;

  2152. 	ret

  2153. .LSEH_end_rsaz_512_gather4:

  2154. .size	rsaz_512_gather4,.-rsaz_512_gather4

  2155. 

  2156. .align	64

  2157. .Linc:

  2158. 	.long	0,0, 1,1

  2159. 	.long	2,2, 2,2

  2160. ___

  2161. }

  2162. 

  2163. # EXCEPTION_DISPOSITION handler (EXCEPTION_RECORD *rec,ULONG64 frame,

  2164. #		CONTEXT *context,DISPATCHER_CONTEXT *disp)

  2165. if ($win64) {

  2166. $rec="%rcx";

  2167. $frame="%rdx";

  2168. $context="%r8";

  2169. $disp="%r9";

  2170. 

  2171. $code.=<<___;

  2172. .extern	__imp_RtlVirtualUnwind

  2173. .type	se_handler,\@abi-omnipotent

  2174. .align	16

  2175. se_handler:

  2176. 	push	%rsi

  2177. 	push	%rdi

  2178. 	push	%rbx

  2179. 	push	%rbp

  2180. 	push	%r12

  2181. 	push	%r13

  2182. 	push	%r14

  2183. 	push	%r15

  2184. 	pushfq

  2185. 	sub	\$64,%rsp

  2186. 

  2187. 	mov	120($context),%rax	# pull context->Rax

  2188. 	mov	248($context),%rbx	# pull context->Rip

  2189. 

  2190. 	mov	8($disp),%rsi		# disp->ImageBase

  2191. 	mov	56($disp),%r11		# disp->HandlerData

  2192. 

  2193. 	mov	0(%r11),%r10d		# HandlerData[0]

  2194. 	lea	(%rsi,%r10),%r10	# end of prologue label

  2195. 	cmp	%r10,%rbx		# context->Rip<end of prologue label

  2196. 	jb	.Lcommon_seh_tail

  2197. 

  2198. 	mov	152($context),%rax	# pull context->Rsp

  2199. 

  2200. 	mov	4(%r11),%r10d		# HandlerData[1]

  2201. 	lea	(%rsi,%r10),%r10	# epilogue label

  2202. 	cmp	%r10,%rbx		# context->Rip>=epilogue label

  2203. 	jae	.Lcommon_seh_tail

  2204. 

  2205. 	lea	128+24+48(%rax),%rax

  2206. 

  2207. 	lea	.Lmul_gather4_epilogue(%rip),%rbx

  2208. 	cmp	%r10,%rbx

  2209. 	jne	.Lse_not_in_mul_gather4

  2210. 

  2211. 	lea	0xb0(%rax),%rax

  2212. 

  2213. 	lea	-48-0xa8(%rax),%rsi

  2214. 	lea	512($context),%rdi

  2215. 	mov	\$20,%ecx

  2216. 	.long	0xa548f3fc		# cld; rep movsq

  2217. 

  2218. .Lse_not_in_mul_gather4:

  2219. 	mov	-8(%rax),%rbx

  2220. 	mov	-16(%rax),%rbp

  2221. 	mov	-24(%rax),%r12

  2222. 	mov	-32(%rax),%r13

  2223. 	mov	-40(%rax),%r14

  2224. 	mov	-48(%rax),%r15

  2225. 	mov	%rbx,144($context)	# restore context->Rbx

  2226. 	mov	%rbp,160($context)	# restore context->Rbp

  2227. 	mov	%r12,216($context)	# restore context->R12

  2228. 	mov	%r13,224($context)	# restore context->R13

  2229. 	mov	%r14,232($context)	# restore context->R14

  2230. 	mov	%r15,240($context)	# restore context->R15

  2231. 

  2232. .Lcommon_seh_tail:

  2233. 	mov	8(%rax),%rdi

  2234. 	mov	16(%rax),%rsi

  2235. 	mov	%rax,152($context)	# restore context->Rsp

  2236. 	mov	%rsi,168($context)	# restore context->Rsi

  2237. 	mov	%rdi,176($context)	# restore context->Rdi

  2238. 

  2239. 	mov	40($disp),%rdi		# disp->ContextRecord

  2240. 	mov	$context,%rsi		# context

  2241. 	mov	\$154,%ecx		# sizeof(CONTEXT)

  2242. 	.long	0xa548f3fc		# cld; rep movsq

  2243. 

  2244. 	mov	$disp,%rsi

  2245. 	xor	%rcx,%rcx		# arg1, UNW_FLAG_NHANDLER

  2246. 	mov	8(%rsi),%rdx		# arg2, disp->ImageBase

  2247. 	mov	0(%rsi),%r8		# arg3, disp->ControlPc

  2248. 	mov	16(%rsi),%r9		# arg4, disp->FunctionEntry

  2249. 	mov	40(%rsi),%r10		# disp->ContextRecord

  2250. 	lea	56(%rsi),%r11		# &disp->HandlerData

  2251. 	lea	24(%rsi),%r12		# &disp->EstablisherFrame

  2252. 	mov	%r10,32(%rsp)		# arg5

  2253. 	mov	%r11,40(%rsp)		# arg6

  2254. 	mov	%r12,48(%rsp)		# arg7

  2255. 	mov	%rcx,56(%rsp)		# arg8, (NULL)

  2256. 	call	*__imp_RtlVirtualUnwind(%rip)

  2257. 

  2258. 	mov	\$1,%eax		# ExceptionContinueSearch

  2259. 	add	\$64,%rsp

  2260. 	popfq

  2261. 	pop	%r15

  2262. 	pop	%r14

  2263. 	pop	%r13

  2264. 	pop	%r12

  2265. 	pop	%rbp

  2266. 	pop	%rbx

  2267. 	pop	%rdi

  2268. 	pop	%rsi

  2269. 	ret

  2270. .size	se_handler,.-se_handler

  2271. 

  2272. .section	.pdata

  2273. .align	4

  2274. 	.rva	.LSEH_begin_rsaz_512_sqr

  2275. 	.rva	.LSEH_end_rsaz_512_sqr

  2276. 	.rva	.LSEH_info_rsaz_512_sqr

  2277. 

  2278. 	.rva	.LSEH_begin_rsaz_512_mul

  2279. 	.rva	.LSEH_end_rsaz_512_mul

  2280. 	.rva	.LSEH_info_rsaz_512_mul

  2281. 

  2282. 	.rva	.LSEH_begin_rsaz_512_mul_gather4

  2283. 	.rva	.LSEH_end_rsaz_512_mul_gather4

  2284. 	.rva	.LSEH_info_rsaz_512_mul_gather4

  2285. 

  2286. 	.rva	.LSEH_begin_rsaz_512_mul_scatter4

  2287. 	.rva	.LSEH_end_rsaz_512_mul_scatter4

  2288. 	.rva	.LSEH_info_rsaz_512_mul_scatter4

  2289. 

  2290. 	.rva	.LSEH_begin_rsaz_512_mul_by_one

  2291. 	.rva	.LSEH_end_rsaz_512_mul_by_one

  2292. 	.rva	.LSEH_info_rsaz_512_mul_by_one

  2293. 

  2294. 	.rva	.LSEH_begin_rsaz_512_gather4

  2295. 	.rva	.LSEH_end_rsaz_512_gather4

  2296. 	.rva	.LSEH_info_rsaz_512_gather4

  2297. 

  2298. .section	.xdata

  2299. .align	8

  2300. .LSEH_info_rsaz_512_sqr:

  2301. 	.byte	9,0,0,0

  2302. 	.rva	se_handler

  2303. 	.rva	.Lsqr_body,.Lsqr_epilogue			# HandlerData[]

  2304. .LSEH_info_rsaz_512_mul:

  2305. 	.byte	9,0,0,0

  2306. 	.rva	se_handler

  2307. 	.rva	.Lmul_body,.Lmul_epilogue			# HandlerData[]

  2308. .LSEH_info_rsaz_512_mul_gather4:

  2309. 	.byte	9,0,0,0

  2310. 	.rva	se_handler

  2311. 	.rva	.Lmul_gather4_body,.Lmul_gather4_epilogue	# HandlerData[]

  2312. .LSEH_info_rsaz_512_mul_scatter4:

  2313. 	.byte	9,0,0,0

  2314. 	.rva	se_handler

  2315. 	.rva	.Lmul_scatter4_body,.Lmul_scatter4_epilogue	# HandlerData[]

  2316. .LSEH_info_rsaz_512_mul_by_one:

  2317. 	.byte	9,0,0,0

  2318. 	.rva	se_handler

  2319. 	.rva	.Lmul_by_one_body,.Lmul_by_one_epilogue		# HandlerData[]

  2320. .LSEH_info_rsaz_512_gather4:

  2321. 	.byte	0x01,0x46,0x16,0x00

  2322. 	.byte	0x46,0xf8,0x09,0x00	# vmovaps 0x90(rsp),xmm15

  2323. 	.byte	0x3d,0xe8,0x08,0x00	# vmovaps 0x80(rsp),xmm14

  2324. 	.byte	0x34,0xd8,0x07,0x00	# vmovaps 0x70(rsp),xmm13

  2325. 	.byte	0x2e,0xc8,0x06,0x00	# vmovaps 0x60(rsp),xmm12

  2326. 	.byte	0x28,0xb8,0x05,0x00	# vmovaps 0x50(rsp),xmm11

  2327. 	.byte	0x22,0xa8,0x04,0x00	# vmovaps 0x40(rsp),xmm10

  2328. 	.byte	0x1c,0x98,0x03,0x00	# vmovaps 0x30(rsp),xmm9

  2329. 	.byte	0x16,0x88,0x02,0x00	# vmovaps 0x20(rsp),xmm8

  2330. 	.byte	0x10,0x78,0x01,0x00	# vmovaps 0x10(rsp),xmm7

  2331. 	.byte	0x0b,0x68,0x00,0x00	# vmovaps 0x00(rsp),xmm6

  2332. 	.byte	0x07,0x01,0x15,0x00	# sub     rsp,0xa8

  2333. ___

  2334. }

  2335. 

  2336. $code =~ s/\`([^\`]*)\`/eval $1/gem;

  2337. print $code;

  2338. close STDOUT;