kris
/
boringssl
1
0
Разклонения 0
Код Задачи 0 Заявки за сливане 0 Версии 0 Уики Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
3169 Ревизии
12 Клонове
38 MiB
 
 
 
 
 
 
ИН на ревизия: 4c4ff02fe8
Клонове Маркери
${ item.name }
Create branch ${ searchTerm }
от '4c4ff02fe8'
${ noResults }
boringssl/crypto/poly1305/asm/poly1305-x86.pl

1794 lines
50 KiB
Директен файл Blame История 

  1. #!/usr/bin/env perl

  2. #

  3. # ====================================================================

  4. # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL

  5. # project. The module is, however, dual licensed under OpenSSL and

  6. # CRYPTOGAMS licenses depending on where you obtain it. For further

  7. # details see http://www.openssl.org/~appro/cryptogams/.

  8. # ====================================================================

  9. #

  10. # This module implements Poly1305 hash for x86.

  11. #

  12. # April 2015

  13. #

  14. # Numbers are cycles per processed byte with poly1305_blocks alone,

  15. # measured with rdtsc at fixed clock frequency.

  16. #

  17. #		IALU/gcc-3.4(*)	SSE2(**)	AVX2

  18. # Pentium	15.7/+80%	-

  19. # PIII		6.21/+90%	-

  20. # P4		19.8/+40%	3.24

  21. # Core 2	4.85/+90%	1.80

  22. # Westmere	4.58/+100%	1.43

  23. # Sandy Bridge	3.90/+100%	1.36

  24. # Haswell	3.88/+70%	1.18		0.72

  25. # Silvermont	11.0/+40%	4.80

  26. # VIA Nano	6.71/+90%	2.47

  27. # Sledgehammer	3.51/+180%	4.27

  28. # Bulldozer	4.53/+140%	1.31

  29. #

  30. # (*)	gcc 4.8 for some reason generated worse code;

  31. # (**)	besides SSE2 there are floating-point and AVX options; FP

  32. #	is deemed unnecessary, because pre-SSE2 processor are too

  33. #	old to care about, while it's not the fastest option on

  34. #	SSE2-capable ones; AVX is omitted, because it doesn't give

  35. #	a lot of improvement, 5-10% depending on processor;

  36. 

  37. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;

  38. push(@INC,"${dir}","${dir}../../perlasm");

  39. require "x86asm.pl";

  40. 

  41. $output=pop;

  42. open STDOUT,">$output";

  43. 

  44. &asm_init($ARGV[0],"poly1305-x86.pl",$ARGV[$#ARGV] eq "386");

  45. 

  46. $sse2=$avx=0;

  47. for (@ARGV) { $sse2=1 if (/-DOPENSSL_IA32_SSE2/); }

  48. 

  49. if ($sse2) {

  50. 	&static_label("const_sse2");

  51. 	&static_label("enter_blocks");

  52. 	&static_label("enter_emit");

  53. 	&external_label("OPENSSL_ia32cap_P");

  54. 

  55. 	# This may be set to 2, but valgrind can't do AVX2 on 32-bit. Without a

  56. 	# way to verify test coverage, keep it disabled.

  57. 	$avx = 0;

  58. }

  59. 

  60. ########################################################################

  61. # Layout of opaque area is following.

  62. #

  63. #	unsigned __int32 h[5];		# current hash value base 2^32

  64. #	unsigned __int32 pad;		# is_base2_26 in vector context

  65. #	unsigned __int32 r[4];		# key value base 2^32

  66. 

  67. &align(64);

  68. &function_begin("poly1305_init");

  69. 	&mov	("edi",&wparam(0));		# context

  70. 	&mov	("esi",&wparam(1));		# key

  71. 	&mov	("ebp",&wparam(2));		# function table

  72. 

  73. 	&xor	("eax","eax");

  74. 	&mov	(&DWP(4*0,"edi"),"eax");	# zero hash value

  75. 	&mov	(&DWP(4*1,"edi"),"eax");

  76. 	&mov	(&DWP(4*2,"edi"),"eax");

  77. 	&mov	(&DWP(4*3,"edi"),"eax");

  78. 	&mov	(&DWP(4*4,"edi"),"eax");

  79. 	&mov	(&DWP(4*5,"edi"),"eax");	# is_base2_26

  80. 

  81. 	&cmp	("esi",0);

  82. 	&je	(&label("nokey"));

  83. 

  84.     if ($sse2) {

  85. 	&call	(&label("pic_point"));

  86.     &set_label("pic_point");

  87. 	&blindpop("ebx");

  88. 

  89. 	&lea	("eax",&DWP("poly1305_blocks-".&label("pic_point"),"ebx"));

  90. 	&lea	("edx",&DWP("poly1305_emit-".&label("pic_point"),"ebx"));

  91. 

  92. 	&picmeup("edi","OPENSSL_ia32cap_P","ebx",&label("pic_point"));

  93. 	&mov	("ecx",&DWP(0,"edi"));

  94. 	&and	("ecx",1<<26|1<<24);

  95. 	&cmp	("ecx",1<<26|1<<24);		# SSE2 and XMM?

  96. 	&jne	(&label("no_sse2"));

  97. 

  98. 	&lea	("eax",&DWP("_poly1305_blocks_sse2-".&label("pic_point"),"ebx"));

  99. 	&lea	("edx",&DWP("_poly1305_emit_sse2-".&label("pic_point"),"ebx"));

  100. 

  101.       if ($avx>1) {

  102. 	&mov	("ecx",&DWP(8,"edi"));

  103. 	&test	("ecx",1<<5);			# AVX2?

  104. 	&jz	(&label("no_sse2"));

  105. 

  106. 	&lea	("eax",&DWP("_poly1305_blocks_avx2-".&label("pic_point"),"ebx"));

  107.       }

  108.     &set_label("no_sse2");

  109. 	&mov	("edi",&wparam(0));		# reload context

  110. 	&mov	(&DWP(0,"ebp"),"eax");		# fill function table

  111. 	&mov	(&DWP(4,"ebp"),"edx");

  112.     }

  113. 

  114. 	&mov	("eax",&DWP(4*0,"esi"));	# load input key

  115. 	&mov	("ebx",&DWP(4*1,"esi"));

  116. 	&mov	("ecx",&DWP(4*2,"esi"));

  117. 	&mov	("edx",&DWP(4*3,"esi"));

  118. 	&and	("eax",0x0fffffff);

  119. 	&and	("ebx",0x0ffffffc);

  120. 	&and	("ecx",0x0ffffffc);

  121. 	&and	("edx",0x0ffffffc);

  122. 	&mov	(&DWP(4*6,"edi"),"eax");

  123. 	&mov	(&DWP(4*7,"edi"),"ebx");

  124. 	&mov	(&DWP(4*8,"edi"),"ecx");

  125. 	&mov	(&DWP(4*9,"edi"),"edx");

  126. 

  127. 	&mov	("eax",$sse2);

  128. &set_label("nokey");

  129. &function_end("poly1305_init");

  130. 

  131. ($h0,$h1,$h2,$h3,$h4,

  132.  $d0,$d1,$d2,$d3,

  133.  $r0,$r1,$r2,$r3,

  134.      $s1,$s2,$s3)=map(4*$_,(0..15));

  135. 

  136. &function_begin("poly1305_blocks");

  137. 	&mov	("edi",&wparam(0));		# ctx

  138. 	&mov	("esi",&wparam(1));		# inp

  139. 	&mov	("ecx",&wparam(2));		# len

  140. &set_label("enter_blocks");

  141. 	&and	("ecx",-15);

  142. 	&jz	(&label("nodata"));

  143. 

  144. 	&stack_push(16);

  145. 	&mov	("eax",&DWP(4*6,"edi"));	# r0

  146. 	&mov	("ebx",&DWP(4*7,"edi"));	# r1

  147. 	 &lea	("ebp",&DWP(0,"esi","ecx"));	# end of input

  148. 	&mov	("ecx",&DWP(4*8,"edi"));	# r2

  149. 	&mov	("edx",&DWP(4*9,"edi"));	# r3

  150. 

  151. 	&mov	(&wparam(2),"ebp");

  152. 	&mov	("ebp","esi");

  153. 

  154. 	&mov	(&DWP($r0,"esp"),"eax");	# r0

  155. 	&mov	("eax","ebx");

  156. 	&shr	("eax",2);

  157. 	&mov	(&DWP($r1,"esp"),"ebx");	# r1

  158. 	&add	("eax","ebx");			# s1

  159. 	&mov	("ebx","ecx");

  160. 	&shr	("ebx",2);

  161. 	&mov	(&DWP($r2,"esp"),"ecx");	# r2

  162. 	&add	("ebx","ecx");			# s2

  163. 	&mov	("ecx","edx");

  164. 	&shr	("ecx",2);

  165. 	&mov	(&DWP($r3,"esp"),"edx");	# r3

  166. 	&add	("ecx","edx");			# s3

  167. 	&mov	(&DWP($s1,"esp"),"eax");	# s1

  168. 	&mov	(&DWP($s2,"esp"),"ebx");	# s2

  169. 	&mov	(&DWP($s3,"esp"),"ecx");	# s3

  170. 

  171. 	&mov	("eax",&DWP(4*0,"edi"));	# load hash value

  172. 	&mov	("ebx",&DWP(4*1,"edi"));

  173. 	&mov	("ecx",&DWP(4*2,"edi"));

  174. 	&mov	("esi",&DWP(4*3,"edi"));

  175. 	&mov	("edi",&DWP(4*4,"edi"));

  176. 	&jmp	(&label("loop"));

  177. 

  178. &set_label("loop",32);

  179. 	&add	("eax",&DWP(4*0,"ebp"));	# accumulate input

  180. 	&adc	("ebx",&DWP(4*1,"ebp"));

  181. 	&adc	("ecx",&DWP(4*2,"ebp"));

  182. 	&adc	("esi",&DWP(4*3,"ebp"));

  183. 	&lea	("ebp",&DWP(4*4,"ebp"));

  184. 	&adc	("edi",&wparam(3));		# padbit

  185. 

  186. 	&mov	(&DWP($h0,"esp"),"eax");	# put aside hash[+inp]

  187. 	&mov	(&DWP($h3,"esp"),"esi");

  188. 

  189. 	&mul	(&DWP($r0,"esp"));		# h0*r0

  190. 	 &mov	(&DWP($h4,"esp"),"edi");

  191. 	&mov	("edi","eax");

  192. 	&mov	("eax","ebx");			# h1

  193. 	&mov	("esi","edx");

  194. 	&mul	(&DWP($s3,"esp"));		# h1*s3

  195. 	&add	("edi","eax");

  196. 	&mov	("eax","ecx");			# h2

  197. 	&adc	("esi","edx");

  198. 	&mul	(&DWP($s2,"esp"));		# h2*s2

  199. 	&add	("edi","eax");

  200. 	&mov	("eax",&DWP($h3,"esp"));

  201. 	&adc	("esi","edx");

  202. 	&mul	(&DWP($s1,"esp"));		# h3*s1

  203. 	&add	("edi","eax");

  204. 	 &mov	("eax",&DWP($h0,"esp"));

  205. 	&adc	("esi","edx");

  206. 

  207. 	&mul	(&DWP($r1,"esp"));		# h0*r1

  208. 	 &mov	(&DWP($d0,"esp"),"edi");

  209. 	&xor	("edi","edi");

  210. 	&add	("esi","eax");

  211. 	&mov	("eax","ebx");			# h1

  212. 	&adc	("edi","edx");

  213. 	&mul	(&DWP($r0,"esp"));		# h1*r0

  214. 	&add	("esi","eax");

  215. 	&mov	("eax","ecx");			# h2

  216. 	&adc	("edi","edx");

  217. 	&mul	(&DWP($s3,"esp"));		# h2*s3

  218. 	&add	("esi","eax");

  219. 	&mov	("eax",&DWP($h3,"esp"));

  220. 	&adc	("edi","edx");

  221. 	&mul	(&DWP($s2,"esp"));		# h3*s2

  222. 	&add	("esi","eax");

  223. 	&mov	("eax",&DWP($h4,"esp"));

  224. 	&adc	("edi","edx");

  225. 	&imul	("eax",&DWP($s1,"esp"));	# h4*s1

  226. 	&add	("esi","eax");

  227. 	 &mov	("eax",&DWP($h0,"esp"));

  228. 	&adc	("edi",0);

  229. 

  230. 	&mul	(&DWP($r2,"esp"));		# h0*r2

  231. 	 &mov	(&DWP($d1,"esp"),"esi");

  232. 	&xor	("esi","esi");

  233. 	&add	("edi","eax");

  234. 	&mov	("eax","ebx");			# h1

  235. 	&adc	("esi","edx");

  236. 	&mul	(&DWP($r1,"esp"));		# h1*r1

  237. 	&add	("edi","eax");

  238. 	&mov	("eax","ecx");			# h2

  239. 	&adc	("esi","edx");

  240. 	&mul	(&DWP($r0,"esp"));		# h2*r0

  241. 	&add	("edi","eax");

  242. 	&mov	("eax",&DWP($h3,"esp"));

  243. 	&adc	("esi","edx");

  244. 	&mul	(&DWP($s3,"esp"));		# h3*s3

  245. 	&add	("edi","eax");

  246. 	&mov	("eax",&DWP($h4,"esp"));

  247. 	&adc	("esi","edx");

  248. 	&imul	("eax",&DWP($s2,"esp"));	# h4*s2

  249. 	&add	("edi","eax");

  250. 	 &mov	("eax",&DWP($h0,"esp"));

  251. 	&adc	("esi",0);

  252. 

  253. 	&mul	(&DWP($r3,"esp"));		# h0*r3

  254. 	 &mov	(&DWP($d2,"esp"),"edi");

  255. 	&xor	("edi","edi");

  256. 	&add	("esi","eax");

  257. 	&mov	("eax","ebx");			# h1

  258. 	&adc	("edi","edx");

  259. 	&mul	(&DWP($r2,"esp"));		# h1*r2

  260. 	&add	("esi","eax");

  261. 	&mov	("eax","ecx");			# h2

  262. 	&adc	("edi","edx");

  263. 	&mul	(&DWP($r1,"esp"));		# h2*r1

  264. 	&add	("esi","eax");

  265. 	&mov	("eax",&DWP($h3,"esp"));

  266. 	&adc	("edi","edx");

  267. 	&mul	(&DWP($r0,"esp"));		# h3*r0

  268. 	&add	("esi","eax");

  269. 	 &mov	("ecx",&DWP($h4,"esp"));

  270. 	&adc	("edi","edx");

  271. 

  272. 	&mov	("edx","ecx");

  273. 	&imul	("ecx",&DWP($s3,"esp"));	# h4*s3

  274. 	&add	("esi","ecx");

  275. 	 &mov	("eax",&DWP($d0,"esp"));

  276. 	&adc	("edi",0);

  277. 

  278. 	&imul	("edx",&DWP($r0,"esp"));	# h4*r0

  279. 	&add	("edx","edi");

  280. 

  281. 	&mov	("ebx",&DWP($d1,"esp"));

  282. 	&mov	("ecx",&DWP($d2,"esp"));

  283. 

  284. 	&mov	("edi","edx");			# last reduction step

  285. 	&shr	("edx",2);

  286. 	&and	("edi",3);

  287. 	&lea	("edx",&DWP(0,"edx","edx",4));	# *5

  288. 	&add	("eax","edx");

  289. 	&adc	("ebx",0);

  290. 	&adc	("ecx",0);

  291. 	&adc	("esi",0);

  292. 

  293. 	&cmp	("ebp",&wparam(2));		# done yet?

  294. 	&jne	(&label("loop"));

  295. 

  296. 	&mov	("edx",&wparam(0));		# ctx

  297. 	&stack_pop(16);

  298. 	&mov	(&DWP(4*0,"edx"),"eax");	# store hash value

  299. 	&mov	(&DWP(4*1,"edx"),"ebx");

  300. 	&mov	(&DWP(4*2,"edx"),"ecx");

  301. 	&mov	(&DWP(4*3,"edx"),"esi");

  302. 	&mov	(&DWP(4*4,"edx"),"edi");

  303. &set_label("nodata");

  304. &function_end("poly1305_blocks");

  305. 

  306. &function_begin("poly1305_emit");

  307. 	&mov	("ebp",&wparam(0));		# context

  308. &set_label("enter_emit");

  309. 	&mov	("edi",&wparam(1));		# output

  310. 	&mov	("eax",&DWP(4*0,"ebp"));	# load hash value

  311. 	&mov	("ebx",&DWP(4*1,"ebp"));

  312. 	&mov	("ecx",&DWP(4*2,"ebp"));

  313. 	&mov	("edx",&DWP(4*3,"ebp"));

  314. 	&mov	("esi",&DWP(4*4,"ebp"));

  315. 

  316. 	&add	("eax",5);			# compare to modulus

  317. 	&adc	("ebx",0);

  318. 	&adc	("ecx",0);

  319. 	&adc	("edx",0);

  320. 	&adc	("esi",0);

  321. 	&shr	("esi",2);			# did it carry/borrow?

  322. 	&neg	("esi");			# do we choose hash-modulus?

  323. 

  324. 	&and	("eax","esi");

  325. 	&and	("ebx","esi");

  326. 	&and	("ecx","esi");

  327. 	&and	("edx","esi");

  328. 	&mov	(&DWP(4*0,"edi"),"eax");

  329. 	&mov	(&DWP(4*1,"edi"),"ebx");

  330. 	&mov	(&DWP(4*2,"edi"),"ecx");

  331. 	&mov	(&DWP(4*3,"edi"),"edx");

  332. 

  333. 	&not	("esi");			# or original hash value?

  334. 	&mov	("eax",&DWP(4*0,"ebp"));

  335. 	&mov	("ebx",&DWP(4*1,"ebp"));

  336. 	&mov	("ecx",&DWP(4*2,"ebp"));

  337. 	&mov	("edx",&DWP(4*3,"ebp"));

  338. 	&mov	("ebp",&wparam(2));

  339. 	&and	("eax","esi");

  340. 	&and	("ebx","esi");

  341. 	&and	("ecx","esi");

  342. 	&and	("edx","esi");

  343. 	&or	("eax",&DWP(4*0,"edi"));

  344. 	&or	("ebx",&DWP(4*1,"edi"));

  345. 	&or	("ecx",&DWP(4*2,"edi"));

  346. 	&or	("edx",&DWP(4*3,"edi"));

  347. 

  348. 	&add	("eax",&DWP(4*0,"ebp"));	# accumulate key

  349. 	&adc	("ebx",&DWP(4*1,"ebp"));

  350. 	&adc	("ecx",&DWP(4*2,"ebp"));

  351. 	&adc	("edx",&DWP(4*3,"ebp"));

  352. 

  353. 	&mov	(&DWP(4*0,"edi"),"eax");

  354. 	&mov	(&DWP(4*1,"edi"),"ebx");

  355. 	&mov	(&DWP(4*2,"edi"),"ecx");

  356. 	&mov	(&DWP(4*3,"edi"),"edx");

  357. &function_end("poly1305_emit");

  358. 

  359. if ($sse2) {

  360. ########################################################################

  361. # Layout of opaque area is following.

  362. #

  363. #	unsigned __int32 h[5];		# current hash value base 2^26

  364. #	unsigned __int32 is_base2_26;

  365. #	unsigned __int32 r[4];		# key value base 2^32

  366. #	unsigned __int32 pad[2];

  367. #	struct { unsigned __int32 r^4, r^3, r^2, r^1; } r[9];

  368. #

  369. # where r^n are base 2^26 digits of degrees of multiplier key. There are

  370. # 5 digits, but last four are interleaved with multiples of 5, totalling

  371. # in 9 elements: r0, r1, 5*r1, r2, 5*r2, r3, 5*r3, r4, 5*r4.

  372. 

  373. my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("xmm$_",(0..7));

  374. my $MASK=$T2;	# borrow and keep in mind

  375. 

  376. &align	(32);

  377. &function_begin_B("_poly1305_init_sse2");

  378. 	&movdqu		($D4,&QWP(4*6,"edi"));		# key base 2^32

  379. 	&lea		("edi",&DWP(16*3,"edi"));	# size optimization

  380. 	&mov		("ebp","esp");

  381. 	&sub		("esp",16*(9+5));

  382. 	&and		("esp",-16);

  383. 

  384. 	#&pand		($D4,&QWP(96,"ebx"));		# magic mask

  385. 	&movq		($MASK,&QWP(64,"ebx"));

  386. 

  387. 	&movdqa		($D0,$D4);

  388. 	&movdqa		($D1,$D4);

  389. 	&movdqa		($D2,$D4);

  390. 

  391. 	&pand		($D0,$MASK);			# -> base 2^26

  392. 	&psrlq		($D1,26);

  393. 	&psrldq		($D2,6);

  394. 	&pand		($D1,$MASK);

  395. 	&movdqa		($D3,$D2);

  396. 	&psrlq		($D2,4)

  397. 	&psrlq		($D3,30);

  398. 	&pand		($D2,$MASK);

  399. 	&pand		($D3,$MASK);

  400. 	&psrldq		($D4,13);

  401. 

  402. 	&lea		("edx",&DWP(16*9,"esp"));	# size optimization

  403. 	&mov		("ecx",2);

  404. &set_label("square");

  405. 	&movdqa		(&QWP(16*0,"esp"),$D0);

  406. 	&movdqa		(&QWP(16*1,"esp"),$D1);

  407. 	&movdqa		(&QWP(16*2,"esp"),$D2);

  408. 	&movdqa		(&QWP(16*3,"esp"),$D3);

  409. 	&movdqa		(&QWP(16*4,"esp"),$D4);

  410. 

  411. 	&movdqa		($T1,$D1);

  412. 	&movdqa		($T0,$D2);

  413. 	&pslld		($T1,2);

  414. 	&pslld		($T0,2);

  415. 	&paddd		($T1,$D1);			# *5

  416. 	&paddd		($T0,$D2);			# *5

  417. 	&movdqa		(&QWP(16*5,"esp"),$T1);

  418. 	&movdqa		(&QWP(16*6,"esp"),$T0);

  419. 	&movdqa		($T1,$D3);

  420. 	&movdqa		($T0,$D4);

  421. 	&pslld		($T1,2);

  422. 	&pslld		($T0,2);

  423. 	&paddd		($T1,$D3);			# *5

  424. 	&paddd		($T0,$D4);			# *5

  425. 	&movdqa		(&QWP(16*7,"esp"),$T1);

  426. 	&movdqa		(&QWP(16*8,"esp"),$T0);

  427. 

  428. 	&pshufd		($T1,$D0,0b01000100);

  429. 	&movdqa		($T0,$D1);

  430. 	&pshufd		($D1,$D1,0b01000100);

  431. 	&pshufd		($D2,$D2,0b01000100);

  432. 	&pshufd		($D3,$D3,0b01000100);

  433. 	&pshufd		($D4,$D4,0b01000100);

  434. 	&movdqa		(&QWP(16*0,"edx"),$T1);

  435. 	&movdqa		(&QWP(16*1,"edx"),$D1);

  436. 	&movdqa		(&QWP(16*2,"edx"),$D2);

  437. 	&movdqa		(&QWP(16*3,"edx"),$D3);

  438. 	&movdqa		(&QWP(16*4,"edx"),$D4);

  439. 

  440. 	################################################################

  441. 	# d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4

  442. 	# d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4

  443. 	# d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4

  444. 	# d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4

  445. 	# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4

  446. 

  447. 	&pmuludq	($D4,$D0);			# h4*r0

  448. 	&pmuludq	($D3,$D0);			# h3*r0

  449. 	&pmuludq	($D2,$D0);			# h2*r0

  450. 	&pmuludq	($D1,$D0);			# h1*r0

  451. 	&pmuludq	($D0,$T1);			# h0*r0

  452. 

  453. sub pmuladd {

  454. my $load = shift;

  455. my $base = shift; $base = "esp" if (!defined($base));

  456. 

  457. 	################################################################

  458. 	# As for choice to "rotate" $T0-$T2 in order to move paddq

  459. 	# past next multiplication. While it makes code harder to read

  460. 	# and doesn't have significant effect on most processors, it

  461. 	# makes a lot of difference on Atom, up to 30% improvement.

  462. 

  463. 	&movdqa		($T1,$T0);

  464. 	&pmuludq	($T0,&QWP(16*3,$base));		# r1*h3

  465. 	&movdqa		($T2,$T1);

  466. 	&pmuludq	($T1,&QWP(16*2,$base));		# r1*h2

  467. 	&paddq		($D4,$T0);

  468. 	&movdqa		($T0,$T2);

  469. 	&pmuludq	($T2,&QWP(16*1,$base));		# r1*h1

  470. 	&paddq		($D3,$T1);

  471. 	&$load		($T1,5);			# s1

  472. 	&pmuludq	($T0,&QWP(16*0,$base));		# r1*h0

  473. 	&paddq		($D2,$T2);

  474. 	&pmuludq	($T1,&QWP(16*4,$base));		# s1*h4

  475. 	 &$load		($T2,2);			# r2^n

  476. 	&paddq		($D1,$T0);

  477. 

  478. 	&movdqa		($T0,$T2);

  479. 	&pmuludq	($T2,&QWP(16*2,$base));		# r2*h2

  480. 	 &paddq		($D0,$T1);

  481. 	&movdqa		($T1,$T0);

  482. 	&pmuludq	($T0,&QWP(16*1,$base));		# r2*h1

  483. 	&paddq		($D4,$T2);

  484. 	&$load		($T2,6);			# s2^n

  485. 	&pmuludq	($T1,&QWP(16*0,$base));		# r2*h0

  486. 	&paddq		($D3,$T0);

  487. 	&movdqa		($T0,$T2);

  488. 	&pmuludq	($T2,&QWP(16*4,$base));		# s2*h4

  489. 	&paddq		($D2,$T1);

  490. 	&pmuludq	($T0,&QWP(16*3,$base));		# s2*h3

  491. 	 &$load		($T1,3);			# r3^n

  492. 	&paddq		($D1,$T2);

  493. 

  494. 	&movdqa		($T2,$T1);

  495. 	&pmuludq	($T1,&QWP(16*1,$base));		# r3*h1

  496. 	 &paddq		($D0,$T0);

  497. 	&$load		($T0,7);			# s3^n

  498. 	&pmuludq	($T2,&QWP(16*0,$base));		# r3*h0

  499. 	&paddq		($D4,$T1);

  500. 	&movdqa		($T1,$T0);

  501. 	&pmuludq	($T0,&QWP(16*4,$base));		# s3*h4

  502. 	&paddq		($D3,$T2);

  503. 	&movdqa		($T2,$T1);

  504. 	&pmuludq	($T1,&QWP(16*3,$base));		# s3*h3

  505. 	&paddq		($D2,$T0);

  506. 	&pmuludq	($T2,&QWP(16*2,$base));		# s3*h2

  507. 	 &$load		($T0,4);			# r4^n

  508. 	&paddq		($D1,$T1);

  509. 

  510. 	&$load		($T1,8);			# s4^n

  511. 	&pmuludq	($T0,&QWP(16*0,$base));		# r4*h0

  512. 	 &paddq		($D0,$T2);

  513. 	&movdqa		($T2,$T1);

  514. 	&pmuludq	($T1,&QWP(16*4,$base));		# s4*h4

  515. 	&paddq		($D4,$T0);

  516. 	&movdqa		($T0,$T2);

  517. 	&pmuludq	($T2,&QWP(16*1,$base));		# s4*h1

  518. 	&paddq		($D3,$T1);

  519. 	&movdqa		($T1,$T0);

  520. 	&pmuludq	($T0,&QWP(16*2,$base));		# s4*h2

  521. 	&paddq		($D0,$T2);

  522. 	&pmuludq	($T1,&QWP(16*3,$base));		# s4*h3

  523. 	 &movdqa	($MASK,&QWP(64,"ebx"));

  524. 	&paddq		($D1,$T0);

  525. 	&paddq		($D2,$T1);

  526. }

  527. 	&pmuladd	(sub {	my ($reg,$i)=@_;

  528. 				&movdqa ($reg,&QWP(16*$i,"esp"));

  529. 			     },"edx");

  530. 

  531. sub lazy_reduction {

  532. my $extra = shift;

  533. my $paddx = defined($extra) ? paddq : paddd;

  534. 

  535. 	################################################################

  536. 	# lazy reduction as discussed in "NEON crypto" by D.J. Bernstein

  537. 	# and P. Schwabe

  538. 

  539. 	 &movdqa	($T0,$D3);

  540. 	 &pand		($D3,$MASK);

  541. 	 &psrlq		($T0,26);

  542. 	 &$extra	()				if (defined($extra));

  543. 	 &paddq		($T0,$D4);			# h3 -> h4

  544. 	&movdqa		($T1,$D0);

  545. 	&pand		($D0,$MASK);

  546. 	&psrlq		($T1,26);

  547. 	 &movdqa	($D4,$T0);

  548. 	&paddq		($T1,$D1);			# h0 -> h1

  549. 	 &psrlq		($T0,26);

  550. 	 &pand		($D4,$MASK);

  551. 	&movdqa		($D1,$T1);

  552. 	&psrlq		($T1,26);

  553. 	 &paddd		($D0,$T0);			# favour paddd when

  554. 							# possible, because

  555. 							# paddq is "broken"

  556. 							# on Atom

  557. 	 &psllq		($T0,2);

  558. 	&paddq		($T1,$D2);			# h1 -> h2

  559. 	 &$paddx	($T0,$D0);			# h4 -> h0

  560. 	&pand		($D1,$MASK);

  561. 	&movdqa		($D2,$T1);

  562. 	&psrlq		($T1,26);

  563. 	&pand		($D2,$MASK);

  564. 	&paddd		($T1,$D3);			# h2 -> h3

  565. 	 &movdqa	($D0,$T0);

  566. 	 &psrlq		($T0,26);

  567. 	&movdqa		($D3,$T1);

  568. 	&psrlq		($T1,26);

  569. 	 &pand		($D0,$MASK);

  570. 	 &paddd		($D1,$T0);			# h0 -> h1

  571. 	&pand		($D3,$MASK);

  572. 	&paddd		($D4,$T1);			# h3 -> h4

  573. }

  574. 	&lazy_reduction	();

  575. 

  576. 	&dec		("ecx");

  577. 	&jz		(&label("square_break"));

  578. 

  579. 	&punpcklqdq	($D0,&QWP(16*0,"esp"));		# 0:r^1:0:r^2

  580. 	&punpcklqdq	($D1,&QWP(16*1,"esp"));

  581. 	&punpcklqdq	($D2,&QWP(16*2,"esp"));

  582. 	&punpcklqdq	($D3,&QWP(16*3,"esp"));

  583. 	&punpcklqdq	($D4,&QWP(16*4,"esp"));

  584. 	&jmp		(&label("square"));

  585. 

  586. &set_label("square_break");

  587. 	&psllq		($D0,32);			# -> r^3:0:r^4:0

  588. 	&psllq		($D1,32);

  589. 	&psllq		($D2,32);

  590. 	&psllq		($D3,32);

  591. 	&psllq		($D4,32);

  592. 	&por		($D0,&QWP(16*0,"esp"));		# r^3:r^1:r^4:r^2

  593. 	&por		($D1,&QWP(16*1,"esp"));

  594. 	&por		($D2,&QWP(16*2,"esp"));

  595. 	&por		($D3,&QWP(16*3,"esp"));

  596. 	&por		($D4,&QWP(16*4,"esp"));

  597. 

  598. 	&pshufd		($D0,$D0,0b10001101);		# -> r^1:r^2:r^3:r^4

  599. 	&pshufd		($D1,$D1,0b10001101);

  600. 	&pshufd		($D2,$D2,0b10001101);

  601. 	&pshufd		($D3,$D3,0b10001101);

  602. 	&pshufd		($D4,$D4,0b10001101);

  603. 

  604. 	&movdqu		(&QWP(16*0,"edi"),$D0);		# save the table

  605. 	&movdqu		(&QWP(16*1,"edi"),$D1);

  606. 	&movdqu		(&QWP(16*2,"edi"),$D2);

  607. 	&movdqu		(&QWP(16*3,"edi"),$D3);

  608. 	&movdqu		(&QWP(16*4,"edi"),$D4);

  609. 

  610. 	&movdqa		($T1,$D1);

  611. 	&movdqa		($T0,$D2);

  612. 	&pslld		($T1,2);

  613. 	&pslld		($T0,2);

  614. 	&paddd		($T1,$D1);			# *5

  615. 	&paddd		($T0,$D2);			# *5

  616. 	&movdqu		(&QWP(16*5,"edi"),$T1);

  617. 	&movdqu		(&QWP(16*6,"edi"),$T0);

  618. 	&movdqa		($T1,$D3);

  619. 	&movdqa		($T0,$D4);

  620. 	&pslld		($T1,2);

  621. 	&pslld		($T0,2);

  622. 	&paddd		($T1,$D3);			# *5

  623. 	&paddd		($T0,$D4);			# *5

  624. 	&movdqu		(&QWP(16*7,"edi"),$T1);

  625. 	&movdqu		(&QWP(16*8,"edi"),$T0);

  626. 

  627. 	&mov		("esp","ebp");

  628. 	&lea		("edi",&DWP(-16*3,"edi"));	# size de-optimization

  629. 	&ret		();

  630. &function_end_B("_poly1305_init_sse2");

  631. 

  632. &align	(32);

  633. &function_begin("_poly1305_blocks_sse2");

  634. 	&mov	("edi",&wparam(0));			# ctx

  635. 	&mov	("esi",&wparam(1));			# inp

  636. 	&mov	("ecx",&wparam(2));			# len

  637. 

  638. 	&mov	("eax",&DWP(4*5,"edi"));		# is_base2_26

  639. 	&and	("ecx",-16);

  640. 	&jz	(&label("nodata"));

  641. 	&cmp	("ecx",64);

  642. 	&jae	(&label("enter_sse2"));

  643. 	&test	("eax","eax");				# is_base2_26?

  644. 	&jz	(&label("enter_blocks"));

  645. 

  646. &set_label("enter_sse2",16);

  647. 	&call	(&label("pic_point"));

  648. &set_label("pic_point");

  649. 	&blindpop("ebx");

  650. 	&lea	("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));

  651. 

  652. 	&test	("eax","eax");				# is_base2_26?

  653. 	&jnz	(&label("base2_26"));

  654. 

  655. 	&call	("_poly1305_init_sse2");

  656. 

  657. 	################################################# base 2^32 -> base 2^26

  658. 	&mov	("eax",&DWP(0,"edi"));

  659. 	&mov	("ecx",&DWP(3,"edi"));

  660. 	&mov	("edx",&DWP(6,"edi"));

  661. 	&mov	("esi",&DWP(9,"edi"));

  662. 	&mov	("ebp",&DWP(13,"edi"));

  663. 	&mov	(&DWP(4*5,"edi"),1);			# is_base2_26

  664. 

  665. 	&shr	("ecx",2);

  666. 	&and	("eax",0x3ffffff);

  667. 	&shr	("edx",4);

  668. 	&and	("ecx",0x3ffffff);

  669. 	&shr	("esi",6);

  670. 	&and	("edx",0x3ffffff);

  671. 

  672. 	&movd	($D0,"eax");

  673. 	&movd	($D1,"ecx");

  674. 	&movd	($D2,"edx");

  675. 	&movd	($D3,"esi");

  676. 	&movd	($D4,"ebp");

  677. 

  678. 	&mov	("esi",&wparam(1));			# [reload] inp

  679. 	&mov	("ecx",&wparam(2));			# [reload] len

  680. 	&jmp	(&label("base2_32"));

  681. 

  682. &set_label("base2_26",16);

  683. 	&movd	($D0,&DWP(4*0,"edi"));			# load hash value

  684. 	&movd	($D1,&DWP(4*1,"edi"));

  685. 	&movd	($D2,&DWP(4*2,"edi"));

  686. 	&movd	($D3,&DWP(4*3,"edi"));

  687. 	&movd	($D4,&DWP(4*4,"edi"));

  688. 	&movdqa	($MASK,&QWP(64,"ebx"));

  689. 

  690. &set_label("base2_32");

  691. 	&mov	("eax",&wparam(3));			# padbit

  692. 	&mov	("ebp","esp");

  693. 

  694. 	&sub	("esp",16*(5+5+5+9+9));

  695. 	&and	("esp",-16);

  696. 

  697. 	&lea	("edi",&DWP(16*3,"edi"));		# size optimization

  698. 	&shl	("eax",24);				# padbit

  699. 

  700. 	&test	("ecx",31);

  701. 	&jz	(&label("even"));

  702. 

  703. 	################################################################

  704. 	# process single block, with SSE2, because it's still faster

  705. 	# even though half of result is discarded

  706. 

  707. 	&movdqu		($T1,&QWP(0,"esi"));		# input

  708. 	&lea		("esi",&DWP(16,"esi"));

  709. 

  710. 	&movdqa		($T0,$T1);			# -> base 2^26 ...

  711. 	&pand		($T1,$MASK);

  712. 	&paddd		($D0,$T1);			# ... and accumuate

  713. 

  714. 	&movdqa		($T1,$T0);

  715. 	&psrlq		($T0,26);

  716. 	&psrldq		($T1,6);

  717. 	&pand		($T0,$MASK);

  718. 	&paddd		($D1,$T0);

  719. 

  720. 	&movdqa		($T0,$T1);

  721. 	&psrlq		($T1,4);

  722. 	&pand		($T1,$MASK);

  723. 	&paddd		($D2,$T1);

  724. 

  725. 	&movdqa		($T1,$T0);

  726. 	&psrlq		($T0,30);

  727. 	&pand		($T0,$MASK);

  728. 	&psrldq		($T1,7);

  729. 	&paddd		($D3,$T0);

  730. 

  731. 	&movd		($T0,"eax");			# padbit

  732. 	&paddd		($D4,$T1);

  733. 	 &movd		($T1,&DWP(16*0+12,"edi"));	# r0

  734. 	&paddd		($D4,$T0);

  735. 

  736. 	&movdqa		(&QWP(16*0,"esp"),$D0);

  737. 	&movdqa		(&QWP(16*1,"esp"),$D1);

  738. 	&movdqa		(&QWP(16*2,"esp"),$D2);

  739. 	&movdqa		(&QWP(16*3,"esp"),$D3);

  740. 	&movdqa		(&QWP(16*4,"esp"),$D4);

  741. 

  742. 	################################################################

  743. 	# d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4

  744. 	# d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4

  745. 	# d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4

  746. 	# d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4

  747. 	# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4

  748. 

  749. 	&pmuludq	($D0,$T1);			# h4*r0

  750. 	&pmuludq	($D1,$T1);			# h3*r0

  751. 	&pmuludq	($D2,$T1);			# h2*r0

  752. 	 &movd		($T0,&DWP(16*1+12,"edi"));	# r1

  753. 	&pmuludq	($D3,$T1);			# h1*r0

  754. 	&pmuludq	($D4,$T1);			# h0*r0

  755. 

  756. 	&pmuladd	(sub {	my ($reg,$i)=@_;

  757. 				&movd ($reg,&DWP(16*$i+12,"edi"));

  758. 			     });

  759. 

  760. 	&lazy_reduction	();

  761. 

  762. 	&sub		("ecx",16);

  763. 	&jz		(&label("done"));

  764. 

  765. &set_label("even");

  766. 	&lea		("edx",&DWP(16*(5+5+5+9),"esp"));# size optimization

  767. 	&lea		("eax",&DWP(-16*2,"esi"));

  768. 	&sub		("ecx",64);

  769. 

  770. 	################################################################

  771. 	# expand and copy pre-calculated table to stack

  772. 

  773. 	&movdqu		($T0,&QWP(16*0,"edi"));		# r^1:r^2:r^3:r^4

  774. 	&pshufd		($T1,$T0,0b01000100);		# duplicate r^3:r^4

  775. 	&cmovb		("esi","eax");

  776. 	&pshufd		($T0,$T0,0b11101110);		# duplicate r^1:r^2

  777. 	&movdqa		(&QWP(16*0,"edx"),$T1);

  778. 	&lea		("eax",&DWP(16*10,"esp"));

  779. 	&movdqu		($T1,&QWP(16*1,"edi"));

  780. 	&movdqa		(&QWP(16*(0-9),"edx"),$T0);

  781. 	&pshufd		($T0,$T1,0b01000100);

  782. 	&pshufd		($T1,$T1,0b11101110);

  783. 	&movdqa		(&QWP(16*1,"edx"),$T0);

  784. 	&movdqu		($T0,&QWP(16*2,"edi"));

  785. 	&movdqa		(&QWP(16*(1-9),"edx"),$T1);

  786. 	&pshufd		($T1,$T0,0b01000100);

  787. 	&pshufd		($T0,$T0,0b11101110);

  788. 	&movdqa		(&QWP(16*2,"edx"),$T1);

  789. 	&movdqu		($T1,&QWP(16*3,"edi"));

  790. 	&movdqa		(&QWP(16*(2-9),"edx"),$T0);

  791. 	&pshufd		($T0,$T1,0b01000100);

  792. 	&pshufd		($T1,$T1,0b11101110);

  793. 	&movdqa		(&QWP(16*3,"edx"),$T0);

  794. 	&movdqu		($T0,&QWP(16*4,"edi"));

  795. 	&movdqa		(&QWP(16*(3-9),"edx"),$T1);

  796. 	&pshufd		($T1,$T0,0b01000100);

  797. 	&pshufd		($T0,$T0,0b11101110);

  798. 	&movdqa		(&QWP(16*4,"edx"),$T1);

  799. 	&movdqu		($T1,&QWP(16*5,"edi"));

  800. 	&movdqa		(&QWP(16*(4-9),"edx"),$T0);

  801. 	&pshufd		($T0,$T1,0b01000100);

  802. 	&pshufd		($T1,$T1,0b11101110);

  803. 	&movdqa		(&QWP(16*5,"edx"),$T0);

  804. 	&movdqu		($T0,&QWP(16*6,"edi"));

  805. 	&movdqa		(&QWP(16*(5-9),"edx"),$T1);

  806. 	&pshufd		($T1,$T0,0b01000100);

  807. 	&pshufd		($T0,$T0,0b11101110);

  808. 	&movdqa		(&QWP(16*6,"edx"),$T1);

  809. 	&movdqu		($T1,&QWP(16*7,"edi"));

  810. 	&movdqa		(&QWP(16*(6-9),"edx"),$T0);

  811. 	&pshufd		($T0,$T1,0b01000100);

  812. 	&pshufd		($T1,$T1,0b11101110);

  813. 	&movdqa		(&QWP(16*7,"edx"),$T0);

  814. 	&movdqu		($T0,&QWP(16*8,"edi"));

  815. 	&movdqa		(&QWP(16*(7-9),"edx"),$T1);

  816. 	&pshufd		($T1,$T0,0b01000100);

  817. 	&pshufd		($T0,$T0,0b11101110);

  818. 	&movdqa		(&QWP(16*8,"edx"),$T1);

  819. 	&movdqa		(&QWP(16*(8-9),"edx"),$T0);

  820. 

  821. sub load_input {

  822. my ($inpbase,$offbase)=@_;

  823. 

  824. 	&movdqu		($T0,&QWP($inpbase+0,"esi"));	# load input

  825. 	&movdqu		($T1,&QWP($inpbase+16,"esi"));

  826. 	&lea		("esi",&DWP(16*2,"esi"));

  827. 

  828. 	&movdqa		(&QWP($offbase+16*2,"esp"),$D2);

  829. 	&movdqa		(&QWP($offbase+16*3,"esp"),$D3);

  830. 	&movdqa		(&QWP($offbase+16*4,"esp"),$D4);

  831. 

  832. 	&movdqa		($D2,$T0);			# splat input

  833. 	&movdqa		($D3,$T1);

  834. 	&psrldq		($D2,6);

  835. 	&psrldq		($D3,6);

  836. 	&movdqa		($D4,$T0);

  837. 	&punpcklqdq	($D2,$D3);			# 2:3

  838. 	&punpckhqdq	($D4,$T1);			# 4

  839. 	&punpcklqdq	($T0,$T1);			# 0:1

  840. 

  841. 	&movdqa		($D3,$D2);

  842. 	&psrlq		($D2,4);

  843. 	&psrlq		($D3,30);

  844. 	&movdqa		($T1,$T0);

  845. 	&psrlq		($D4,40);			# 4

  846. 	&psrlq		($T1,26);

  847. 	&pand		($T0,$MASK);			# 0

  848. 	&pand		($T1,$MASK);			# 1

  849. 	&pand		($D2,$MASK);			# 2

  850. 	&pand		($D3,$MASK);			# 3

  851. 	&por		($D4,&QWP(0,"ebx"));		# padbit, yes, always

  852. 

  853. 	&movdqa		(&QWP($offbase+16*0,"esp"),$D0)	if ($offbase);

  854. 	&movdqa		(&QWP($offbase+16*1,"esp"),$D1)	if ($offbase);

  855. }

  856. 	&load_input	(16*2,16*5);

  857. 

  858. 	&jbe		(&label("skip_loop"));

  859. 	&jmp		(&label("loop"));

  860. 

  861. &set_label("loop",32);

  862. 	################################################################

  863. 	# ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2

  864. 	# ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^3+inp[7]*r

  865. 	#   \___________________/

  866. 	# ((inp[0]*r^4+inp[2]*r^2+inp[4])*r^4+inp[6]*r^2+inp[8])*r^2

  867. 	# ((inp[1]*r^4+inp[3]*r^2+inp[5])*r^4+inp[7]*r^2+inp[9])*r

  868. 	#   \___________________/ \____________________/

  869. 	################################################################

  870. 

  871. 	&movdqa		($T2,&QWP(16*(0-9),"edx"));	# r0^2

  872. 	&movdqa		(&QWP(16*1,"eax"),$T1);

  873. 	&movdqa		(&QWP(16*2,"eax"),$D2);

  874. 	&movdqa		(&QWP(16*3,"eax"),$D3);

  875. 	&movdqa		(&QWP(16*4,"eax"),$D4);

  876. 

  877. 	################################################################

  878. 	# d4 = h4*r0 + h0*r4   + h1*r3   + h2*r2   + h3*r1

  879. 	# d3 = h3*r0 + h0*r3   + h1*r2   + h2*r1   + h4*5*r4

  880. 	# d2 = h2*r0 + h0*r2   + h1*r1   + h3*5*r4 + h4*5*r3

  881. 	# d1 = h1*r0 + h0*r1   + h2*5*r4 + h3*5*r3 + h4*5*r2

  882. 	# d0 = h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1

  883. 

  884. 	&movdqa		($D1,$T0);

  885. 	&pmuludq	($T0,$T2);			# h0*r0

  886. 	&movdqa		($D0,$T1);

  887. 	&pmuludq	($T1,$T2);			# h1*r0

  888. 	&pmuludq	($D2,$T2);			# h2*r0

  889. 	&pmuludq	($D3,$T2);			# h3*r0

  890. 	&pmuludq	($D4,$T2);			# h4*r0

  891. 

  892. sub pmuladd_alt {

  893. my $addr = shift;

  894. 

  895. 	&pmuludq	($D0,&$addr(8));		# h1*s4

  896. 	&movdqa		($T2,$D1);

  897. 	&pmuludq	($D1,&$addr(1));		# h0*r1

  898. 	&paddq		($D0,$T0);

  899. 	&movdqa		($T0,$T2);

  900. 	&pmuludq	($T2,&$addr(2));		# h0*r2

  901. 	&paddq		($D1,$T1);

  902. 	&movdqa		($T1,$T0);

  903. 	&pmuludq	($T0,&$addr(3));		# h0*r3

  904. 	&paddq		($D2,$T2);

  905. 	 &movdqa	($T2,&QWP(16*1,"eax"));		# pull h1

  906. 	&pmuludq	($T1,&$addr(4));		# h0*r4

  907. 	&paddq		($D3,$T0);

  908. 

  909. 	&movdqa		($T0,$T2);

  910. 	&pmuludq	($T2,&$addr(1));		# h1*r1

  911. 	 &paddq		($D4,$T1);

  912. 	&movdqa		($T1,$T0);

  913. 	&pmuludq	($T0,&$addr(2));		# h1*r2

  914. 	&paddq		($D2,$T2);

  915. 	&movdqa		($T2,&QWP(16*2,"eax"));		# pull h2

  916. 	&pmuludq	($T1,&$addr(3));		# h1*r3

  917. 	&paddq		($D3,$T0);

  918. 	&movdqa		($T0,$T2);

  919. 	&pmuludq	($T2,&$addr(7));		# h2*s3

  920. 	&paddq		($D4,$T1);

  921. 	&movdqa		($T1,$T0);

  922. 	&pmuludq	($T0,&$addr(8));		# h2*s4

  923. 	&paddq		($D0,$T2);

  924. 

  925. 	&movdqa		($T2,$T1);

  926. 	&pmuludq	($T1,&$addr(1));		# h2*r1

  927. 	 &paddq		($D1,$T0);

  928. 	&movdqa		($T0,&QWP(16*3,"eax"));		# pull h3

  929. 	&pmuludq	($T2,&$addr(2));		# h2*r2

  930. 	&paddq		($D3,$T1);

  931. 	&movdqa		($T1,$T0);

  932. 	&pmuludq	($T0,&$addr(6));		# h3*s2

  933. 	&paddq		($D4,$T2);

  934. 	&movdqa		($T2,$T1);

  935. 	&pmuludq	($T1,&$addr(7));		# h3*s3

  936. 	&paddq		($D0,$T0);

  937. 	&movdqa		($T0,$T2);

  938. 	&pmuludq	($T2,&$addr(8));		# h3*s4

  939. 	&paddq		($D1,$T1);

  940. 

  941. 	&movdqa		($T1,&QWP(16*4,"eax"));		# pull h4

  942. 	&pmuludq	($T0,&$addr(1));		# h3*r1

  943. 	 &paddq		($D2,$T2);

  944. 	&movdqa		($T2,$T1);

  945. 	&pmuludq	($T1,&$addr(8));		# h4*s4

  946. 	&paddq		($D4,$T0);

  947. 	&movdqa		($T0,$T2);

  948. 	&pmuludq	($T2,&$addr(5));		# h4*s1

  949. 	&paddq		($D3,$T1);

  950. 	&movdqa		($T1,$T0);

  951. 	&pmuludq	($T0,&$addr(6));		# h4*s2

  952. 	&paddq		($D0,$T2);

  953. 	 &movdqa	($MASK,&QWP(64,"ebx"));

  954. 	&pmuludq	($T1,&$addr(7));		# h4*s3

  955. 	&paddq		($D1,$T0);

  956. 	&paddq		($D2,$T1);

  957. }

  958. 	&pmuladd_alt	(sub {	my $i=shift; &QWP(16*($i-9),"edx");	});

  959. 

  960. 	&load_input	(-16*2,0);

  961. 	&lea		("eax",&DWP(-16*2,"esi"));

  962. 	&sub		("ecx",64);

  963. 

  964. 	&paddd		($T0,&QWP(16*(5+0),"esp"));	# add hash value

  965. 	&paddd		($T1,&QWP(16*(5+1),"esp"));

  966. 	&paddd		($D2,&QWP(16*(5+2),"esp"));

  967. 	&paddd		($D3,&QWP(16*(5+3),"esp"));

  968. 	&paddd		($D4,&QWP(16*(5+4),"esp"));

  969. 

  970. 	&cmovb		("esi","eax");

  971. 	&lea		("eax",&DWP(16*10,"esp"));

  972. 

  973. 	&movdqa		($T2,&QWP(16*0,"edx"));		# r0^4

  974. 	&movdqa		(&QWP(16*1,"esp"),$D1);

  975. 	&movdqa		(&QWP(16*1,"eax"),$T1);

  976. 	&movdqa		(&QWP(16*2,"eax"),$D2);

  977. 	&movdqa		(&QWP(16*3,"eax"),$D3);

  978. 	&movdqa		(&QWP(16*4,"eax"),$D4);

  979. 

  980. 	################################################################

  981. 	# d4 += h4*r0 + h0*r4   + h1*r3   + h2*r2   + h3*r1

  982. 	# d3 += h3*r0 + h0*r3   + h1*r2   + h2*r1   + h4*5*r4

  983. 	# d2 += h2*r0 + h0*r2   + h1*r1   + h3*5*r4 + h4*5*r3

  984. 	# d1 += h1*r0 + h0*r1   + h2*5*r4 + h3*5*r3 + h4*5*r2

  985. 	# d0 += h0*r0 + h1*5*r4 + h2*5*r3 + h3*5*r2 + h4*5*r1

  986. 

  987. 	&movdqa		($D1,$T0);

  988. 	&pmuludq	($T0,$T2);			# h0*r0

  989. 	&paddq		($T0,$D0);

  990. 	&movdqa		($D0,$T1);

  991. 	&pmuludq	($T1,$T2);			# h1*r0

  992. 	&pmuludq	($D2,$T2);			# h2*r0

  993. 	&pmuludq	($D3,$T2);			# h3*r0

  994. 	&pmuludq	($D4,$T2);			# h4*r0

  995. 

  996. 	&paddq		($T1,&QWP(16*1,"esp"));

  997. 	&paddq		($D2,&QWP(16*2,"esp"));

  998. 	&paddq		($D3,&QWP(16*3,"esp"));

  999. 	&paddq		($D4,&QWP(16*4,"esp"));

  1000. 

  1001. 	&pmuladd_alt	(sub {	my $i=shift; &QWP(16*$i,"edx");	});

  1002. 

  1003. 	&lazy_reduction	();

  1004. 

  1005. 	&load_input	(16*2,16*5);

  1006. 

  1007. 	&ja		(&label("loop"));

  1008. 

  1009. &set_label("skip_loop");

  1010. 	################################################################

  1011. 	# multiply (inp[0:1]+hash) or inp[2:3] by r^2:r^1

  1012. 

  1013. 	 &pshufd	($T2,&QWP(16*(0-9),"edx"),0x10);# r0^n

  1014. 	&add		("ecx",32);

  1015. 	&jnz		(&label("long_tail"));

  1016. 

  1017. 	&paddd		($T0,$D0);			# add hash value

  1018. 	&paddd		($T1,$D1);

  1019. 	&paddd		($D2,&QWP(16*7,"esp"));

  1020. 	&paddd		($D3,&QWP(16*8,"esp"));

  1021. 	&paddd		($D4,&QWP(16*9,"esp"));

  1022. 

  1023. &set_label("long_tail");

  1024. 

  1025. 	&movdqa		(&QWP(16*0,"eax"),$T0);

  1026. 	&movdqa		(&QWP(16*1,"eax"),$T1);

  1027. 	&movdqa		(&QWP(16*2,"eax"),$D2);

  1028. 	&movdqa		(&QWP(16*3,"eax"),$D3);

  1029. 	&movdqa		(&QWP(16*4,"eax"),$D4);

  1030. 

  1031. 	################################################################

  1032. 	# d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4

  1033. 	# d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4

  1034. 	# d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4

  1035. 	# d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4

  1036. 	# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4

  1037. 

  1038. 	&pmuludq	($T0,$T2);			# h0*r0

  1039. 	&pmuludq	($T1,$T2);			# h1*r0

  1040. 	&pmuludq	($D2,$T2);			# h2*r0

  1041. 	&movdqa		($D0,$T0);

  1042. 	 &pshufd	($T0,&QWP(16*(1-9),"edx"),0x10);# r1^n

  1043. 	&pmuludq	($D3,$T2);			# h3*r0

  1044. 	&movdqa		($D1,$T1);

  1045. 	&pmuludq	($D4,$T2);			# h4*r0

  1046. 

  1047. 	&pmuladd	(sub {	my ($reg,$i)=@_;

  1048. 				&pshufd ($reg,&QWP(16*($i-9),"edx"),0x10);

  1049. 			     },"eax");

  1050. 

  1051. 	&jz		(&label("short_tail"));

  1052. 

  1053. 	&load_input	(-16*2,0);

  1054. 

  1055. 	 &pshufd	($T2,&QWP(16*0,"edx"),0x10);	# r0^n

  1056. 	&paddd		($T0,&QWP(16*5,"esp"));		# add hash value

  1057. 	&paddd		($T1,&QWP(16*6,"esp"));

  1058. 	&paddd		($D2,&QWP(16*7,"esp"));

  1059. 	&paddd		($D3,&QWP(16*8,"esp"));

  1060. 	&paddd		($D4,&QWP(16*9,"esp"));

  1061. 

  1062. 	################################################################

  1063. 	# multiply inp[0:1] by r^4:r^3 and accumulate

  1064. 

  1065. 	&movdqa		(&QWP(16*0,"esp"),$T0);

  1066. 	&pmuludq	($T0,$T2);			# h0*r0

  1067. 	&movdqa		(&QWP(16*1,"esp"),$T1);

  1068. 	&pmuludq	($T1,$T2);			# h1*r0

  1069. 	&paddq		($D0,$T0);

  1070. 	&movdqa		($T0,$D2);

  1071. 	&pmuludq	($D2,$T2);			# h2*r0

  1072. 	&paddq		($D1,$T1);

  1073. 	&movdqa		($T1,$D3);

  1074. 	&pmuludq	($D3,$T2);			# h3*r0

  1075. 	&paddq		($D2,&QWP(16*2,"esp"));

  1076. 	&movdqa		(&QWP(16*2,"esp"),$T0);

  1077. 	 &pshufd	($T0,&QWP(16*1,"edx"),0x10);	# r1^n

  1078. 	&paddq		($D3,&QWP(16*3,"esp"));

  1079. 	&movdqa		(&QWP(16*3,"esp"),$T1);

  1080. 	&movdqa		($T1,$D4);

  1081. 	&pmuludq	($D4,$T2);			# h4*r0

  1082. 	&paddq		($D4,&QWP(16*4,"esp"));

  1083. 	&movdqa		(&QWP(16*4,"esp"),$T1);

  1084. 

  1085. 	&pmuladd	(sub {	my ($reg,$i)=@_;

  1086. 				&pshufd ($reg,&QWP(16*$i,"edx"),0x10);

  1087. 			     });

  1088. 

  1089. &set_label("short_tail");

  1090. 

  1091. 	################################################################

  1092. 	# horizontal addition

  1093. 

  1094. 	&pshufd		($T1,$D4,0b01001110);

  1095. 	&pshufd		($T0,$D3,0b01001110);

  1096. 	&paddq		($D4,$T1);

  1097. 	&paddq		($D3,$T0);

  1098. 	&pshufd		($T1,$D0,0b01001110);

  1099. 	&pshufd		($T0,$D1,0b01001110);

  1100. 	&paddq		($D0,$T1);

  1101. 	&paddq		($D1,$T0);

  1102. 	&pshufd		($T1,$D2,0b01001110);

  1103. 	#&paddq		($D2,$T1);

  1104. 

  1105. 	&lazy_reduction	(sub { &paddq ($D2,$T1) });

  1106. 

  1107. &set_label("done");

  1108. 	&movd		(&DWP(-16*3+4*0,"edi"),$D0);	# store hash value

  1109. 	&movd		(&DWP(-16*3+4*1,"edi"),$D1);

  1110. 	&movd		(&DWP(-16*3+4*2,"edi"),$D2);

  1111. 	&movd		(&DWP(-16*3+4*3,"edi"),$D3);

  1112. 	&movd		(&DWP(-16*3+4*4,"edi"),$D4);

  1113. 	&mov	("esp","ebp");

  1114. &set_label("nodata");

  1115. &function_end("_poly1305_blocks_sse2");

  1116. 

  1117. &align	(32);

  1118. &function_begin("_poly1305_emit_sse2");

  1119. 	&mov	("ebp",&wparam(0));		# context

  1120. 

  1121. 	&cmp	(&DWP(4*5,"ebp"),0);		# is_base2_26?

  1122. 	&je	(&label("enter_emit"));

  1123. 

  1124. 	&mov	("eax",&DWP(4*0,"ebp"));	# load hash value

  1125. 	&mov	("edi",&DWP(4*1,"ebp"));

  1126. 	&mov	("ecx",&DWP(4*2,"ebp"));

  1127. 	&mov	("edx",&DWP(4*3,"ebp"));

  1128. 	&mov	("esi",&DWP(4*4,"ebp"));

  1129. 

  1130. 	&mov	("ebx","edi");			# base 2^26 -> base 2^32

  1131. 	&shl	("edi",26);

  1132. 	&shr	("ebx",6);

  1133. 	&add	("eax","edi");

  1134. 	&mov	("edi","ecx");

  1135. 	&adc	("ebx",0);

  1136. 

  1137. 	&shl	("edi",20);

  1138. 	&shr	("ecx",12);

  1139. 	&add	("ebx","edi");

  1140. 	&mov	("edi","edx");

  1141. 	&adc	("ecx",0);

  1142. 

  1143. 	&shl	("edi",14);

  1144. 	&shr	("edx",18);

  1145. 	&add	("ecx","edi");

  1146. 	&mov	("edi","esi");

  1147. 	&adc	("edx",0);

  1148. 

  1149. 	&shl	("edi",8);

  1150. 	&shr	("esi",24);

  1151. 	&add	("edx","edi");

  1152. 	&adc	("esi",0);			# can be partially reduced

  1153. 

  1154. 	&mov	("edi","esi");			# final reduction

  1155. 	&and	("esi",3);

  1156. 	&shr	("edi",2);

  1157. 	&lea	("ebp",&DWP(0,"edi","edi",4));	# *5

  1158. 	 &mov	("edi",&wparam(1));		# output

  1159. 	add	("eax","ebp");

  1160. 	 &mov	("ebp",&wparam(2));		# key

  1161. 	adc	("ebx",0);

  1162. 	adc	("ecx",0);

  1163. 	adc	("edx",0);

  1164. 

  1165. 	&movd	($D0,"eax");			# offload original hash value

  1166. 	&add	("eax",5);			# compare to modulus

  1167. 	&movd	($D1,"ebx");

  1168. 	&adc	("ebx",0);

  1169. 	&movd	($D2,"ecx");

  1170. 	&adc	("ecx",0);

  1171. 	&movd	($D3,"edx");

  1172. 	&adc	("edx",0);

  1173. 	&adc	("esi",0);

  1174. 	&shr	("esi",2);			# did it carry/borrow?

  1175. 

  1176. 	&neg	("esi");			# do we choose (hash-modulus) ...

  1177. 	&and	("eax","esi");

  1178. 	&and	("ebx","esi");

  1179. 	&and	("ecx","esi");

  1180. 	&and	("edx","esi");

  1181. 	&mov	(&DWP(4*0,"edi"),"eax");

  1182. 	&movd	("eax",$D0);

  1183. 	&mov	(&DWP(4*1,"edi"),"ebx");

  1184. 	&movd	("ebx",$D1);

  1185. 	&mov	(&DWP(4*2,"edi"),"ecx");

  1186. 	&movd	("ecx",$D2);

  1187. 	&mov	(&DWP(4*3,"edi"),"edx");

  1188. 	&movd	("edx",$D3);

  1189. 

  1190. 	&not	("esi");			# ... or original hash value?

  1191. 	&and	("eax","esi");

  1192. 	&and	("ebx","esi");

  1193. 	&or	("eax",&DWP(4*0,"edi"));

  1194. 	&and	("ecx","esi");

  1195. 	&or	("ebx",&DWP(4*1,"edi"));

  1196. 	&and	("edx","esi");

  1197. 	&or	("ecx",&DWP(4*2,"edi"));

  1198. 	&or	("edx",&DWP(4*3,"edi"));

  1199. 

  1200. 	&add	("eax",&DWP(4*0,"ebp"));	# accumulate key

  1201. 	&adc	("ebx",&DWP(4*1,"ebp"));

  1202. 	&mov	(&DWP(4*0,"edi"),"eax");

  1203. 	&adc	("ecx",&DWP(4*2,"ebp"));

  1204. 	&mov	(&DWP(4*1,"edi"),"ebx");

  1205. 	&adc	("edx",&DWP(4*3,"ebp"));

  1206. 	&mov	(&DWP(4*2,"edi"),"ecx");

  1207. 	&mov	(&DWP(4*3,"edi"),"edx");

  1208. &function_end("_poly1305_emit_sse2");

  1209. 

  1210. if ($avx>1) {

  1211. ########################################################################

  1212. # Note that poly1305_init_avx2 operates on %xmm, I could have used

  1213. # poly1305_init_sse2...

  1214. 

  1215. &align	(32);

  1216. &function_begin_B("_poly1305_init_avx2");

  1217. 	&vmovdqu	($D4,&QWP(4*6,"edi"));		# key base 2^32

  1218. 	&lea		("edi",&DWP(16*3,"edi"));	# size optimization

  1219. 	&mov		("ebp","esp");

  1220. 	&sub		("esp",16*(9+5));

  1221. 	&and		("esp",-16);

  1222. 

  1223. 	#&vpand		($D4,$D4,&QWP(96,"ebx"));	# magic mask

  1224. 	&vmovdqa	($MASK,&QWP(64,"ebx"));

  1225. 

  1226. 	&vpand		($D0,$D4,$MASK);		# -> base 2^26

  1227. 	&vpsrlq		($D1,$D4,26);

  1228. 	&vpsrldq	($D3,$D4,6);

  1229. 	&vpand		($D1,$D1,$MASK);

  1230. 	&vpsrlq		($D2,$D3,4)

  1231. 	&vpsrlq		($D3,$D3,30);

  1232. 	&vpand		($D2,$D2,$MASK);

  1233. 	&vpand		($D3,$D3,$MASK);

  1234. 	&vpsrldq	($D4,$D4,13);

  1235. 

  1236. 	&lea		("edx",&DWP(16*9,"esp"));	# size optimization

  1237. 	&mov		("ecx",2);

  1238. &set_label("square");

  1239. 	&vmovdqa	(&QWP(16*0,"esp"),$D0);

  1240. 	&vmovdqa	(&QWP(16*1,"esp"),$D1);

  1241. 	&vmovdqa	(&QWP(16*2,"esp"),$D2);

  1242. 	&vmovdqa	(&QWP(16*3,"esp"),$D3);

  1243. 	&vmovdqa	(&QWP(16*4,"esp"),$D4);

  1244. 

  1245. 	&vpslld		($T1,$D1,2);

  1246. 	&vpslld		($T0,$D2,2);

  1247. 	&vpaddd		($T1,$T1,$D1);			# *5

  1248. 	&vpaddd		($T0,$T0,$D2);			# *5

  1249. 	&vmovdqa	(&QWP(16*5,"esp"),$T1);

  1250. 	&vmovdqa	(&QWP(16*6,"esp"),$T0);

  1251. 	&vpslld		($T1,$D3,2);

  1252. 	&vpslld		($T0,$D4,2);

  1253. 	&vpaddd		($T1,$T1,$D3);			# *5

  1254. 	&vpaddd		($T0,$T0,$D4);			# *5

  1255. 	&vmovdqa	(&QWP(16*7,"esp"),$T1);

  1256. 	&vmovdqa	(&QWP(16*8,"esp"),$T0);

  1257. 

  1258. 	&vpshufd	($T0,$D0,0b01000100);

  1259. 	&vmovdqa	($T1,$D1);

  1260. 	&vpshufd	($D1,$D1,0b01000100);

  1261. 	&vpshufd	($D2,$D2,0b01000100);

  1262. 	&vpshufd	($D3,$D3,0b01000100);

  1263. 	&vpshufd	($D4,$D4,0b01000100);

  1264. 	&vmovdqa	(&QWP(16*0,"edx"),$T0);

  1265. 	&vmovdqa	(&QWP(16*1,"edx"),$D1);

  1266. 	&vmovdqa	(&QWP(16*2,"edx"),$D2);

  1267. 	&vmovdqa	(&QWP(16*3,"edx"),$D3);

  1268. 	&vmovdqa	(&QWP(16*4,"edx"),$D4);

  1269. 

  1270. 	################################################################

  1271. 	# d4 = h4*r0 + h3*r1   + h2*r2   + h1*r3   + h0*r4

  1272. 	# d3 = h3*r0 + h2*r1   + h1*r2   + h0*r3   + h4*5*r4

  1273. 	# d2 = h2*r0 + h1*r1   + h0*r2   + h4*5*r3 + h3*5*r4

  1274. 	# d1 = h1*r0 + h0*r1   + h4*5*r2 + h3*5*r3 + h2*5*r4

  1275. 	# d0 = h0*r0 + h4*5*r1 + h3*5*r2 + h2*5*r3 + h1*5*r4

  1276. 

  1277. 	&vpmuludq	($D4,$D4,$D0);			# h4*r0

  1278. 	&vpmuludq	($D3,$D3,$D0);			# h3*r0

  1279. 	&vpmuludq	($D2,$D2,$D0);			# h2*r0

  1280. 	&vpmuludq	($D1,$D1,$D0);			# h1*r0

  1281. 	&vpmuludq	($D0,$T0,$D0);			# h0*r0

  1282. 

  1283. 	&vpmuludq	($T0,$T1,&QWP(16*3,"edx"));	# r1*h3

  1284. 	&vpaddq		($D4,$D4,$T0);

  1285. 	&vpmuludq	($T2,$T1,&QWP(16*2,"edx"));	# r1*h2

  1286. 	&vpaddq		($D3,$D3,$T2);

  1287. 	&vpmuludq	($T0,$T1,&QWP(16*1,"edx"));	# r1*h1

  1288. 	&vpaddq		($D2,$D2,$T0);

  1289. 	&vmovdqa	($T2,&QWP(16*5,"esp"));		# s1

  1290. 	&vpmuludq	($T1,$T1,&QWP(16*0,"edx"));	# r1*h0

  1291. 	&vpaddq		($D1,$D1,$T1);

  1292. 	 &vmovdqa	($T0,&QWP(16*2,"esp"));		# r2

  1293. 	&vpmuludq	($T2,$T2,&QWP(16*4,"edx"));	# s1*h4

  1294. 	&vpaddq		($D0,$D0,$T2);

  1295. 

  1296. 	&vpmuludq	($T1,$T0,&QWP(16*2,"edx"));	# r2*h2

  1297. 	&vpaddq		($D4,$D4,$T1);

  1298. 	&vpmuludq	($T2,$T0,&QWP(16*1,"edx"));	# r2*h1

  1299. 	&vpaddq		($D3,$D3,$T2);

  1300. 	&vmovdqa	($T1,&QWP(16*6,"esp"));		# s2

  1301. 	&vpmuludq	($T0,$T0,&QWP(16*0,"edx"));	# r2*h0

  1302. 	&vpaddq		($D2,$D2,$T0);

  1303. 	&vpmuludq	($T2,$T1,&QWP(16*4,"edx"));	# s2*h4

  1304. 	&vpaddq		($D1,$D1,$T2);

  1305. 	 &vmovdqa	($T0,&QWP(16*3,"esp"));		# r3

  1306. 	&vpmuludq	($T1,$T1,&QWP(16*3,"edx"));	# s2*h3

  1307. 	&vpaddq		($D0,$D0,$T1);

  1308. 

  1309. 	&vpmuludq	($T2,$T0,&QWP(16*1,"edx"));	# r3*h1

  1310. 	&vpaddq		($D4,$D4,$T2);

  1311. 	&vmovdqa	($T1,&QWP(16*7,"esp"));		# s3

  1312. 	&vpmuludq	($T0,$T0,&QWP(16*0,"edx"));	# r3*h0

  1313. 	&vpaddq		($D3,$D3,$T0);

  1314. 	&vpmuludq	($T2,$T1,&QWP(16*4,"edx"));	# s3*h4

  1315. 	&vpaddq		($D2,$D2,$T2);

  1316. 	&vpmuludq	($T0,$T1,&QWP(16*3,"edx"));	# s3*h3

  1317. 	&vpaddq		($D1,$D1,$T0);

  1318. 	 &vmovdqa	($T2,&QWP(16*4,"esp"));		# r4

  1319. 	&vpmuludq	($T1,$T1,&QWP(16*2,"edx"));	# s3*h2

  1320. 	&vpaddq		($D0,$D0,$T1);

  1321. 

  1322. 	&vmovdqa	($T0,&QWP(16*8,"esp"));		# s4

  1323. 	&vpmuludq	($T2,$T2,&QWP(16*0,"edx"));	# r4*h0

  1324. 	&vpaddq		($D4,$D4,$T2);

  1325. 	&vpmuludq	($T1,$T0,&QWP(16*4,"edx"));	# s4*h4

  1326. 	&vpaddq		($D3,$D3,$T1);

  1327. 	&vpmuludq	($T2,$T0,&QWP(16*1,"edx"));	# s4*h1

  1328. 	&vpaddq		($D0,$D0,$T2);

  1329. 	&vpmuludq	($T1,$T0,&QWP(16*2,"edx"));	# s4*h2

  1330. 	&vpaddq		($D1,$D1,$T1);

  1331. 	 &vmovdqa	($MASK,&QWP(64,"ebx"));

  1332. 	&vpmuludq	($T0,$T0,&QWP(16*3,"edx"));	# s4*h3

  1333. 	&vpaddq		($D2,$D2,$T0);

  1334. 

  1335. 	################################################################

  1336. 	# lazy reduction

  1337. 	 &vpsrlq	($T0,$D3,26);

  1338. 	 &vpand		($D3,$D3,$MASK);

  1339. 	&vpsrlq		($T1,$D0,26);

  1340. 	&vpand		($D0,$D0,$MASK);

  1341. 	 &vpaddq	($D4,$D4,$T0);			# h3 -> h4

  1342. 	&vpaddq		($D1,$D1,$T1);			# h0 -> h1

  1343. 	 &vpsrlq	($T0,$D4,26);

  1344. 	 &vpand		($D4,$D4,$MASK);

  1345. 	&vpsrlq		($T1,$D1,26);

  1346. 	&vpand		($D1,$D1,$MASK);

  1347. 	&vpaddq		($D2,$D2,$T1);			# h1 -> h2

  1348. 	 &vpaddd	($D0,$D0,$T0);

  1349. 	 &vpsllq	($T0,$T0,2);

  1350. 	&vpsrlq		($T1,$D2,26);

  1351. 	&vpand		($D2,$D2,$MASK);

  1352. 	 &vpaddd	($D0,$D0,$T0);			# h4 -> h0

  1353. 	&vpaddd		($D3,$D3,$T1);			# h2 -> h3

  1354. 	&vpsrlq		($T1,$D3,26);

  1355. 	 &vpsrlq	($T0,$D0,26);

  1356. 	 &vpand		($D0,$D0,$MASK);

  1357. 	&vpand		($D3,$D3,$MASK);

  1358. 	 &vpaddd	($D1,$D1,$T0);			# h0 -> h1

  1359. 	&vpaddd		($D4,$D4,$T1);			# h3 -> h4

  1360. 

  1361. 	&dec		("ecx");

  1362. 	&jz		(&label("square_break"));

  1363. 

  1364. 	&vpunpcklqdq	($D0,$D0,&QWP(16*0,"esp"));	# 0:r^1:0:r^2

  1365. 	&vpunpcklqdq	($D1,$D1,&QWP(16*1,"esp"));

  1366. 	&vpunpcklqdq	($D2,$D2,&QWP(16*2,"esp"));

  1367. 	&vpunpcklqdq	($D3,$D3,&QWP(16*3,"esp"));

  1368. 	&vpunpcklqdq	($D4,$D4,&QWP(16*4,"esp"));

  1369. 	&jmp		(&label("square"));

  1370. 

  1371. &set_label("square_break");

  1372. 	&vpsllq		($D0,$D0,32);			# -> r^3:0:r^4:0

  1373. 	&vpsllq		($D1,$D1,32);

  1374. 	&vpsllq		($D2,$D2,32);

  1375. 	&vpsllq		($D3,$D3,32);

  1376. 	&vpsllq		($D4,$D4,32);

  1377. 	&vpor		($D0,$D0,&QWP(16*0,"esp"));	# r^3:r^1:r^4:r^2

  1378. 	&vpor		($D1,$D1,&QWP(16*1,"esp"));

  1379. 	&vpor		($D2,$D2,&QWP(16*2,"esp"));

  1380. 	&vpor		($D3,$D3,&QWP(16*3,"esp"));

  1381. 	&vpor		($D4,$D4,&QWP(16*4,"esp"));

  1382. 

  1383. 	&vpshufd	($D0,$D0,0b10001101);		# -> r^1:r^2:r^3:r^4

  1384. 	&vpshufd	($D1,$D1,0b10001101);

  1385. 	&vpshufd	($D2,$D2,0b10001101);

  1386. 	&vpshufd	($D3,$D3,0b10001101);

  1387. 	&vpshufd	($D4,$D4,0b10001101);

  1388. 

  1389. 	&vmovdqu	(&QWP(16*0,"edi"),$D0);		# save the table

  1390. 	&vmovdqu	(&QWP(16*1,"edi"),$D1);

  1391. 	&vmovdqu	(&QWP(16*2,"edi"),$D2);

  1392. 	&vmovdqu	(&QWP(16*3,"edi"),$D3);

  1393. 	&vmovdqu	(&QWP(16*4,"edi"),$D4);

  1394. 

  1395. 	&vpslld		($T1,$D1,2);

  1396. 	&vpslld		($T0,$D2,2);

  1397. 	&vpaddd		($T1,$T1,$D1);			# *5

  1398. 	&vpaddd		($T0,$T0,$D2);			# *5

  1399. 	&vmovdqu	(&QWP(16*5,"edi"),$T1);

  1400. 	&vmovdqu	(&QWP(16*6,"edi"),$T0);

  1401. 	&vpslld		($T1,$D3,2);

  1402. 	&vpslld		($T0,$D4,2);

  1403. 	&vpaddd		($T1,$T1,$D3);			# *5

  1404. 	&vpaddd		($T0,$T0,$D4);			# *5

  1405. 	&vmovdqu	(&QWP(16*7,"edi"),$T1);

  1406. 	&vmovdqu	(&QWP(16*8,"edi"),$T0);

  1407. 

  1408. 	&mov		("esp","ebp");

  1409. 	&lea		("edi",&DWP(-16*3,"edi"));	# size de-optimization

  1410. 	&ret		();

  1411. &function_end_B("_poly1305_init_avx2");

  1412. 

  1413. ########################################################################

  1414. # now it's time to switch to %ymm

  1415. 

  1416. my ($D0,$D1,$D2,$D3,$D4,$T0,$T1,$T2)=map("ymm$_",(0..7));

  1417. my $MASK=$T2;

  1418. 

  1419. sub X { my $reg=shift; $reg=~s/^ymm/xmm/; $reg; }

  1420. 

  1421. &align	(32);

  1422. &function_begin("_poly1305_blocks_avx2");

  1423. 	&mov	("edi",&wparam(0));			# ctx

  1424. 	&mov	("esi",&wparam(1));			# inp

  1425. 	&mov	("ecx",&wparam(2));			# len

  1426. 

  1427. 	&mov	("eax",&DWP(4*5,"edi"));		# is_base2_26

  1428. 	&and	("ecx",-16);

  1429. 	&jz	(&label("nodata"));

  1430. 	&cmp	("ecx",64);

  1431. 	&jae	(&label("enter_avx2"));

  1432. 	&test	("eax","eax");				# is_base2_26?

  1433. 	&jz	(&label("enter_blocks"));

  1434. 

  1435. &set_label("enter_avx2");

  1436. 	&vzeroupper	();

  1437. 

  1438. 	&call	(&label("pic_point"));

  1439. &set_label("pic_point");

  1440. 	&blindpop("ebx");

  1441. 	&lea	("ebx",&DWP(&label("const_sse2")."-".&label("pic_point"),"ebx"));

  1442. 

  1443. 	&test	("eax","eax");				# is_base2_26?

  1444. 	&jnz	(&label("base2_26"));

  1445. 

  1446. 	&call	("_poly1305_init_avx2");

  1447. 

  1448. 	################################################# base 2^32 -> base 2^26

  1449. 	&mov	("eax",&DWP(0,"edi"));

  1450. 	&mov	("ecx",&DWP(3,"edi"));

  1451. 	&mov	("edx",&DWP(6,"edi"));

  1452. 	&mov	("esi",&DWP(9,"edi"));

  1453. 	&mov	("ebp",&DWP(13,"edi"));

  1454. 

  1455. 	&shr	("ecx",2);

  1456. 	&and	("eax",0x3ffffff);

  1457. 	&shr	("edx",4);

  1458. 	&and	("ecx",0x3ffffff);

  1459. 	&shr	("esi",6);

  1460. 	&and	("edx",0x3ffffff);

  1461. 

  1462. 	&mov	(&DWP(4*0,"edi"),"eax");

  1463. 	&mov	(&DWP(4*1,"edi"),"ecx");

  1464. 	&mov	(&DWP(4*2,"edi"),"edx");

  1465. 	&mov	(&DWP(4*3,"edi"),"esi");

  1466. 	&mov	(&DWP(4*4,"edi"),"ebp");

  1467. 	&mov	(&DWP(4*5,"edi"),1);			# is_base2_26

  1468. 

  1469. 	&mov	("esi",&wparam(1));			# [reload] inp

  1470. 	&mov	("ecx",&wparam(2));			# [reload] len

  1471. 

  1472. &set_label("base2_26");

  1473. 	&mov	("eax",&wparam(3));			# padbit

  1474. 	&mov	("ebp","esp");

  1475. 

  1476. 	&sub	("esp",32*(5+9));

  1477. 	&and	("esp",-512);				# ensure that frame

  1478. 							# doesn't cross page

  1479. 							# boundary, which is

  1480. 							# essential for

  1481. 							# misaligned 32-byte

  1482. 							# loads

  1483. 

  1484. 	################################################################

  1485.         # expand and copy pre-calculated table to stack

  1486. 

  1487. 	&vmovdqu	(&X($D0),&QWP(16*(3+0),"edi"));

  1488. 	&lea		("edx",&DWP(32*5+128,"esp"));	# +128 size optimization

  1489. 	&vmovdqu	(&X($D1),&QWP(16*(3+1),"edi"));

  1490. 	&vmovdqu	(&X($D2),&QWP(16*(3+2),"edi"));

  1491. 	&vmovdqu	(&X($D3),&QWP(16*(3+3),"edi"));

  1492. 	&vmovdqu	(&X($D4),&QWP(16*(3+4),"edi"));

  1493. 	&lea		("edi",&DWP(16*3,"edi"));	# size optimization

  1494. 	&vpermq		($D0,$D0,0b01000000);		# 00001234 -> 12343434

  1495. 	&vpermq		($D1,$D1,0b01000000);

  1496. 	&vpermq		($D2,$D2,0b01000000);

  1497. 	&vpermq		($D3,$D3,0b01000000);

  1498. 	&vpermq		($D4,$D4,0b01000000);

  1499. 	&vpshufd	($D0,$D0,0b11001000);		# 12343434 -> 14243444

  1500. 	&vpshufd	($D1,$D1,0b11001000);

  1501. 	&vpshufd	($D2,$D2,0b11001000);

  1502. 	&vpshufd	($D3,$D3,0b11001000);

  1503. 	&vpshufd	($D4,$D4,0b11001000);

  1504. 	&vmovdqa	(&QWP(32*0-128,"edx"),$D0);

  1505. 	&vmovdqu	(&X($D0),&QWP(16*5,"edi"));

  1506. 	&vmovdqa	(&QWP(32*1-128,"edx"),$D1);

  1507. 	&vmovdqu	(&X($D1),&QWP(16*6,"edi"));

  1508. 	&vmovdqa	(&QWP(32*2-128,"edx"),$D2);

  1509. 	&vmovdqu	(&X($D2),&QWP(16*7,"edi"));

  1510. 	&vmovdqa	(&QWP(32*3-128,"edx"),$D3);

  1511. 	&vmovdqu	(&X($D3),&QWP(16*8,"edi"));

  1512. 	&vmovdqa	(&QWP(32*4-128,"edx"),$D4);

  1513. 	&vpermq		($D0,$D0,0b01000000);

  1514. 	&vpermq		($D1,$D1,0b01000000);

  1515. 	&vpermq		($D2,$D2,0b01000000);

  1516. 	&vpermq		($D3,$D3,0b01000000);

  1517. 	&vpshufd	($D0,$D0,0b11001000);

  1518. 	&vpshufd	($D1,$D1,0b11001000);

  1519. 	&vpshufd	($D2,$D2,0b11001000);

  1520. 	&vpshufd	($D3,$D3,0b11001000);

  1521. 	&vmovdqa	(&QWP(32*5-128,"edx"),$D0);

  1522. 	&vmovd		(&X($D0),&DWP(-16*3+4*0,"edi"));# load hash value

  1523. 	&vmovdqa	(&QWP(32*6-128,"edx"),$D1);

  1524. 	&vmovd		(&X($D1),&DWP(-16*3+4*1,"edi"));

  1525. 	&vmovdqa	(&QWP(32*7-128,"edx"),$D2);

  1526. 	&vmovd		(&X($D2),&DWP(-16*3+4*2,"edi"));

  1527. 	&vmovdqa	(&QWP(32*8-128,"edx"),$D3);

  1528. 	&vmovd		(&X($D3),&DWP(-16*3+4*3,"edi"));

  1529. 	&vmovd		(&X($D4),&DWP(-16*3+4*4,"edi"));

  1530. 	&vmovdqa	($MASK,&QWP(64,"ebx"));

  1531. 	&neg		("eax");			# padbit

  1532. 

  1533. 	&test		("ecx",63);

  1534. 	&jz		(&label("even"));

  1535. 

  1536. 	&mov		("edx","ecx");

  1537. 	&and		("ecx",-64);

  1538. 	&and		("edx",63);

  1539. 

  1540. 	&vmovdqu	(&X($T0),&QWP(16*0,"esi"));

  1541. 	&cmp		("edx",32);

  1542. 	&jb		(&label("one"));

  1543. 

  1544. 	&vmovdqu	(&X($T1),&QWP(16*1,"esi"));

  1545. 	&je		(&label("two"));

  1546. 

  1547. 	&vinserti128	($T0,$T0,&QWP(16*2,"esi"),1);

  1548. 	&lea		("esi",&DWP(16*3,"esi"));

  1549. 	&lea		("ebx",&DWP(8,"ebx"));		# three padbits

  1550. 	&lea		("edx",&DWP(32*5+128+8,"esp"));	# --:r^1:r^2:r^3 (*)

  1551. 	&jmp		(&label("tail"));

  1552. 

  1553. &set_label("two");

  1554. 	&lea		("esi",&DWP(16*2,"esi"));

  1555. 	&lea		("ebx",&DWP(16,"ebx"));		# two padbits

  1556. 	&lea		("edx",&DWP(32*5+128+16,"esp"));# --:--:r^1:r^2 (*)

  1557. 	&jmp		(&label("tail"));

  1558. 

  1559. &set_label("one");

  1560. 	&lea		("esi",&DWP(16*1,"esi"));

  1561. 	&vpxor		($T1,$T1,$T1);

  1562. 	&lea		("ebx",&DWP(32,"ebx","eax",8));	# one or no padbits

  1563. 	&lea		("edx",&DWP(32*5+128+24,"esp"));# --:--:--:r^1 (*)

  1564. 	&jmp		(&label("tail"));

  1565. 

  1566. # (*)	spots marked with '--' are data from next table entry, but they

  1567. #	are multiplied by 0 and therefore rendered insignificant

  1568. 

  1569. &set_label("even",32);

  1570. 	&vmovdqu	(&X($T0),&QWP(16*0,"esi"));	# load input

  1571. 	&vmovdqu	(&X($T1),&QWP(16*1,"esi"));

  1572. 	&vinserti128	($T0,$T0,&QWP(16*2,"esi"),1);

  1573. 	&vinserti128	($T1,$T1,&QWP(16*3,"esi"),1);

  1574. 	&lea		("esi",&DWP(16*4,"esi"));

  1575. 	&sub		("ecx",64);

  1576. 	&jz		(&label("tail"));

  1577. 

  1578. &set_label("loop");

  1579. 	################################################################

  1580. 	# ((inp[0]*r^4+r[4])*r^4+r[8])*r^4

  1581. 	# ((inp[1]*r^4+r[5])*r^4+r[9])*r^3

  1582. 	# ((inp[2]*r^4+r[6])*r^4+r[10])*r^2

  1583. 	# ((inp[3]*r^4+r[7])*r^4+r[11])*r^1

  1584. 	#   \________/ \_______/

  1585. 	################################################################

  1586. 

  1587. sub vsplat_input {

  1588. 	&vmovdqa	(&QWP(32*2,"esp"),$D2);

  1589. 	&vpsrldq	($D2,$T0,6);			# splat input

  1590. 	&vmovdqa	(&QWP(32*0,"esp"),$D0);

  1591. 	&vpsrldq	($D0,$T1,6);

  1592. 	&vmovdqa	(&QWP(32*1,"esp"),$D1);

  1593. 	&vpunpckhqdq	($D1,$T0,$T1);			# 4

  1594. 	&vpunpcklqdq	($T0,$T0,$T1);			# 0:1

  1595. 	&vpunpcklqdq	($D2,$D2,$D0);			# 2:3

  1596. 

  1597. 	&vpsrlq		($D0,$D2,30);

  1598. 	&vpsrlq		($D2,$D2,4);

  1599. 	&vpsrlq		($T1,$T0,26);

  1600. 	&vpsrlq		($D1,$D1,40);			# 4

  1601. 	&vpand		($D2,$D2,$MASK);		# 2

  1602. 	&vpand		($T0,$T0,$MASK);		# 0

  1603. 	&vpand		($T1,$T1,$MASK);		# 1

  1604. 	&vpand		($D0,$D0,$MASK);		# 3 (*)

  1605. 	&vpor		($D1,$D1,&QWP(0,"ebx"));	# padbit, yes, always

  1606. 

  1607. 	# (*)	note that output is counterintuitive, inp[3:4] is

  1608. 	#	returned in $D1-2, while $D3-4 are preserved;

  1609. }

  1610. 	&vsplat_input	();

  1611. 

  1612. sub vpmuladd {

  1613. my $addr = shift;

  1614. 

  1615. 	&vpaddq		($D2,$D2,&QWP(32*2,"esp"));	# add hash value

  1616. 	&vpaddq		($T0,$T0,&QWP(32*0,"esp"));

  1617. 	&vpaddq		($T1,$T1,&QWP(32*1,"esp"));

  1618. 	&vpaddq		($D0,$D0,$D3);

  1619. 	&vpaddq		($D1,$D1,$D4);

  1620. 

  1621. 	################################################################

  1622. 	# d3 = h2*r1   + h0*r3 + h1*r2   + h3*r0   + h4*5*r4

  1623. 	# d4 = h2*r2   + h0*r4 + h1*r3   + h3*r1   + h4*r0

  1624. 	# d0 = h2*5*r3 + h0*r0 + h1*5*r4 + h3*5*r2 + h4*5*r1

  1625. 	# d1 = h2*5*r4 + h0*r1 + h1*r0   + h3*5*r3 + h4*5*r2

  1626. 	# d2 = h2*r0   + h0*r2 + h1*r1   + h3*5*r4 + h4*5*r3

  1627. 

  1628. 	&vpmuludq	($D3,$D2,&$addr(1));		# d3 = h2*r1

  1629. 	 &vmovdqa	(QWP(32*1,"esp"),$T1);

  1630. 	&vpmuludq	($D4,$D2,&$addr(2));		# d4 = h2*r2

  1631. 	 &vmovdqa	(QWP(32*3,"esp"),$D0);

  1632. 	&vpmuludq	($D0,$D2,&$addr(7));		# d0 = h2*s3

  1633. 	 &vmovdqa	(QWP(32*4,"esp"),$D1);

  1634. 	&vpmuludq	($D1,$D2,&$addr(8));		# d1 = h2*s4

  1635. 	&vpmuludq	($D2,$D2,&$addr(0));		# d2 = h2*r0

  1636. 

  1637. 	&vpmuludq	($T2,$T0,&$addr(3));		# h0*r3

  1638. 	&vpaddq		($D3,$D3,$T2);			# d3 += h0*r3

  1639. 	&vpmuludq	($T1,$T0,&$addr(4));		# h0*r4

  1640. 	&vpaddq		($D4,$D4,$T1);			# d4 + h0*r4

  1641. 	&vpmuludq	($T2,$T0,&$addr(0));		# h0*r0

  1642. 	&vpaddq		($D0,$D0,$T2);			# d0 + h0*r0

  1643. 	 &vmovdqa	($T2,&QWP(32*1,"esp"));		# h1

  1644. 	&vpmuludq	($T1,$T0,&$addr(1));		# h0*r1

  1645. 	&vpaddq		($D1,$D1,$T1);			# d1 += h0*r1

  1646. 	&vpmuludq	($T0,$T0,&$addr(2));		# h0*r2

  1647. 	&vpaddq		($D2,$D2,$T0);			# d2 += h0*r2

  1648. 

  1649. 	&vpmuludq	($T1,$T2,&$addr(2));		# h1*r2

  1650. 	&vpaddq		($D3,$D3,$T1);			# d3 += h1*r2

  1651. 	&vpmuludq	($T0,$T2,&$addr(3));		# h1*r3

  1652. 	&vpaddq		($D4,$D4,$T0);			# d4 += h1*r3

  1653. 	&vpmuludq	($T1,$T2,&$addr(8));		# h1*s4

  1654. 	&vpaddq		($D0,$D0,$T1);			# d0 += h1*s4

  1655. 	 &vmovdqa	($T1,&QWP(32*3,"esp"));		# h3

  1656. 	&vpmuludq	($T0,$T2,&$addr(0));		# h1*r0

  1657. 	&vpaddq		($D1,$D1,$T0);			# d1 += h1*r0

  1658. 	&vpmuludq	($T2,$T2,&$addr(1));		# h1*r1

  1659. 	&vpaddq		($D2,$D2,$T2);			# d2 += h1*r1

  1660. 

  1661. 	&vpmuludq	($T0,$T1,&$addr(0));		# h3*r0

  1662. 	&vpaddq		($D3,$D3,$T0);			# d3 += h3*r0

  1663. 	&vpmuludq	($T2,$T1,&$addr(1));		# h3*r1

  1664. 	&vpaddq		($D4,$D4,$T2);			# d4 += h3*r1

  1665. 	&vpmuludq	($T0,$T1,&$addr(6));		# h3*s2

  1666. 	&vpaddq		($D0,$D0,$T0);			# d0 += h3*s2

  1667. 	 &vmovdqa	($T0,&QWP(32*4,"esp"));		# h4

  1668. 	&vpmuludq	($T2,$T1,&$addr(7));		# h3*s3

  1669. 	&vpaddq		($D1,$D1,$T2);			# d1+= h3*s3

  1670. 	&vpmuludq	($T1,$T1,&$addr(8));		# h3*s4

  1671. 	&vpaddq		($D2,$D2,$T1);			# d2 += h3*s4

  1672. 

  1673. 	&vpmuludq	($T2,$T0,&$addr(8));		# h4*s4

  1674. 	&vpaddq		($D3,$D3,$T2);			# d3 += h4*s4

  1675. 	&vpmuludq	($T1,$T0,&$addr(5));		# h4*s1

  1676. 	&vpaddq		($D0,$D0,$T1);			# d0 += h4*s1

  1677. 	&vpmuludq	($T2,$T0,&$addr(0));		# h4*r0

  1678. 	&vpaddq		($D4,$D4,$T2);			# d4 += h4*r0

  1679. 	 &vmovdqa	($MASK,&QWP(64,"ebx"));

  1680. 	&vpmuludq	($T1,$T0,&$addr(6));		# h4*s2

  1681. 	&vpaddq		($D1,$D1,$T1);			# d1 += h4*s2

  1682. 	&vpmuludq	($T0,$T0,&$addr(7));		# h4*s3

  1683. 	&vpaddq		($D2,$D2,$T0);			# d2 += h4*s3

  1684. }

  1685. 	&vpmuladd	(sub {	my $i=shift; &QWP(32*$i-128,"edx");	});

  1686. 

  1687. sub vlazy_reduction {

  1688. 	################################################################

  1689. 	# lazy reduction

  1690. 

  1691. 	 &vpsrlq	($T0,$D3,26);

  1692. 	 &vpand		($D3,$D3,$MASK);

  1693. 	&vpsrlq		($T1,$D0,26);

  1694. 	&vpand		($D0,$D0,$MASK);

  1695. 	 &vpaddq	($D4,$D4,$T0);			# h3 -> h4

  1696. 	&vpaddq		($D1,$D1,$T1);			# h0 -> h1

  1697. 	 &vpsrlq	($T0,$D4,26);

  1698. 	 &vpand		($D4,$D4,$MASK);

  1699. 	&vpsrlq		($T1,$D1,26);

  1700. 	&vpand		($D1,$D1,$MASK);

  1701. 	&vpaddq		($D2,$D2,$T1);			# h1 -> h2

  1702. 	 &vpaddq	($D0,$D0,$T0);

  1703. 	 &vpsllq	($T0,$T0,2);

  1704. 	&vpsrlq		($T1,$D2,26);

  1705. 	&vpand		($D2,$D2,$MASK);

  1706. 	 &vpaddq	($D0,$D0,$T0);			# h4 -> h0

  1707. 	&vpaddq		($D3,$D3,$T1);			# h2 -> h3

  1708. 	&vpsrlq		($T1,$D3,26);

  1709. 	 &vpsrlq	($T0,$D0,26);

  1710. 	 &vpand		($D0,$D0,$MASK);

  1711. 	&vpand		($D3,$D3,$MASK);

  1712. 	 &vpaddq	($D1,$D1,$T0);			# h0 -> h1

  1713. 	&vpaddq		($D4,$D4,$T1);			# h3 -> h4

  1714. }

  1715. 	&vlazy_reduction();

  1716. 

  1717. 	&vmovdqu	(&X($T0),&QWP(16*0,"esi"));	# load input

  1718. 	&vmovdqu	(&X($T1),&QWP(16*1,"esi"));

  1719. 	&vinserti128	($T0,$T0,&QWP(16*2,"esi"),1);

  1720. 	&vinserti128	($T1,$T1,&QWP(16*3,"esi"),1);

  1721. 	&lea		("esi",&DWP(16*4,"esi"));

  1722. 	&sub		("ecx",64);

  1723. 	&jnz		(&label("loop"));

  1724. 

  1725. &set_label("tail");

  1726. 	&vsplat_input	();

  1727. 	&and		("ebx",-64);			# restore pointer

  1728. 

  1729. 	&vpmuladd	(sub {	my $i=shift; &QWP(4+32*$i-128,"edx");	});

  1730. 

  1731. 	################################################################

  1732. 	# horizontal addition

  1733. 

  1734. 	&vpsrldq	($T0,$D4,8);

  1735. 	&vpsrldq	($T1,$D3,8);

  1736. 	&vpaddq		($D4,$D4,$T0);

  1737. 	&vpsrldq	($T0,$D0,8);

  1738. 	&vpaddq		($D3,$D3,$T1);

  1739. 	&vpsrldq	($T1,$D1,8);

  1740. 	&vpaddq		($D0,$D0,$T0);

  1741. 	&vpsrldq	($T0,$D2,8);

  1742. 	&vpaddq		($D1,$D1,$T1);

  1743. 	&vpermq		($T1,$D4,2);			# keep folding

  1744. 	&vpaddq		($D2,$D2,$T0);

  1745. 	&vpermq		($T0,$D3,2);

  1746. 	&vpaddq		($D4,$D4,$T1);

  1747. 	&vpermq		($T1,$D0,2);

  1748. 	&vpaddq		($D3,$D3,$T0);

  1749. 	&vpermq		($T0,$D1,2);

  1750. 	&vpaddq		($D0,$D0,$T1);

  1751. 	&vpermq		($T1,$D2,2);

  1752. 	&vpaddq		($D1,$D1,$T0);

  1753. 	&vpaddq		($D2,$D2,$T1);

  1754. 

  1755. 	&vlazy_reduction();

  1756. 

  1757. 	&cmp		("ecx",0);

  1758. 	&je		(&label("done"));

  1759. 

  1760. 	################################################################

  1761. 	# clear all but single word

  1762. 

  1763. 	&vpshufd	(&X($D0),&X($D0),0b11111100);

  1764. 	&lea		("edx",&DWP(32*5+128,"esp"));	# restore pointer

  1765. 	&vpshufd	(&X($D1),&X($D1),0b11111100);

  1766. 	&vpshufd	(&X($D2),&X($D2),0b11111100);

  1767. 	&vpshufd	(&X($D3),&X($D3),0b11111100);

  1768. 	&vpshufd	(&X($D4),&X($D4),0b11111100);

  1769. 	&jmp		(&label("even"));

  1770. 

  1771. &set_label("done",16);

  1772. 	&vmovd		(&DWP(-16*3+4*0,"edi"),&X($D0));# store hash value

  1773. 	&vmovd		(&DWP(-16*3+4*1,"edi"),&X($D1));

  1774. 	&vmovd		(&DWP(-16*3+4*2,"edi"),&X($D2));

  1775. 	&vmovd		(&DWP(-16*3+4*3,"edi"),&X($D3));

  1776. 	&vmovd		(&DWP(-16*3+4*4,"edi"),&X($D4));

  1777. 	&vzeroupper	();

  1778. 	&mov	("esp","ebp");

  1779. &set_label("nodata");

  1780. &function_end("_poly1305_blocks_avx2");

  1781. }

  1782. &set_label("const_sse2",64);

  1783. 	&data_word(1<<24,0,	1<<24,0,	1<<24,0,	1<<24,0);

  1784. 	&data_word(0,0,		0,0,		0,0,		0,0);

  1785. 	&data_word(0x03ffffff,0,0x03ffffff,0,	0x03ffffff,0,	0x03ffffff,0);

  1786. 	&data_word(0x0fffffff,0x0ffffffc,0x0ffffffc,0x0ffffffc);

  1787. }

  1788. &asciz	("Poly1305 for x86, CRYPTOGAMS by <appro\@openssl.org>");

  1789. &align	(4);

  1790. 

  1791. &asm_finish();

  1792. 

  1793. close STDOUT;