kris
/
boringssl
1
0
Bifurcation 0
Code Tickets 0 Demandes d'ajout 0 Versions 0 Wiki Activité
Vous ne pouvez pas sélectionner plus de 25 sujets Les noms de sujets doivent commencer par une lettre ou un nombre, peuvent contenir des tirets ('-') et peuvent comporter jusqu'à 35 caractères.
255 Révisions
12 Branches
38 MiB
 
 
 
 
 
 
Aborescence: 019c3cc64a
Branches Tags
${ item.name }
Créer la branche ${ searchTerm }
de '019c3cc64a'
${ noResults }
boringssl/crypto/sha/asm/sha256-586.pl

1277 lignes
35 KiB
Brut Annotations Historique 

  1. #!/usr/bin/env perl

  2. #

  3. # ====================================================================

  4. # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL

  5. # project. The module is, however, dual licensed under OpenSSL and

  6. # CRYPTOGAMS licenses depending on where you obtain it. For further

  7. # details see http://www.openssl.org/~appro/cryptogams/.

  8. # ====================================================================

  9. #

  10. # SHA256 block transform for x86. September 2007.

  11. #

  12. # Performance improvement over compiler generated code varies from

  13. # 10% to 40% [see below]. Not very impressive on some µ-archs, but

  14. # it's 5 times smaller and optimizies amount of writes.

  15. #

  16. # May 2012.

  17. #

  18. # Optimization including two of Pavel Semjanov's ideas, alternative

  19. # Maj and full unroll, resulted in ~20-25% improvement on most CPUs,

  20. # ~7% on Pentium, ~40% on Atom. As fully unrolled loop body is almost

  21. # 15x larger, 8KB vs. 560B, it's fired only for longer inputs. But not

  22. # on P4, where it kills performance, nor Sandy Bridge, where folded

  23. # loop is approximately as fast...

  24. #

  25. # June 2012.

  26. #

  27. # Add AMD XOP-specific code path, >30% improvement on Bulldozer over

  28. # May version, >60% over original. Add AVX+shrd code path, >25%

  29. # improvement on Sandy Bridge over May version, 60% over original.

  30. #

  31. # May 2013.

  32. #

  33. # Replace AMD XOP code path with SSSE3 to cover more processors.

  34. # (Biggest improvement coefficient is on upcoming Atom Silvermont,

  35. # not shown.) Add AVX+BMI code path.

  36. #

  37. # March 2014.

  38. #

  39. # Add support for Intel SHA Extensions.

  40. #

  41. # Performance in clock cycles per processed byte (less is better):

  42. #

  43. #		gcc	icc	x86 asm(*)	SIMD	x86_64 asm(**)	

  44. # Pentium	46	57	40/38		-	-

  45. # PIII		36	33	27/24		-	-

  46. # P4		41	38	28		-	17.3

  47. # AMD K8	27	25	19/15.5		-	14.9

  48. # Core2		26	23	18/15.6		14.3	13.8

  49. # Westmere	27	-	19/15.7		13.4	12.3

  50. # Sandy Bridge	25	-	15.9		12.4	11.6

  51. # Ivy Bridge	24	-	15.0		11.4	10.3

  52. # Haswell	22	-	13.9		9.46	7.80

  53. # Bulldozer	36	-	27/22		17.0	13.6

  54. # VIA Nano	36	-	25/22		16.8	16.5

  55. # Atom		50	-	30/25		21.9	18.9

  56. #

  57. # (*)	numbers after slash are for unrolled loop, where applicable;

  58. # (**)	x86_64 assembly performance is presented for reference

  59. #	purposes, results are best-available;

  60. 

  61. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;

  62. push(@INC,"${dir}","${dir}../../perlasm");

  63. require "x86asm.pl";

  64. 

  65. &asm_init($ARGV[0],"sha512-586.pl",$ARGV[$#ARGV] eq "386");

  66. 

  67. $xmm=$avx=0;

  68. for (@ARGV) { $xmm=1 if (/-DOPENSSL_IA32_SSE2/); }

  69. 

  70. if ($xmm &&	`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`

  71. 			=~ /GNU assembler version ([2-9]\.[0-9]+)/) {

  72. 	$avx = ($1>=2.19) + ($1>=2.22);

  73. }

  74. 

  75. if ($xmm && !$avx && $ARGV[0] eq "win32n" &&

  76. 		`nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {

  77. 	$avx = ($1>=2.03) + ($1>=2.10);

  78. }

  79. 

  80. if ($xmm && !$avx && $ARGV[0] eq "win32" &&

  81. 		`ml 2>&1` =~ /Version ([0-9]+)\./) {

  82. 	$avx = ($1>=10) + ($1>=11);

  83. }

  84. 

  85. $shaext=$xmm;	### set to zero if compiling for 1.0.1

  86. 

  87. $unroll_after = 64*4;	# If pre-evicted from L1P cache first spin of

  88. 			# fully unrolled loop was measured to run about

  89. 			# 3-4x slower. If slowdown coefficient is N and

  90. 			# unrolled loop is m times faster, then you break

  91. 			# even at (N-1)/(m-1) blocks. Then it needs to be

  92. 			# adjusted for probability of code being evicted,

  93. 			# code size/cache size=1/4. Typical m is 1.15...

  94. 

  95. $A="eax";

  96. $E="edx";

  97. $T="ebx";

  98. $Aoff=&DWP(4,"esp");

  99. $Boff=&DWP(8,"esp");

  100. $Coff=&DWP(12,"esp");

  101. $Doff=&DWP(16,"esp");

  102. $Eoff=&DWP(20,"esp");

  103. $Foff=&DWP(24,"esp");

  104. $Goff=&DWP(28,"esp");

  105. $Hoff=&DWP(32,"esp");

  106. $Xoff=&DWP(36,"esp");

  107. $K256="ebp";

  108. 

  109. sub BODY_16_63() {

  110. 	&mov	($T,"ecx");			# "ecx" is preloaded

  111. 	 &mov	("esi",&DWP(4*(9+15+16-14),"esp"));

  112. 	&ror	("ecx",18-7);

  113. 	 &mov	("edi","esi");

  114. 	&ror	("esi",19-17);

  115. 	 &xor	("ecx",$T);

  116. 	 &shr	($T,3);

  117. 	&ror	("ecx",7);

  118. 	 &xor	("esi","edi");

  119. 	 &xor	($T,"ecx");			# T = sigma0(X[-15])

  120. 	&ror	("esi",17);

  121. 	 &add	($T,&DWP(4*(9+15+16),"esp"));	# T += X[-16]

  122. 	&shr	("edi",10);

  123. 	 &add	($T,&DWP(4*(9+15+16-9),"esp"));	# T += X[-7]

  124. 	#&xor	("edi","esi")			# sigma1(X[-2])

  125. 	# &add	($T,"edi");			# T += sigma1(X[-2])

  126. 	# &mov	(&DWP(4*(9+15),"esp"),$T);	# save X[0]

  127. 

  128. 	&BODY_00_15(1);

  129. }

  130. sub BODY_00_15() {

  131.     my $in_16_63=shift;

  132. 

  133. 	&mov	("ecx",$E);

  134. 	 &xor	("edi","esi")			if ($in_16_63);	# sigma1(X[-2])

  135. 	 &mov	("esi",$Foff);

  136. 	&ror	("ecx",25-11);

  137. 	 &add	($T,"edi")			if ($in_16_63);	# T += sigma1(X[-2])

  138. 	 &mov	("edi",$Goff);

  139. 	&xor	("ecx",$E);

  140. 	 &xor	("esi","edi");

  141. 	 &mov	($T,&DWP(4*(9+15),"esp"))	if (!$in_16_63);

  142. 	 &mov	(&DWP(4*(9+15),"esp"),$T)	if ($in_16_63);	# save X[0]

  143. 	&ror	("ecx",11-6);

  144. 	 &and	("esi",$E);

  145. 	 &mov	($Eoff,$E);		# modulo-scheduled

  146. 	&xor	($E,"ecx");

  147. 	 &add	($T,$Hoff);		# T += h

  148. 	 &xor	("esi","edi");		# Ch(e,f,g)

  149. 	&ror	($E,6);			# Sigma1(e)

  150. 	 &mov	("ecx",$A);

  151. 	 &add	($T,"esi");		# T += Ch(e,f,g)

  152. 

  153. 	&ror	("ecx",22-13);

  154. 	 &add	($T,$E);		# T += Sigma1(e)

  155. 	 &mov	("edi",$Boff);

  156. 	&xor	("ecx",$A);

  157. 	 &mov	($Aoff,$A);		# modulo-scheduled

  158. 	 &lea	("esp",&DWP(-4,"esp"));

  159. 	&ror	("ecx",13-2);

  160. 	 &mov	("esi",&DWP(0,$K256));

  161. 	&xor	("ecx",$A);

  162. 	 &mov	($E,$Eoff);		# e in next iteration, d in this one

  163. 	 &xor	($A,"edi");		# a ^= b

  164. 	&ror	("ecx",2);		# Sigma0(a)

  165. 

  166. 	 &add	($T,"esi");		# T+= K[i]

  167. 	 &mov	(&DWP(0,"esp"),$A);	# (b^c) in next round

  168. 	&add	($E,$T);		# d += T

  169. 	 &and	($A,&DWP(4,"esp"));	# a &= (b^c)

  170. 	&add	($T,"ecx");		# T += Sigma0(a)

  171. 	 &xor	($A,"edi");		# h = Maj(a,b,c) = Ch(a^b,c,b)

  172. 	 &mov	("ecx",&DWP(4*(9+15+16-1),"esp"))	if ($in_16_63);	# preload T

  173. 	&add	($K256,4);

  174. 	 &add	($A,$T);		# h += T

  175. }

  176. 

  177. &external_label("OPENSSL_ia32cap_P")		if (!$i386);

  178. 

  179. &function_begin("sha256_block_data_order");

  180. 	&mov	("esi",wparam(0));	# ctx

  181. 	&mov	("edi",wparam(1));	# inp

  182. 	&mov	("eax",wparam(2));	# num

  183. 	&mov	("ebx","esp");		# saved sp

  184. 

  185. 	&call	(&label("pic_point"));	# make it PIC!

  186. &set_label("pic_point");

  187. 	&blindpop($K256);

  188. 	&lea	($K256,&DWP(&label("K256")."-".&label("pic_point"),$K256));

  189. 

  190. 	&sub	("esp",16);

  191. 	&and	("esp",-64);

  192. 

  193. 	&shl	("eax",6);

  194. 	&add	("eax","edi");

  195. 	&mov	(&DWP(0,"esp"),"esi");	# ctx

  196. 	&mov	(&DWP(4,"esp"),"edi");	# inp

  197. 	&mov	(&DWP(8,"esp"),"eax");	# inp+num*128

  198. 	&mov	(&DWP(12,"esp"),"ebx");	# saved sp

  199. 						if (!$i386 && $xmm) {

  200. 	&picmeup("edx","OPENSSL_ia32cap_P",$K256,&label("K256"));

  201. 	&mov	("ecx",&DWP(0,"edx"));

  202. 	&mov	("ebx",&DWP(4,"edx"));

  203. 	&test	("ecx",1<<20);		# check for P4

  204. 	&jnz	(&label("loop"));

  205. 	&mov	("edx",&DWP(8,"edx"))	if ($xmm);

  206. 	&test	("ecx",1<<24);		# check for FXSR

  207. 	&jz	($unroll_after?&label("no_xmm"):&label("loop"));

  208. 	&and	("ecx",1<<30);		# mask "Intel CPU" bit

  209. 	&and	("ebx",1<<28|1<<9);	# mask AVX and SSSE3 bits

  210. 	&test	("edx",1<<29)		if ($shaext);	# check for SHA

  211. 	&jnz	(&label("shaext"))	if ($shaext);

  212. 	&or	("ecx","ebx");

  213. 	&and	("ecx",1<<28|1<<30);

  214. 	&cmp	("ecx",1<<28|1<<30);

  215. 					if ($xmm) {

  216. 	&je	(&label("AVX"))		if ($avx);

  217. 	&test	("ebx",1<<9);		# check for SSSE3

  218. 	&jnz	(&label("SSSE3"));

  219. 					} else {

  220. 	&je	(&label("loop_shrd"));

  221. 					}

  222. 						if ($unroll_after) {

  223. &set_label("no_xmm");

  224. 	&sub	("eax","edi");

  225. 	&cmp	("eax",$unroll_after);

  226. 	&jae	(&label("unrolled"));

  227. 						} }

  228. 	&jmp	(&label("loop"));

  229. 

  230. sub COMPACT_LOOP() {

  231. my $suffix=shift;

  232. 

  233. &set_label("loop$suffix",$suffix?32:16);

  234.     # copy input block to stack reversing byte and dword order

  235.     for($i=0;$i<4;$i++) {

  236. 	&mov	("eax",&DWP($i*16+0,"edi"));

  237. 	&mov	("ebx",&DWP($i*16+4,"edi"));

  238. 	&mov	("ecx",&DWP($i*16+8,"edi"));

  239. 	&bswap	("eax");

  240. 	&mov	("edx",&DWP($i*16+12,"edi"));

  241. 	&bswap	("ebx");

  242. 	&push	("eax");

  243. 	&bswap	("ecx");

  244. 	&push	("ebx");

  245. 	&bswap	("edx");

  246. 	&push	("ecx");

  247. 	&push	("edx");

  248.     }

  249. 	&add	("edi",64);

  250. 	&lea	("esp",&DWP(-4*9,"esp"));# place for A,B,C,D,E,F,G,H

  251. 	&mov	(&DWP(4*(9+16)+4,"esp"),"edi");

  252. 

  253. 	# copy ctx->h[0-7] to A,B,C,D,E,F,G,H on stack

  254. 	&mov	($A,&DWP(0,"esi"));

  255. 	&mov	("ebx",&DWP(4,"esi"));

  256. 	&mov	("ecx",&DWP(8,"esi"));

  257. 	&mov	("edi",&DWP(12,"esi"));

  258. 	# &mov	($Aoff,$A);

  259. 	&mov	($Boff,"ebx");

  260. 	&xor	("ebx","ecx");

  261. 	&mov	($Coff,"ecx");

  262. 	&mov	($Doff,"edi");

  263. 	&mov	(&DWP(0,"esp"),"ebx");	# magic

  264. 	&mov	($E,&DWP(16,"esi"));	

  265. 	&mov	("ebx",&DWP(20,"esi"));

  266. 	&mov	("ecx",&DWP(24,"esi"));

  267. 	&mov	("edi",&DWP(28,"esi"));

  268. 	# &mov	($Eoff,$E);

  269. 	&mov	($Foff,"ebx");

  270. 	&mov	($Goff,"ecx");

  271. 	&mov	($Hoff,"edi");

  272. 

  273. &set_label("00_15$suffix",16);

  274. 

  275. 	&BODY_00_15();

  276. 

  277. 	&cmp	("esi",0xc19bf174);

  278. 	&jne	(&label("00_15$suffix"));

  279. 

  280. 	&mov	("ecx",&DWP(4*(9+15+16-1),"esp"));	# preloaded in BODY_00_15(1)

  281. 	&jmp	(&label("16_63$suffix"));

  282. 

  283. &set_label("16_63$suffix",16);

  284. 

  285. 	&BODY_16_63();

  286. 

  287. 	&cmp	("esi",0xc67178f2);

  288. 	&jne	(&label("16_63$suffix"));

  289. 

  290. 	&mov	("esi",&DWP(4*(9+16+64)+0,"esp"));#ctx

  291. 	# &mov	($A,$Aoff);

  292. 	&mov	("ebx",$Boff);

  293. 	# &mov	("edi",$Coff);

  294. 	&mov	("ecx",$Doff);

  295. 	&add	($A,&DWP(0,"esi"));

  296. 	&add	("ebx",&DWP(4,"esi"));

  297. 	&add	("edi",&DWP(8,"esi"));

  298. 	&add	("ecx",&DWP(12,"esi"));

  299. 	&mov	(&DWP(0,"esi"),$A);

  300. 	&mov	(&DWP(4,"esi"),"ebx");

  301. 	&mov	(&DWP(8,"esi"),"edi");

  302. 	&mov	(&DWP(12,"esi"),"ecx");

  303. 	# &mov	($E,$Eoff);

  304. 	&mov	("eax",$Foff);

  305. 	&mov	("ebx",$Goff);

  306. 	&mov	("ecx",$Hoff);

  307. 	&mov	("edi",&DWP(4*(9+16+64)+4,"esp"));#inp

  308. 	&add	($E,&DWP(16,"esi"));

  309. 	&add	("eax",&DWP(20,"esi"));

  310. 	&add	("ebx",&DWP(24,"esi"));

  311. 	&add	("ecx",&DWP(28,"esi"));

  312. 	&mov	(&DWP(16,"esi"),$E);

  313. 	&mov	(&DWP(20,"esi"),"eax");

  314. 	&mov	(&DWP(24,"esi"),"ebx");

  315. 	&mov	(&DWP(28,"esi"),"ecx");

  316. 

  317. 	&lea	("esp",&DWP(4*(9+16+64),"esp"));# destroy frame

  318. 	&sub	($K256,4*64);			# rewind K

  319. 

  320. 	&cmp	("edi",&DWP(8,"esp"));		# are we done yet?

  321. 	&jb	(&label("loop$suffix"));

  322. }

  323. 	&COMPACT_LOOP();

  324. 	&mov	("esp",&DWP(12,"esp"));		# restore sp

  325. &function_end_A();

  326. 						if (!$i386 && !$xmm) {

  327. 	# ~20% improvement on Sandy Bridge

  328. 	local *ror = sub { &shrd(@_[0],@_) };

  329. 	&COMPACT_LOOP("_shrd");

  330. 	&mov	("esp",&DWP(12,"esp"));		# restore sp

  331. &function_end_A();

  332. 						}

  333. 

  334. &set_label("K256",64);	# Yes! I keep it in the code segment!

  335. @K256=(	0x428a2f98,0x71374491,0xb5c0fbcf,0xe9b5dba5,

  336. 	0x3956c25b,0x59f111f1,0x923f82a4,0xab1c5ed5,

  337. 	0xd807aa98,0x12835b01,0x243185be,0x550c7dc3,

  338. 	0x72be5d74,0x80deb1fe,0x9bdc06a7,0xc19bf174,

  339. 	0xe49b69c1,0xefbe4786,0x0fc19dc6,0x240ca1cc,

  340. 	0x2de92c6f,0x4a7484aa,0x5cb0a9dc,0x76f988da,

  341. 	0x983e5152,0xa831c66d,0xb00327c8,0xbf597fc7,

  342. 	0xc6e00bf3,0xd5a79147,0x06ca6351,0x14292967,

  343. 	0x27b70a85,0x2e1b2138,0x4d2c6dfc,0x53380d13,

  344. 	0x650a7354,0x766a0abb,0x81c2c92e,0x92722c85,

  345. 	0xa2bfe8a1,0xa81a664b,0xc24b8b70,0xc76c51a3,

  346. 	0xd192e819,0xd6990624,0xf40e3585,0x106aa070,

  347. 	0x19a4c116,0x1e376c08,0x2748774c,0x34b0bcb5,

  348. 	0x391c0cb3,0x4ed8aa4a,0x5b9cca4f,0x682e6ff3,

  349. 	0x748f82ee,0x78a5636f,0x84c87814,0x8cc70208,

  350. 	0x90befffa,0xa4506ceb,0xbef9a3f7,0xc67178f2	);

  351. &data_word(@K256);

  352. &data_word(0x00010203,0x04050607,0x08090a0b,0x0c0d0e0f);	# byte swap mask

  353. &asciz("SHA256 block transform for x86, CRYPTOGAMS by <appro\@openssl.org>");

  354. 

  355. ($a,$b,$c,$d,$e,$f,$g,$h)=(0..7);	# offsets

  356. sub off { &DWP(4*(((shift)-$i)&7),"esp"); }

  357. 

  358. if (!$i386 && $unroll_after) {

  359. my @AH=($A,$K256);

  360. 

  361. &set_label("unrolled",16);

  362. 	&lea	("esp",&DWP(-96,"esp"));

  363. 	# copy ctx->h[0-7] to A,B,C,D,E,F,G,H on stack

  364. 	&mov	($AH[0],&DWP(0,"esi"));

  365. 	&mov	($AH[1],&DWP(4,"esi"));

  366. 	&mov	("ecx",&DWP(8,"esi"));

  367. 	&mov	("ebx",&DWP(12,"esi"));

  368. 	#&mov	(&DWP(0,"esp"),$AH[0]);

  369. 	&mov	(&DWP(4,"esp"),$AH[1]);

  370. 	&xor	($AH[1],"ecx");		# magic

  371. 	&mov	(&DWP(8,"esp"),"ecx");

  372. 	&mov	(&DWP(12,"esp"),"ebx");

  373. 	&mov	($E,&DWP(16,"esi"));	

  374. 	&mov	("ebx",&DWP(20,"esi"));

  375. 	&mov	("ecx",&DWP(24,"esi"));

  376. 	&mov	("esi",&DWP(28,"esi"));

  377. 	#&mov	(&DWP(16,"esp"),$E);

  378. 	&mov	(&DWP(20,"esp"),"ebx");

  379. 	&mov	(&DWP(24,"esp"),"ecx");

  380. 	&mov	(&DWP(28,"esp"),"esi");

  381. 	&jmp	(&label("grand_loop"));

  382. 

  383. &set_label("grand_loop",16);

  384.     # copy input block to stack reversing byte order

  385.     for($i=0;$i<5;$i++) {

  386. 	&mov	("ebx",&DWP(12*$i+0,"edi"));

  387. 	&mov	("ecx",&DWP(12*$i+4,"edi"));

  388. 	&bswap	("ebx");

  389. 	&mov	("esi",&DWP(12*$i+8,"edi"));

  390. 	&bswap	("ecx");

  391. 	&mov	(&DWP(32+12*$i+0,"esp"),"ebx");

  392. 	&bswap	("esi");

  393. 	&mov	(&DWP(32+12*$i+4,"esp"),"ecx");

  394. 	&mov	(&DWP(32+12*$i+8,"esp"),"esi");

  395.     }

  396. 	&mov	("ebx",&DWP($i*12,"edi"));

  397. 	&add	("edi",64);

  398. 	&bswap	("ebx");

  399. 	&mov	(&DWP(96+4,"esp"),"edi");

  400. 	&mov	(&DWP(32+12*$i,"esp"),"ebx");

  401. 

  402.     my ($t1,$t2) = ("ecx","esi");

  403. 

  404.     for ($i=0;$i<64;$i++) {

  405. 

  406.       if ($i>=16) {

  407. 	&mov	($T,$t1);			# $t1 is preloaded

  408. 	# &mov	($t2,&DWP(32+4*(($i+14)&15),"esp"));

  409. 	&ror	($t1,18-7);

  410. 	 &mov	("edi",$t2);

  411. 	&ror	($t2,19-17);

  412. 	 &xor	($t1,$T);

  413. 	 &shr	($T,3);

  414. 	&ror	($t1,7);

  415. 	 &xor	($t2,"edi");

  416. 	 &xor	($T,$t1);			# T = sigma0(X[-15])

  417. 	&ror	($t2,17);

  418. 	 &add	($T,&DWP(32+4*($i&15),"esp"));	# T += X[-16]

  419. 	&shr	("edi",10);

  420. 	 &add	($T,&DWP(32+4*(($i+9)&15),"esp"));	# T += X[-7]

  421. 	#&xor	("edi",$t2)			# sigma1(X[-2])

  422. 	# &add	($T,"edi");			# T += sigma1(X[-2])

  423. 	# &mov	(&DWP(4*(9+15),"esp"),$T);	# save X[0]

  424.       }

  425. 	&mov	($t1,$E);

  426. 	 &xor	("edi",$t2)			if ($i>=16);	# sigma1(X[-2])

  427. 	 &mov	($t2,&off($f));

  428. 	&ror	($E,25-11);

  429. 	 &add	($T,"edi")			if ($i>=16);	# T += sigma1(X[-2])

  430. 	 &mov	("edi",&off($g));

  431. 	&xor	($E,$t1);

  432. 	 &mov	($T,&DWP(32+4*($i&15),"esp"))	if ($i<16);	# X[i]

  433. 	 &mov	(&DWP(32+4*($i&15),"esp"),$T)	if ($i>=16 && $i<62);	# save X[0]

  434. 	 &xor	($t2,"edi");

  435. 	&ror	($E,11-6);

  436. 	 &and	($t2,$t1);

  437. 	 &mov	(&off($e),$t1);		# save $E, modulo-scheduled

  438. 	&xor	($E,$t1);

  439. 	 &add	($T,&off($h));		# T += h

  440. 	 &xor	("edi",$t2);		# Ch(e,f,g)

  441. 	&ror	($E,6);			# Sigma1(e)

  442. 	 &mov	($t1,$AH[0]);

  443. 	 &add	($T,"edi");		# T += Ch(e,f,g)

  444. 

  445. 	&ror	($t1,22-13);

  446. 	 &mov	($t2,$AH[0]);

  447. 	 &mov	("edi",&off($b));

  448. 	&xor	($t1,$AH[0]);

  449. 	 &mov	(&off($a),$AH[0]);	# save $A, modulo-scheduled

  450. 	 &xor	($AH[0],"edi");		# a ^= b, (b^c) in next round

  451. 	&ror	($t1,13-2);

  452. 	 &and	($AH[1],$AH[0]);	# (b^c) &= (a^b)

  453. 	 &lea	($E,&DWP(@K256[$i],$T,$E));	# T += Sigma1(1)+K[i]

  454. 	&xor	($t1,$t2);

  455. 	 &xor	($AH[1],"edi");		# h = Maj(a,b,c) = Ch(a^b,c,b)

  456. 	 &mov	($t2,&DWP(32+4*(($i+2)&15),"esp"))	if ($i>=15 && $i<63);

  457. 	&ror	($t1,2);		# Sigma0(a)

  458. 

  459. 	 &add	($AH[1],$E);		# h += T

  460. 	 &add	($E,&off($d));		# d += T

  461. 	&add	($AH[1],$t1);		# h += Sigma0(a)

  462. 	 &mov	($t1,&DWP(32+4*(($i+15)&15),"esp"))	if ($i>=15 && $i<63);

  463. 

  464. 	@AH = reverse(@AH);		# rotate(a,h)

  465. 	($t1,$t2) = ($t2,$t1);		# rotate(t1,t2)

  466.     }

  467. 	&mov	("esi",&DWP(96,"esp"));	#ctx

  468. 					#&mov	($AH[0],&DWP(0,"esp"));

  469. 	&xor	($AH[1],"edi");		#&mov	($AH[1],&DWP(4,"esp"));

  470. 					#&mov	("edi", &DWP(8,"esp"));

  471. 	&mov	("ecx",&DWP(12,"esp"));

  472. 	&add	($AH[0],&DWP(0,"esi"));

  473. 	&add	($AH[1],&DWP(4,"esi"));

  474. 	&add	("edi",&DWP(8,"esi"));

  475. 	&add	("ecx",&DWP(12,"esi"));

  476. 	&mov	(&DWP(0,"esi"),$AH[0]);

  477. 	&mov	(&DWP(4,"esi"),$AH[1]);

  478. 	&mov	(&DWP(8,"esi"),"edi");

  479. 	&mov	(&DWP(12,"esi"),"ecx");

  480. 	 #&mov	(&DWP(0,"esp"),$AH[0]);

  481. 	 &mov	(&DWP(4,"esp"),$AH[1]);

  482. 	 &xor	($AH[1],"edi");		# magic

  483. 	 &mov	(&DWP(8,"esp"),"edi");

  484. 	 &mov	(&DWP(12,"esp"),"ecx");

  485. 	#&mov	($E,&DWP(16,"esp"));

  486. 	&mov	("edi",&DWP(20,"esp"));

  487. 	&mov	("ebx",&DWP(24,"esp"));

  488. 	&mov	("ecx",&DWP(28,"esp"));

  489. 	&add	($E,&DWP(16,"esi"));

  490. 	&add	("edi",&DWP(20,"esi"));

  491. 	&add	("ebx",&DWP(24,"esi"));

  492. 	&add	("ecx",&DWP(28,"esi"));

  493. 	&mov	(&DWP(16,"esi"),$E);

  494. 	&mov	(&DWP(20,"esi"),"edi");

  495. 	&mov	(&DWP(24,"esi"),"ebx");

  496. 	&mov	(&DWP(28,"esi"),"ecx");

  497. 	 #&mov	(&DWP(16,"esp"),$E);

  498. 	 &mov	(&DWP(20,"esp"),"edi");

  499. 	&mov	("edi",&DWP(96+4,"esp"));	# inp

  500. 	 &mov	(&DWP(24,"esp"),"ebx");

  501. 	 &mov	(&DWP(28,"esp"),"ecx");

  502. 

  503. 	&cmp	("edi",&DWP(96+8,"esp"));	# are we done yet?

  504. 	&jb	(&label("grand_loop"));

  505. 

  506. 	&mov	("esp",&DWP(96+12,"esp"));	# restore sp

  507. &function_end_A();

  508. }

  509. 						if (!$i386 && $xmm) {{{

  510. if ($shaext) {

  511. ######################################################################

  512. # Intel SHA Extensions implementation of SHA256 update function.

  513. #

  514. my ($ctx,$inp,$end)=("esi","edi","eax");

  515. my ($Wi,$ABEF,$CDGH,$TMP)=map("xmm$_",(0..2,7));

  516. my @MSG=map("xmm$_",(3..6));

  517. 

  518. sub sha256op38 {

  519.  my ($opcodelet,$dst,$src)=@_;

  520.     if ("$dst:$src" =~ /xmm([0-7]):xmm([0-7])/)

  521.     {	&data_byte(0x0f,0x38,$opcodelet,0xc0|($1<<3)|$2);	}

  522. }

  523. sub sha256rnds2	{ sha256op38(0xcb,@_); }

  524. sub sha256msg1	{ sha256op38(0xcc,@_); }

  525. sub sha256msg2	{ sha256op38(0xcd,@_); }

  526. 

  527. &set_label("shaext",32);

  528. 	&sub		("esp",32);

  529. 

  530. 	&movdqu		($ABEF,&QWP(0,$ctx));		# DCBA

  531. 	&lea		($K256,&DWP(0x80,$K256));

  532. 	&movdqu		($CDGH,&QWP(16,$ctx));		# HGFE

  533. 	&movdqa		($TMP,&QWP(0x100-0x80,$K256));	# byte swap mask

  534. 

  535. 	&pshufd		($Wi,$ABEF,0x1b);		# ABCD

  536. 	&pshufd		($ABEF,$ABEF,0xb1);		# CDAB

  537. 	&pshufd		($CDGH,$CDGH,0x1b);		# EFGH

  538. 	&palignr	($ABEF,$CDGH,8);		# ABEF

  539. 	&punpcklqdq	($CDGH,$Wi);			# CDGH

  540. 	&jmp		(&label("loop_shaext"));

  541. 

  542. &set_label("loop_shaext",16);

  543. 	&movdqu		(@MSG[0],&QWP(0,$inp));

  544. 	&movdqu		(@MSG[1],&QWP(0x10,$inp));

  545. 	&movdqu		(@MSG[2],&QWP(0x20,$inp));

  546. 	&pshufb		(@MSG[0],$TMP);

  547. 	&movdqu		(@MSG[3],&QWP(0x30,$inp));

  548. 	&movdqa		(&QWP(16,"esp"),$CDGH);		# offload

  549. 

  550. 	&movdqa		($Wi,&QWP(0*16-0x80,$K256));

  551. 	&paddd		($Wi,@MSG[0]);

  552. 	&pshufb		(@MSG[1],$TMP);

  553. 	&sha256rnds2	($CDGH,$ABEF);			# 0-3

  554. 	&pshufd		($Wi,$Wi,0x0e);

  555. 	&nop		();

  556. 	&movdqa		(&QWP(0,"esp"),$ABEF);		# offload

  557. 	&sha256rnds2	($ABEF,$CDGH);

  558. 

  559. 	&movdqa		($Wi,&QWP(1*16-0x80,$K256));

  560. 	&paddd		($Wi,@MSG[1]);

  561. 	&pshufb		(@MSG[2],$TMP);

  562. 	&sha256rnds2	($CDGH,$ABEF);			# 4-7

  563. 	&pshufd		($Wi,$Wi,0x0e);

  564. 	&lea		($inp,&DWP(0x40,$inp));

  565. 	&sha256msg1	(@MSG[0],@MSG[1]);

  566. 	&sha256rnds2	($ABEF,$CDGH);

  567. 

  568. 	&movdqa		($Wi,&QWP(2*16-0x80,$K256));

  569. 	&paddd		($Wi,@MSG[2]);

  570. 	&pshufb		(@MSG[3],$TMP);

  571. 	&sha256rnds2	($CDGH,$ABEF);			# 8-11

  572. 	&pshufd		($Wi,$Wi,0x0e);

  573. 	&movdqa		($TMP,@MSG[3]);

  574. 	&palignr	($TMP,@MSG[2],4);

  575. 	&nop		();

  576. 	&paddd		(@MSG[0],$TMP);

  577. 	&sha256msg1	(@MSG[1],@MSG[2]);

  578. 	&sha256rnds2	($ABEF,$CDGH);

  579. 

  580. 	&movdqa		($Wi,&QWP(3*16-0x80,$K256));

  581. 	&paddd		($Wi,@MSG[3]);

  582. 	&sha256msg2	(@MSG[0],@MSG[3]);

  583. 	&sha256rnds2	($CDGH,$ABEF);			# 12-15

  584. 	&pshufd		($Wi,$Wi,0x0e);

  585. 	&movdqa		($TMP,@MSG[0]);

  586. 	&palignr	($TMP,@MSG[3],4);

  587. 	&nop		();

  588. 	&paddd		(@MSG[1],$TMP);

  589. 	&sha256msg1	(@MSG[2],@MSG[3]);

  590. 	&sha256rnds2	($ABEF,$CDGH);

  591. 

  592. for($i=4;$i<16-3;$i++) {

  593. 	&movdqa		($Wi,&QWP($i*16-0x80,$K256));

  594. 	&paddd		($Wi,@MSG[0]);

  595. 	&sha256msg2	(@MSG[1],@MSG[0]);

  596. 	&sha256rnds2	($CDGH,$ABEF);			# 16-19...

  597. 	&pshufd		($Wi,$Wi,0x0e);

  598. 	&movdqa		($TMP,@MSG[1]);

  599. 	&palignr	($TMP,@MSG[0],4);

  600. 	&nop		();

  601. 	&paddd		(@MSG[2],$TMP);

  602. 	&sha256msg1	(@MSG[3],@MSG[0]);

  603. 	&sha256rnds2	($ABEF,$CDGH);

  604. 

  605. 	push(@MSG,shift(@MSG));

  606. }

  607. 	&movdqa		($Wi,&QWP(13*16-0x80,$K256));

  608. 	&paddd		($Wi,@MSG[0]);

  609. 	&sha256msg2	(@MSG[1],@MSG[0]);

  610. 	&sha256rnds2	($CDGH,$ABEF);			# 52-55

  611. 	&pshufd		($Wi,$Wi,0x0e);

  612. 	&movdqa		($TMP,@MSG[1])

  613. 	&palignr	($TMP,@MSG[0],4);

  614. 	&sha256rnds2	($ABEF,$CDGH);

  615. 	&paddd		(@MSG[2],$TMP);

  616. 

  617. 	&movdqa		($Wi,&QWP(14*16-0x80,$K256));

  618. 	&paddd		($Wi,@MSG[1]);

  619. 	&sha256rnds2	($CDGH,$ABEF);			# 56-59

  620. 	&pshufd		($Wi,$Wi,0x0e);

  621. 	&sha256msg2	(@MSG[2],@MSG[1]);

  622. 	&movdqa		($TMP,&QWP(0x100-0x80,$K256));	# byte swap mask

  623. 	&sha256rnds2	($ABEF,$CDGH);

  624. 

  625. 	&movdqa		($Wi,&QWP(15*16-0x80,$K256));

  626. 	&paddd		($Wi,@MSG[2]);

  627. 	&nop		();

  628. 	&sha256rnds2	($CDGH,$ABEF);			# 60-63

  629. 	&pshufd		($Wi,$Wi,0x0e);

  630. 	&cmp		($end,$inp);

  631. 	&nop		();

  632. 	&sha256rnds2	($ABEF,$CDGH);

  633. 

  634. 	&paddd		($CDGH,&QWP(16,"esp"));

  635. 	&paddd		($ABEF,&QWP(0,"esp"));

  636. 	&jnz		(&label("loop_shaext"));

  637. 

  638. 	&pshufd		($CDGH,$CDGH,0xb1);		# DCHG

  639. 	&pshufd		($TMP,$ABEF,0x1b);		# FEBA

  640. 	&pshufd		($ABEF,$ABEF,0xb1);		# BAFE

  641. 	&punpckhqdq	($ABEF,$CDGH);			# DCBA

  642. 	&palignr	($CDGH,$TMP,8);			# HGFE

  643. 

  644. 	&mov		("esp",&DWP(32+12,"esp"));

  645. 	&movdqu		(&QWP(0,$ctx),$ABEF);

  646. 	&movdqu		(&QWP(16,$ctx),$CDGH);

  647. &function_end_A();

  648. }

  649. 

  650. my @X = map("xmm$_",(0..3));

  651. my ($t0,$t1,$t2,$t3) = map("xmm$_",(4..7));

  652. my @AH = ($A,$T);

  653. 

  654. &set_label("SSSE3",32);

  655. 	&lea	("esp",&DWP(-96,"esp"));

  656. 	# copy ctx->h[0-7] to A,B,C,D,E,F,G,H on stack

  657. 	&mov	($AH[0],&DWP(0,"esi"));

  658. 	&mov	($AH[1],&DWP(4,"esi"));

  659. 	&mov	("ecx",&DWP(8,"esi"));

  660. 	&mov	("edi",&DWP(12,"esi"));

  661. 	#&mov	(&DWP(0,"esp"),$AH[0]);

  662. 	&mov	(&DWP(4,"esp"),$AH[1]);

  663. 	&xor	($AH[1],"ecx");			# magic

  664. 	&mov	(&DWP(8,"esp"),"ecx");

  665. 	&mov	(&DWP(12,"esp"),"edi");

  666. 	&mov	($E,&DWP(16,"esi"));

  667. 	&mov	("edi",&DWP(20,"esi"));

  668. 	&mov	("ecx",&DWP(24,"esi"));

  669. 	&mov	("esi",&DWP(28,"esi"));

  670. 	#&mov	(&DWP(16,"esp"),$E);

  671. 	&mov	(&DWP(20,"esp"),"edi");

  672. 	&mov	("edi",&DWP(96+4,"esp"));	# inp

  673. 	&mov	(&DWP(24,"esp"),"ecx");

  674. 	&mov	(&DWP(28,"esp"),"esi");

  675. 	&movdqa	($t3,&QWP(256,$K256));

  676. 	&jmp	(&label("grand_ssse3"));

  677. 

  678. &set_label("grand_ssse3",16);

  679. 	# load input, reverse byte order, add K256[0..15], save to stack

  680. 	&movdqu	(@X[0],&QWP(0,"edi"));

  681. 	&movdqu	(@X[1],&QWP(16,"edi"));

  682. 	&movdqu	(@X[2],&QWP(32,"edi"));

  683. 	&movdqu	(@X[3],&QWP(48,"edi"));

  684. 	&add	("edi",64);

  685. 	&pshufb	(@X[0],$t3);

  686. 	&mov	(&DWP(96+4,"esp"),"edi");

  687. 	&pshufb	(@X[1],$t3);

  688. 	&movdqa	($t0,&QWP(0,$K256));

  689. 	&pshufb	(@X[2],$t3);

  690. 	&movdqa	($t1,&QWP(16,$K256));

  691. 	&paddd	($t0,@X[0]);

  692. 	&pshufb	(@X[3],$t3);

  693. 	&movdqa	($t2,&QWP(32,$K256));

  694. 	&paddd	($t1,@X[1]);

  695. 	&movdqa	($t3,&QWP(48,$K256));

  696. 	&movdqa	(&QWP(32+0,"esp"),$t0);

  697. 	&paddd	($t2,@X[2]);

  698. 	&movdqa	(&QWP(32+16,"esp"),$t1);

  699. 	&paddd	($t3,@X[3]);

  700. 	&movdqa	(&QWP(32+32,"esp"),$t2);

  701. 	&movdqa	(&QWP(32+48,"esp"),$t3);

  702. 	&jmp	(&label("ssse3_00_47"));

  703. 

  704. &set_label("ssse3_00_47",16);

  705. 	&add		($K256,64);

  706. 

  707. sub SSSE3_00_47 () {

  708. my $j = shift;

  709. my $body = shift;

  710. my @X = @_;

  711. my @insns = (&$body,&$body,&$body,&$body);	# 120 instructions

  712. 

  713. 	  eval(shift(@insns));

  714. 	&movdqa		($t0,@X[1]);

  715. 	  eval(shift(@insns));			# @

  716. 	  eval(shift(@insns));

  717. 	&movdqa		($t3,@X[3]);

  718. 	  eval(shift(@insns));

  719. 	  eval(shift(@insns));

  720. 	&palignr	($t0,@X[0],4);		# X[1..4]

  721. 	  eval(shift(@insns));

  722. 	  eval(shift(@insns));			# @

  723. 	  eval(shift(@insns));

  724. 	 &palignr	($t3,@X[2],4);		# X[9..12]

  725. 	  eval(shift(@insns));

  726. 	  eval(shift(@insns));

  727. 	  eval(shift(@insns));

  728. 	&movdqa		($t1,$t0);

  729. 	  eval(shift(@insns));			# @

  730. 	  eval(shift(@insns));

  731. 	&movdqa		($t2,$t0);

  732. 	  eval(shift(@insns));

  733. 	  eval(shift(@insns));

  734. 	&psrld		($t0,3);

  735. 	  eval(shift(@insns));

  736. 	  eval(shift(@insns));			# @

  737. 	 &paddd		(@X[0],$t3);		# X[0..3] += X[9..12]

  738. 	  eval(shift(@insns));

  739. 	  eval(shift(@insns));

  740. 	&psrld		($t2,7);

  741. 	  eval(shift(@insns));

  742. 	  eval(shift(@insns));

  743. 	  eval(shift(@insns));			# @

  744. 	  eval(shift(@insns));

  745. 	 &pshufd	($t3,@X[3],0b11111010);	# X[14..15]

  746. 	  eval(shift(@insns));

  747. 	  eval(shift(@insns));

  748. 	&pslld		($t1,32-18);

  749. 	  eval(shift(@insns));

  750. 	  eval(shift(@insns));			# @

  751. 	&pxor		($t0,$t2);

  752. 	  eval(shift(@insns));

  753. 	  eval(shift(@insns));

  754. 	&psrld		($t2,18-7);

  755. 	  eval(shift(@insns));

  756. 	  eval(shift(@insns));

  757. 	  eval(shift(@insns));			# @

  758. 	&pxor		($t0,$t1);

  759. 	  eval(shift(@insns));

  760. 	  eval(shift(@insns));

  761. 	&pslld		($t1,18-7);

  762. 	  eval(shift(@insns));

  763. 	  eval(shift(@insns));

  764. 	  eval(shift(@insns));			# @

  765. 	&pxor		($t0,$t2);

  766. 	  eval(shift(@insns));

  767. 	  eval(shift(@insns));

  768. 	 &movdqa	($t2,$t3);

  769. 	  eval(shift(@insns));

  770. 	  eval(shift(@insns));

  771. 	  eval(shift(@insns));			# @

  772. 	&pxor		($t0,$t1);		# sigma0(X[1..4])

  773. 	  eval(shift(@insns));

  774. 	  eval(shift(@insns));

  775. 	 &psrld		($t3,10);

  776. 	  eval(shift(@insns));

  777. 	  eval(shift(@insns));

  778. 	  eval(shift(@insns));			# @

  779. 	&paddd		(@X[0],$t0);		# X[0..3] += sigma0(X[1..4])

  780. 	  eval(shift(@insns));

  781. 	  eval(shift(@insns));

  782. 	 &psrlq		($t2,17);

  783. 	  eval(shift(@insns));

  784. 	  eval(shift(@insns));

  785. 	  eval(shift(@insns));			# @

  786. 	 &pxor		($t3,$t2);

  787. 	  eval(shift(@insns));

  788. 	  eval(shift(@insns));

  789. 	 &psrlq		($t2,19-17);

  790. 	  eval(shift(@insns));

  791. 	  eval(shift(@insns));

  792. 	  eval(shift(@insns));			# @

  793. 	 &pxor		($t3,$t2);

  794. 	  eval(shift(@insns));

  795. 	  eval(shift(@insns));

  796. 	 &pshufd	($t3,$t3,0b10000000);

  797. 	  eval(shift(@insns));

  798. 	  eval(shift(@insns));

  799. 	  eval(shift(@insns));			# @

  800. 	  eval(shift(@insns));

  801. 	  eval(shift(@insns));

  802. 	  eval(shift(@insns));

  803. 	  eval(shift(@insns));

  804. 	  eval(shift(@insns));			# @

  805. 	  eval(shift(@insns));

  806. 	 &psrldq	($t3,8);

  807. 	  eval(shift(@insns));

  808. 	  eval(shift(@insns));

  809. 	  eval(shift(@insns));

  810. 	&paddd		(@X[0],$t3);		# X[0..1] += sigma1(X[14..15])

  811. 	  eval(shift(@insns));			# @

  812. 	  eval(shift(@insns));

  813. 	  eval(shift(@insns));

  814. 	  eval(shift(@insns));

  815. 	  eval(shift(@insns));

  816. 	  eval(shift(@insns));			# @

  817. 	  eval(shift(@insns));

  818. 	 &pshufd	($t3,@X[0],0b01010000);	# X[16..17]

  819. 	  eval(shift(@insns));

  820. 	  eval(shift(@insns));

  821. 	  eval(shift(@insns));

  822. 	 &movdqa	($t2,$t3);

  823. 	  eval(shift(@insns));			# @

  824. 	 &psrld		($t3,10);

  825. 	  eval(shift(@insns));

  826. 	 &psrlq		($t2,17);

  827. 	  eval(shift(@insns));

  828. 	  eval(shift(@insns));

  829. 	  eval(shift(@insns));

  830. 	  eval(shift(@insns));			# @

  831. 	 &pxor		($t3,$t2);

  832. 	  eval(shift(@insns));

  833. 	  eval(shift(@insns));

  834. 	 &psrlq		($t2,19-17);

  835. 	  eval(shift(@insns));

  836. 	  eval(shift(@insns));

  837. 	  eval(shift(@insns));			# @

  838. 	 &pxor		($t3,$t2);

  839. 	  eval(shift(@insns));

  840. 	  eval(shift(@insns));

  841. 	  eval(shift(@insns));

  842. 	 &pshufd	($t3,$t3,0b00001000);

  843. 	  eval(shift(@insns));

  844. 	  eval(shift(@insns));			# @

  845. 	&movdqa		($t2,&QWP(16*$j,$K256));

  846. 	  eval(shift(@insns));

  847. 	  eval(shift(@insns));

  848. 	 &pslldq	($t3,8);

  849. 	  eval(shift(@insns));

  850. 	  eval(shift(@insns));

  851. 	  eval(shift(@insns));			# @

  852. 	  eval(shift(@insns));

  853. 	  eval(shift(@insns));

  854. 	  eval(shift(@insns));

  855. 	  eval(shift(@insns));

  856. 	  eval(shift(@insns));			# @

  857. 	&paddd		(@X[0],$t3);		# X[2..3] += sigma1(X[16..17])

  858. 	  eval(shift(@insns));

  859. 	  eval(shift(@insns));

  860. 	  eval(shift(@insns));

  861. 	  eval(shift(@insns));

  862. 	&paddd		($t2,@X[0]);

  863. 	  eval(shift(@insns));			# @

  864. 

  865. 	foreach (@insns) { eval; }		# remaining instructions

  866. 

  867. 	&movdqa		(&QWP(32+16*$j,"esp"),$t2);

  868. }

  869. 

  870. sub body_00_15 () {

  871. 	(

  872. 	'&mov	("ecx",$E);',

  873. 	'&ror	($E,25-11);',

  874. 	 '&mov	("esi",&off($f));',

  875. 	'&xor	($E,"ecx");',

  876. 	 '&mov	("edi",&off($g));',

  877. 	 '&xor	("esi","edi");',

  878. 	'&ror	($E,11-6);',

  879. 	 '&and	("esi","ecx");',

  880. 	 '&mov	(&off($e),"ecx");',	# save $E, modulo-scheduled

  881. 	'&xor	($E,"ecx");',

  882. 	 '&xor	("edi","esi");',	# Ch(e,f,g)

  883. 	'&ror	($E,6);',		# T = Sigma1(e)

  884. 	 '&mov	("ecx",$AH[0]);',

  885. 	 '&add	($E,"edi");',		# T += Ch(e,f,g)

  886. 	 '&mov	("edi",&off($b));',

  887. 	'&mov	("esi",$AH[0]);',

  888. 

  889. 	'&ror	("ecx",22-13);',

  890. 	 '&mov	(&off($a),$AH[0]);',	# save $A, modulo-scheduled

  891. 	'&xor	("ecx",$AH[0]);',

  892. 	 '&xor	($AH[0],"edi");',	# a ^= b, (b^c) in next round

  893. 	 '&add	($E,&off($h));',	# T += h

  894. 	'&ror	("ecx",13-2);',

  895. 	 '&and	($AH[1],$AH[0]);',	# (b^c) &= (a^b)

  896. 	'&xor	("ecx","esi");',

  897. 	 '&add	($E,&DWP(32+4*($i&15),"esp"));',	# T += K[i]+X[i]

  898. 	 '&xor	($AH[1],"edi");',	# h = Maj(a,b,c) = Ch(a^b,c,b)

  899. 	'&ror	("ecx",2);',		# Sigma0(a)

  900. 

  901. 	 '&add	($AH[1],$E);',		# h += T

  902. 	 '&add	($E,&off($d));',	# d += T

  903. 	'&add	($AH[1],"ecx");'.	# h += Sigma0(a)

  904. 

  905. 	'@AH = reverse(@AH); $i++;'	# rotate(a,h)

  906. 	);

  907. }

  908. 

  909.     for ($i=0,$j=0; $j<4; $j++) {

  910. 	&SSSE3_00_47($j,\&body_00_15,@X);

  911. 	push(@X,shift(@X));		# rotate(@X)

  912.     }

  913. 	&cmp	(&DWP(16*$j,$K256),0x00010203);

  914. 	&jne	(&label("ssse3_00_47"));

  915. 

  916.     for ($i=0; $i<16; ) {

  917. 	foreach(body_00_15()) { eval; }

  918.     }

  919. 

  920. 	&mov	("esi",&DWP(96,"esp"));	#ctx

  921. 					#&mov	($AH[0],&DWP(0,"esp"));

  922. 	&xor	($AH[1],"edi");		#&mov	($AH[1],&DWP(4,"esp"));

  923. 					#&mov	("edi", &DWP(8,"esp"));

  924. 	&mov	("ecx",&DWP(12,"esp"));

  925. 	&add	($AH[0],&DWP(0,"esi"));

  926. 	&add	($AH[1],&DWP(4,"esi"));

  927. 	&add	("edi",&DWP(8,"esi"));

  928. 	&add	("ecx",&DWP(12,"esi"));

  929. 	&mov	(&DWP(0,"esi"),$AH[0]);

  930. 	&mov	(&DWP(4,"esi"),$AH[1]);

  931. 	&mov	(&DWP(8,"esi"),"edi");

  932. 	&mov	(&DWP(12,"esi"),"ecx");

  933. 	 #&mov	(&DWP(0,"esp"),$AH[0]);

  934. 	 &mov	(&DWP(4,"esp"),$AH[1]);

  935. 	 &xor	($AH[1],"edi");			# magic

  936. 	 &mov	(&DWP(8,"esp"),"edi");

  937. 	 &mov	(&DWP(12,"esp"),"ecx");

  938. 	#&mov	($E,&DWP(16,"esp"));

  939. 	&mov	("edi",&DWP(20,"esp"));

  940. 	&mov	("ecx",&DWP(24,"esp"));

  941. 	&add	($E,&DWP(16,"esi"));

  942. 	&add	("edi",&DWP(20,"esi"));

  943. 	&add	("ecx",&DWP(24,"esi"));

  944. 	&mov	(&DWP(16,"esi"),$E);

  945. 	&mov	(&DWP(20,"esi"),"edi");

  946. 	 &mov	(&DWP(20,"esp"),"edi");

  947. 	&mov	("edi",&DWP(28,"esp"));

  948. 	&mov	(&DWP(24,"esi"),"ecx");

  949. 	 #&mov	(&DWP(16,"esp"),$E);

  950. 	&add	("edi",&DWP(28,"esi"));

  951. 	 &mov	(&DWP(24,"esp"),"ecx");

  952. 	&mov	(&DWP(28,"esi"),"edi");

  953. 	 &mov	(&DWP(28,"esp"),"edi");

  954. 	&mov	("edi",&DWP(96+4,"esp"));	# inp

  955. 

  956. 	&movdqa	($t3,&QWP(64,$K256));

  957. 	&sub	($K256,3*64);			# rewind K

  958. 	&cmp	("edi",&DWP(96+8,"esp"));	# are we done yet?

  959. 	&jb	(&label("grand_ssse3"));

  960. 

  961. 	&mov	("esp",&DWP(96+12,"esp"));	# restore sp

  962. &function_end_A();

  963. 						if ($avx) {

  964. &set_label("AVX",32);

  965. 						if ($avx>1) {

  966. 	&and	("edx",1<<8|1<<3);		# check for BMI2+BMI1

  967. 	&cmp	("edx",1<<8|1<<3);

  968. 	&je	(&label("AVX_BMI"));

  969. 						}

  970. 	&lea	("esp",&DWP(-96,"esp"));

  971. 	&vzeroall	();

  972. 	# copy ctx->h[0-7] to A,B,C,D,E,F,G,H on stack

  973. 	&mov	($AH[0],&DWP(0,"esi"));

  974. 	&mov	($AH[1],&DWP(4,"esi"));

  975. 	&mov	("ecx",&DWP(8,"esi"));

  976. 	&mov	("edi",&DWP(12,"esi"));

  977. 	#&mov	(&DWP(0,"esp"),$AH[0]);

  978. 	&mov	(&DWP(4,"esp"),$AH[1]);

  979. 	&xor	($AH[1],"ecx");			# magic

  980. 	&mov	(&DWP(8,"esp"),"ecx");

  981. 	&mov	(&DWP(12,"esp"),"edi");

  982. 	&mov	($E,&DWP(16,"esi"));

  983. 	&mov	("edi",&DWP(20,"esi"));

  984. 	&mov	("ecx",&DWP(24,"esi"));

  985. 	&mov	("esi",&DWP(28,"esi"));

  986. 	#&mov	(&DWP(16,"esp"),$E);

  987. 	&mov	(&DWP(20,"esp"),"edi");

  988. 	&mov	("edi",&DWP(96+4,"esp"));	# inp

  989. 	&mov	(&DWP(24,"esp"),"ecx");

  990. 	&mov	(&DWP(28,"esp"),"esi");

  991. 	&vmovdqa	($t3,&QWP(256,$K256));

  992. 	&jmp	(&label("grand_avx"));

  993. 

  994. &set_label("grand_avx",32);

  995. 	# load input, reverse byte order, add K256[0..15], save to stack

  996. 	&vmovdqu	(@X[0],&QWP(0,"edi"));

  997. 	&vmovdqu	(@X[1],&QWP(16,"edi"));

  998. 	&vmovdqu	(@X[2],&QWP(32,"edi"));

  999. 	&vmovdqu	(@X[3],&QWP(48,"edi"));

  1000. 	&add		("edi",64);

  1001. 	&vpshufb	(@X[0],@X[0],$t3);

  1002. 	&mov		(&DWP(96+4,"esp"),"edi");

  1003. 	&vpshufb	(@X[1],@X[1],$t3);

  1004. 	&vpshufb	(@X[2],@X[2],$t3);

  1005. 	&vpaddd		($t0,@X[0],&QWP(0,$K256));

  1006. 	&vpshufb	(@X[3],@X[3],$t3);

  1007. 	&vpaddd		($t1,@X[1],&QWP(16,$K256));

  1008. 	&vpaddd		($t2,@X[2],&QWP(32,$K256));

  1009. 	&vpaddd		($t3,@X[3],&QWP(48,$K256));

  1010. 	&vmovdqa	(&QWP(32+0,"esp"),$t0);

  1011. 	&vmovdqa	(&QWP(32+16,"esp"),$t1);

  1012. 	&vmovdqa	(&QWP(32+32,"esp"),$t2);

  1013. 	&vmovdqa	(&QWP(32+48,"esp"),$t3);

  1014. 	&jmp		(&label("avx_00_47"));

  1015. 

  1016. &set_label("avx_00_47",16);

  1017. 	&add		($K256,64);

  1018. 

  1019. sub Xupdate_AVX () {

  1020. 	(

  1021. 	'&vpalignr	($t0,@X[1],@X[0],4);',	# X[1..4]

  1022. 	 '&vpalignr	($t3,@X[3],@X[2],4);',	# X[9..12]

  1023. 	'&vpsrld	($t2,$t0,7);',

  1024. 	 '&vpaddd	(@X[0],@X[0],$t3);',	# X[0..3] += X[9..16]

  1025. 	'&vpsrld	($t3,$t0,3);',

  1026. 	'&vpslld	($t1,$t0,14);',

  1027. 	'&vpxor		($t0,$t3,$t2);',

  1028. 	 '&vpshufd	($t3,@X[3],0b11111010)',# X[14..15]

  1029. 	'&vpsrld	($t2,$t2,18-7);',

  1030. 	'&vpxor		($t0,$t0,$t1);',

  1031. 	'&vpslld	($t1,$t1,25-14);',

  1032. 	'&vpxor		($t0,$t0,$t2);',

  1033. 	 '&vpsrld	($t2,$t3,10);',

  1034. 	'&vpxor		($t0,$t0,$t1);',	# sigma0(X[1..4])

  1035. 	 '&vpsrlq	($t1,$t3,17);',

  1036. 	'&vpaddd	(@X[0],@X[0],$t0);',	# X[0..3] += sigma0(X[1..4])

  1037. 	 '&vpxor	($t2,$t2,$t1);',

  1038. 	 '&vpsrlq	($t3,$t3,19);',

  1039. 	 '&vpxor	($t2,$t2,$t3);',	# sigma1(X[14..15]

  1040. 	 '&vpshufd	($t3,$t2,0b10000100);',

  1041. 	'&vpsrldq	($t3,$t3,8);',

  1042. 	'&vpaddd	(@X[0],@X[0],$t3);',	# X[0..1] += sigma1(X[14..15])

  1043. 	 '&vpshufd	($t3,@X[0],0b01010000)',# X[16..17]

  1044. 	 '&vpsrld	($t2,$t3,10);',

  1045. 	 '&vpsrlq	($t1,$t3,17);',

  1046. 	 '&vpxor	($t2,$t2,$t1);',

  1047. 	 '&vpsrlq	($t3,$t3,19);',

  1048. 	 '&vpxor	($t2,$t2,$t3);',	# sigma1(X[16..17]

  1049. 	 '&vpshufd	($t3,$t2,0b11101000);',

  1050. 	'&vpslldq	($t3,$t3,8);',

  1051. 	'&vpaddd	(@X[0],@X[0],$t3);'	# X[2..3] += sigma1(X[16..17])

  1052. 	);

  1053. }

  1054. 

  1055. local *ror = sub { &shrd(@_[0],@_) };

  1056. sub AVX_00_47 () {

  1057. my $j = shift;

  1058. my $body = shift;

  1059. my @X = @_;

  1060. my @insns = (&$body,&$body,&$body,&$body);	# 120 instructions

  1061. my $insn;

  1062. 

  1063. 	foreach (Xupdate_AVX()) {		# 31 instructions

  1064. 	    eval;

  1065. 	    eval(shift(@insns));

  1066. 	    eval(shift(@insns));

  1067. 	    eval($insn = shift(@insns));

  1068. 	    eval(shift(@insns)) if ($insn =~ /rorx/ && @insns[0] =~ /rorx/);

  1069. 	}

  1070. 	&vpaddd		($t2,@X[0],&QWP(16*$j,$K256));

  1071. 	foreach (@insns) { eval; }		# remaining instructions

  1072. 	&vmovdqa	(&QWP(32+16*$j,"esp"),$t2);

  1073. }

  1074. 

  1075.     for ($i=0,$j=0; $j<4; $j++) {

  1076. 	&AVX_00_47($j,\&body_00_15,@X);

  1077. 	push(@X,shift(@X));		# rotate(@X)

  1078.     }

  1079. 	&cmp	(&DWP(16*$j,$K256),0x00010203);

  1080. 	&jne	(&label("avx_00_47"));

  1081. 

  1082.     for ($i=0; $i<16; ) {

  1083. 	foreach(body_00_15()) { eval; }

  1084.     }

  1085. 

  1086. 	&mov	("esi",&DWP(96,"esp"));	#ctx

  1087. 					#&mov	($AH[0],&DWP(0,"esp"));

  1088. 	&xor	($AH[1],"edi");		#&mov	($AH[1],&DWP(4,"esp"));

  1089. 					#&mov	("edi", &DWP(8,"esp"));

  1090. 	&mov	("ecx",&DWP(12,"esp"));

  1091. 	&add	($AH[0],&DWP(0,"esi"));

  1092. 	&add	($AH[1],&DWP(4,"esi"));

  1093. 	&add	("edi",&DWP(8,"esi"));

  1094. 	&add	("ecx",&DWP(12,"esi"));

  1095. 	&mov	(&DWP(0,"esi"),$AH[0]);

  1096. 	&mov	(&DWP(4,"esi"),$AH[1]);

  1097. 	&mov	(&DWP(8,"esi"),"edi");

  1098. 	&mov	(&DWP(12,"esi"),"ecx");

  1099. 	 #&mov	(&DWP(0,"esp"),$AH[0]);

  1100. 	 &mov	(&DWP(4,"esp"),$AH[1]);

  1101. 	 &xor	($AH[1],"edi");			# magic

  1102. 	 &mov	(&DWP(8,"esp"),"edi");

  1103. 	 &mov	(&DWP(12,"esp"),"ecx");

  1104. 	#&mov	($E,&DWP(16,"esp"));

  1105. 	&mov	("edi",&DWP(20,"esp"));

  1106. 	&mov	("ecx",&DWP(24,"esp"));

  1107. 	&add	($E,&DWP(16,"esi"));

  1108. 	&add	("edi",&DWP(20,"esi"));

  1109. 	&add	("ecx",&DWP(24,"esi"));

  1110. 	&mov	(&DWP(16,"esi"),$E);

  1111. 	&mov	(&DWP(20,"esi"),"edi");

  1112. 	 &mov	(&DWP(20,"esp"),"edi");

  1113. 	&mov	("edi",&DWP(28,"esp"));

  1114. 	&mov	(&DWP(24,"esi"),"ecx");

  1115. 	 #&mov	(&DWP(16,"esp"),$E);

  1116. 	&add	("edi",&DWP(28,"esi"));

  1117. 	 &mov	(&DWP(24,"esp"),"ecx");

  1118. 	&mov	(&DWP(28,"esi"),"edi");

  1119. 	 &mov	(&DWP(28,"esp"),"edi");

  1120. 	&mov	("edi",&DWP(96+4,"esp"));	# inp

  1121. 

  1122. 	&vmovdqa	($t3,&QWP(64,$K256));

  1123. 	&sub	($K256,3*64);			# rewind K

  1124. 	&cmp	("edi",&DWP(96+8,"esp"));	# are we done yet?

  1125. 	&jb	(&label("grand_avx"));

  1126. 

  1127. 	&mov	("esp",&DWP(96+12,"esp"));	# restore sp

  1128. 	&vzeroall	();

  1129. &function_end_A();

  1130. 						if ($avx>1) {

  1131. sub bodyx_00_15 () {			# +10%

  1132. 	(

  1133. 	'&rorx	("ecx",$E,6)',

  1134. 	'&rorx	("esi",$E,11)',

  1135. 	 '&mov	(&off($e),$E)',		# save $E, modulo-scheduled

  1136. 	'&rorx	("edi",$E,25)',

  1137. 	'&xor	("ecx","esi")',

  1138. 	 '&andn	("esi",$E,&off($g))',

  1139. 	'&xor	("ecx","edi")',		# Sigma1(e)

  1140. 	 '&and	($E,&off($f))',

  1141. 	 '&mov	(&off($a),$AH[0]);',	# save $A, modulo-scheduled

  1142. 	 '&or	($E,"esi")',		# T = Ch(e,f,g)

  1143. 

  1144. 	'&rorx	("edi",$AH[0],2)',

  1145. 	'&rorx	("esi",$AH[0],13)',

  1146. 	 '&lea	($E,&DWP(0,$E,"ecx"))',	# T += Sigma1(e)

  1147. 	'&rorx	("ecx",$AH[0],22)',

  1148. 	'&xor	("esi","edi")',

  1149. 	 '&mov	("edi",&off($b))',

  1150. 	'&xor	("ecx","esi")',		# Sigma0(a)

  1151. 

  1152. 	 '&xor	($AH[0],"edi")',	# a ^= b, (b^c) in next round

  1153. 	 '&add	($E,&off($h))',		# T += h

  1154. 	 '&and	($AH[1],$AH[0])',	# (b^c) &= (a^b)

  1155. 	 '&add	($E,&DWP(32+4*($i&15),"esp"))',	# T += K[i]+X[i]

  1156. 	 '&xor	($AH[1],"edi")',	# h = Maj(a,b,c) = Ch(a^b,c,b)

  1157. 

  1158. 	 '&add	("ecx",$E)',		# h += T

  1159. 	 '&add	($E,&off($d))',		# d += T

  1160. 	'&lea	($AH[1],&DWP(0,$AH[1],"ecx"));'.	# h += Sigma0(a)

  1161. 

  1162. 	'@AH = reverse(@AH); $i++;'	# rotate(a,h)

  1163. 	);

  1164. }

  1165. 

  1166. &set_label("AVX_BMI",32);

  1167. 	&lea	("esp",&DWP(-96,"esp"));

  1168. 	&vzeroall	();

  1169. 	# copy ctx->h[0-7] to A,B,C,D,E,F,G,H on stack

  1170. 	&mov	($AH[0],&DWP(0,"esi"));

  1171. 	&mov	($AH[1],&DWP(4,"esi"));

  1172. 	&mov	("ecx",&DWP(8,"esi"));

  1173. 	&mov	("edi",&DWP(12,"esi"));

  1174. 	#&mov	(&DWP(0,"esp"),$AH[0]);

  1175. 	&mov	(&DWP(4,"esp"),$AH[1]);

  1176. 	&xor	($AH[1],"ecx");			# magic

  1177. 	&mov	(&DWP(8,"esp"),"ecx");

  1178. 	&mov	(&DWP(12,"esp"),"edi");

  1179. 	&mov	($E,&DWP(16,"esi"));

  1180. 	&mov	("edi",&DWP(20,"esi"));

  1181. 	&mov	("ecx",&DWP(24,"esi"));

  1182. 	&mov	("esi",&DWP(28,"esi"));

  1183. 	#&mov	(&DWP(16,"esp"),$E);

  1184. 	&mov	(&DWP(20,"esp"),"edi");

  1185. 	&mov	("edi",&DWP(96+4,"esp"));	# inp

  1186. 	&mov	(&DWP(24,"esp"),"ecx");

  1187. 	&mov	(&DWP(28,"esp"),"esi");

  1188. 	&vmovdqa	($t3,&QWP(256,$K256));

  1189. 	&jmp	(&label("grand_avx_bmi"));

  1190. 

  1191. &set_label("grand_avx_bmi",32);

  1192. 	# load input, reverse byte order, add K256[0..15], save to stack

  1193. 	&vmovdqu	(@X[0],&QWP(0,"edi"));

  1194. 	&vmovdqu	(@X[1],&QWP(16,"edi"));

  1195. 	&vmovdqu	(@X[2],&QWP(32,"edi"));

  1196. 	&vmovdqu	(@X[3],&QWP(48,"edi"));

  1197. 	&add		("edi",64);

  1198. 	&vpshufb	(@X[0],@X[0],$t3);

  1199. 	&mov		(&DWP(96+4,"esp"),"edi");

  1200. 	&vpshufb	(@X[1],@X[1],$t3);

  1201. 	&vpshufb	(@X[2],@X[2],$t3);

  1202. 	&vpaddd		($t0,@X[0],&QWP(0,$K256));

  1203. 	&vpshufb	(@X[3],@X[3],$t3);

  1204. 	&vpaddd		($t1,@X[1],&QWP(16,$K256));

  1205. 	&vpaddd		($t2,@X[2],&QWP(32,$K256));

  1206. 	&vpaddd		($t3,@X[3],&QWP(48,$K256));

  1207. 	&vmovdqa	(&QWP(32+0,"esp"),$t0);

  1208. 	&vmovdqa	(&QWP(32+16,"esp"),$t1);

  1209. 	&vmovdqa	(&QWP(32+32,"esp"),$t2);

  1210. 	&vmovdqa	(&QWP(32+48,"esp"),$t3);

  1211. 	&jmp		(&label("avx_bmi_00_47"));

  1212. 

  1213. &set_label("avx_bmi_00_47",16);

  1214. 	&add		($K256,64);

  1215. 

  1216.     for ($i=0,$j=0; $j<4; $j++) {

  1217. 	&AVX_00_47($j,\&bodyx_00_15,@X);

  1218. 	push(@X,shift(@X));		# rotate(@X)

  1219.     }

  1220. 	&cmp	(&DWP(16*$j,$K256),0x00010203);

  1221. 	&jne	(&label("avx_bmi_00_47"));

  1222. 

  1223.     for ($i=0; $i<16; ) {

  1224. 	foreach(bodyx_00_15()) { eval; }

  1225.     }

  1226. 

  1227. 	&mov	("esi",&DWP(96,"esp"));	#ctx

  1228. 					#&mov	($AH[0],&DWP(0,"esp"));

  1229. 	&xor	($AH[1],"edi");		#&mov	($AH[1],&DWP(4,"esp"));

  1230. 					#&mov	("edi", &DWP(8,"esp"));

  1231. 	&mov	("ecx",&DWP(12,"esp"));

  1232. 	&add	($AH[0],&DWP(0,"esi"));

  1233. 	&add	($AH[1],&DWP(4,"esi"));

  1234. 	&add	("edi",&DWP(8,"esi"));

  1235. 	&add	("ecx",&DWP(12,"esi"));

  1236. 	&mov	(&DWP(0,"esi"),$AH[0]);

  1237. 	&mov	(&DWP(4,"esi"),$AH[1]);

  1238. 	&mov	(&DWP(8,"esi"),"edi");

  1239. 	&mov	(&DWP(12,"esi"),"ecx");

  1240. 	 #&mov	(&DWP(0,"esp"),$AH[0]);

  1241. 	 &mov	(&DWP(4,"esp"),$AH[1]);

  1242. 	 &xor	($AH[1],"edi");			# magic

  1243. 	 &mov	(&DWP(8,"esp"),"edi");

  1244. 	 &mov	(&DWP(12,"esp"),"ecx");

  1245. 	#&mov	($E,&DWP(16,"esp"));

  1246. 	&mov	("edi",&DWP(20,"esp"));

  1247. 	&mov	("ecx",&DWP(24,"esp"));

  1248. 	&add	($E,&DWP(16,"esi"));

  1249. 	&add	("edi",&DWP(20,"esi"));

  1250. 	&add	("ecx",&DWP(24,"esi"));

  1251. 	&mov	(&DWP(16,"esi"),$E);

  1252. 	&mov	(&DWP(20,"esi"),"edi");

  1253. 	 &mov	(&DWP(20,"esp"),"edi");

  1254. 	&mov	("edi",&DWP(28,"esp"));

  1255. 	&mov	(&DWP(24,"esi"),"ecx");

  1256. 	 #&mov	(&DWP(16,"esp"),$E);

  1257. 	&add	("edi",&DWP(28,"esi"));

  1258. 	 &mov	(&DWP(24,"esp"),"ecx");

  1259. 	&mov	(&DWP(28,"esi"),"edi");

  1260. 	 &mov	(&DWP(28,"esp"),"edi");

  1261. 	&mov	("edi",&DWP(96+4,"esp"));	# inp

  1262. 

  1263. 	&vmovdqa	($t3,&QWP(64,$K256));

  1264. 	&sub	($K256,3*64);			# rewind K

  1265. 	&cmp	("edi",&DWP(96+8,"esp"));	# are we done yet?

  1266. 	&jb	(&label("grand_avx_bmi"));

  1267. 

  1268. 	&mov	("esp",&DWP(96+12,"esp"));	# restore sp

  1269. 	&vzeroall	();

  1270. &function_end_A();

  1271. 						}

  1272. 						}

  1273. 						}}}

  1274. &function_end_B("sha256_block_data_order");

  1275. 

  1276. &asm_finish();