kris
/
boringssl
1
0
複製 0
程式碼 問題 0 合併請求 0 版本發佈 0 Wiki 活動
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
528 提交
12 分支
38 MiB
 
 
 
 
 
 
目錄樹: f4b4952719
分支列表 標籤
${ item.name }
建立分支 ${ searchTerm }
從 'f4b4952719'
${ noResults }
boringssl/crypto/modes/asm/aesni-gcm-x86_64.pl

1055 line
28 KiB
原始文件 Blame 歷史記錄 

  1. #!/usr/bin/env perl

  2. #

  3. # ====================================================================

  4. # Written by Andy Polyakov <appro@openssl.org> for the OpenSSL

  5. # project. The module is, however, dual licensed under OpenSSL and

  6. # CRYPTOGAMS licenses depending on where you obtain it. For further

  7. # details see http://www.openssl.org/~appro/cryptogams/.

  8. # ====================================================================

  9. #

  10. #

  11. # AES-NI-CTR+GHASH stitch.

  12. #

  13. # February 2013

  14. #

  15. # OpenSSL GCM implementation is organized in such way that its

  16. # performance is rather close to the sum of its streamed components,

  17. # in the context parallelized AES-NI CTR and modulo-scheduled

  18. # PCLMULQDQ-enabled GHASH. Unfortunately, as no stitch implementation

  19. # was observed to perform significantly better than the sum of the

  20. # components on contemporary CPUs, the effort was deemed impossible to

  21. # justify. This module is based on combination of Intel submissions,

  22. # [1] and [2], with MOVBE twist suggested by Ilya Albrekht and Max

  23. # Locktyukhin of Intel Corp. who verified that it reduces shuffles

  24. # pressure with notable relative improvement, achieving 1.0 cycle per

  25. # byte processed with 128-bit key on Haswell processor.

  26. #

  27. # [1] http://rt.openssl.org/Ticket/Display.html?id=2900&user=guest&pass=guest

  28. # [2] http://www.intel.com/content/dam/www/public/us/en/documents/software-support/enabling-high-performance-gcm.pdf

  29. 

  30. $flavour = shift;

  31. $output  = shift;

  32. if ($flavour =~ /\./) { $output = $flavour; undef $flavour; }

  33. 

  34. $win64=0; $win64=1 if ($flavour =~ /[nm]asm|mingw64/ || $output =~ /\.asm$/);

  35. 

  36. $0 =~ m/(.*[\/\\])[^\/\\]+$/; $dir=$1;

  37. ( $xlate="${dir}x86_64-xlate.pl" and -f $xlate ) or

  38. ( $xlate="${dir}../../perlasm/x86_64-xlate.pl" and -f $xlate) or

  39. die "can't locate x86_64-xlate.pl";

  40. 

  41. if (`$ENV{CC} -Wa,-v -c -o /dev/null -x assembler /dev/null 2>&1`

  42. 		=~ /GNU assembler version ([2-9]\.[0-9]+)/) {

  43. 	$avx = ($1>=2.19) + ($1>=2.22);

  44. }

  45. 

  46. if (!$avx && $win64 && ($flavour =~ /nasm/ || $ENV{ASM} =~ /nasm/) &&

  47. 	    `nasm -v 2>&1` =~ /NASM version ([2-9]\.[0-9]+)/) {

  48. 	$avx = ($1>=2.09) + ($1>=2.10);

  49. }

  50. 

  51. if (!$avx && $win64 && ($flavour =~ /masm/ || $ENV{ASM} =~ /ml64/) &&

  52. 	    `ml64 2>&1` =~ /Version ([0-9]+)\./) {

  53. 	$avx = ($1>=10) + ($1>=11);

  54. }

  55. 

  56. if (!$avx && `$ENV{CC} -v 2>&1` =~ /(^clang version|based on LLVM) ([3-9]\.[0-9]+)/) {

  57. 	$avx = ($2>=3.0) + ($2>3.0);

  58. }

  59. 

  60. open OUT,"| \"$^X\" $xlate $flavour $output";

  61. *STDOUT=*OUT;

  62. 

  63. if ($avx>1) {{{

  64. 

  65. ($inp,$out,$len,$key,$ivp,$Xip)=("%rdi","%rsi","%rdx","%rcx","%r8","%r9");

  66. 

  67. ($Ii,$T1,$T2,$Hkey,

  68.  $Z0,$Z1,$Z2,$Z3,$Xi) = map("%xmm$_",(0..8));

  69. 

  70. ($inout0,$inout1,$inout2,$inout3,$inout4,$inout5,$rndkey) = map("%xmm$_",(9..15));

  71. 

  72. ($counter,$rounds,$ret,$const,$in0,$end0)=("%ebx","%ebp","%r10","%r11","%r14","%r15");

  73. 

  74. $code=<<___;

  75. .text

  76. 

  77. .type	_aesni_ctr32_ghash_6x,\@abi-omnipotent

  78. .align	32

  79. _aesni_ctr32_ghash_6x:

  80. 	vmovdqu		0x20($const),$T2	# borrow $T2, .Lone_msb

  81. 	sub		\$6,$len

  82. 	vpxor		$Z0,$Z0,$Z0		# $Z0   = 0

  83. 	vmovdqu		0x00-0x80($key),$rndkey

  84. 	vpaddb		$T2,$T1,$inout1

  85. 	vpaddb		$T2,$inout1,$inout2

  86. 	vpaddb		$T2,$inout2,$inout3

  87. 	vpaddb		$T2,$inout3,$inout4

  88. 	vpaddb		$T2,$inout4,$inout5

  89. 	vpxor		$rndkey,$T1,$inout0

  90. 	vmovdqu		$Z0,16+8(%rsp)		# "$Z3" = 0

  91. 	jmp		.Loop6x

  92. 

  93. .align	32

  94. .Loop6x:

  95. 	add		\$`6<<24`,$counter

  96. 	jc		.Lhandle_ctr32		# discard $inout[1-5]?

  97. 	vmovdqu		0x00-0x20($Xip),$Hkey	# $Hkey^1

  98. 	  vpaddb	$T2,$inout5,$T1		# next counter value

  99. 	  vpxor		$rndkey,$inout1,$inout1

  100. 	  vpxor		$rndkey,$inout2,$inout2

  101. 

  102. .Lresume_ctr32:

  103. 	vmovdqu		$T1,($ivp)		# save next counter value

  104. 	vpclmulqdq	\$0x10,$Hkey,$Z3,$Z1

  105. 	  vpxor		$rndkey,$inout3,$inout3

  106. 	  vmovups	0x10-0x80($key),$T2	# borrow $T2 for $rndkey

  107. 	vpclmulqdq	\$0x01,$Hkey,$Z3,$Z2

  108. 	xor		%r12,%r12

  109. 	cmp		$in0,$end0

  110. 

  111. 	  vaesenc	$T2,$inout0,$inout0

  112. 	vmovdqu		0x30+8(%rsp),$Ii	# I[4]

  113. 	  vpxor		$rndkey,$inout4,$inout4

  114. 	vpclmulqdq	\$0x00,$Hkey,$Z3,$T1

  115. 	  vaesenc	$T2,$inout1,$inout1

  116. 	  vpxor		$rndkey,$inout5,$inout5

  117. 	setnc		%r12b

  118. 	vpclmulqdq	\$0x11,$Hkey,$Z3,$Z3

  119. 	  vaesenc	$T2,$inout2,$inout2

  120. 	vmovdqu		0x10-0x20($Xip),$Hkey	# $Hkey^2

  121. 	neg		%r12

  122. 	  vaesenc	$T2,$inout3,$inout3

  123. 	 vpxor		$Z1,$Z2,$Z2

  124. 	vpclmulqdq	\$0x00,$Hkey,$Ii,$Z1

  125. 	 vpxor		$Z0,$Xi,$Xi		# modulo-scheduled

  126. 	  vaesenc	$T2,$inout4,$inout4

  127. 	 vpxor		$Z1,$T1,$Z0

  128. 	and		\$0x60,%r12

  129. 	  vmovups	0x20-0x80($key),$rndkey

  130. 	vpclmulqdq	\$0x10,$Hkey,$Ii,$T1

  131. 	  vaesenc	$T2,$inout5,$inout5

  132. 

  133. 	vpclmulqdq	\$0x01,$Hkey,$Ii,$T2

  134. 	lea		($in0,%r12),$in0

  135. 	  vaesenc	$rndkey,$inout0,$inout0

  136. 	 vpxor		16+8(%rsp),$Xi,$Xi	# modulo-scheduled [vpxor $Z3,$Xi,$Xi]

  137. 	vpclmulqdq	\$0x11,$Hkey,$Ii,$Hkey

  138. 	 vmovdqu	0x40+8(%rsp),$Ii	# I[3]

  139. 	  vaesenc	$rndkey,$inout1,$inout1

  140. 	movbe		0x58($in0),%r13

  141. 	  vaesenc	$rndkey,$inout2,$inout2

  142. 	movbe		0x50($in0),%r12

  143. 	  vaesenc	$rndkey,$inout3,$inout3

  144. 	mov		%r13,0x20+8(%rsp)

  145. 	  vaesenc	$rndkey,$inout4,$inout4

  146. 	mov		%r12,0x28+8(%rsp)

  147. 	vmovdqu		0x30-0x20($Xip),$Z1	# borrow $Z1 for $Hkey^3

  148. 	  vaesenc	$rndkey,$inout5,$inout5

  149. 

  150. 	  vmovups	0x30-0x80($key),$rndkey

  151. 	 vpxor		$T1,$Z2,$Z2

  152. 	vpclmulqdq	\$0x00,$Z1,$Ii,$T1

  153. 	  vaesenc	$rndkey,$inout0,$inout0

  154. 	 vpxor		$T2,$Z2,$Z2

  155. 	vpclmulqdq	\$0x10,$Z1,$Ii,$T2

  156. 	  vaesenc	$rndkey,$inout1,$inout1

  157. 	 vpxor		$Hkey,$Z3,$Z3

  158. 	vpclmulqdq	\$0x01,$Z1,$Ii,$Hkey

  159. 	  vaesenc	$rndkey,$inout2,$inout2

  160. 	vpclmulqdq	\$0x11,$Z1,$Ii,$Z1

  161. 	 vmovdqu	0x50+8(%rsp),$Ii	# I[2]

  162. 	  vaesenc	$rndkey,$inout3,$inout3

  163. 	  vaesenc	$rndkey,$inout4,$inout4

  164. 	 vpxor		$T1,$Z0,$Z0

  165. 	vmovdqu		0x40-0x20($Xip),$T1	# borrow $T1 for $Hkey^4

  166. 	  vaesenc	$rndkey,$inout5,$inout5

  167. 

  168. 	  vmovups	0x40-0x80($key),$rndkey

  169. 	 vpxor		$T2,$Z2,$Z2

  170. 	vpclmulqdq	\$0x00,$T1,$Ii,$T2

  171. 	  vaesenc	$rndkey,$inout0,$inout0

  172. 	 vpxor		$Hkey,$Z2,$Z2

  173. 	vpclmulqdq	\$0x10,$T1,$Ii,$Hkey

  174. 	  vaesenc	$rndkey,$inout1,$inout1

  175. 	movbe		0x48($in0),%r13

  176. 	 vpxor		$Z1,$Z3,$Z3

  177. 	vpclmulqdq	\$0x01,$T1,$Ii,$Z1

  178. 	  vaesenc	$rndkey,$inout2,$inout2

  179. 	movbe		0x40($in0),%r12

  180. 	vpclmulqdq	\$0x11,$T1,$Ii,$T1

  181. 	 vmovdqu	0x60+8(%rsp),$Ii	# I[1]

  182. 	  vaesenc	$rndkey,$inout3,$inout3

  183. 	mov		%r13,0x30+8(%rsp)

  184. 	  vaesenc	$rndkey,$inout4,$inout4

  185. 	mov		%r12,0x38+8(%rsp)

  186. 	 vpxor		$T2,$Z0,$Z0

  187. 	vmovdqu		0x60-0x20($Xip),$T2	# borrow $T2 for $Hkey^5

  188. 	  vaesenc	$rndkey,$inout5,$inout5

  189. 

  190. 	  vmovups	0x50-0x80($key),$rndkey

  191. 	 vpxor		$Hkey,$Z2,$Z2

  192. 	vpclmulqdq	\$0x00,$T2,$Ii,$Hkey

  193. 	  vaesenc	$rndkey,$inout0,$inout0

  194. 	 vpxor		$Z1,$Z2,$Z2

  195. 	vpclmulqdq	\$0x10,$T2,$Ii,$Z1

  196. 	  vaesenc	$rndkey,$inout1,$inout1

  197. 	movbe		0x38($in0),%r13

  198. 	 vpxor		$T1,$Z3,$Z3

  199. 	vpclmulqdq	\$0x01,$T2,$Ii,$T1

  200. 	 vpxor		0x70+8(%rsp),$Xi,$Xi	# accumulate I[0]

  201. 	  vaesenc	$rndkey,$inout2,$inout2

  202. 	movbe		0x30($in0),%r12

  203. 	vpclmulqdq	\$0x11,$T2,$Ii,$T2

  204. 	  vaesenc	$rndkey,$inout3,$inout3

  205. 	mov		%r13,0x40+8(%rsp)

  206. 	  vaesenc	$rndkey,$inout4,$inout4

  207. 	mov		%r12,0x48+8(%rsp)

  208. 	 vpxor		$Hkey,$Z0,$Z0

  209. 	 vmovdqu	0x70-0x20($Xip),$Hkey	# $Hkey^6

  210. 	  vaesenc	$rndkey,$inout5,$inout5

  211. 

  212. 	  vmovups	0x60-0x80($key),$rndkey

  213. 	 vpxor		$Z1,$Z2,$Z2

  214. 	vpclmulqdq	\$0x10,$Hkey,$Xi,$Z1

  215. 	  vaesenc	$rndkey,$inout0,$inout0

  216. 	 vpxor		$T1,$Z2,$Z2

  217. 	vpclmulqdq	\$0x01,$Hkey,$Xi,$T1

  218. 	  vaesenc	$rndkey,$inout1,$inout1

  219. 	movbe		0x28($in0),%r13

  220. 	 vpxor		$T2,$Z3,$Z3

  221. 	vpclmulqdq	\$0x00,$Hkey,$Xi,$T2

  222. 	  vaesenc	$rndkey,$inout2,$inout2

  223. 	movbe		0x20($in0),%r12

  224. 	vpclmulqdq	\$0x11,$Hkey,$Xi,$Xi

  225. 	  vaesenc	$rndkey,$inout3,$inout3

  226. 	mov		%r13,0x50+8(%rsp)

  227. 	  vaesenc	$rndkey,$inout4,$inout4

  228. 	mov		%r12,0x58+8(%rsp)

  229. 	vpxor		$Z1,$Z2,$Z2

  230. 	  vaesenc	$rndkey,$inout5,$inout5

  231. 	vpxor		$T1,$Z2,$Z2

  232. 

  233. 	  vmovups	0x70-0x80($key),$rndkey

  234. 	vpslldq		\$8,$Z2,$Z1

  235. 	vpxor		$T2,$Z0,$Z0

  236. 	vmovdqu		0x10($const),$Hkey	# .Lpoly

  237. 

  238. 	  vaesenc	$rndkey,$inout0,$inout0

  239. 	vpxor		$Xi,$Z3,$Z3

  240. 	  vaesenc	$rndkey,$inout1,$inout1

  241. 	vpxor		$Z1,$Z0,$Z0

  242. 	movbe		0x18($in0),%r13

  243. 	  vaesenc	$rndkey,$inout2,$inout2

  244. 	movbe		0x10($in0),%r12

  245. 	vpalignr	\$8,$Z0,$Z0,$Ii		# 1st phase

  246. 	vpclmulqdq	\$0x10,$Hkey,$Z0,$Z0

  247. 	mov		%r13,0x60+8(%rsp)

  248. 	  vaesenc	$rndkey,$inout3,$inout3

  249. 	mov		%r12,0x68+8(%rsp)

  250. 	  vaesenc	$rndkey,$inout4,$inout4

  251. 	  vmovups	0x80-0x80($key),$T1	# borrow $T1 for $rndkey

  252. 	  vaesenc	$rndkey,$inout5,$inout5

  253. 

  254. 	  vaesenc	$T1,$inout0,$inout0

  255. 	  vmovups	0x90-0x80($key),$rndkey

  256. 	  vaesenc	$T1,$inout1,$inout1

  257. 	vpsrldq		\$8,$Z2,$Z2

  258. 	  vaesenc	$T1,$inout2,$inout2

  259. 	vpxor		$Z2,$Z3,$Z3

  260. 	  vaesenc	$T1,$inout3,$inout3

  261. 	vpxor		$Ii,$Z0,$Z0

  262. 	movbe		0x08($in0),%r13

  263. 	  vaesenc	$T1,$inout4,$inout4

  264. 	movbe		0x00($in0),%r12

  265. 	  vaesenc	$T1,$inout5,$inout5

  266. 	  vmovups	0xa0-0x80($key),$T1

  267. 	  cmp		\$11,$rounds

  268. 	  jb		.Lenc_tail		# 128-bit key

  269. 

  270. 	  vaesenc	$rndkey,$inout0,$inout0

  271. 	  vaesenc	$rndkey,$inout1,$inout1

  272. 	  vaesenc	$rndkey,$inout2,$inout2

  273. 	  vaesenc	$rndkey,$inout3,$inout3

  274. 	  vaesenc	$rndkey,$inout4,$inout4

  275. 	  vaesenc	$rndkey,$inout5,$inout5

  276. 

  277. 	  vaesenc	$T1,$inout0,$inout0

  278. 	  vaesenc	$T1,$inout1,$inout1

  279. 	  vaesenc	$T1,$inout2,$inout2

  280. 	  vaesenc	$T1,$inout3,$inout3

  281. 	  vaesenc	$T1,$inout4,$inout4

  282. 	  vmovups	0xb0-0x80($key),$rndkey

  283. 	  vaesenc	$T1,$inout5,$inout5

  284. 	  vmovups	0xc0-0x80($key),$T1

  285. 	  je		.Lenc_tail		# 192-bit key

  286. 

  287. 	  vaesenc	$rndkey,$inout0,$inout0

  288. 	  vaesenc	$rndkey,$inout1,$inout1

  289. 	  vaesenc	$rndkey,$inout2,$inout2

  290. 	  vaesenc	$rndkey,$inout3,$inout3

  291. 	  vaesenc	$rndkey,$inout4,$inout4

  292. 	  vaesenc	$rndkey,$inout5,$inout5

  293. 

  294. 	  vaesenc	$T1,$inout0,$inout0

  295. 	  vaesenc	$T1,$inout1,$inout1

  296. 	  vaesenc	$T1,$inout2,$inout2

  297. 	  vaesenc	$T1,$inout3,$inout3

  298. 	  vaesenc	$T1,$inout4,$inout4

  299. 	  vmovups	0xd0-0x80($key),$rndkey

  300. 	  vaesenc	$T1,$inout5,$inout5

  301. 	  vmovups	0xe0-0x80($key),$T1

  302. 	  jmp		.Lenc_tail		# 256-bit key

  303. 

  304. .align	32

  305. .Lhandle_ctr32:

  306. 	vmovdqu		($const),$Ii		# borrow $Ii for .Lbswap_mask

  307. 	  vpshufb	$Ii,$T1,$Z2		# byte-swap counter

  308. 	  vmovdqu	0x30($const),$Z1	# borrow $Z1, .Ltwo_lsb

  309. 	  vpaddd	0x40($const),$Z2,$inout1	# .Lone_lsb

  310. 	  vpaddd	$Z1,$Z2,$inout2

  311. 	vmovdqu		0x00-0x20($Xip),$Hkey	# $Hkey^1

  312. 	  vpaddd	$Z1,$inout1,$inout3

  313. 	  vpshufb	$Ii,$inout1,$inout1

  314. 	  vpaddd	$Z1,$inout2,$inout4

  315. 	  vpshufb	$Ii,$inout2,$inout2

  316. 	  vpxor		$rndkey,$inout1,$inout1

  317. 	  vpaddd	$Z1,$inout3,$inout5

  318. 	  vpshufb	$Ii,$inout3,$inout3

  319. 	  vpxor		$rndkey,$inout2,$inout2

  320. 	  vpaddd	$Z1,$inout4,$T1		# byte-swapped next counter value

  321. 	  vpshufb	$Ii,$inout4,$inout4

  322. 	  vpshufb	$Ii,$inout5,$inout5

  323. 	  vpshufb	$Ii,$T1,$T1		# next counter value

  324. 	jmp		.Lresume_ctr32

  325. 

  326. .align	32

  327. .Lenc_tail:

  328. 	  vaesenc	$rndkey,$inout0,$inout0

  329. 	vmovdqu		$Z3,16+8(%rsp)		# postpone vpxor $Z3,$Xi,$Xi

  330. 	vpalignr	\$8,$Z0,$Z0,$Xi		# 2nd phase

  331. 	  vaesenc	$rndkey,$inout1,$inout1

  332. 	vpclmulqdq	\$0x10,$Hkey,$Z0,$Z0

  333. 	  vpxor		0x00($inp),$T1,$T2

  334. 	  vaesenc	$rndkey,$inout2,$inout2

  335. 	  vpxor		0x10($inp),$T1,$Ii

  336. 	  vaesenc	$rndkey,$inout3,$inout3

  337. 	  vpxor		0x20($inp),$T1,$Z1

  338. 	  vaesenc	$rndkey,$inout4,$inout4

  339. 	  vpxor		0x30($inp),$T1,$Z2

  340. 	  vaesenc	$rndkey,$inout5,$inout5

  341. 	  vpxor		0x40($inp),$T1,$Z3

  342. 	  vpxor		0x50($inp),$T1,$Hkey

  343. 	  vmovdqu	($ivp),$T1		# load next counter value

  344. 

  345. 	  vaesenclast	$T2,$inout0,$inout0

  346. 	  vmovdqu	0x20($const),$T2	# borrow $T2, .Lone_msb

  347. 	  vaesenclast	$Ii,$inout1,$inout1

  348. 	 vpaddb		$T2,$T1,$Ii

  349. 	mov		%r13,0x70+8(%rsp)

  350. 	lea		0x60($inp),$inp

  351. 	  vaesenclast	$Z1,$inout2,$inout2

  352. 	 vpaddb		$T2,$Ii,$Z1

  353. 	mov		%r12,0x78+8(%rsp)

  354. 	lea		0x60($out),$out

  355. 	  vmovdqu	0x00-0x80($key),$rndkey

  356. 	  vaesenclast	$Z2,$inout3,$inout3

  357. 	 vpaddb		$T2,$Z1,$Z2

  358. 	  vaesenclast	$Z3, $inout4,$inout4

  359. 	 vpaddb		$T2,$Z2,$Z3

  360. 	  vaesenclast	$Hkey,$inout5,$inout5

  361. 	 vpaddb		$T2,$Z3,$Hkey

  362. 

  363. 	add		\$0x60,$ret

  364. 	sub		\$0x6,$len

  365. 	jc		.L6x_done

  366. 

  367. 	  vmovups	$inout0,-0x60($out)	# save output

  368. 	 vpxor		$rndkey,$T1,$inout0

  369. 	  vmovups	$inout1,-0x50($out)

  370. 	 vmovdqa	$Ii,$inout1		# 0 latency

  371. 	  vmovups	$inout2,-0x40($out)

  372. 	 vmovdqa	$Z1,$inout2		# 0 latency

  373. 	  vmovups	$inout3,-0x30($out)

  374. 	 vmovdqa	$Z2,$inout3		# 0 latency

  375. 	  vmovups	$inout4,-0x20($out)

  376. 	 vmovdqa	$Z3,$inout4		# 0 latency

  377. 	  vmovups	$inout5,-0x10($out)

  378. 	 vmovdqa	$Hkey,$inout5		# 0 latency

  379. 	vmovdqu		0x20+8(%rsp),$Z3	# I[5]

  380. 	jmp		.Loop6x

  381. 

  382. .L6x_done:

  383. 	vpxor		16+8(%rsp),$Xi,$Xi	# modulo-scheduled

  384. 	vpxor		$Z0,$Xi,$Xi		# modulo-scheduled

  385. 

  386. 	ret

  387. .size	_aesni_ctr32_ghash_6x,.-_aesni_ctr32_ghash_6x

  388. ___

  389. ######################################################################

  390. #

  391. # size_t aesni_gcm_[en|de]crypt(const void *inp, void *out, size_t len,

  392. #		const AES_KEY *key, unsigned char iv[16],

  393. #		struct { u128 Xi,H,Htbl[9]; } *Xip);

  394. $code.=<<___;

  395. .globl	aesni_gcm_decrypt

  396. .type	aesni_gcm_decrypt,\@function,6

  397. .align	32

  398. aesni_gcm_decrypt:

  399. 	xor	$ret,$ret

  400. 	cmp	\$0x60,$len			# minimal accepted length

  401. 	jb	.Lgcm_dec_abort

  402. 

  403. 	lea	(%rsp),%rax			# save stack pointer

  404. 	push	%rbx

  405. 	push	%rbp

  406. 	push	%r12

  407. 	push	%r13

  408. 	push	%r14

  409. 	push	%r15

  410. ___

  411. $code.=<<___ if ($win64);

  412. 	lea	-0xa8(%rsp),%rsp

  413. 	movaps	%xmm6,-0xd8(%rax)

  414. 	movaps	%xmm7,-0xc8(%rax)

  415. 	movaps	%xmm8,-0xb8(%rax)

  416. 	movaps	%xmm9,-0xa8(%rax)

  417. 	movaps	%xmm10,-0x98(%rax)

  418. 	movaps	%xmm11,-0x88(%rax)

  419. 	movaps	%xmm12,-0x78(%rax)

  420. 	movaps	%xmm13,-0x68(%rax)

  421. 	movaps	%xmm14,-0x58(%rax)

  422. 	movaps	%xmm15,-0x48(%rax)

  423. .Lgcm_dec_body:

  424. ___

  425. $code.=<<___;

  426. 	vzeroupper

  427. 

  428. 	vmovdqu		($ivp),$T1		# input counter value

  429. 	add		\$-128,%rsp

  430. 	mov		12($ivp),$counter

  431. 	lea		.Lbswap_mask(%rip),$const

  432. 	lea		-0x80($key),$in0	# borrow $in0

  433. 	mov		\$0xf80,$end0		# borrow $end0

  434. 	vmovdqu		($Xip),$Xi		# load Xi

  435. 	and		\$-128,%rsp		# ensure stack alignment

  436. 	vmovdqu		($const),$Ii		# borrow $Ii for .Lbswap_mask

  437. 	lea		0x80($key),$key		# size optimization

  438. 	lea		0x20+0x20($Xip),$Xip	# size optimization

  439. 	mov		0xf0-0x80($key),$rounds

  440. 	vpshufb		$Ii,$Xi,$Xi

  441. 

  442. 	and		$end0,$in0

  443. 	and		%rsp,$end0

  444. 	sub		$in0,$end0

  445. 	jc		.Ldec_no_key_aliasing

  446. 	cmp		\$768,$end0

  447. 	jnc		.Ldec_no_key_aliasing

  448. 	sub		$end0,%rsp		# avoid aliasing with key

  449. .Ldec_no_key_aliasing:

  450. 

  451. 	vmovdqu		0x50($inp),$Z3		# I[5]

  452. 	lea		($inp),$in0

  453. 	vmovdqu		0x40($inp),$Z0

  454. 	lea		-0xc0($inp,$len),$end0

  455. 	vmovdqu		0x30($inp),$Z1

  456. 	shr		\$4,$len

  457. 	xor		$ret,$ret

  458. 	vmovdqu		0x20($inp),$Z2

  459. 	 vpshufb	$Ii,$Z3,$Z3		# passed to _aesni_ctr32_ghash_6x

  460. 	vmovdqu		0x10($inp),$T2

  461. 	 vpshufb	$Ii,$Z0,$Z0

  462. 	vmovdqu		($inp),$Hkey

  463. 	 vpshufb	$Ii,$Z1,$Z1

  464. 	vmovdqu		$Z0,0x30(%rsp)

  465. 	 vpshufb	$Ii,$Z2,$Z2

  466. 	vmovdqu		$Z1,0x40(%rsp)

  467. 	 vpshufb	$Ii,$T2,$T2

  468. 	vmovdqu		$Z2,0x50(%rsp)

  469. 	 vpshufb	$Ii,$Hkey,$Hkey

  470. 	vmovdqu		$T2,0x60(%rsp)

  471. 	vmovdqu		$Hkey,0x70(%rsp)

  472. 

  473. 	call		_aesni_ctr32_ghash_6x

  474. 

  475. 	vmovups		$inout0,-0x60($out)	# save output

  476. 	vmovups		$inout1,-0x50($out)

  477. 	vmovups		$inout2,-0x40($out)

  478. 	vmovups		$inout3,-0x30($out)

  479. 	vmovups		$inout4,-0x20($out)

  480. 	vmovups		$inout5,-0x10($out)

  481. 

  482. 	vpshufb		($const),$Xi,$Xi	# .Lbswap_mask

  483. 	vmovdqu		$Xi,-0x40($Xip)		# output Xi

  484. 

  485. 	vzeroupper

  486. ___

  487. $code.=<<___ if ($win64);

  488. 	movaps	-0xd8(%rax),%xmm6

  489. 	movaps	-0xd8(%rax),%xmm7

  490. 	movaps	-0xb8(%rax),%xmm8

  491. 	movaps	-0xa8(%rax),%xmm9

  492. 	movaps	-0x98(%rax),%xmm10

  493. 	movaps	-0x88(%rax),%xmm11

  494. 	movaps	-0x78(%rax),%xmm12

  495. 	movaps	-0x68(%rax),%xmm13

  496. 	movaps	-0x58(%rax),%xmm14

  497. 	movaps	-0x48(%rax),%xmm15

  498. ___

  499. $code.=<<___;

  500. 	mov	-48(%rax),%r15

  501. 	mov	-40(%rax),%r14

  502. 	mov	-32(%rax),%r13

  503. 	mov	-24(%rax),%r12

  504. 	mov	-16(%rax),%rbp

  505. 	mov	-8(%rax),%rbx

  506. 	lea	(%rax),%rsp		# restore %rsp

  507. .Lgcm_dec_abort:

  508. 	mov	$ret,%rax		# return value

  509. 	ret

  510. .size	aesni_gcm_decrypt,.-aesni_gcm_decrypt

  511. ___

  512. 

  513. $code.=<<___;

  514. .type	_aesni_ctr32_6x,\@abi-omnipotent

  515. .align	32

  516. _aesni_ctr32_6x:

  517. 	vmovdqu		0x00-0x80($key),$Z0	# borrow $Z0 for $rndkey

  518. 	vmovdqu		0x20($const),$T2	# borrow $T2, .Lone_msb

  519. 	lea		-1($rounds),%r13

  520. 	vmovups		0x10-0x80($key),$rndkey

  521. 	lea		0x20-0x80($key),%r12

  522. 	vpxor		$Z0,$T1,$inout0

  523. 	add		\$`6<<24`,$counter

  524. 	jc		.Lhandle_ctr32_2

  525. 	vpaddb		$T2,$T1,$inout1

  526. 	vpaddb		$T2,$inout1,$inout2

  527. 	vpxor		$Z0,$inout1,$inout1

  528. 	vpaddb		$T2,$inout2,$inout3

  529. 	vpxor		$Z0,$inout2,$inout2

  530. 	vpaddb		$T2,$inout3,$inout4

  531. 	vpxor		$Z0,$inout3,$inout3

  532. 	vpaddb		$T2,$inout4,$inout5

  533. 	vpxor		$Z0,$inout4,$inout4

  534. 	vpaddb		$T2,$inout5,$T1

  535. 	vpxor		$Z0,$inout5,$inout5

  536. 	jmp		.Loop_ctr32

  537. 

  538. .align	16

  539. .Loop_ctr32:

  540. 	vaesenc		$rndkey,$inout0,$inout0

  541. 	vaesenc		$rndkey,$inout1,$inout1

  542. 	vaesenc		$rndkey,$inout2,$inout2

  543. 	vaesenc		$rndkey,$inout3,$inout3

  544. 	vaesenc		$rndkey,$inout4,$inout4

  545. 	vaesenc		$rndkey,$inout5,$inout5

  546. 	vmovups		(%r12),$rndkey

  547. 	lea		0x10(%r12),%r12

  548. 	dec		%r13d

  549. 	jnz		.Loop_ctr32

  550. 

  551. 	vmovdqu		(%r12),$Hkey		# last round key

  552. 	vaesenc		$rndkey,$inout0,$inout0

  553. 	vpxor		0x00($inp),$Hkey,$Z0

  554. 	vaesenc		$rndkey,$inout1,$inout1

  555. 	vpxor		0x10($inp),$Hkey,$Z1

  556. 	vaesenc		$rndkey,$inout2,$inout2

  557. 	vpxor		0x20($inp),$Hkey,$Z2

  558. 	vaesenc		$rndkey,$inout3,$inout3

  559. 	vpxor		0x30($inp),$Hkey,$Xi

  560. 	vaesenc		$rndkey,$inout4,$inout4

  561. 	vpxor		0x40($inp),$Hkey,$T2

  562. 	vaesenc		$rndkey,$inout5,$inout5

  563. 	vpxor		0x50($inp),$Hkey,$Hkey

  564. 	lea		0x60($inp),$inp

  565. 

  566. 	vaesenclast	$Z0,$inout0,$inout0

  567. 	vaesenclast	$Z1,$inout1,$inout1

  568. 	vaesenclast	$Z2,$inout2,$inout2

  569. 	vaesenclast	$Xi,$inout3,$inout3

  570. 	vaesenclast	$T2,$inout4,$inout4

  571. 	vaesenclast	$Hkey,$inout5,$inout5

  572. 	vmovups		$inout0,0x00($out)

  573. 	vmovups		$inout1,0x10($out)

  574. 	vmovups		$inout2,0x20($out)

  575. 	vmovups		$inout3,0x30($out)

  576. 	vmovups		$inout4,0x40($out)

  577. 	vmovups		$inout5,0x50($out)

  578. 	lea		0x60($out),$out

  579. 

  580. 	ret

  581. .align	32

  582. .Lhandle_ctr32_2:

  583. 	vpshufb		$Ii,$T1,$Z2		# byte-swap counter

  584. 	vmovdqu		0x30($const),$Z1	# borrow $Z1, .Ltwo_lsb

  585. 	vpaddd		0x40($const),$Z2,$inout1	# .Lone_lsb

  586. 	vpaddd		$Z1,$Z2,$inout2

  587. 	vpaddd		$Z1,$inout1,$inout3

  588. 	vpshufb		$Ii,$inout1,$inout1

  589. 	vpaddd		$Z1,$inout2,$inout4

  590. 	vpshufb		$Ii,$inout2,$inout2

  591. 	vpxor		$Z0,$inout1,$inout1

  592. 	vpaddd		$Z1,$inout3,$inout5

  593. 	vpshufb		$Ii,$inout3,$inout3

  594. 	vpxor		$Z0,$inout2,$inout2

  595. 	vpaddd		$Z1,$inout4,$T1		# byte-swapped next counter value

  596. 	vpshufb		$Ii,$inout4,$inout4

  597. 	vpxor		$Z0,$inout3,$inout3

  598. 	vpshufb		$Ii,$inout5,$inout5

  599. 	vpxor		$Z0,$inout4,$inout4

  600. 	vpshufb		$Ii,$T1,$T1		# next counter value

  601. 	vpxor		$Z0,$inout5,$inout5

  602. 	jmp	.Loop_ctr32

  603. .size	_aesni_ctr32_6x,.-_aesni_ctr32_6x

  604. 

  605. .globl	aesni_gcm_encrypt

  606. .type	aesni_gcm_encrypt,\@function,6

  607. .align	32

  608. aesni_gcm_encrypt:

  609. 	xor	$ret,$ret

  610. 	cmp	\$0x60*3,$len			# minimal accepted length

  611. 	jb	.Lgcm_enc_abort

  612. 

  613. 	lea	(%rsp),%rax			# save stack pointer

  614. 	push	%rbx

  615. 	push	%rbp

  616. 	push	%r12

  617. 	push	%r13

  618. 	push	%r14

  619. 	push	%r15

  620. ___

  621. $code.=<<___ if ($win64);

  622. 	lea	-0xa8(%rsp),%rsp

  623. 	movaps	%xmm6,-0xd8(%rax)

  624. 	movaps	%xmm7,-0xc8(%rax)

  625. 	movaps	%xmm8,-0xb8(%rax)

  626. 	movaps	%xmm9,-0xa8(%rax)

  627. 	movaps	%xmm10,-0x98(%rax)

  628. 	movaps	%xmm11,-0x88(%rax)

  629. 	movaps	%xmm12,-0x78(%rax)

  630. 	movaps	%xmm13,-0x68(%rax)

  631. 	movaps	%xmm14,-0x58(%rax)

  632. 	movaps	%xmm15,-0x48(%rax)

  633. .Lgcm_enc_body:

  634. ___

  635. $code.=<<___;

  636. 	vzeroupper

  637. 

  638. 	vmovdqu		($ivp),$T1		# input counter value

  639. 	add		\$-128,%rsp

  640. 	mov		12($ivp),$counter

  641. 	lea		.Lbswap_mask(%rip),$const

  642. 	lea		-0x80($key),$in0	# borrow $in0

  643. 	mov		\$0xf80,$end0		# borrow $end0

  644. 	lea		0x80($key),$key		# size optimization

  645. 	vmovdqu		($const),$Ii		# borrow $Ii for .Lbswap_mask

  646. 	and		\$-128,%rsp		# ensure stack alignment

  647. 	mov		0xf0-0x80($key),$rounds

  648. 

  649. 	and		$end0,$in0

  650. 	and		%rsp,$end0

  651. 	sub		$in0,$end0

  652. 	jc		.Lenc_no_key_aliasing

  653. 	cmp		\$768,$end0

  654. 	jnc		.Lenc_no_key_aliasing

  655. 	sub		$end0,%rsp		# avoid aliasing with key

  656. .Lenc_no_key_aliasing:

  657. 

  658. 	lea		($out),$in0

  659. 	lea		-0xc0($out,$len),$end0

  660. 	shr		\$4,$len

  661. 

  662. 	call		_aesni_ctr32_6x

  663. 	vpshufb		$Ii,$inout0,$Xi		# save bswapped output on stack

  664. 	vpshufb		$Ii,$inout1,$T2

  665. 	vmovdqu		$Xi,0x70(%rsp)

  666. 	vpshufb		$Ii,$inout2,$Z0

  667. 	vmovdqu		$T2,0x60(%rsp)

  668. 	vpshufb		$Ii,$inout3,$Z1

  669. 	vmovdqu		$Z0,0x50(%rsp)

  670. 	vpshufb		$Ii,$inout4,$Z2

  671. 	vmovdqu		$Z1,0x40(%rsp)

  672. 	vpshufb		$Ii,$inout5,$Z3		# passed to _aesni_ctr32_ghash_6x

  673. 	vmovdqu		$Z2,0x30(%rsp)

  674. 

  675. 	call		_aesni_ctr32_6x

  676. 

  677. 	vmovdqu		($Xip),$Xi		# load Xi

  678. 	lea		0x20+0x20($Xip),$Xip	# size optimization

  679. 	sub		\$12,$len

  680. 	mov		\$0x60*2,$ret

  681. 	vpshufb		$Ii,$Xi,$Xi

  682. 

  683. 	call		_aesni_ctr32_ghash_6x

  684. 	vmovdqu		0x20(%rsp),$Z3		# I[5]

  685. 	 vmovdqu	($const),$Ii		# borrow $Ii for .Lbswap_mask

  686. 	vmovdqu		0x00-0x20($Xip),$Hkey	# $Hkey^1

  687. 	vpunpckhqdq	$Z3,$Z3,$T1

  688. 	vmovdqu		0x20-0x20($Xip),$rndkey	# borrow $rndkey for $HK

  689. 	 vmovups	$inout0,-0x60($out)	# save output

  690. 	 vpshufb	$Ii,$inout0,$inout0	# but keep bswapped copy

  691. 	vpxor		$Z3,$T1,$T1

  692. 	 vmovups	$inout1,-0x50($out)

  693. 	 vpshufb	$Ii,$inout1,$inout1

  694. 	 vmovups	$inout2,-0x40($out)

  695. 	 vpshufb	$Ii,$inout2,$inout2

  696. 	 vmovups	$inout3,-0x30($out)

  697. 	 vpshufb	$Ii,$inout3,$inout3

  698. 	 vmovups	$inout4,-0x20($out)

  699. 	 vpshufb	$Ii,$inout4,$inout4

  700. 	 vmovups	$inout5,-0x10($out)

  701. 	 vpshufb	$Ii,$inout5,$inout5

  702. 	 vmovdqu	$inout0,0x10(%rsp)	# free $inout0

  703. ___

  704. { my ($HK,$T3)=($rndkey,$inout0);

  705. 

  706. $code.=<<___;

  707. 	 vmovdqu	0x30(%rsp),$Z2		# I[4]

  708. 	 vmovdqu	0x10-0x20($Xip),$Ii	# borrow $Ii for $Hkey^2

  709. 	 vpunpckhqdq	$Z2,$Z2,$T2

  710. 	vpclmulqdq	\$0x00,$Hkey,$Z3,$Z1

  711. 	 vpxor		$Z2,$T2,$T2

  712. 	vpclmulqdq	\$0x11,$Hkey,$Z3,$Z3

  713. 	vpclmulqdq	\$0x00,$HK,$T1,$T1

  714. 

  715. 	 vmovdqu	0x40(%rsp),$T3		# I[3]

  716. 	vpclmulqdq	\$0x00,$Ii,$Z2,$Z0

  717. 	 vmovdqu	0x30-0x20($Xip),$Hkey	# $Hkey^3

  718. 	vpxor		$Z1,$Z0,$Z0

  719. 	 vpunpckhqdq	$T3,$T3,$Z1

  720. 	vpclmulqdq	\$0x11,$Ii,$Z2,$Z2

  721. 	 vpxor		$T3,$Z1,$Z1

  722. 	vpxor		$Z3,$Z2,$Z2

  723. 	vpclmulqdq	\$0x10,$HK,$T2,$T2

  724. 	 vmovdqu	0x50-0x20($Xip),$HK

  725. 	vpxor		$T1,$T2,$T2

  726. 

  727. 	 vmovdqu	0x50(%rsp),$T1		# I[2]

  728. 	vpclmulqdq	\$0x00,$Hkey,$T3,$Z3

  729. 	 vmovdqu	0x40-0x20($Xip),$Ii	# borrow $Ii for $Hkey^4

  730. 	vpxor		$Z0,$Z3,$Z3

  731. 	 vpunpckhqdq	$T1,$T1,$Z0

  732. 	vpclmulqdq	\$0x11,$Hkey,$T3,$T3

  733. 	 vpxor		$T1,$Z0,$Z0

  734. 	vpxor		$Z2,$T3,$T3

  735. 	vpclmulqdq	\$0x00,$HK,$Z1,$Z1

  736. 	vpxor		$T2,$Z1,$Z1

  737. 

  738. 	 vmovdqu	0x60(%rsp),$T2		# I[1]

  739. 	vpclmulqdq	\$0x00,$Ii,$T1,$Z2

  740. 	 vmovdqu	0x60-0x20($Xip),$Hkey	# $Hkey^5

  741. 	vpxor		$Z3,$Z2,$Z2

  742. 	 vpunpckhqdq	$T2,$T2,$Z3

  743. 	vpclmulqdq	\$0x11,$Ii,$T1,$T1

  744. 	 vpxor		$T2,$Z3,$Z3

  745. 	vpxor		$T3,$T1,$T1

  746. 	vpclmulqdq	\$0x10,$HK,$Z0,$Z0

  747. 	 vmovdqu	0x80-0x20($Xip),$HK

  748. 	vpxor		$Z1,$Z0,$Z0

  749. 

  750. 	 vpxor		0x70(%rsp),$Xi,$Xi	# accumulate I[0]

  751. 	vpclmulqdq	\$0x00,$Hkey,$T2,$Z1

  752. 	 vmovdqu	0x70-0x20($Xip),$Ii	# borrow $Ii for $Hkey^6

  753. 	 vpunpckhqdq	$Xi,$Xi,$T3

  754. 	vpxor		$Z2,$Z1,$Z1

  755. 	vpclmulqdq	\$0x11,$Hkey,$T2,$T2

  756. 	 vpxor		$Xi,$T3,$T3

  757. 	vpxor		$T1,$T2,$T2

  758. 	vpclmulqdq	\$0x00,$HK,$Z3,$Z3

  759. 	vpxor		$Z0,$Z3,$Z0

  760. 

  761. 	vpclmulqdq	\$0x00,$Ii,$Xi,$Z2

  762. 	 vmovdqu	0x00-0x20($Xip),$Hkey	# $Hkey^1

  763. 	 vpunpckhqdq	$inout5,$inout5,$T1

  764. 	vpclmulqdq	\$0x11,$Ii,$Xi,$Xi

  765. 	 vpxor		$inout5,$T1,$T1

  766. 	vpxor		$Z1,$Z2,$Z1

  767. 	vpclmulqdq	\$0x10,$HK,$T3,$T3

  768. 	 vmovdqu	0x20-0x20($Xip),$HK

  769. 	vpxor		$T2,$Xi,$Z3

  770. 	vpxor		$Z0,$T3,$Z2

  771. 

  772. 	 vmovdqu	0x10-0x20($Xip),$Ii	# borrow $Ii for $Hkey^2

  773. 	  vpxor		$Z1,$Z3,$T3		# aggregated Karatsuba post-processing

  774. 	vpclmulqdq	\$0x00,$Hkey,$inout5,$Z0

  775. 	  vpxor		$T3,$Z2,$Z2

  776. 	 vpunpckhqdq	$inout4,$inout4,$T2

  777. 	vpclmulqdq	\$0x11,$Hkey,$inout5,$inout5

  778. 	 vpxor		$inout4,$T2,$T2

  779. 	  vpslldq	\$8,$Z2,$T3

  780. 	vpclmulqdq	\$0x00,$HK,$T1,$T1

  781. 	  vpxor		$T3,$Z1,$Xi

  782. 	  vpsrldq	\$8,$Z2,$Z2

  783. 	  vpxor		$Z2,$Z3,$Z3

  784. 

  785. 	vpclmulqdq	\$0x00,$Ii,$inout4,$Z1

  786. 	 vmovdqu	0x30-0x20($Xip),$Hkey	# $Hkey^3

  787. 	vpxor		$Z0,$Z1,$Z1

  788. 	 vpunpckhqdq	$inout3,$inout3,$T3

  789. 	vpclmulqdq	\$0x11,$Ii,$inout4,$inout4

  790. 	 vpxor		$inout3,$T3,$T3

  791. 	vpxor		$inout5,$inout4,$inout4

  792. 	  vpalignr	\$8,$Xi,$Xi,$inout5	# 1st phase

  793. 	vpclmulqdq	\$0x10,$HK,$T2,$T2

  794. 	 vmovdqu	0x50-0x20($Xip),$HK

  795. 	vpxor		$T1,$T2,$T2

  796. 

  797. 	vpclmulqdq	\$0x00,$Hkey,$inout3,$Z0

  798. 	 vmovdqu	0x40-0x20($Xip),$Ii	# borrow $Ii for $Hkey^4

  799. 	vpxor		$Z1,$Z0,$Z0

  800. 	 vpunpckhqdq	$inout2,$inout2,$T1

  801. 	vpclmulqdq	\$0x11,$Hkey,$inout3,$inout3

  802. 	 vpxor		$inout2,$T1,$T1

  803. 	vpxor		$inout4,$inout3,$inout3

  804. 	  vxorps	0x10(%rsp),$Z3,$Z3	# accumulate $inout0

  805. 	vpclmulqdq	\$0x00,$HK,$T3,$T3

  806. 	vpxor		$T2,$T3,$T3

  807. 

  808. 	  vpclmulqdq	\$0x10,0x10($const),$Xi,$Xi

  809. 	  vxorps	$inout5,$Xi,$Xi

  810. 

  811. 	vpclmulqdq	\$0x00,$Ii,$inout2,$Z1

  812. 	 vmovdqu	0x60-0x20($Xip),$Hkey	# $Hkey^5

  813. 	vpxor		$Z0,$Z1,$Z1

  814. 	 vpunpckhqdq	$inout1,$inout1,$T2

  815. 	vpclmulqdq	\$0x11,$Ii,$inout2,$inout2

  816. 	 vpxor		$inout1,$T2,$T2

  817. 	  vpalignr	\$8,$Xi,$Xi,$inout5	# 2nd phase

  818. 	vpxor		$inout3,$inout2,$inout2

  819. 	vpclmulqdq	\$0x10,$HK,$T1,$T1

  820. 	 vmovdqu	0x80-0x20($Xip),$HK

  821. 	vpxor		$T3,$T1,$T1

  822. 

  823. 	  vxorps	$Z3,$inout5,$inout5

  824. 	  vpclmulqdq	\$0x10,0x10($const),$Xi,$Xi

  825. 	  vxorps	$inout5,$Xi,$Xi

  826. 

  827. 	vpclmulqdq	\$0x00,$Hkey,$inout1,$Z0

  828. 	 vmovdqu	0x70-0x20($Xip),$Ii	# borrow $Ii for $Hkey^6

  829. 	vpxor		$Z1,$Z0,$Z0

  830. 	 vpunpckhqdq	$Xi,$Xi,$T3

  831. 	vpclmulqdq	\$0x11,$Hkey,$inout1,$inout1

  832. 	 vpxor		$Xi,$T3,$T3

  833. 	vpxor		$inout2,$inout1,$inout1

  834. 	vpclmulqdq	\$0x00,$HK,$T2,$T2

  835. 	vpxor		$T1,$T2,$T2

  836. 

  837. 	vpclmulqdq	\$0x00,$Ii,$Xi,$Z1

  838. 	vpclmulqdq	\$0x11,$Ii,$Xi,$Z3

  839. 	vpxor		$Z0,$Z1,$Z1

  840. 	vpclmulqdq	\$0x10,$HK,$T3,$Z2

  841. 	vpxor		$inout1,$Z3,$Z3

  842. 	vpxor		$T2,$Z2,$Z2

  843. 

  844. 	vpxor		$Z1,$Z3,$Z0		# aggregated Karatsuba post-processing

  845. 	vpxor		$Z0,$Z2,$Z2

  846. 	vpslldq		\$8,$Z2,$T1

  847. 	vmovdqu		0x10($const),$Hkey	# .Lpoly

  848. 	vpsrldq		\$8,$Z2,$Z2

  849. 	vpxor		$T1,$Z1,$Xi

  850. 	vpxor		$Z2,$Z3,$Z3

  851. 

  852. 	vpalignr	\$8,$Xi,$Xi,$T2		# 1st phase

  853. 	vpclmulqdq	\$0x10,$Hkey,$Xi,$Xi

  854. 	vpxor		$T2,$Xi,$Xi

  855. 

  856. 	vpalignr	\$8,$Xi,$Xi,$T2		# 2nd phase

  857. 	vpclmulqdq	\$0x10,$Hkey,$Xi,$Xi

  858. 	vpxor		$Z3,$T2,$T2

  859. 	vpxor		$T2,$Xi,$Xi

  860. ___

  861. }

  862. $code.=<<___;

  863. 	vpshufb		($const),$Xi,$Xi	# .Lbswap_mask

  864. 	vmovdqu		$Xi,-0x40($Xip)		# output Xi

  865. 

  866. 	vzeroupper

  867. ___

  868. $code.=<<___ if ($win64);

  869. 	movaps	-0xd8(%rax),%xmm6

  870. 	movaps	-0xc8(%rax),%xmm7

  871. 	movaps	-0xb8(%rax),%xmm8

  872. 	movaps	-0xa8(%rax),%xmm9

  873. 	movaps	-0x98(%rax),%xmm10

  874. 	movaps	-0x88(%rax),%xmm11

  875. 	movaps	-0x78(%rax),%xmm12

  876. 	movaps	-0x68(%rax),%xmm13

  877. 	movaps	-0x58(%rax),%xmm14

  878. 	movaps	-0x48(%rax),%xmm15

  879. ___

  880. $code.=<<___;

  881. 	mov	-48(%rax),%r15

  882. 	mov	-40(%rax),%r14

  883. 	mov	-32(%rax),%r13

  884. 	mov	-24(%rax),%r12

  885. 	mov	-16(%rax),%rbp

  886. 	mov	-8(%rax),%rbx

  887. 	lea	(%rax),%rsp		# restore %rsp

  888. .Lgcm_enc_abort:

  889. 	mov	$ret,%rax		# return value

  890. 	ret

  891. .size	aesni_gcm_encrypt,.-aesni_gcm_encrypt

  892. ___

  893. 

  894. $code.=<<___;

  895. .align	64

  896. .Lbswap_mask:

  897. 	.byte	15,14,13,12,11,10,9,8,7,6,5,4,3,2,1,0

  898. .Lpoly:

  899. 	.byte	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0xc2

  900. .Lone_msb:

  901. 	.byte	0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1

  902. .Ltwo_lsb:

  903. 	.byte	2,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0

  904. .Lone_lsb:

  905. 	.byte	1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0

  906. .asciz	"AES-NI GCM module for x86_64, CRYPTOGAMS by <appro\@openssl.org>"

  907. .align	64

  908. ___

  909. if ($win64) {

  910. $rec="%rcx";

  911. $frame="%rdx";

  912. $context="%r8";

  913. $disp="%r9";

  914. 

  915. $code.=<<___

  916. .extern	__imp_RtlVirtualUnwind

  917. .type	gcm_se_handler,\@abi-omnipotent

  918. .align	16

  919. gcm_se_handler:

  920. 	push	%rsi

  921. 	push	%rdi

  922. 	push	%rbx

  923. 	push	%rbp

  924. 	push	%r12

  925. 	push	%r13

  926. 	push	%r14

  927. 	push	%r15

  928. 	pushfq

  929. 	sub	\$64,%rsp

  930. 

  931. 	mov	120($context),%rax	# pull context->Rax

  932. 	mov	248($context),%rbx	# pull context->Rip

  933. 

  934. 	mov	8($disp),%rsi		# disp->ImageBase

  935. 	mov	56($disp),%r11		# disp->HandlerData

  936. 

  937. 	mov	0(%r11),%r10d		# HandlerData[0]

  938. 	lea	(%rsi,%r10),%r10	# prologue label

  939. 	cmp	%r10,%rbx		# context->Rip<prologue label

  940. 	jb	.Lcommon_seh_tail

  941. 

  942. 	mov	152($context),%rax	# pull context->Rsp

  943. 

  944. 	mov	4(%r11),%r10d		# HandlerData[1]

  945. 	lea	(%rsi,%r10),%r10	# epilogue label

  946. 	cmp	%r10,%rbx		# context->Rip>=epilogue label

  947. 	jae	.Lcommon_seh_tail

  948. 

  949. 	mov	120($context),%rax	# pull context->Rax

  950. 

  951. 	mov	-48(%rax),%r15

  952. 	mov	-40(%rax),%r14

  953. 	mov	-32(%rax),%r13

  954. 	mov	-24(%rax),%r12

  955. 	mov	-16(%rax),%rbp

  956. 	mov	-8(%rax),%rbx

  957. 	mov	%r15,240($context)

  958. 	mov	%r14,232($context)

  959. 	mov	%r13,224($context)

  960. 	mov	%r12,216($context)

  961. 	mov	%rbp,160($context)

  962. 	mov	%rbx,144($context)

  963. 

  964. 	lea	-0xd8(%rax),%rsi	# %xmm save area

  965. 	lea	512($context),%rdi	# & context.Xmm6

  966. 	mov	\$20,%ecx		# 10*sizeof(%xmm0)/sizeof(%rax)

  967. 	.long	0xa548f3fc		# cld; rep movsq

  968. 

  969. .Lcommon_seh_tail:

  970. 	mov	8(%rax),%rdi

  971. 	mov	16(%rax),%rsi

  972. 	mov	%rax,152($context)	# restore context->Rsp

  973. 	mov	%rsi,168($context)	# restore context->Rsi

  974. 	mov	%rdi,176($context)	# restore context->Rdi

  975. 

  976. 	mov	40($disp),%rdi		# disp->ContextRecord

  977. 	mov	$context,%rsi		# context

  978. 	mov	\$154,%ecx		# sizeof(CONTEXT)

  979. 	.long	0xa548f3fc		# cld; rep movsq

  980. 

  981. 	mov	$disp,%rsi

  982. 	xor	%rcx,%rcx		# arg1, UNW_FLAG_NHANDLER

  983. 	mov	8(%rsi),%rdx		# arg2, disp->ImageBase

  984. 	mov	0(%rsi),%r8		# arg3, disp->ControlPc

  985. 	mov	16(%rsi),%r9		# arg4, disp->FunctionEntry

  986. 	mov	40(%rsi),%r10		# disp->ContextRecord

  987. 	lea	56(%rsi),%r11		# &disp->HandlerData

  988. 	lea	24(%rsi),%r12		# &disp->EstablisherFrame

  989. 	mov	%r10,32(%rsp)		# arg5

  990. 	mov	%r11,40(%rsp)		# arg6

  991. 	mov	%r12,48(%rsp)		# arg7

  992. 	mov	%rcx,56(%rsp)		# arg8, (NULL)

  993. 	call	*__imp_RtlVirtualUnwind(%rip)

  994. 

  995. 	mov	\$1,%eax		# ExceptionContinueSearch

  996. 	add	\$64,%rsp

  997. 	popfq

  998. 	pop	%r15

  999. 	pop	%r14

  1000. 	pop	%r13

  1001. 	pop	%r12

  1002. 	pop	%rbp

  1003. 	pop	%rbx

  1004. 	pop	%rdi

  1005. 	pop	%rsi

  1006. 	ret

  1007. .size	gcm_se_handler,.-gcm_se_handler

  1008. 

  1009. .section	.pdata

  1010. .align	4

  1011. 	.rva	.LSEH_begin_aesni_gcm_decrypt

  1012. 	.rva	.LSEH_end_aesni_gcm_decrypt

  1013. 	.rva	.LSEH_gcm_dec_info

  1014. 

  1015. 	.rva	.LSEH_begin_aesni_gcm_encrypt

  1016. 	.rva	.LSEH_end_aesni_gcm_encrypt

  1017. 	.rva	.LSEH_gcm_enc_info

  1018. .section	.xdata

  1019. .align	8

  1020. .LSEH_gcm_dec_info:

  1021. 	.byte	9,0,0,0

  1022. 	.rva	gcm_se_handler

  1023. 	.rva	.Lgcm_dec_body,.Lgcm_dec_abort

  1024. .LSEH_gcm_enc_info:

  1025. 	.byte	9,0,0,0

  1026. 	.rva	gcm_se_handler

  1027. 	.rva	.Lgcm_enc_body,.Lgcm_enc_abort

  1028. ___

  1029. }

  1030. }}} else {{{

  1031. $code=<<___;	# assembler is too old

  1032. .text

  1033. 

  1034. .globl	aesni_gcm_encrypt

  1035. .type	aesni_gcm_encrypt,\@abi-omnipotent

  1036. aesni_gcm_encrypt:

  1037. 	xor	%eax,%eax

  1038. 	ret

  1039. .size	aesni_gcm_encrypt,.-aesni_gcm_encrypt

  1040. 

  1041. .globl	aesni_gcm_decrypt

  1042. .type	aesni_gcm_decrypt,\@abi-omnipotent

  1043. aesni_gcm_decrypt:

  1044. 	xor	%eax,%eax

  1045. 	ret

  1046. .size	aesni_gcm_decrypt,.-aesni_gcm_decrypt

  1047. ___

  1048. }}}

  1049. 

  1050. $code =~ s/\`([^\`]*)\`/eval($1)/gem;

  1051. 

  1052. print $code;

  1053. 

  1054. close STDOUT;