kris
/
SIDH_p751
1
0
Fork 0
Code Issues 0 Pull Requests 0 Releases 0 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
126 Commits
14 Branches
653 KiB
 
 
 
Branch: trials/PERF_escp
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'trials/PERF_escp'
${ noResults }
SIDH_p751/p503/arith_amd64.s

1724 lines
39 KiB
Raw Permalink Blame History 

  1. // +build amd64,!noasm

  2. 

  3. #include "textflag.h"

  4. 

  5. // p503

  6. #define P503_0     $0xFFFFFFFFFFFFFFFF

  7. #define P503_1     $0xFFFFFFFFFFFFFFFF

  8. #define P503_2     $0xFFFFFFFFFFFFFFFF

  9. #define P503_3     $0xABFFFFFFFFFFFFFF

  10. #define P503_4     $0x13085BDA2211E7A0

  11. #define P503_5     $0x1B9BF6C87B7E7DAF

  12. #define P503_6     $0x6045C6BDDA77A4D0

  13. #define P503_7     $0x004066F541811E1E

  14. 

  15. // p503+1

  16. #define P503P1_3   $0xAC00000000000000

  17. #define P503P1_4   $0x13085BDA2211E7A0

  18. #define P503P1_5   $0x1B9BF6C87B7E7DAF

  19. #define P503P1_6   $0x6045C6BDDA77A4D0

  20. #define P503P1_7   $0x004066F541811E1E

  21. 

  22. // p503x2

  23. #define P503X2_0   $0xFFFFFFFFFFFFFFFE

  24. #define P503X2_1   $0xFFFFFFFFFFFFFFFF

  25. #define P503X2_2   $0xFFFFFFFFFFFFFFFF

  26. #define P503X2_3   $0x57FFFFFFFFFFFFFF

  27. #define P503X2_4   $0x2610B7B44423CF41

  28. #define P503X2_5   $0x3737ED90F6FCFB5E

  29. #define P503X2_6   $0xC08B8D7BB4EF49A0

  30. #define P503X2_7   $0x0080CDEA83023C3C

  31. 

  32. #define REG_P1 DI

  33. #define REG_P2 SI

  34. #define REG_P3 DX

  35. 

  36. // Performs schoolbook multiplication of 2 256-bit numbers. This optimized version

  37. // uses MULX instruction. Macro smashes value in DX.

  38. // Input: I0 and I1.

  39. // Output: O

  40. // All the other arguments are resgisters, used for storing temporary values

  41. #define MULS256_MULX(O, I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9) \

  42. 	MOVQ    I0, DX          \

  43. 	MULXQ   I1, T1, T0      \   // T0:T1 = A0*B0

  44. 	MOVQ    T1, O           \   // O[0]

  45. 	MULXQ   8+I1, T2, T1    \   // T1:T2 = U0*V1

  46. 	ADDQ    T2, T0          \

  47. 	MULXQ   16+I1, T3, T2   \   // T2:T3 = U0*V2

  48. 	ADCQ    T3, T1          \

  49. 	MULXQ   24+I1, T4, T3   \   // T3:T4 = U0*V3

  50. 	ADCQ    T4, T2          \

  51. 	\ // Column U1

  52. 	MOVQ    8+I0, DX        \

  53. 	ADCQ    $0, T3          \

  54. 	MULXQ   0+I1, T4, T5    \   // T5:T4 = U1*V0

  55. 	MULXQ   8+I1, T7, T6    \   // T6:T7 = U1*V1

  56. 	ADDQ    T7, T5          \

  57. 	MULXQ   16+I1, T8, T7   \   // T7:T8 = U1*V2

  58. 	ADCQ    T8, T6          \

  59. 	MULXQ   24+I1, T9, T8   \   // T8:T9 = U1*V3

  60. 	ADCQ    T9, T7          \

  61. 	ADCQ    $0, T8          \

  62. 	ADDQ    T0, T4          \

  63. 	MOVQ    T4, 8+O         \   // O[1]

  64. 	ADCQ    T1, T5          \

  65. 	ADCQ    T2, T6          \

  66. 	ADCQ    T3, T7          \

  67. 	\ // Column U2

  68. 	MOVQ    16+I0, DX       \

  69. 	ADCQ    $0, T8          \

  70. 	MULXQ   0+I1, T0, T1    \   // T1:T0 = U2*V0

  71. 	MULXQ   8+I1, T3, T2    \   // T2:T3 = U2*V1

  72. 	ADDQ    T3, T1          \

  73. 	MULXQ   16+I1, T4, T3   \   // T3:T4 = U2*V2

  74. 	ADCQ    T4, T2          \

  75. 	MULXQ   24+I1, T9, T4   \   // T4:T9 = U2*V3

  76. 	ADCQ    T9, T3          \

  77. 	\ // Column U3

  78. 	MOVQ    24+I0, DX       \

  79. 	ADCQ    $0, T4          \

  80. 	ADDQ    T5, T0          \

  81. 	MOVQ    T0, 16+O        \   // O[2]

  82. 	ADCQ    T6, T1          \

  83. 	ADCQ    T7, T2          \

  84. 	ADCQ    T8, T3          \

  85. 	ADCQ    $0, T4          \

  86. 	MULXQ   0+I1, T0, T5    \   // T5:T0 = U3*V0

  87. 	MULXQ   8+I1, T7, T6    \   // T6:T7 = U3*V1

  88. 	ADDQ    T7, T5          \

  89. 	MULXQ   16+I1, T8, T7   \   // T7:T8 = U3*V2

  90. 	ADCQ    T8, T6          \

  91. 	MULXQ   24+I1, T9, T8   \   // T8:T9 = U3*V3

  92. 	ADCQ    T9, T7          \

  93. 	ADCQ    $0, T8          \

  94. 	\ // Add values in remaining columns

  95. 	ADDQ    T0, T1          \

  96. 	MOVQ    T1, 24+O        \   // O[3]

  97. 	ADCQ    T5, T2          \

  98. 	MOVQ    T2, 32+O        \   // O[4]

  99. 	ADCQ    T6, T3          \

  100. 	MOVQ    T3, 40+O        \   // O[5]

  101. 	ADCQ    T7, T4          \

  102. 	MOVQ    T4, 48+O        \   // O[6]

  103. 	ADCQ    $0, T8          \   // O[7]

  104. 	MOVQ    T8, 56+O

  105. 

  106. // Performs schoolbook multiplication of 2 256-bit numbers. This optimized version

  107. // uses ADOX, ADCX and MULX instructions. Macro smashes values in AX and DX.

  108. // Input: I0 and I1.

  109. // Output: O

  110. // All the other arguments resgisters are used for storing temporary values

  111. #define MULS256_MULXADX(O, I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9) \

  112. 							\   // U0[0]

  113. 	MOVQ     0+I0, DX       \   // MULX requires multiplayer in DX

  114. 							\   // T0:T1 = I1*DX

  115. 	MULXQ    I1, T1, T0     \   // T0:T1 = U0*V0 (low:high)

  116. 	MOVQ     T1, O          \   // O0[0]

  117. 	MULXQ     8+I1, T2, T1  \   // T2:T1 = U0*V1

  118. 	XORQ     AX, AX         \

  119. 	ADOXQ    T2, T0         \

  120. 	MULXQ    16+I1, T3, T2  \   // T2:T3 = U0*V2

  121. 	ADOXQ    T3, T1         \

  122. 	MULXQ    24+I1, T4, T3  \   // T3:T4 = U0*V3

  123. 	ADOXQ    T4, T2         \

  124. 	\  // Column U1

  125. 	MOVQ      8+I0, DX      \

  126. 	MULXQ    I1, T4, T5     \   // T5:T4 = U1*V0

  127. 	ADOXQ    AX, T3         \

  128. 	XORQ     AX, AX         \

  129. 	MULXQ     8+I1, T7, T6  \   // T6:T7 = U1*V1

  130. 	ADOXQ    T0, T4         \

  131. 	MOVQ     T4, 8+O        \   // O[1]

  132. 	ADCXQ    T7, T5         \

  133. 	MULXQ    16+I1, T8, T7  \   // T7:T8 = U1*V2

  134. 	ADCXQ    T8, T6         \

  135. 	ADOXQ    T1, T5 \

  136. 	MULXQ    24+I1, T9, T8  \   // T8:T9 = U1*V3

  137. 	ADCXQ    T9, T7         \

  138. 	ADCXQ    AX, T8         \

  139. 	ADOXQ    T2, T6         \

  140. 	\ // Column U2

  141. 	MOVQ     16+I0, DX      \

  142. 	MULXQ    I1, T0, T1     \   // T1:T0 = U2*V0

  143. 	ADOXQ    T3, T7         \

  144. 	ADOXQ    AX, T8         \

  145. 	XORQ     AX, AX         \

  146. 	MULXQ    8+I1, T3, T2   \   // T2:T3 = U2*V1

  147. 	ADOXQ    T5, T0         \

  148. 	MOVQ     T0, 16+O       \   // O[2]

  149. 	ADCXQ    T3, T1         \

  150. 	MULXQ    16+I1, T4, T3  \   // T3:T4 = U2*V2

  151. 	ADCXQ    T4, T2         \

  152. 	ADOXQ    T6, T1         \

  153. 	MULXQ    24+I1, T9, T4  \   // T9:T4 = U2*V3

  154. 	ADCXQ    T9, T3         \

  155. 	MOVQ     24+I0, DX      \

  156. 	ADCXQ    AX, T4         \

  157. 	\

  158. 	ADOXQ    T7, T2         \

  159. 	ADOXQ    T8, T3         \

  160. 	ADOXQ    AX, T4         \

  161. 	\ // Column U3

  162. 	MULXQ    I1, T0, T5     \   // T5:T0 = U3*B0

  163. 	XORQ     AX, AX         \

  164. 	MULXQ    8+I1, T7, T6   \   // T6:T7 = U3*B1

  165. 	ADCXQ    T7, T5         \

  166. 	ADOXQ    T0, T1         \

  167. 	MULXQ    16+I1, T8, T7  \   // T7:T8 = U3*V2

  168. 	ADCXQ    T8, T6         \

  169. 	ADOXQ    T5, T2         \

  170. 	MULXQ    24+I1, T9, T8  \   // T8:T9 = U3*V3

  171. 	ADCXQ    T9, T7         \

  172. 	ADCXQ    AX, T8         \

  173. 	\

  174. 	ADOXQ   T6, T3          \

  175. 	ADOXQ   T7, T4          \

  176. 	ADOXQ   AX, T8          \

  177. 	MOVQ    T1, 24+O        \   // O[3]

  178. 	MOVQ    T2, 32+O        \   // O[4]

  179. 	MOVQ    T3, 40+O        \   // O[5]

  180. 	MOVQ    T4, 48+O        \   // O[6] and O[7] below

  181. 	MOVQ    T8, 56+O

  182. 

  183. // Template of a macro that performs schoolbook multiplication of 128-bit with 320-bit

  184. // number. It uses MULX instruction This template must be customized with functions

  185. // performing ADD (add1, add2) and ADD-with-carry (adc1, adc2). addX/adcX may or may

  186. // not be instructions that use two independent carry chains.

  187. // Input:

  188. //   * I0 128-bit number

  189. //   * I1 320-bit number

  190. //   * add1, add2: instruction performing integer addition and starting carry chain

  191. //   * adc1, adc2: instruction performing integer addition with carry

  192. // Output: T[0-6] registers

  193. #define MULS_128x320(I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, add1, add2, adc1, adc2) \

  194. 	\ // Column 0

  195. 	MOVQ    I0, DX              \

  196. 	MULXQ   I1+24(SB), T0, T1   \

  197. 	MULXQ   I1+32(SB), T4, T2   \

  198. 	XORQ    AX, AX              \

  199. 	MULXQ   I1+40(SB), T5, T3   \

  200. 	add1    T4, T1              \

  201. 	adc1    T5, T2              \

  202. 	MULXQ   I1+48(SB), T7, T4   \

  203. 	adc1    T7, T3              \

  204. 	MULXQ   I1+56(SB), T6, T5   \

  205. 	adc1    T6, T4              \

  206. 	adc1    AX, T5              \

  207. 	\ // Column 1

  208. 	MOVQ    8+I0, DX            \

  209. 	MULXQ   I1+24(SB), T6, T7   \

  210. 	add2    T6, T1              \

  211. 	adc2    T7, T2              \

  212. 	MULXQ   I1+32(SB), T8, T6   \

  213. 	adc2    T6, T3              \

  214. 	MULXQ   I1+40(SB), T7, T9   \

  215. 	adc2    T9, T4              \

  216. 	MULXQ   I1+48(SB), T9, T6   \

  217. 	adc2    T6, T5              \

  218. 	MULXQ   I1+56(SB), DX, T6   \

  219. 	adc2    AX, T6              \

  220. 	\ // Output

  221. 	XORQ    AX, AX              \

  222. 	add1    T8, T2              \

  223. 	adc1    T7, T3              \

  224. 	adc1    T9, T4              \

  225. 	adc1    DX, T5              \

  226. 	adc1    AX, T6

  227. 

  228. // Multiplies 128-bit with 320-bit integer. Optimized with MULX instruction.

  229. #define MULS_128x320_MULX(I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9) \

  230. 	MULS_128x320(I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, ADDQ, ADDQ, ADCQ, ADCQ)

  231. 

  232. // Multiplies 128-bit with 320-bit integer. Optimized with  MULX, ADOX and ADCX instructions

  233. #define MULS_128x320_MULXADX(I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9) \

  234. 	MULS_128x320(I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, ADOXQ, ADCXQ, ADOXQ, ADCXQ)

  235. 

  236. // Template of a macro performing multiplication of two 512-bit numbers. It uses one

  237. // level of Karatsuba and one level of schoolbook multiplication. Template must be

  238. // customized with macro performing schoolbook multiplication.

  239. // Input:

  240. //  * I0, I1 - two 512-bit numbers

  241. //  * MULS - either MULS256_MULX or MULS256_MULXADX

  242. // Output: OUT - 1024-bit long

  243. #define MUL(OUT, I0, I1, MULS) \

  244. 	\ // R[8-11]: U1+U0

  245. 	XORQ    AX, AX  \

  246. 	MOVQ    ( 0)(I0), R8    \

  247. 	MOVQ    ( 8)(I0), R9    \

  248. 	MOVQ    (16)(I0), R10   \

  249. 	MOVQ    (24)(I0), R11   \

  250. 	ADDQ    (32)(I0), R8    \

  251. 	ADCQ    (40)(I0), R9    \

  252. 	ADCQ    (48)(I0), R10   \

  253. 	ADCQ    (56)(I0), R11   \

  254. 	SBBQ    $0, AX          \ // store mask

  255. 	MOVQ    R8,  ( 0)(SP)   \

  256. 	MOVQ    R9,  ( 8)(SP)   \

  257. 	MOVQ    R10, (16)(SP)   \

  258. 	MOVQ    R11, (24)(SP)   \

  259. 	\

  260. 	\ // R[12-15]: V1+V0

  261. 	XORQ    BX, BX          \

  262. 	MOVQ    ( 0)(I1), R12   \

  263. 	MOVQ    ( 8)(I1), R13   \

  264. 	MOVQ    (16)(I1), R14   \

  265. 	MOVQ    (24)(I1), R15   \

  266. 	ADDQ    (32)(I1), R12   \

  267. 	ADCQ    (40)(I1), R13   \

  268. 	ADCQ    (48)(I1), R14   \

  269. 	ADCQ    (56)(I1), R15   \

  270. 	SBBQ    $0, BX          \ // store mask

  271. 	MOVQ    R12, (32)(SP)   \

  272. 	MOVQ    R13, (40)(SP)   \

  273. 	MOVQ    R14, (48)(SP)   \

  274. 	MOVQ    R15, (56)(SP)   \

  275. 	\ // Prepare mask for U0+U1 (U1+U0 mod 256^4 if U1+U0 sets carry flag, otherwise 0)

  276. 	ANDQ    AX, R12         \

  277. 	ANDQ    AX, R13         \

  278. 	ANDQ    AX, R14         \

  279. 	ANDQ    AX, R15         \

  280. 	\ // Prepare mask for V0+V1 (V1+V0 mod 256^4 if U1+U0 sets carry flag, otherwise 0)

  281. 	ANDQ    BX, R8          \

  282. 	ANDQ    BX, R9          \

  283. 	ANDQ    BX, R10         \

  284. 	ANDQ    BX, R11         \

  285. 	\ // res = masked(U0+U1) + masked(V0 + V1)

  286. 	ADDQ    R12, R8         \

  287. 	ADCQ    R13, R9         \

  288. 	ADCQ    R14, R10        \

  289. 	ADCQ    R15, R11        \

  290. 	\ // SP[64-96] <- res

  291. 	MOVQ     R8, (64)(SP)   \

  292. 	MOVQ     R9, (72)(SP)   \

  293. 	MOVQ    R10, (80)(SP)   \

  294. 	MOVQ    R11, (88)(SP)   \

  295. 	\ // BP will be used for schoolbook multiplication below

  296. 	MOVQ    BP, 96(SP)  \

  297. 	\ // (U1+U0)*(V1+V0)

  298. 	MULS((64)(OUT), 0(SP), 32(SP), R8, R9, R10, R11, R12, R13, R14, R15, BX, BP)    \

  299. 	\ // U0 x V0

  300. 	MULS(0(OUT), 0(I0), 0(I1), R8, R9, R10, R11, R12, R13, R14, R15, BX, BP)    \

  301. 	\ // U1 x V1

  302. 	MULS(0(SP), 32(I0), 32(I1), R8, R9, R10, R11, R12, R13, R14, R15, BX, BP)  \

  303. 	\ // Recover BP

  304. 	MOVQ    96(SP), BP  \

  305. 	\ // Final part of schoolbook multiplication; R[8-11] = (U0+U1) x (V0+V1)

  306. 	MOVQ    (64)(SP), R8    \

  307. 	MOVQ    (72)(SP), R9    \

  308. 	MOVQ    (80)(SP), R10   \

  309. 	MOVQ    (88)(SP), R11   \

  310. 	MOVQ    (96)(OUT), AX   \

  311. 	ADDQ    AX, R8          \

  312. 	MOVQ    (104)(OUT), AX  \

  313. 	ADCQ    AX, R9          \

  314. 	MOVQ    (112)(OUT), AX  \

  315. 	ADCQ    AX, R10         \

  316. 	MOVQ    (120)(OUT), AX  \

  317. 	ADCQ    AX, R11 \

  318. 	\ // R[12-15, 8-11] = (U0+U1) x (V0+V1) - U0xV0

  319. 	MOVQ    (64)(OUT), R12  \

  320. 	MOVQ    (72)(OUT), R13  \

  321. 	MOVQ    (80)(OUT), R14  \

  322. 	MOVQ    (88)(OUT), R15  \

  323. 	SUBQ    ( 0)(OUT), R12  \

  324. 	SBBQ    ( 8)(OUT), R13  \

  325. 	SBBQ    (16)(OUT), R14  \

  326. 	SBBQ    (24)(OUT), R15  \

  327. 	SBBQ    (32)(OUT), R8   \

  328. 	SBBQ    (40)(OUT), R9   \

  329. 	SBBQ    (48)(OUT), R10  \

  330. 	SBBQ    (56)(OUT), R11  \

  331. 	\ // r8-r15 <- (U0+U1) x (V0+V1) - U0xV0 - U1xV1

  332. 	SUBQ    ( 0)(SP), R12   \

  333. 	SBBQ    ( 8)(SP), R13   \

  334. 	SBBQ    (16)(SP), R14   \

  335. 	SBBQ    (24)(SP), R15   \

  336. 	SBBQ    (32)(SP), R8    \

  337. 	SBBQ    (40)(SP), R9    \

  338. 	SBBQ    (48)(SP), R10   \

  339. 	SBBQ    (56)(SP), R11   \

  340. 	\

  341. 	;                       ADDQ   (32)(OUT), R12; MOVQ    R12, ( 32)(OUT) \

  342. 	;                       ADCQ   (40)(OUT), R13; MOVQ    R13, ( 40)(OUT) \

  343. 	;                       ADCQ   (48)(OUT), R14; MOVQ    R14, ( 48)(OUT) \

  344. 	;                       ADCQ   (56)(OUT), R15; MOVQ    R15, ( 56)(OUT) \

  345. 	MOVQ    ( 0)(SP), AX;   ADCQ    AX,  R8;       MOVQ     R8, ( 64)(OUT) \

  346. 	MOVQ    ( 8)(SP), AX;   ADCQ    AX,  R9;       MOVQ     R9, ( 72)(OUT) \

  347. 	MOVQ    (16)(SP), AX;   ADCQ    AX, R10;       MOVQ    R10, ( 80)(OUT) \

  348. 	MOVQ    (24)(SP), AX;   ADCQ    AX, R11;       MOVQ    R11, ( 88)(OUT) \

  349. 	MOVQ    (32)(SP), R12;  ADCQ    $0, R12;       MOVQ    R12, ( 96)(OUT) \

  350. 	MOVQ    (40)(SP), R13;  ADCQ    $0, R13;       MOVQ    R13, (104)(OUT) \

  351. 	MOVQ    (48)(SP), R14;  ADCQ    $0, R14;       MOVQ    R14, (112)(OUT) \

  352. 	MOVQ    (56)(SP), R15;  ADCQ    $0, R15;       MOVQ    R15, (120)(OUT)

  353. 

  354. // Template for calculating the Montgomery reduction algorithm described in

  355. // section 5.2.3 of https://eprint.iacr.org/2017/1015.pdf. Template must be

  356. // customized with schoolbook multiplicaton for 128 x 320-bit number.

  357. // This macro reuses memory of IN value and *changes* it.

  358. // Input:

  359. //    * IN: 1024-bit number to be reduced

  360. //    * MULS: either MULS_128x320_MULX or MULS_128x320_MULXADX

  361. // Output: OUT 512-bit

  362. #define REDC(OUT, IN, MULS) \

  363. 	MULS(0(IN), ·p503p1, R8, R9, R10, R11, R12, R13, R14, BX, CX, R15) \

  364. 	XORQ    R15, R15        \

  365. 	ADDQ    (24)(IN), R8    \

  366. 	ADCQ    (32)(IN), R9    \

  367. 	ADCQ    (40)(IN), R10   \

  368. 	ADCQ    (48)(IN), R11   \

  369. 	ADCQ    (56)(IN), R12   \

  370. 	ADCQ    (64)(IN), R13   \

  371. 	ADCQ    (72)(IN), R14   \

  372. 	ADCQ    (80)(IN), R15   \

  373. 	MOVQ    R8, (24)(IN)    \

  374. 	MOVQ    R9, (32)(IN)    \

  375. 	MOVQ    R10, (40)(IN)   \

  376. 	MOVQ    R11, (48)(IN)   \

  377. 	MOVQ    R12, (56)(IN)   \

  378. 	MOVQ    R13, (64)(IN)   \

  379. 	MOVQ    R14, (72)(IN)   \

  380. 	MOVQ    R15, (80)(IN)   \

  381. 	MOVQ    (88)(IN), R8    \

  382. 	MOVQ    (96)(IN), R9    \

  383. 	MOVQ    (104)(IN), R10  \

  384. 	MOVQ    (112)(IN), R11  \

  385. 	MOVQ    (120)(IN), R12  \

  386. 	ADCQ    $0, R8          \

  387. 	ADCQ    $0, R9          \

  388. 	ADCQ    $0, R10         \

  389. 	ADCQ    $0, R11         \

  390. 	ADCQ    $0, R12         \

  391. 	MOVQ    R8, (88)(IN)    \

  392. 	MOVQ    R9, (96)(IN)    \

  393. 	MOVQ    R10, (104)(IN)  \

  394. 	MOVQ    R11, (112)(IN)  \

  395. 	MOVQ    R12, (120)(IN)  \

  396. 	\

  397. 	MULS(16(IN), ·p503p1, R8, R9, R10, R11, R12, R13, R14, BX, CX, R15)    \

  398. 	XORQ    R15, R15        \

  399. 	ADDQ    (40)(IN), R8    \

  400. 	ADCQ    (48)(IN), R9    \

  401. 	ADCQ    (56)(IN), R10   \

  402. 	ADCQ    (64)(IN), R11   \

  403. 	ADCQ    (72)(IN), R12   \

  404. 	ADCQ    (80)(IN), R13   \

  405. 	ADCQ    (88)(IN), R14   \

  406. 	ADCQ    (96)(IN), R15   \

  407. 	MOVQ    R8, (40)(IN)    \

  408. 	MOVQ    R9, (48)(IN)    \

  409. 	MOVQ    R10, (56)(IN)   \

  410. 	MOVQ    R11, (64)(IN)   \

  411. 	MOVQ    R12, (72)(IN)   \

  412. 	MOVQ    R13, (80)(IN)   \

  413. 	MOVQ    R14, (88)(IN)   \

  414. 	MOVQ    R15, (96)(IN)   \

  415. 	MOVQ    (104)(IN), R8   \

  416. 	MOVQ    (112)(IN), R9   \

  417. 	MOVQ    (120)(IN), R10  \

  418. 	ADCQ    $0, R8          \

  419. 	ADCQ    $0, R9          \

  420. 	ADCQ    $0, R10         \

  421. 	MOVQ    R8, (104)(IN)   \

  422. 	MOVQ    R9, (112)(IN)   \

  423. 	MOVQ    R10, (120)(IN)  \

  424. 	\

  425. 	MULS(32(IN), ·p503p1, R8, R9, R10, R11, R12, R13, R14, BX, CX, R15)    \

  426. 	XORQ    R15, R15        \

  427. 	XORQ    BX, BX          \

  428. 	ADDQ    ( 56)(IN), R8   \

  429. 	ADCQ    ( 64)(IN), R9   \

  430. 	ADCQ    ( 72)(IN), R10  \

  431. 	ADCQ    ( 80)(IN), R11  \

  432. 	ADCQ    ( 88)(IN), R12  \

  433. 	ADCQ    ( 96)(IN), R13  \

  434. 	ADCQ    (104)(IN), R14  \

  435. 	ADCQ    (112)(IN), R15  \

  436. 	ADCQ    (120)(IN), BX   \

  437. 	MOVQ    R8,  ( 56)(IN)  \

  438. 	MOVQ    R10, ( 72)(IN)  \

  439. 	MOVQ    R11, ( 80)(IN)  \

  440. 	MOVQ    R12, ( 88)(IN)  \

  441. 	MOVQ    R13, ( 96)(IN)  \

  442. 	MOVQ    R14, (104)(IN)  \

  443. 	MOVQ    R15, (112)(IN)  \

  444. 	MOVQ    BX,  (120)(IN)  \

  445. 	MOVQ    R9,  (  0)(OUT) \ // Result: OUT[0]

  446. 	\

  447. 	MULS(48(IN), ·p503p1, R8, R9, R10, R11, R12, R13, R14, BX, CX, R15)    \

  448. 	ADDQ    ( 72)(IN), R8   \

  449. 	ADCQ    ( 80)(IN), R9   \

  450. 	ADCQ    ( 88)(IN), R10  \

  451. 	ADCQ    ( 96)(IN), R11  \

  452. 	ADCQ    (104)(IN), R12  \

  453. 	ADCQ    (112)(IN), R13  \

  454. 	ADCQ    (120)(IN), R14  \

  455. 	MOVQ    R8,  ( 8)(OUT)  \ // Result: OUT[1]

  456. 	MOVQ    R9,  (16)(OUT)  \ // Result: OUT[2]

  457. 	MOVQ    R10, (24)(OUT)  \ // Result: OUT[3]

  458. 	MOVQ    R11, (32)(OUT)  \ // Result: OUT[4]

  459. 	MOVQ    R12, (40)(OUT)  \ // Result: OUT[5]

  460. 	MOVQ    R13, (48)(OUT)  \ // Result: OUT[6] and OUT[7]

  461. 	MOVQ    R14, (56)(OUT)

  462. 

  463. TEXT ·fp503StrongReduce(SB), NOSPLIT, $0-8

  464. 	MOVQ	x+0(FP), REG_P1

  465. 

  466. 	// Zero AX for later use:

  467. 	XORQ	AX, AX

  468. 

  469. 	// Load p into registers:

  470. 	MOVQ	P503_0, R8

  471. 	// P503_{1,2} = P503_0, so reuse R8

  472. 	MOVQ	P503_3, R9

  473. 	MOVQ	P503_4, R10

  474. 	MOVQ	P503_5, R11

  475. 	MOVQ	P503_6, R12

  476. 	MOVQ	P503_7, R13

  477. 

  478. 	// Set x <- x - p

  479. 	SUBQ	R8,  ( 0)(REG_P1)

  480. 	SBBQ	R8,  ( 8)(REG_P1)

  481. 	SBBQ	R8,  (16)(REG_P1)

  482. 	SBBQ	R9,  (24)(REG_P1)

  483. 	SBBQ	R10, (32)(REG_P1)

  484. 	SBBQ	R11, (40)(REG_P1)

  485. 	SBBQ	R12, (48)(REG_P1)

  486. 	SBBQ	R13, (56)(REG_P1)

  487. 

  488. 	// Save carry flag indicating x-p < 0 as a mask

  489. 	SBBQ	$0, AX

  490. 

  491. 	// Conditionally add p to x if x-p < 0

  492. 	ANDQ	AX, R8

  493. 	ANDQ	AX, R9

  494. 	ANDQ	AX, R10

  495. 	ANDQ	AX, R11

  496. 	ANDQ	AX, R12

  497. 	ANDQ	AX, R13

  498. 

  499. 	ADDQ	R8, ( 0)(REG_P1)

  500. 	ADCQ	R8, ( 8)(REG_P1)

  501. 	ADCQ	R8, (16)(REG_P1)

  502. 	ADCQ	R9, (24)(REG_P1)

  503. 	ADCQ	R10,(32)(REG_P1)

  504. 	ADCQ	R11,(40)(REG_P1)

  505. 	ADCQ	R12,(48)(REG_P1)

  506. 	ADCQ	R13,(56)(REG_P1)

  507. 

  508. 	RET

  509. 

  510. TEXT ·fp503ConditionalSwap(SB),NOSPLIT,$0-17

  511. 

  512. 	MOVQ	x+0(FP), REG_P1

  513. 	MOVQ	y+8(FP), REG_P2

  514. 	MOVB	choice+16(FP), AL	// AL = 0 or 1

  515. 	MOVBLZX	AL, AX				// AX = 0 or 1

  516. 	NEGQ	AX					// AX = 0x00..00 or 0xff..ff

  517. 

  518. #ifndef CSWAP_BLOCK

  519. #define CSWAP_BLOCK(idx) 	\

  520. 	MOVQ	(idx*8)(REG_P1), BX	\ // BX = x[idx]

  521. 	MOVQ 	(idx*8)(REG_P2), CX	\ // CX = y[idx]

  522. 	MOVQ	CX, DX				\ // DX = y[idx]

  523. 	XORQ	BX, DX				\ // DX = y[idx] ^ x[idx]

  524. 	ANDQ	AX, DX				\ // DX = (y[idx] ^ x[idx]) & mask

  525. 	XORQ	DX, BX				\ // BX = (y[idx] ^ x[idx]) & mask) ^ x[idx] = x[idx] or y[idx]

  526. 	XORQ	DX, CX				\ // CX = (y[idx] ^ x[idx]) & mask) ^ y[idx] = y[idx] or x[idx]

  527. 	MOVQ	BX, (idx*8)(REG_P1)	\

  528. 	MOVQ	CX, (idx*8)(REG_P2)

  529. #endif

  530. 

  531. 	CSWAP_BLOCK(0)

  532. 	CSWAP_BLOCK(1)

  533. 	CSWAP_BLOCK(2)

  534. 	CSWAP_BLOCK(3)

  535. 	CSWAP_BLOCK(4)

  536. 	CSWAP_BLOCK(5)

  537. 	CSWAP_BLOCK(6)

  538. 	CSWAP_BLOCK(7)

  539. 

  540. #ifdef CSWAP_BLOCK

  541. #undef CSWAP_BLOCK

  542. #endif

  543. 

  544. 	RET

  545. 

  546. TEXT ·fp503AddReduced(SB),NOSPLIT,$0-24

  547. 

  548. 	MOVQ	z+0(FP), REG_P3

  549. 	MOVQ	x+8(FP), REG_P1

  550. 	MOVQ	y+16(FP), REG_P2

  551. 

  552. 	// Used later to calculate a mask

  553. 	XORQ    CX, CX

  554. 

  555. 	// [R8-R15]: z = x + y

  556. 	MOVQ	( 0)(REG_P1), R8

  557. 	MOVQ	( 8)(REG_P1), R9

  558. 	MOVQ	(16)(REG_P1), R10

  559. 	MOVQ	(24)(REG_P1), R11

  560. 	MOVQ	(32)(REG_P1), R12

  561. 	MOVQ	(40)(REG_P1), R13

  562. 	MOVQ	(48)(REG_P1), R14

  563. 	MOVQ	(56)(REG_P1), R15

  564. 	ADDQ	( 0)(REG_P2), R8

  565. 	ADCQ	( 8)(REG_P2), R9

  566. 	ADCQ	(16)(REG_P2), R10

  567. 	ADCQ	(24)(REG_P2), R11

  568. 	ADCQ	(32)(REG_P2), R12

  569. 	ADCQ	(40)(REG_P2), R13

  570. 	ADCQ	(48)(REG_P2), R14

  571. 	ADCQ	(56)(REG_P2), R15

  572. 

  573. 	MOVQ    P503X2_0, AX

  574. 	SUBQ    AX, R8

  575. 	MOVQ    P503X2_1, AX

  576. 	SBBQ    AX, R9

  577. 	SBBQ    AX, R10

  578. 	MOVQ    P503X2_3, AX

  579. 	SBBQ    AX, R11

  580. 	MOVQ    P503X2_4, AX

  581. 	SBBQ    AX, R12

  582. 	MOVQ    P503X2_5, AX

  583. 	SBBQ    AX, R13

  584. 	MOVQ    P503X2_6, AX

  585. 	SBBQ    AX, R14

  586. 	MOVQ    P503X2_7, AX

  587. 	SBBQ    AX, R15

  588. 

  589. 	// mask

  590. 	SBBQ    $0, CX

  591. 

  592. 	// move z to REG_P3

  593. 	MOVQ    R8,  ( 0)(REG_P3)

  594. 	MOVQ    R9,  ( 8)(REG_P3)

  595. 	MOVQ    R10, (16)(REG_P3)

  596. 	MOVQ    R11, (24)(REG_P3)

  597. 	MOVQ    R12, (32)(REG_P3)

  598. 	MOVQ    R13, (40)(REG_P3)

  599. 	MOVQ    R14, (48)(REG_P3)

  600. 	MOVQ    R15, (56)(REG_P3)

  601. 

  602. 	// if z<0 add p503x2 back

  603. 	MOVQ    P503X2_0,   R8

  604. 	MOVQ    P503X2_1,   R9

  605. 	MOVQ    P503X2_3,   R10

  606. 	MOVQ    P503X2_4,   R11

  607. 	MOVQ    P503X2_5,   R12

  608. 	MOVQ    P503X2_6,   R13

  609. 	MOVQ    P503X2_7,   R14

  610. 	ANDQ    CX, R8

  611. 	ANDQ    CX, R9

  612. 	ANDQ    CX, R10

  613. 	ANDQ    CX, R11

  614. 	ANDQ    CX, R12

  615. 	ANDQ    CX, R13

  616. 	ANDQ    CX, R14

  617. 	MOVQ    ( 0)(REG_P3), AX; ADDQ    R8,  AX; MOVQ    AX, ( 0)(REG_P3)

  618. 	MOVQ    ( 8)(REG_P3), AX; ADCQ    R9,  AX; MOVQ    AX, ( 8)(REG_P3)

  619. 	MOVQ    (16)(REG_P3), AX; ADCQ    R9,  AX; MOVQ    AX, (16)(REG_P3)

  620. 	MOVQ    (24)(REG_P3), AX; ADCQ    R10, AX; MOVQ    AX, (24)(REG_P3)

  621. 	MOVQ    (32)(REG_P3), AX; ADCQ    R11, AX; MOVQ    AX, (32)(REG_P3)

  622. 	MOVQ    (40)(REG_P3), AX; ADCQ    R12, AX; MOVQ    AX, (40)(REG_P3)

  623. 	MOVQ    (48)(REG_P3), AX; ADCQ    R13, AX; MOVQ    AX, (48)(REG_P3)

  624. 	MOVQ    (56)(REG_P3), AX; ADCQ    R14, AX; MOVQ    AX, (56)(REG_P3)

  625. 	RET

  626. 

  627. TEXT ·fp503SubReduced(SB), NOSPLIT, $0-24

  628. 

  629. 	MOVQ    z+0(FP), REG_P3

  630. 	MOVQ    x+8(FP), REG_P1

  631. 	MOVQ    y+16(FP), REG_P2

  632. 

  633. 	// Used later to calculate a mask

  634. 	XORQ    CX, CX

  635. 

  636. 	MOVQ    ( 0)(REG_P1), R8

  637. 	MOVQ    ( 8)(REG_P1), R9

  638. 	MOVQ    (16)(REG_P1), R10

  639. 	MOVQ    (24)(REG_P1), R11

  640. 	MOVQ    (32)(REG_P1), R12

  641. 	MOVQ    (40)(REG_P1), R13

  642. 	MOVQ    (48)(REG_P1), R14

  643. 	MOVQ    (56)(REG_P1), R15

  644. 

  645. 	SUBQ    ( 0)(REG_P2), R8

  646. 	SBBQ    ( 8)(REG_P2), R9

  647. 	SBBQ    (16)(REG_P2), R10

  648. 	SBBQ    (24)(REG_P2), R11

  649. 	SBBQ    (32)(REG_P2), R12

  650. 	SBBQ    (40)(REG_P2), R13

  651. 	SBBQ    (48)(REG_P2), R14

  652. 	SBBQ    (56)(REG_P2), R15

  653. 

  654. 	// mask

  655. 	SBBQ    $0, CX

  656. 

  657. 	// store x-y in REG_P3

  658. 	MOVQ    R8,  ( 0)(REG_P3)

  659. 	MOVQ    R9,  ( 8)(REG_P3)

  660. 	MOVQ    R10, (16)(REG_P3)

  661. 	MOVQ    R11, (24)(REG_P3)

  662. 	MOVQ    R12, (32)(REG_P3)

  663. 	MOVQ    R13, (40)(REG_P3)

  664. 	MOVQ    R14, (48)(REG_P3)

  665. 	MOVQ    R15, (56)(REG_P3)

  666. 

  667. 	// if z<0 add p503x2 back

  668. 	MOVQ    P503X2_0, R8

  669. 	MOVQ    P503X2_1, R9

  670. 	MOVQ    P503X2_3, R10

  671. 	MOVQ    P503X2_4, R11

  672. 	MOVQ    P503X2_5, R12

  673. 	MOVQ    P503X2_6, R13

  674. 	MOVQ    P503X2_7, R14

  675. 	ANDQ    CX, R8

  676. 	ANDQ    CX, R9

  677. 	ANDQ    CX, R10

  678. 	ANDQ    CX, R11

  679. 	ANDQ    CX, R12

  680. 	ANDQ    CX, R13

  681. 	ANDQ    CX, R14

  682. 	MOVQ    ( 0)(REG_P3), AX; ADDQ    R8,  AX; MOVQ    AX, ( 0)(REG_P3)

  683. 	MOVQ    ( 8)(REG_P3), AX; ADCQ    R9,  AX; MOVQ    AX, ( 8)(REG_P3)

  684. 	MOVQ    (16)(REG_P3), AX; ADCQ    R9,  AX; MOVQ    AX, (16)(REG_P3)

  685. 	MOVQ    (24)(REG_P3), AX; ADCQ    R10, AX; MOVQ    AX, (24)(REG_P3)

  686. 	MOVQ    (32)(REG_P3), AX; ADCQ    R11, AX; MOVQ    AX, (32)(REG_P3)

  687. 	MOVQ    (40)(REG_P3), AX; ADCQ    R12, AX; MOVQ    AX, (40)(REG_P3)

  688. 	MOVQ    (48)(REG_P3), AX; ADCQ    R13, AX; MOVQ    AX, (48)(REG_P3)

  689. 	MOVQ    (56)(REG_P3), AX; ADCQ    R14, AX; MOVQ    AX, (56)(REG_P3)

  690. 

  691. 	RET

  692. 

  693. TEXT ·mulWithMULXADX(SB), NOSPLIT, $104-24

  694. 	// Actual implementation

  695. 	MOVQ    z+ 0(FP), CX

  696. 	MOVQ    x+ 8(FP), REG_P2

  697. 	MOVQ    y+16(FP), REG_P1

  698. 	MUL(CX, REG_P2, REG_P1, MULS256_MULXADX)

  699. 	RET

  700. 

  701. // TEXT ·fp503Mul(SB), NOSPLIT, $104-24

  702. // 	// Actual implementation

  703. // 	MOVQ    z+ 0(FP), CX

  704. // 	MOVQ    x+ 8(FP), REG_P2

  705. // 	MOVQ    y+16(FP), REG_P1

  706. // 	MUL(CX, REG_P2, REG_P1, MULS256_MULXADX)

  707. // 	RET

  708. 

  709. TEXT ·fp503MulXXX(SB), NOSPLIT, $104-72

  710. 	// Actual implementation

  711. 	MOVQ    z+ 0(FP), CX

  712. 	MOVQ    x+24(FP), REG_P2

  713. 	MOVQ    y+48(FP), REG_P1

  714. 	MUL(CX, REG_P2, REG_P1, MULS256_MULXADX)

  715. 	RET

  716. 

  717. TEXT ·mulWithMULX(SB), NOSPLIT, $104-24

  718. 	// Actual implementation

  719. 	MOVQ    z+ 0(FP), CX

  720. 	MOVQ    x+ 8(FP), REG_P2

  721. 	MOVQ    y+16(FP), REG_P1

  722. 	MUL(CX, REG_P2, REG_P1, MULS256_MULX)

  723. 	RET

  724. 

  725. TEXT ·mul(SB), $96-24

  726. 	// Uses variant of Karatsuba method.

  727. 	//

  728. 	// Here we store the destination in CX instead of in REG_P3 because the

  729. 	// multiplication instructions use DX as an implicit destination

  730. 	// operand: MULQ $REG sets DX:AX <-- AX * $REG.

  731. 

  732. 	// Actual implementation

  733. 	MOVQ	z+0(FP), CX

  734. 	MOVQ	x+8(FP), REG_P1

  735. 	MOVQ	y+16(FP), REG_P2

  736. 

  737. 	// RAX and RDX will be used for a mask (0-borrow)

  738. 	XORQ	AX, AX

  739. 

  740. 	// RCX[0-3]: U1+U0

  741. 	MOVQ	(32)(REG_P1), R8

  742. 	MOVQ	(40)(REG_P1), R9

  743. 	MOVQ	(48)(REG_P1), R10

  744. 	MOVQ	(56)(REG_P1), R11

  745. 	ADDQ	( 0)(REG_P1), R8

  746. 	ADCQ	( 8)(REG_P1), R9

  747. 	ADCQ	(16)(REG_P1), R10

  748. 	ADCQ	(24)(REG_P1), R11

  749. 	MOVQ	R8,  ( 0)(CX)

  750. 	MOVQ	R9,  ( 8)(CX)

  751. 	MOVQ	R10, (16)(CX)

  752. 	MOVQ	R11, (24)(CX)

  753. 

  754. 	SBBQ	$0, AX

  755. 

  756. 	// R12-R15: V1+V0

  757. 	XORQ	DX, DX

  758. 	MOVQ	(32)(REG_P2), R12

  759. 	MOVQ	(40)(REG_P2), R13

  760. 	MOVQ	(48)(REG_P2), R14

  761. 	MOVQ	(56)(REG_P2), R15

  762. 	ADDQ	( 0)(REG_P2), R12

  763. 	ADCQ	( 8)(REG_P2), R13

  764. 	ADCQ	(16)(REG_P2), R14

  765. 	ADCQ	(24)(REG_P2), R15

  766. 

  767. 	SBBQ	$0, DX

  768. 

  769. 	// Store carries on stack

  770. 	MOVQ	AX, (64)(SP)

  771. 	MOVQ	DX, (72)(SP)

  772. 

  773. 	// (SP[0-3],R8,R9,R10,R11) <- (U0+U1)*(V0+V1).

  774. 	// MUL using comba; In comments below U=U0+U1 V=V0+V1

  775. 

  776. 	// U0*V0

  777. 	MOVQ    (CX), AX

  778. 	MULQ    R12

  779. 	MOVQ    AX, (SP)        // C0

  780. 	MOVQ    DX, R8

  781. 

  782. 	// U0*V1

  783. 	XORQ    R9, R9

  784. 	MOVQ    (CX), AX

  785. 	MULQ    R13

  786. 	ADDQ    AX, R8

  787. 	ADCQ    DX, R9

  788. 

  789. 	// U1*V0

  790. 	XORQ    R10, R10

  791. 	MOVQ    (8)(CX), AX

  792. 	MULQ    R12

  793. 	ADDQ    AX, R8

  794. 	MOVQ    R8, (8)(SP)     // C1

  795. 	ADCQ    DX, R9

  796. 	ADCQ    $0, R10

  797. 

  798. 	// U0*V2

  799. 	XORQ    R8, R8

  800. 	MOVQ    (CX), AX

  801. 	MULQ    R14

  802. 	ADDQ    AX, R9

  803. 	ADCQ    DX, R10

  804. 	ADCQ    $0, R8

  805. 

  806. 	// U2*V0

  807. 	MOVQ    (16)(CX), AX

  808. 	MULQ    R12

  809. 	ADDQ    AX, R9

  810. 	ADCQ    DX, R10

  811. 	ADCQ    $0, R8

  812. 

  813. 	// U1*V1

  814. 	MOVQ    (8)(CX), AX

  815. 	MULQ    R13

  816. 	ADDQ    AX, R9

  817. 	MOVQ    R9, (16)(SP)        // C2

  818. 	ADCQ    DX, R10

  819. 	ADCQ    $0, R8

  820. 

  821. 	// U0*V3

  822. 	XORQ    R9, R9

  823. 	MOVQ    (CX), AX

  824. 	MULQ    R15

  825. 	ADDQ    AX, R10

  826. 	ADCQ    DX, R8

  827. 	ADCQ    $0, R9

  828. 

  829. 	// U3*V0

  830. 	MOVQ    (24)(CX), AX

  831. 	MULQ    R12

  832. 	ADDQ    AX, R10

  833. 	ADCQ    DX, R8

  834. 	ADCQ    $0, R9

  835. 

  836. 	// U1*V2

  837. 	MOVQ    (8)(CX), AX

  838. 	MULQ    R14

  839. 	ADDQ    AX, R10

  840. 	ADCQ    DX, R8

  841. 	ADCQ    $0, R9

  842. 

  843. 	// U2*V1

  844. 	MOVQ    (16)(CX), AX

  845. 	MULQ    R13

  846. 	ADDQ    AX, R10

  847. 	MOVQ    R10, (24)(SP)       // C3

  848. 	ADCQ    DX, R8

  849. 	ADCQ    $0, R9

  850. 

  851. 	// U1*V3

  852. 	XORQ    R10, R10

  853. 	MOVQ    (8)(CX), AX

  854. 	MULQ    R15

  855. 	ADDQ    AX, R8

  856. 	ADCQ    DX, R9

  857. 	ADCQ    $0, R10

  858. 

  859. 	// U3*V1

  860. 	MOVQ    (24)(CX), AX

  861. 	MULQ    R13

  862. 	ADDQ    AX, R8

  863. 	ADCQ    DX, R9

  864. 	ADCQ    $0, R10

  865. 

  866. 	// U2*V2

  867. 	MOVQ    (16)(CX), AX

  868. 	MULQ    R14

  869. 	ADDQ    AX, R8

  870. 	MOVQ    R8, (32)(SP)        // C4

  871. 	ADCQ    DX, R9

  872. 	ADCQ    $0, R10

  873. 

  874. 	// U2*V3

  875. 	XORQ    R11, R11

  876. 	MOVQ    (16)(CX), AX

  877. 	MULQ    R15

  878. 	ADDQ    AX, R9

  879. 	ADCQ    DX, R10

  880. 	ADCQ    $0, R11

  881. 

  882. 	// U3*V2

  883. 	MOVQ    (24)(CX), AX

  884. 	MULQ    R14

  885. 	ADDQ    AX, R9              // C5

  886. 	ADCQ    DX, R10

  887. 	ADCQ    $0, R11

  888. 

  889. 	// U3*V3

  890. 	MOVQ    (24)(CX), AX

  891. 	MULQ    R15

  892. 	ADDQ    AX, R10             // C6

  893. 	ADCQ    DX, R11             // C7

  894. 

  895. 	MOVQ    (64)(SP), AX

  896. 	ANDQ    AX, R12

  897. 	ANDQ    AX, R13

  898. 	ANDQ    AX, R14

  899. 	ANDQ    AX, R15

  900. 	ADDQ    R8, R12

  901. 	ADCQ    R9, R13

  902. 	ADCQ    R10, R14

  903. 	ADCQ    R11, R15

  904. 

  905. 	MOVQ    (72)(SP), AX

  906. 	MOVQ    (CX), R8

  907. 	MOVQ    (8)(CX), R9

  908. 	MOVQ    (16)(CX), R10

  909. 	MOVQ    (24)(CX), R11

  910. 	ANDQ    AX, R8

  911. 	ANDQ    AX, R9

  912. 	ANDQ    AX, R10

  913. 	ANDQ    AX, R11

  914. 	ADDQ    R12, R8

  915. 	ADCQ    R13, R9

  916. 	ADCQ    R14, R10

  917. 	ADCQ    R15, R11

  918. 	MOVQ    R8, (32)(SP)

  919. 	MOVQ    R9, (40)(SP)

  920. 	MOVQ    R10, (48)(SP)

  921. 	MOVQ    R11, (56)(SP)

  922. 

  923. 	// CX[0-7] <- AL*BL

  924. 

  925. 	// U0*V0

  926. 	MOVQ    (REG_P1), R11

  927. 	MOVQ    (REG_P2), AX

  928. 	MULQ    R11

  929. 	XORQ    R9, R9

  930. 	MOVQ    AX, (CX)            // C0

  931. 	MOVQ    DX, R8

  932. 

  933. 	// U0*V1

  934. 	MOVQ    (16)(REG_P1), R14

  935. 	MOVQ    (8)(REG_P2), AX

  936. 	MULQ    R11

  937. 	XORQ    R10, R10

  938. 	ADDQ    AX, R8

  939. 	ADCQ    DX, R9

  940. 

  941. 	// U1*V0

  942. 	MOVQ    (8)(REG_P1), R12

  943. 	MOVQ    (REG_P2), AX

  944. 	MULQ    R12

  945. 	ADDQ    AX, R8

  946. 	MOVQ    R8, (8)(CX)         // C1

  947. 	ADCQ    DX, R9

  948. 	ADCQ    $0, R10

  949. 

  950. 	// U0*V2

  951. 	XORQ    R8, R8

  952. 	MOVQ    (16)(REG_P2), AX

  953. 	MULQ    R11

  954. 	ADDQ    AX, R9

  955. 	ADCQ    DX, R10

  956. 	ADCQ    $0, R8

  957. 

  958. 	// U2*V0

  959. 	MOVQ    (REG_P2), R13

  960. 	MOVQ    R14, AX

  961. 	MULQ    R13

  962. 	ADDQ    AX, R9

  963. 	ADCQ    DX, R10

  964. 	ADCQ    $0, R8

  965. 

  966. 	// U1*V1

  967. 	MOVQ    (8)(REG_P2), AX

  968. 	MULQ    R12

  969. 	ADDQ    AX, R9

  970. 	MOVQ    R9, (16)(CX)        // C2

  971. 	ADCQ    DX, R10

  972. 	ADCQ    $0, R8

  973. 

  974. 	// U0*V3

  975. 	XORQ    R9, R9

  976. 	MOVQ    (24)(REG_P2), AX

  977. 	MULQ    R11

  978. 	MOVQ    (24)(REG_P1), R15

  979. 	ADDQ    AX, R10

  980. 	ADCQ    DX, R8

  981. 	ADCQ    $0, R9

  982. 

  983. 	// U3*V1

  984. 	MOVQ    R15, AX

  985. 	MULQ    R13

  986. 	ADDQ    AX, R10

  987. 	ADCQ    DX, R8

  988. 	ADCQ    $0, R9

  989. 

  990. 	// U2*V2

  991. 	MOVQ    (16)(REG_P2), AX

  992. 	MULQ    R12

  993. 	ADDQ    AX, R10

  994. 	ADCQ    DX, R8

  995. 	ADCQ    $0, R9

  996. 

  997. 	// U2*V3

  998. 	MOVQ    (8)(REG_P2), AX

  999. 	MULQ    R14

  1000. 	ADDQ    AX, R10

  1001. 	MOVQ    R10, (24)(CX)       // C3

  1002. 	ADCQ    DX, R8

  1003. 	ADCQ    $0, R9

  1004. 

  1005. 	// U3*V2

  1006. 	XORQ    R10, R10

  1007. 	MOVQ    (24)(REG_P2), AX

  1008. 	MULQ    R12

  1009. 	ADDQ    AX, R8

  1010. 	ADCQ    DX, R9

  1011. 	ADCQ    $0, R10

  1012. 

  1013. 	// U3*V1

  1014. 	MOVQ    (8)(REG_P2), AX

  1015. 	MULQ    R15

  1016. 	ADDQ    AX, R8

  1017. 	ADCQ    DX, R9

  1018. 	ADCQ    $0, R10

  1019. 

  1020. 	// U2*V2

  1021. 	MOVQ    (16)(REG_P2), AX

  1022. 	MULQ    R14

  1023. 	ADDQ    AX, R8

  1024. 	MOVQ    R8, (32)(CX)		// C4

  1025. 	ADCQ    DX, R9

  1026. 	ADCQ    $0, R10

  1027. 

  1028. 	// U2*V3

  1029. 	XORQ    R8, R8

  1030. 	MOVQ    (24)(REG_P2), AX

  1031. 	MULQ    R14

  1032. 	ADDQ    AX, R9

  1033. 	ADCQ    DX, R10

  1034. 	ADCQ    $0, R8

  1035. 

  1036. 	// U3*V2

  1037. 	MOVQ    (16)(REG_P2), AX

  1038. 	MULQ    R15

  1039. 	ADDQ    AX, R9

  1040. 	MOVQ    R9, (40)(CX)		// C5

  1041. 	ADCQ    DX, R10

  1042. 	ADCQ    $0, R8

  1043. 

  1044. 	// U3*V3

  1045. 	MOVQ    (24)(REG_P2), AX

  1046. 	MULQ    R15

  1047. 	ADDQ    AX, R10

  1048. 	MOVQ    R10, (48)(CX)		// C6

  1049. 	ADCQ    DX, R8

  1050. 	MOVQ    R8, (56)(CX)		// C7

  1051. 

  1052. 	// CX[8-15] <- U1*V1

  1053. 	MOVQ    (32)(REG_P1), R11

  1054. 	MOVQ    (32)(REG_P2), AX

  1055. 	MULQ    R11

  1056. 	XORQ    R9, R9

  1057. 	MOVQ    AX, (64)(CX)        // C0

  1058. 	MOVQ    DX, R8

  1059. 

  1060. 	MOVQ    (48)(REG_P1), R14

  1061. 	MOVQ    (40)(REG_P2), AX

  1062. 	MULQ    R11

  1063. 	XORQ    R10, R10

  1064. 	ADDQ    AX, R8

  1065. 	ADCQ    DX, R9

  1066. 

  1067. 	MOVQ    (40)(REG_P1), R12

  1068. 	MOVQ    (32)(REG_P2), AX

  1069. 	MULQ    R12

  1070. 	ADDQ    AX, R8

  1071. 	MOVQ    R8, (72)(CX)        // C1

  1072. 	ADCQ    DX, R9

  1073. 	ADCQ    $0, R10

  1074. 

  1075. 	XORQ    R8, R8

  1076. 	MOVQ    (48)(REG_P2), AX

  1077. 	MULQ    R11

  1078. 	ADDQ    AX, R9

  1079. 	ADCQ    DX, R10

  1080. 	ADCQ    $0, R8

  1081. 

  1082. 	MOVQ    (32)(REG_P2), R13

  1083. 	MOVQ    R14, AX

  1084. 	MULQ    R13

  1085. 	ADDQ    AX, R9

  1086. 	ADCQ    DX, R10

  1087. 	ADCQ    $0, R8

  1088. 

  1089. 	MOVQ    (40)(REG_P2), AX

  1090. 	MULQ    R12

  1091. 	ADDQ    AX, R9

  1092. 	MOVQ    R9, (80)(CX)        // C2

  1093. 	ADCQ    DX, R10

  1094. 	ADCQ    $0, R8

  1095. 

  1096. 	XORQ    R9, R9

  1097. 	MOVQ    (56)(REG_P2), AX

  1098. 	MULQ    R11

  1099. 	MOVQ    (56)(REG_P1), R15

  1100. 	ADDQ    AX, R10

  1101. 	ADCQ    DX, R8

  1102. 	ADCQ    $0, R9

  1103. 

  1104. 	MOVQ    R15, AX

  1105. 	MULQ    R13

  1106. 	ADDQ    AX, R10

  1107. 	ADCQ    DX, R8

  1108. 	ADCQ    $0, R9

  1109. 

  1110. 	MOVQ    (48)(REG_P2), AX

  1111. 	MULQ    R12

  1112. 	ADDQ    AX, R10

  1113. 	ADCQ    DX, R8

  1114. 	ADCQ    $0, R9

  1115. 

  1116. 	MOVQ    (40)(REG_P2), AX

  1117. 	MULQ    R14

  1118. 	ADDQ    AX, R10

  1119. 	MOVQ    R10, (88)(CX)       // C3

  1120. 	ADCQ    DX, R8

  1121. 	ADCQ    $0, R9

  1122. 

  1123. 	XORQ    R10, R10

  1124. 	MOVQ    (56)(REG_P2), AX

  1125. 	MULQ    R12

  1126. 	ADDQ    AX, R8

  1127. 	ADCQ    DX, R9

  1128. 	ADCQ    $0, R10

  1129. 

  1130. 	MOVQ    (40)(REG_P2), AX

  1131. 	MULQ    R15

  1132. 	ADDQ    AX, R8

  1133. 	ADCQ    DX, R9

  1134. 	ADCQ    $0, R10

  1135. 

  1136. 	MOVQ    (48)(REG_P2), AX

  1137. 	MULQ    R14

  1138. 	ADDQ    AX, R8

  1139. 	MOVQ    R8, (96)(CX)        // C4

  1140. 	ADCQ    DX, R9

  1141. 	ADCQ    $0, R10

  1142. 

  1143. 	XORQ    R8, R8

  1144. 	MOVQ    (56)(REG_P2), AX

  1145. 	MULQ    R14

  1146. 	ADDQ    AX, R9

  1147. 	ADCQ    DX, R10

  1148. 	ADCQ    $0, R8

  1149. 

  1150. 	MOVQ    (48)(REG_P2), AX

  1151. 	MULQ    R15

  1152. 	ADDQ    AX, R9

  1153. 	MOVQ    R9, (104)(CX)       // C5

  1154. 	ADCQ    DX, R10

  1155. 	ADCQ    $0, R8

  1156. 

  1157. 	MOVQ    (56)(REG_P2), AX

  1158. 	MULQ    R15

  1159. 	ADDQ    AX, R10

  1160. 	MOVQ    R10, (112)(CX)      // C6

  1161. 	ADCQ    DX, R8

  1162. 	MOVQ    R8, (120)(CX)       // C7

  1163. 

  1164. 	// [R8-R15] <- (U0+U1)*(V0+V1) - U1*V1

  1165. 	MOVQ    (SP), R8

  1166. 	SUBQ    (CX), R8

  1167. 	MOVQ    (8)(SP), R9

  1168. 	SBBQ    (8)(CX), R9

  1169. 	MOVQ    (16)(SP), R10

  1170. 	SBBQ    (16)(CX), R10

  1171. 	MOVQ    (24)(SP), R11

  1172. 	SBBQ    (24)(CX), R11

  1173. 	MOVQ    (32)(SP), R12

  1174. 	SBBQ    (32)(CX), R12

  1175. 	MOVQ    (40)(SP), R13

  1176. 	SBBQ    (40)(CX), R13

  1177. 	MOVQ    (48)(SP), R14

  1178. 	SBBQ    (48)(CX), R14

  1179. 	MOVQ    (56)(SP), R15

  1180. 	SBBQ    (56)(CX), R15

  1181. 

  1182. 	// [R8-R15] <- (U0+U1)*(V0+V1) - U1*V0 - U0*U1

  1183. 	MOVQ    ( 64)(CX), AX;	SUBQ    AX, R8

  1184. 	MOVQ    ( 72)(CX), AX;	SBBQ    AX, R9

  1185. 	MOVQ    ( 80)(CX), AX;	SBBQ    AX, R10

  1186. 	MOVQ    ( 88)(CX), AX;	SBBQ    AX, R11

  1187. 	MOVQ    ( 96)(CX), AX;	SBBQ    AX, R12

  1188. 	MOVQ    (104)(CX), DX;	SBBQ    DX, R13

  1189. 	MOVQ    (112)(CX), DI;	SBBQ    DI, R14

  1190. 	MOVQ    (120)(CX), SI;	SBBQ    SI, R15

  1191. 

  1192. 	// Final result

  1193. 	ADDQ    (32)(CX), R8;	MOVQ    R8,  (32)(CX)

  1194. 	ADCQ    (40)(CX), R9;	MOVQ    R9,  (40)(CX)

  1195. 	ADCQ    (48)(CX), R10;	MOVQ    R10, (48)(CX)

  1196. 	ADCQ    (56)(CX), R11;	MOVQ    R11, (56)(CX)

  1197. 	ADCQ    (64)(CX), R12;	MOVQ    R12, (64)(CX)

  1198. 	ADCQ    (72)(CX), R13;	MOVQ    R13, (72)(CX)

  1199. 	ADCQ    (80)(CX), R14;	MOVQ    R14, (80)(CX)

  1200. 	ADCQ    (88)(CX), R15;	MOVQ    R15, (88)(CX)

  1201. 	ADCQ    $0, AX;        	MOVQ    AX,  (96)(CX)

  1202. 	ADCQ    $0, DX;        	MOVQ    DX, (104)(CX)

  1203. 	ADCQ    $0, DI;         MOVQ    DI, (112)(CX)

  1204. 	ADCQ    $0, SI;     	MOVQ    SI, (120)(CX)

  1205. 

  1206. 	RET

  1207. 

  1208. TEXT ·redc(SB), $0-16

  1209. 	MOVQ	z+0(FP), REG_P2

  1210. 	MOVQ	x+8(FP), REG_P1

  1211. 

  1212. 	MOVQ    (REG_P1), R11

  1213. 	MOVQ    P503P1_3, AX

  1214. 	MULQ    R11

  1215. 	XORQ    R8, R8

  1216. 	ADDQ    (24)(REG_P1), AX

  1217. 	MOVQ    AX, (24)(REG_P2)

  1218. 	ADCQ    DX, R8

  1219. 

  1220. 	XORQ    R9, R9

  1221. 	MOVQ    P503P1_4, AX

  1222. 	MULQ    R11

  1223. 	XORQ    R10, R10

  1224. 	ADDQ    AX, R8

  1225. 	ADCQ    DX, R9

  1226. 

  1227. 	MOVQ    (8)(REG_P1), R12

  1228. 	MOVQ    P503P1_3, AX

  1229. 	MULQ    R12

  1230. 	ADDQ    AX, R8

  1231. 	ADCQ    DX, R9

  1232. 	ADCQ    $0, R10

  1233. 	ADDQ    (32)(REG_P1), R8

  1234. 	MOVQ    R8, (32)(REG_P2)       // Z4

  1235. 	ADCQ    $0, R9

  1236. 	ADCQ    $0, R10

  1237. 

  1238. 	XORQ    R8, R8

  1239. 	MOVQ    P503P1_5, AX

  1240. 	MULQ    R11

  1241. 	ADDQ    AX, R9

  1242. 	ADCQ    DX, R10

  1243. 	ADCQ    $0, R8

  1244. 

  1245. 	MOVQ    P503P1_4, AX

  1246. 	MULQ    R12

  1247. 	ADDQ    AX, R9

  1248. 	ADCQ    DX, R10

  1249. 	ADCQ    $0, R8

  1250. 

  1251. 	MOVQ    (16)(REG_P1), R13

  1252. 	MOVQ    P503P1_3, AX

  1253. 	MULQ    R13

  1254. 	ADDQ    AX, R9

  1255. 	ADCQ    DX, R10

  1256. 	ADCQ    $0, R8

  1257. 	ADDQ    (40)(REG_P1), R9

  1258. 	MOVQ    R9, (40)(REG_P2)       // Z5

  1259. 	ADCQ    $0, R10

  1260. 	ADCQ    $0, R8

  1261. 

  1262. 	XORQ    R9, R9

  1263. 	MOVQ    P503P1_6, AX

  1264. 	MULQ    R11

  1265. 	ADDQ    AX, R10

  1266. 	ADCQ    DX, R8

  1267. 	ADCQ    $0, R9

  1268. 

  1269. 	MOVQ    P503P1_5, AX

  1270. 	MULQ    R12

  1271. 	ADDQ    AX, R10

  1272. 	ADCQ    DX, R8

  1273. 	ADCQ    $0, R9

  1274. 

  1275. 	MOVQ    P503P1_4, AX

  1276. 	MULQ    R13

  1277. 	ADDQ    AX, R10

  1278. 	ADCQ    DX, R8

  1279. 	ADCQ    $0, R9

  1280. 

  1281. 	MOVQ    (24)(REG_P2), R14

  1282. 	MOVQ    P503P1_3, AX

  1283. 	MULQ    R14

  1284. 	ADDQ    AX, R10

  1285. 	ADCQ    DX, R8

  1286. 	ADCQ    $0, R9

  1287. 	ADDQ    (48)(REG_P1), R10

  1288. 	MOVQ    R10, (48)(REG_P2)      // Z6

  1289. 	ADCQ    $0, R8

  1290. 	ADCQ    $0, R9

  1291. 

  1292. 	XORQ    R10, R10

  1293. 	MOVQ    P503P1_7, AX

  1294. 	MULQ    R11

  1295. 	ADDQ    AX, R8

  1296. 	ADCQ    DX, R9

  1297. 	ADCQ    $0, R10

  1298. 

  1299. 	MOVQ    P503P1_6, AX

  1300. 	MULQ    R12

  1301. 	ADDQ    AX, R8

  1302. 	ADCQ    DX, R9

  1303. 	ADCQ    $0, R10

  1304. 

  1305. 	MOVQ    P503P1_5, AX

  1306. 	MULQ    R13

  1307. 	ADDQ    AX, R8

  1308. 	ADCQ    DX, R9

  1309. 	ADCQ    $0, R10

  1310. 

  1311. 	MOVQ    P503P1_4, AX

  1312. 	MULQ    R14

  1313. 	ADDQ    AX, R8

  1314. 	ADCQ    DX, R9

  1315. 	ADCQ    $0, R10

  1316. 

  1317. 	MOVQ    (32)(REG_P2), R15

  1318. 	MOVQ    P503P1_3, AX

  1319. 	MULQ    R15

  1320. 	ADDQ    AX, R8

  1321. 	ADCQ    DX, R9

  1322. 	ADCQ    $0, R10

  1323. 	ADDQ    (56)(REG_P1), R8

  1324. 	MOVQ    R8, (56)(REG_P2)       // Z7

  1325. 	ADCQ    $0, R9

  1326. 	ADCQ    $0, R10

  1327. 

  1328. 	XORQ    R8, R8

  1329. 	MOVQ    P503P1_7, AX

  1330. 	MULQ    R12

  1331. 	ADDQ    AX, R9

  1332. 	ADCQ    DX, R10

  1333. 	ADCQ    $0, R8

  1334. 

  1335. 	MOVQ    P503P1_6, AX

  1336. 	MULQ    R13

  1337. 	ADDQ    AX, R9

  1338. 	ADCQ    DX, R10

  1339. 	ADCQ    $0, R8

  1340. 

  1341. 	MOVQ    P503P1_5, AX

  1342. 	MULQ    R14

  1343. 	ADDQ    AX, R9

  1344. 	ADCQ    DX, R10

  1345. 	ADCQ    $0, R8

  1346. 

  1347. 	MOVQ    P503P1_4, AX

  1348. 	MULQ    R15

  1349. 	ADDQ    AX, R9

  1350. 	ADCQ    DX, R10

  1351. 	ADCQ    $0, R8

  1352. 

  1353. 	MOVQ    (40)(REG_P2), CX

  1354. 	MOVQ    P503P1_3, AX

  1355. 	MULQ    CX

  1356. 	ADDQ    AX, R9

  1357. 	ADCQ    DX, R10

  1358. 	ADCQ    $0, R8

  1359. 	ADDQ    (64)(REG_P1), R9

  1360. 	MOVQ    R9, (REG_P2)           // Z0

  1361. 	ADCQ    $0, R10

  1362. 	ADCQ    $0, R8

  1363. 

  1364. 	XORQ    R9, R9

  1365. 	MOVQ    P503P1_7, AX

  1366. 	MULQ    R13

  1367. 	ADDQ    AX, R10

  1368. 	ADCQ    DX, R8

  1369. 	ADCQ    $0, R9

  1370. 

  1371. 	MOVQ    P503P1_6, AX

  1372. 	MULQ    R14

  1373. 	ADDQ    AX, R10

  1374. 	ADCQ    DX, R8

  1375. 	ADCQ    $0, R9

  1376. 

  1377. 	MOVQ    P503P1_5, AX

  1378. 	MULQ    R15

  1379. 	ADDQ    AX, R10

  1380. 	ADCQ    DX, R8

  1381. 	ADCQ    $0, R9

  1382. 

  1383. 	MOVQ    P503P1_4, AX

  1384. 	MULQ    CX

  1385. 	ADDQ    AX, R10

  1386. 	ADCQ    DX, R8

  1387. 	ADCQ    $0, R9

  1388. 

  1389. 	MOVQ    (48)(REG_P2), R13

  1390. 	MOVQ    P503P1_3, AX

  1391. 	MULQ    R13

  1392. 	ADDQ    AX, R10

  1393. 	ADCQ    DX, R8

  1394. 	ADCQ    $0, R9

  1395. 	ADDQ    (72)(REG_P1), R10

  1396. 	MOVQ    R10, (8)(REG_P2)       // Z1

  1397. 	ADCQ    $0, R8

  1398. 	ADCQ    $0, R9

  1399. 

  1400. 	XORQ    R10, R10

  1401. 	MOVQ    P503P1_7, AX

  1402. 	MULQ    R14

  1403. 	ADDQ    AX, R8

  1404. 	ADCQ    DX, R9

  1405. 	ADCQ    $0, R10

  1406. 

  1407. 	MOVQ    P503P1_6, AX

  1408. 	MULQ    R15

  1409. 	ADDQ    AX, R8

  1410. 	ADCQ    DX, R9

  1411. 	ADCQ    $0, R10

  1412. 

  1413. 	MOVQ    P503P1_5, AX

  1414. 	MULQ    CX

  1415. 	ADDQ    AX, R8

  1416. 	ADCQ    DX, R9

  1417. 	ADCQ    $0, R10

  1418. 

  1419. 	MOVQ    P503P1_4, AX

  1420. 	MULQ    R13

  1421. 	ADDQ    AX, R8

  1422. 	ADCQ    DX, R9

  1423. 	ADCQ    $0, R10

  1424. 

  1425. 	MOVQ    (56)(REG_P2), R14

  1426. 	MOVQ    P503P1_3, AX

  1427. 	MULQ    R14

  1428. 	ADDQ    AX, R8

  1429. 	ADCQ    DX, R9

  1430. 	ADCQ    $0, R10

  1431. 	ADDQ    (80)(REG_P1), R8

  1432. 	MOVQ    R8, (16)(REG_P2)       // Z2

  1433. 	ADCQ    $0, R9

  1434. 	ADCQ    $0, R10

  1435. 

  1436. 	XORQ    R8, R8

  1437. 	MOVQ    P503P1_7, AX

  1438. 	MULQ    R15

  1439. 	ADDQ    AX, R9

  1440. 	ADCQ    DX, R10

  1441. 	ADCQ    $0, R8

  1442. 

  1443. 	MOVQ    P503P1_6, AX

  1444. 	MULQ    CX

  1445. 	ADDQ    AX, R9

  1446. 	ADCQ    DX, R10

  1447. 	ADCQ    $0, R8

  1448. 

  1449. 	MOVQ    P503P1_5, AX

  1450. 	MULQ    R13

  1451. 	ADDQ    AX, R9

  1452. 	ADCQ    DX, R10

  1453. 	ADCQ    $0, R8

  1454. 

  1455. 	MOVQ    P503P1_4, AX

  1456. 	MULQ    R14

  1457. 	ADDQ    AX, R9

  1458. 	ADCQ    DX, R10

  1459. 	ADCQ    $0, R8

  1460. 	ADDQ    (88)(REG_P1), R9

  1461. 	MOVQ    R9, (24)(REG_P2)       // Z3

  1462. 	ADCQ    $0, R10

  1463. 	ADCQ    $0, R8

  1464. 

  1465. 	XORQ    R9, R9

  1466. 	MOVQ    P503P1_7, AX

  1467. 	MULQ    CX

  1468. 	ADDQ    AX, R10

  1469. 	ADCQ    DX, R8

  1470. 	ADCQ    $0, R9

  1471. 

  1472. 	MOVQ    P503P1_6, AX

  1473. 	MULQ    R13

  1474. 	ADDQ    AX, R10

  1475. 	ADCQ    DX, R8

  1476. 	ADCQ    $0, R9

  1477. 

  1478. 	MOVQ    P503P1_5, AX

  1479. 	MULQ    R14

  1480. 	ADDQ    AX, R10

  1481. 	ADCQ    DX, R8

  1482. 	ADCQ    $0, R9

  1483. 	ADDQ    (96)(REG_P1), R10

  1484. 	MOVQ    R10, (32)(REG_P2)      // Z4

  1485. 	ADCQ    $0, R8

  1486. 	ADCQ    $0, R9

  1487. 

  1488. 	XORQ    R10, R10

  1489. 	MOVQ    P503P1_7, AX

  1490. 	MULQ    R13

  1491. 	ADDQ    AX, R8

  1492. 	ADCQ    DX, R9

  1493. 	ADCQ    $0, R10

  1494. 

  1495. 	MOVQ    P503P1_6, AX

  1496. 	MULQ    R14

  1497. 	ADDQ    AX, R8

  1498. 	ADCQ    DX, R9

  1499. 	ADCQ    $0, R10

  1500. 	ADDQ    (104)(REG_P1), R8      // Z5

  1501. 	MOVQ    R8, (40)(REG_P2)       // Z5

  1502. 	ADCQ    $0, R9

  1503. 	ADCQ    $0, R10

  1504. 

  1505. 	MOVQ    P503P1_7, AX

  1506. 	MULQ    R14

  1507. 	ADDQ    AX, R9

  1508. 	ADCQ    DX, R10

  1509. 	ADDQ    (112)(REG_P1), R9      // Z6

  1510. 	MOVQ    R9, (48)(REG_P2)       // Z6

  1511. 	ADCQ    $0, R10

  1512. 	ADDQ    (120)(REG_P1), R10     // Z7

  1513. 	MOVQ    R10, (56)(REG_P2)      // Z7

  1514. 

  1515. 	RET

  1516. 

  1517. TEXT ·redcWithMULX(SB), $0-16

  1518. 	MOVQ    z+0(FP), DI

  1519. 	MOVQ    x+8(FP), SI

  1520. 	REDC(DI, SI, MULS_128x320_MULX)

  1521. 	RET

  1522. 

  1523. TEXT ·redcWithMULXADX(SB), $0-16

  1524. 	MOVQ    z+0(FP), DI

  1525. 	MOVQ    x+8(FP), SI

  1526. 	REDC(DI, SI, MULS_128x320_MULXADX)

  1527. 	RET

  1528. TEXT ·fp503MontgomeryReduce(SB), $0-16

  1529. 	MOVQ    z+0(FP), DI

  1530. 	MOVQ    x+8(FP), SI

  1531. 	REDC(DI, SI, MULS_128x320_MULXADX)

  1532. 	RET

  1533. 

  1534. TEXT ·fp503AddLazy(SB), NOSPLIT, $0-24

  1535. 

  1536. 	MOVQ z+0(FP), REG_P3

  1537. 	MOVQ x+8(FP), REG_P1

  1538. 	MOVQ y+16(FP), REG_P2

  1539. 

  1540. 	MOVQ	(REG_P1), R8

  1541. 	MOVQ	(8)(REG_P1), R9

  1542. 	MOVQ	(16)(REG_P1), R10

  1543. 	MOVQ	(24)(REG_P1), R11

  1544. 	MOVQ	(32)(REG_P1), R12

  1545. 	MOVQ	(40)(REG_P1), R13

  1546. 	MOVQ	(48)(REG_P1), R14

  1547. 	MOVQ	(56)(REG_P1), R15

  1548. 

  1549. 	ADDQ	(REG_P2), R8

  1550. 	ADCQ	(8)(REG_P2), R9

  1551. 	ADCQ	(16)(REG_P2), R10

  1552. 	ADCQ	(24)(REG_P2), R11

  1553. 	ADCQ	(32)(REG_P2), R12

  1554. 	ADCQ	(40)(REG_P2), R13

  1555. 	ADCQ	(48)(REG_P2), R14

  1556. 	ADCQ	(56)(REG_P2), R15

  1557. 

  1558. 	MOVQ	R8, (REG_P3)

  1559. 	MOVQ	R9, (8)(REG_P3)

  1560. 	MOVQ	R10, (16)(REG_P3)

  1561. 	MOVQ	R11, (24)(REG_P3)

  1562. 	MOVQ	R12, (32)(REG_P3)

  1563. 	MOVQ	R13, (40)(REG_P3)

  1564. 	MOVQ	R14, (48)(REG_P3)

  1565. 	MOVQ	R15, (56)(REG_P3)

  1566. 

  1567. 	RET

  1568. 

  1569. TEXT ·fp503X2AddLazy(SB), NOSPLIT, $0-24

  1570. 

  1571. 	MOVQ	z+0(FP), REG_P3

  1572. 	MOVQ	x+8(FP), REG_P1

  1573. 	MOVQ	y+16(FP), REG_P2

  1574. 

  1575. 	MOVQ	(REG_P1), R8

  1576. 	MOVQ	(8)(REG_P1), R9

  1577. 	MOVQ	(16)(REG_P1), R10

  1578. 	MOVQ	(24)(REG_P1), R11

  1579. 	MOVQ	(32)(REG_P1), R12

  1580. 	MOVQ	(40)(REG_P1), R13

  1581. 	MOVQ	(48)(REG_P1), R14

  1582. 	MOVQ	(56)(REG_P1), R15

  1583. 	MOVQ	(64)(REG_P1), AX

  1584. 	MOVQ	(72)(REG_P1), BX

  1585. 	MOVQ	(80)(REG_P1), CX

  1586. 

  1587. 	ADDQ	(REG_P2), R8

  1588. 	ADCQ	(8)(REG_P2), R9

  1589. 	ADCQ	(16)(REG_P2), R10

  1590. 	ADCQ	(24)(REG_P2), R11

  1591. 	ADCQ	(32)(REG_P2), R12

  1592. 	ADCQ	(40)(REG_P2), R13

  1593. 	ADCQ	(48)(REG_P2), R14

  1594. 	ADCQ	(56)(REG_P2), R15

  1595. 	ADCQ	(64)(REG_P2), AX

  1596. 	ADCQ	(72)(REG_P2), BX

  1597. 	ADCQ	(80)(REG_P2), CX

  1598. 

  1599. 	MOVQ	R8, (REG_P3)

  1600. 	MOVQ	R9, (8)(REG_P3)

  1601. 	MOVQ	R10, (16)(REG_P3)

  1602. 	MOVQ	R11, (24)(REG_P3)

  1603. 	MOVQ	R12, (32)(REG_P3)

  1604. 	MOVQ	R13, (40)(REG_P3)

  1605. 	MOVQ	R14, (48)(REG_P3)

  1606. 	MOVQ	R15, (56)(REG_P3)

  1607. 	MOVQ	AX, (64)(REG_P3)

  1608. 	MOVQ	BX, (72)(REG_P3)

  1609. 	MOVQ	CX, (80)(REG_P3)

  1610. 

  1611. 	MOVQ	(88)(REG_P1), R8

  1612. 	MOVQ	(96)(REG_P1), R9

  1613. 	MOVQ	(104)(REG_P1), R10

  1614. 	MOVQ	(112)(REG_P1), R11

  1615. 	MOVQ	(120)(REG_P1), R12

  1616. 

  1617. 	ADCQ	(88)(REG_P2), R8

  1618. 	ADCQ	(96)(REG_P2), R9

  1619. 	ADCQ	(104)(REG_P2), R10

  1620. 	ADCQ	(112)(REG_P2), R11

  1621. 	ADCQ	(120)(REG_P2), R12

  1622. 

  1623. 	MOVQ	R8, (88)(REG_P3)

  1624. 	MOVQ	R9, (96)(REG_P3)

  1625. 	MOVQ	R10, (104)(REG_P3)

  1626. 	MOVQ	R11, (112)(REG_P3)

  1627. 	MOVQ	R12, (120)(REG_P3)

  1628. 

  1629. 	RET

  1630. 

  1631. TEXT ·fp503X2SubLazy(SB), NOSPLIT, $0-24

  1632. 

  1633. 	MOVQ z+0(FP), REG_P3

  1634. 	MOVQ x+8(FP), REG_P1

  1635. 	MOVQ y+16(FP), REG_P2

  1636. 	// Used later to store result of 0-borrow

  1637. 	XORQ CX, CX

  1638. 

  1639. 	// SUBC for first 11 limbs

  1640. 	MOVQ	(REG_P1), R8

  1641. 	MOVQ	(8)(REG_P1), R9

  1642. 	MOVQ	(16)(REG_P1), R10

  1643. 	MOVQ	(24)(REG_P1), R11

  1644. 	MOVQ	(32)(REG_P1), R12

  1645. 	MOVQ	(40)(REG_P1), R13

  1646. 	MOVQ	(48)(REG_P1), R14

  1647. 	MOVQ	(56)(REG_P1), R15

  1648. 	MOVQ	(64)(REG_P1), AX

  1649. 	MOVQ	(72)(REG_P1), BX

  1650. 

  1651. 	SUBQ	(REG_P2), R8

  1652. 	SBBQ	(8)(REG_P2), R9

  1653. 	SBBQ	(16)(REG_P2), R10

  1654. 	SBBQ	(24)(REG_P2), R11

  1655. 	SBBQ	(32)(REG_P2), R12

  1656. 	SBBQ	(40)(REG_P2), R13

  1657. 	SBBQ	(48)(REG_P2), R14

  1658. 	SBBQ	(56)(REG_P2), R15

  1659. 	SBBQ	(64)(REG_P2), AX

  1660. 	SBBQ	(72)(REG_P2), BX

  1661. 

  1662. 	MOVQ	R8, (REG_P3)

  1663. 	MOVQ	R9, (8)(REG_P3)

  1664. 	MOVQ	R10, (16)(REG_P3)

  1665. 	MOVQ	R11, (24)(REG_P3)

  1666. 	MOVQ	R12, (32)(REG_P3)

  1667. 	MOVQ	R13, (40)(REG_P3)

  1668. 	MOVQ	R14, (48)(REG_P3)

  1669. 	MOVQ	R15, (56)(REG_P3)

  1670. 	MOVQ	AX, (64)(REG_P3)

  1671. 	MOVQ	BX, (72)(REG_P3)

  1672. 

  1673. 	// SUBC for last 5 limbs

  1674. 	MOVQ	(80)(REG_P1), 	R8

  1675. 	MOVQ	(88)(REG_P1), 	R9

  1676. 	MOVQ	(96)(REG_P1), 	R10

  1677. 	MOVQ	(104)(REG_P1), 	R11

  1678. 	MOVQ	(112)(REG_P1), 	R12

  1679. 	MOVQ	(120)(REG_P1), 	R13

  1680. 

  1681. 	SBBQ	(80)(REG_P2), R8

  1682. 	SBBQ	(88)(REG_P2), R9

  1683. 	SBBQ	(96)(REG_P2), R10

  1684. 	SBBQ	(104)(REG_P2), R11

  1685. 	SBBQ	(112)(REG_P2), R12

  1686. 	SBBQ	(120)(REG_P2), R13

  1687. 

  1688. 	MOVQ	R8, (80)(REG_P3)

  1689. 	MOVQ	R9, (88)(REG_P3)

  1690. 	MOVQ	R10, (96)(REG_P3)

  1691. 	MOVQ	R11, (104)(REG_P3)

  1692. 	MOVQ	R12, (112)(REG_P3)

  1693. 	MOVQ	R13, (120)(REG_P3)

  1694. 

  1695. 	// Now the carry flag is 1 if x-y < 0.  If so, add p*2^512.

  1696. 	SBBQ	$0, CX

  1697. 

  1698. 	// Load p into registers:

  1699. 	MOVQ	P503_0, R8

  1700. 	// P503_{1,2} = P503_0, so reuse R8

  1701. 	MOVQ	P503_3, R9

  1702. 	MOVQ	P503_4, R10

  1703. 	MOVQ	P503_5, R11

  1704. 	MOVQ	P503_6, R12

  1705. 	MOVQ	P503_7, R13

  1706. 

  1707. 	ANDQ	CX, R8

  1708. 	ANDQ	CX, R9

  1709. 	ANDQ	CX, R10

  1710. 	ANDQ	CX, R11

  1711. 	ANDQ	CX, R12

  1712. 	ANDQ	CX, R13

  1713. 

  1714. 	MOVQ   (64   )(REG_P3), AX; ADDQ R8,  AX; MOVQ AX, (64   )(REG_P3)

  1715. 	MOVQ   (64+ 8)(REG_P3), AX; ADCQ R8,  AX; MOVQ AX, (64+ 8)(REG_P3)

  1716. 	MOVQ   (64+16)(REG_P3), AX; ADCQ R8,  AX; MOVQ AX, (64+16)(REG_P3)

  1717. 	MOVQ   (64+24)(REG_P3), AX; ADCQ R9,  AX; MOVQ AX, (64+24)(REG_P3)

  1718. 	MOVQ   (64+32)(REG_P3), AX; ADCQ R10, AX; MOVQ AX, (64+32)(REG_P3)

  1719. 	MOVQ   (64+40)(REG_P3), AX; ADCQ R11, AX; MOVQ AX, (64+40)(REG_P3)

  1720. 	MOVQ   (64+48)(REG_P3), AX; ADCQ R12, AX; MOVQ AX, (64+48)(REG_P3)

  1721. 	MOVQ   (64+56)(REG_P3), AX; ADCQ R13, AX; MOVQ AX, (64+56)(REG_P3)

  1722. 

  1723. 	RET