kris
/
SIDH_p751
1
0
Fork 0
Code Issues 0 Pull-Requests 0 Releases 0 Wiki Aktivität
Du kannst nicht mehr als 25 Themen auswählen Themen müssen entweder mit einem Buchstaben oder einer Ziffer beginnen. Sie können Bindestriche („-“) enthalten und bis zu 35 Zeichen lang sein.
126 Commits
14 Branches
653 KiB
 
 
 
Branch: trials/PERF_try1
Branches Tags
${ item.name }
Erstelle Branch ${ searchTerm }
von „trials/PERF_try1“
${ noResults }
SIDH_p751/p503/arith_amd64.s

1703 Zeilen
38 KiB
Originalformat Permalink Blame Verlauf 

  1. // +build amd64,!noasm

  2. 

  3. #include "textflag.h"

  4. 

  5. // p503

  6. #define P503_0     $0xFFFFFFFFFFFFFFFF

  7. #define P503_1     $0xFFFFFFFFFFFFFFFF

  8. #define P503_2     $0xFFFFFFFFFFFFFFFF

  9. #define P503_3     $0xABFFFFFFFFFFFFFF

  10. #define P503_4     $0x13085BDA2211E7A0

  11. #define P503_5     $0x1B9BF6C87B7E7DAF

  12. #define P503_6     $0x6045C6BDDA77A4D0

  13. #define P503_7     $0x004066F541811E1E

  14. 

  15. // p503+1

  16. #define P503P1_3   $0xAC00000000000000

  17. #define P503P1_4   $0x13085BDA2211E7A0

  18. #define P503P1_5   $0x1B9BF6C87B7E7DAF

  19. #define P503P1_6   $0x6045C6BDDA77A4D0

  20. #define P503P1_7   $0x004066F541811E1E

  21. 

  22. // p503x2

  23. #define P503X2_0   $0xFFFFFFFFFFFFFFFE

  24. #define P503X2_1   $0xFFFFFFFFFFFFFFFF

  25. #define P503X2_2   $0xFFFFFFFFFFFFFFFF

  26. #define P503X2_3   $0x57FFFFFFFFFFFFFF

  27. #define P503X2_4   $0x2610B7B44423CF41

  28. #define P503X2_5   $0x3737ED90F6FCFB5E

  29. #define P503X2_6   $0xC08B8D7BB4EF49A0

  30. #define P503X2_7   $0x0080CDEA83023C3C

  31. 

  32. #define REG_P1 DI

  33. #define REG_P2 SI

  34. #define REG_P3 DX

  35. 

  36. // Performs schoolbook multiplication of 2 256-bit numbers. This optimized version

  37. // uses MULX instruction. Macro smashes value in DX.

  38. // Input: I0 and I1.

  39. // Output: O

  40. // All the other arguments are resgisters, used for storing temporary values

  41. #define MULS256_MULX(O, I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9) \

  42. 	MOVQ    I0, DX          \

  43. 	MULXQ   I1, T1, T0      \   // T0:T1 = A0*B0

  44. 	MOVQ    T1, O           \   // O[0]

  45. 	MULXQ   8+I1, T2, T1    \   // T1:T2 = U0*V1

  46. 	ADDQ    T2, T0          \

  47. 	MULXQ   16+I1, T3, T2   \   // T2:T3 = U0*V2

  48. 	ADCQ    T3, T1          \

  49. 	MULXQ   24+I1, T4, T3   \   // T3:T4 = U0*V3

  50. 	ADCQ    T4, T2          \

  51. 	\ // Column U1

  52. 	MOVQ    8+I0, DX        \

  53. 	ADCQ    $0, T3          \

  54. 	MULXQ   0+I1, T4, T5    \   // T5:T4 = U1*V0

  55. 	MULXQ   8+I1, T7, T6    \   // T6:T7 = U1*V1

  56. 	ADDQ    T7, T5          \

  57. 	MULXQ   16+I1, T8, T7   \   // T7:T8 = U1*V2

  58. 	ADCQ    T8, T6          \

  59. 	MULXQ   24+I1, T9, T8   \   // T8:T9 = U1*V3

  60. 	ADCQ    T9, T7          \

  61. 	ADCQ    $0, T8          \

  62. 	ADDQ    T0, T4          \

  63. 	MOVQ    T4, 8+O         \   // O[1]

  64. 	ADCQ    T1, T5          \

  65. 	ADCQ    T2, T6          \

  66. 	ADCQ    T3, T7          \

  67. 	\ // Column U2

  68. 	MOVQ    16+I0, DX       \

  69. 	ADCQ    $0, T8          \

  70. 	MULXQ   0+I1, T0, T1    \   // T1:T0 = U2*V0

  71. 	MULXQ   8+I1, T3, T2    \   // T2:T3 = U2*V1

  72. 	ADDQ    T3, T1          \

  73. 	MULXQ   16+I1, T4, T3   \   // T3:T4 = U2*V2

  74. 	ADCQ    T4, T2          \

  75. 	MULXQ   24+I1, T9, T4   \   // T4:T9 = U2*V3

  76. 	ADCQ    T9, T3          \

  77. 	\ // Column U3

  78. 	MOVQ    24+I0, DX       \

  79. 	ADCQ    $0, T4          \

  80. 	ADDQ    T5, T0          \

  81. 	MOVQ    T0, 16+O        \   // O[2]

  82. 	ADCQ    T6, T1          \

  83. 	ADCQ    T7, T2          \

  84. 	ADCQ    T8, T3          \

  85. 	ADCQ    $0, T4          \

  86. 	MULXQ   0+I1, T0, T5    \   // T5:T0 = U3*V0

  87. 	MULXQ   8+I1, T7, T6    \   // T6:T7 = U3*V1

  88. 	ADDQ    T7, T5          \

  89. 	MULXQ   16+I1, T8, T7   \   // T7:T8 = U3*V2

  90. 	ADCQ    T8, T6          \

  91. 	MULXQ   24+I1, T9, T8   \   // T8:T9 = U3*V3

  92. 	ADCQ    T9, T7          \

  93. 	ADCQ    $0, T8          \

  94. 	\ // Add values in remaining columns

  95. 	ADDQ    T0, T1          \

  96. 	MOVQ    T1, 24+O        \   // O[3]

  97. 	ADCQ    T5, T2          \

  98. 	MOVQ    T2, 32+O        \   // O[4]

  99. 	ADCQ    T6, T3          \

  100. 	MOVQ    T3, 40+O        \   // O[5]

  101. 	ADCQ    T7, T4          \

  102. 	MOVQ    T4, 48+O        \   // O[6]

  103. 	ADCQ    $0, T8          \   // O[7]

  104. 	MOVQ    T8, 56+O

  105. 

  106. // Performs schoolbook multiplication of 2 256-bit numbers. This optimized version

  107. // uses ADOX, ADCX and MULX instructions. Macro smashes values in AX and DX.

  108. // Input: I0 and I1.

  109. // Output: O

  110. // All the other arguments resgisters are used for storing temporary values

  111. #define MULS256_MULXADX(O, I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9) \

  112. 							\   // U0[0]

  113. 	MOVQ     0+I0, DX       \   // MULX requires multiplayer in DX

  114. 							\   // T0:T1 = I1*DX

  115. 	MULXQ    I1, T1, T0     \   // T0:T1 = U0*V0 (low:high)

  116. 	MOVQ     T1, O          \   // O0[0]

  117. 	MULXQ     8+I1, T2, T1  \   // T2:T1 = U0*V1

  118. 	XORQ     AX, AX         \

  119. 	ADOXQ    T2, T0         \

  120. 	MULXQ    16+I1, T3, T2  \   // T2:T3 = U0*V2

  121. 	ADOXQ    T3, T1         \

  122. 	MULXQ    24+I1, T4, T3  \   // T3:T4 = U0*V3

  123. 	ADOXQ    T4, T2         \

  124. 	\  // Column U1

  125. 	MOVQ      8+I0, DX      \

  126. 	MULXQ    I1, T4, T5     \   // T5:T4 = U1*V0

  127. 	ADOXQ    AX, T3         \

  128. 	XORQ     AX, AX         \

  129. 	MULXQ     8+I1, T7, T6  \   // T6:T7 = U1*V1

  130. 	ADOXQ    T0, T4         \

  131. 	MOVQ     T4, 8+O        \   // O[1]

  132. 	ADCXQ    T7, T5         \

  133. 	MULXQ    16+I1, T8, T7  \   // T7:T8 = U1*V2

  134. 	ADCXQ    T8, T6         \

  135. 	ADOXQ    T1, T5 \

  136. 	MULXQ    24+I1, T9, T8  \   // T8:T9 = U1*V3

  137. 	ADCXQ    T9, T7         \

  138. 	ADCXQ    AX, T8         \

  139. 	ADOXQ    T2, T6         \

  140. 	\ // Column U2

  141. 	MOVQ     16+I0, DX      \

  142. 	MULXQ    I1, T0, T1     \   // T1:T0 = U2*V0

  143. 	ADOXQ    T3, T7         \

  144. 	ADOXQ    AX, T8         \

  145. 	XORQ     AX, AX         \

  146. 	MULXQ    8+I1, T3, T2   \   // T2:T3 = U2*V1

  147. 	ADOXQ    T5, T0         \

  148. 	MOVQ     T0, 16+O       \   // O[2]

  149. 	ADCXQ    T3, T1         \

  150. 	MULXQ    16+I1, T4, T3  \   // T3:T4 = U2*V2

  151. 	ADCXQ    T4, T2         \

  152. 	ADOXQ    T6, T1         \

  153. 	MULXQ    24+I1, T9, T4  \   // T9:T4 = U2*V3

  154. 	ADCXQ    T9, T3         \

  155. 	MOVQ     24+I0, DX      \

  156. 	ADCXQ    AX, T4         \

  157. 	\

  158. 	ADOXQ    T7, T2         \

  159. 	ADOXQ    T8, T3         \

  160. 	ADOXQ    AX, T4         \

  161. 	\ // Column U3

  162. 	MULXQ    I1, T0, T5     \   // T5:T0 = U3*B0

  163. 	XORQ     AX, AX         \

  164. 	MULXQ    8+I1, T7, T6   \   // T6:T7 = U3*B1

  165. 	ADCXQ    T7, T5         \

  166. 	ADOXQ    T0, T1         \

  167. 	MULXQ    16+I1, T8, T7  \   // T7:T8 = U3*V2

  168. 	ADCXQ    T8, T6         \

  169. 	ADOXQ    T5, T2         \

  170. 	MULXQ    24+I1, T9, T8  \   // T8:T9 = U3*V3

  171. 	ADCXQ    T9, T7         \

  172. 	ADCXQ    AX, T8         \

  173. 	\

  174. 	ADOXQ   T6, T3          \

  175. 	ADOXQ   T7, T4          \

  176. 	ADOXQ   AX, T8          \

  177. 	MOVQ    T1, 24+O        \   // O[3]

  178. 	MOVQ    T2, 32+O        \   // O[4]

  179. 	MOVQ    T3, 40+O        \   // O[5]

  180. 	MOVQ    T4, 48+O        \   // O[6] and O[7] below

  181. 	MOVQ    T8, 56+O

  182. 

  183. // Template of a macro that performs schoolbook multiplication of 128-bit with 320-bit

  184. // number. It uses MULX instruction This template must be customized with functions

  185. // performing ADD (add1, add2) and ADD-with-carry (adc1, adc2). addX/adcX may or may

  186. // not be instructions that use two independent carry chains.

  187. // Input:

  188. //   * I0 128-bit number

  189. //   * I1 320-bit number

  190. //   * add1, add2: instruction performing integer addition and starting carry chain

  191. //   * adc1, adc2: instruction performing integer addition with carry

  192. // Output: T[0-6] registers

  193. #define MULS_128x320(I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, add1, add2, adc1, adc2) \

  194. 	\ // Column 0

  195. 	MOVQ    I0, DX              \

  196. 	MULXQ   I1+24(SB), T0, T1   \

  197. 	MULXQ   I1+32(SB), T4, T2   \

  198. 	XORQ    AX, AX              \

  199. 	MULXQ   I1+40(SB), T5, T3   \

  200. 	add1    T4, T1              \

  201. 	adc1    T5, T2              \

  202. 	MULXQ   I1+48(SB), T7, T4   \

  203. 	adc1    T7, T3              \

  204. 	MULXQ   I1+56(SB), T6, T5   \

  205. 	adc1    T6, T4              \

  206. 	adc1    AX, T5              \

  207. 	\ // Column 1

  208. 	MOVQ    8+I0, DX            \

  209. 	MULXQ   I1+24(SB), T6, T7   \

  210. 	add2    T6, T1              \

  211. 	adc2    T7, T2              \

  212. 	MULXQ   I1+32(SB), T8, T6   \

  213. 	adc2    T6, T3              \

  214. 	MULXQ   I1+40(SB), T7, T9   \

  215. 	adc2    T9, T4              \

  216. 	MULXQ   I1+48(SB), T9, T6   \

  217. 	adc2    T6, T5              \

  218. 	MULXQ   I1+56(SB), DX, T6   \

  219. 	adc2    AX, T6              \

  220. 	\ // Output

  221. 	XORQ    AX, AX              \

  222. 	add1    T8, T2              \

  223. 	adc1    T7, T3              \

  224. 	adc1    T9, T4              \

  225. 	adc1    DX, T5              \

  226. 	adc1    AX, T6

  227. 

  228. // Multiplies 128-bit with 320-bit integer. Optimized with MULX instruction.

  229. #define MULS_128x320_MULX(I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9) \

  230. 	MULS_128x320(I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, ADDQ, ADDQ, ADCQ, ADCQ)

  231. 

  232. // Multiplies 128-bit with 320-bit integer. Optimized with  MULX, ADOX and ADCX instructions

  233. #define MULS_128x320_MULXADX(I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9) \

  234. 	MULS_128x320(I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, ADOXQ, ADCXQ, ADOXQ, ADCXQ)

  235. 

  236. // Template of a macro performing multiplication of two 512-bit numbers. It uses one

  237. // level of Karatsuba and one level of schoolbook multiplication. Template must be

  238. // customized with macro performing schoolbook multiplication.

  239. // Input:

  240. //  * I0, I1 - two 512-bit numbers

  241. //  * MULS - either MULS256_MULX or MULS256_MULXADX

  242. // Output: OUT - 1024-bit long

  243. #define MUL(OUT, I0, I1, MULS) \

  244. 	\ // R[8-11]: U1+U0

  245. 	XORQ    AX, AX  \

  246. 	MOVQ    ( 0)(I0), R8    \

  247. 	MOVQ    ( 8)(I0), R9    \

  248. 	MOVQ    (16)(I0), R10   \

  249. 	MOVQ    (24)(I0), R11   \

  250. 	ADDQ    (32)(I0), R8    \

  251. 	ADCQ    (40)(I0), R9    \

  252. 	ADCQ    (48)(I0), R10   \

  253. 	ADCQ    (56)(I0), R11   \

  254. 	SBBQ    $0, AX          \ // store mask

  255. 	MOVQ    R8,  ( 0)(SP)   \

  256. 	MOVQ    R9,  ( 8)(SP)   \

  257. 	MOVQ    R10, (16)(SP)   \

  258. 	MOVQ    R11, (24)(SP)   \

  259. 	\

  260. 	\ // R[12-15]: V1+V0

  261. 	XORQ    BX, BX          \

  262. 	MOVQ    ( 0)(I1), R12   \

  263. 	MOVQ    ( 8)(I1), R13   \

  264. 	MOVQ    (16)(I1), R14   \

  265. 	MOVQ    (24)(I1), R15   \

  266. 	ADDQ    (32)(I1), R12   \

  267. 	ADCQ    (40)(I1), R13   \

  268. 	ADCQ    (48)(I1), R14   \

  269. 	ADCQ    (56)(I1), R15   \

  270. 	SBBQ    $0, BX          \ // store mask

  271. 	MOVQ    R12, (32)(SP)   \

  272. 	MOVQ    R13, (40)(SP)   \

  273. 	MOVQ    R14, (48)(SP)   \

  274. 	MOVQ    R15, (56)(SP)   \

  275. 	\ // Prepare mask for U0+U1 (U1+U0 mod 256^4 if U1+U0 sets carry flag, otherwise 0)

  276. 	ANDQ    AX, R12         \

  277. 	ANDQ    AX, R13         \

  278. 	ANDQ    AX, R14         \

  279. 	ANDQ    AX, R15         \

  280. 	\ // Prepare mask for V0+V1 (V1+V0 mod 256^4 if U1+U0 sets carry flag, otherwise 0)

  281. 	ANDQ    BX, R8          \

  282. 	ANDQ    BX, R9          \

  283. 	ANDQ    BX, R10         \

  284. 	ANDQ    BX, R11         \

  285. 	\ // res = masked(U0+U1) + masked(V0 + V1)

  286. 	ADDQ    R12, R8         \

  287. 	ADCQ    R13, R9         \

  288. 	ADCQ    R14, R10        \

  289. 	ADCQ    R15, R11        \

  290. 	\ // SP[64-96] <- res

  291. 	MOVQ     R8, (64)(SP)   \

  292. 	MOVQ     R9, (72)(SP)   \

  293. 	MOVQ    R10, (80)(SP)   \

  294. 	MOVQ    R11, (88)(SP)   \

  295. 	\ // BP will be used for schoolbook multiplication below

  296. 	MOVQ    BP, 96(SP)  \

  297. 	\ // (U1+U0)*(V1+V0)

  298. 	MULS((64)(OUT), 0(SP), 32(SP), R8, R9, R10, R11, R12, R13, R14, R15, BX, BP)    \

  299. 	\ // U0 x V0

  300. 	MULS(0(OUT), 0(I0), 0(I1), R8, R9, R10, R11, R12, R13, R14, R15, BX, BP)    \

  301. 	\ // U1 x V1

  302. 	MULS(0(SP), 32(I0), 32(I1), R8, R9, R10, R11, R12, R13, R14, R15, BX, BP)  \

  303. 	\ // Recover BP

  304. 	MOVQ    96(SP), BP  \

  305. 	\ // Final part of schoolbook multiplication; R[8-11] = (U0+U1) x (V0+V1)

  306. 	MOVQ    (64)(SP), R8    \

  307. 	MOVQ    (72)(SP), R9    \

  308. 	MOVQ    (80)(SP), R10   \

  309. 	MOVQ    (88)(SP), R11   \

  310. 	MOVQ    (96)(OUT), AX   \

  311. 	ADDQ    AX, R8          \

  312. 	MOVQ    (104)(OUT), AX  \

  313. 	ADCQ    AX, R9          \

  314. 	MOVQ    (112)(OUT), AX  \

  315. 	ADCQ    AX, R10         \

  316. 	MOVQ    (120)(OUT), AX  \

  317. 	ADCQ    AX, R11 \

  318. 	\ // R[12-15, 8-11] = (U0+U1) x (V0+V1) - U0xV0

  319. 	MOVQ    (64)(OUT), R12  \

  320. 	MOVQ    (72)(OUT), R13  \

  321. 	MOVQ    (80)(OUT), R14  \

  322. 	MOVQ    (88)(OUT), R15  \

  323. 	SUBQ    ( 0)(OUT), R12  \

  324. 	SBBQ    ( 8)(OUT), R13  \

  325. 	SBBQ    (16)(OUT), R14  \

  326. 	SBBQ    (24)(OUT), R15  \

  327. 	SBBQ    (32)(OUT), R8   \

  328. 	SBBQ    (40)(OUT), R9   \

  329. 	SBBQ    (48)(OUT), R10  \

  330. 	SBBQ    (56)(OUT), R11  \

  331. 	\ // r8-r15 <- (U0+U1) x (V0+V1) - U0xV0 - U1xV1

  332. 	SUBQ    ( 0)(SP), R12   \

  333. 	SBBQ    ( 8)(SP), R13   \

  334. 	SBBQ    (16)(SP), R14   \

  335. 	SBBQ    (24)(SP), R15   \

  336. 	SBBQ    (32)(SP), R8    \

  337. 	SBBQ    (40)(SP), R9    \

  338. 	SBBQ    (48)(SP), R10   \

  339. 	SBBQ    (56)(SP), R11   \

  340. 	\

  341. 	;                       ADDQ   (32)(OUT), R12; MOVQ    R12, ( 32)(OUT) \

  342. 	;                       ADCQ   (40)(OUT), R13; MOVQ    R13, ( 40)(OUT) \

  343. 	;                       ADCQ   (48)(OUT), R14; MOVQ    R14, ( 48)(OUT) \

  344. 	;                       ADCQ   (56)(OUT), R15; MOVQ    R15, ( 56)(OUT) \

  345. 	MOVQ    ( 0)(SP), AX;   ADCQ    AX,  R8;       MOVQ     R8, ( 64)(OUT) \

  346. 	MOVQ    ( 8)(SP), AX;   ADCQ    AX,  R9;       MOVQ     R9, ( 72)(OUT) \

  347. 	MOVQ    (16)(SP), AX;   ADCQ    AX, R10;       MOVQ    R10, ( 80)(OUT) \

  348. 	MOVQ    (24)(SP), AX;   ADCQ    AX, R11;       MOVQ    R11, ( 88)(OUT) \

  349. 	MOVQ    (32)(SP), R12;  ADCQ    $0, R12;       MOVQ    R12, ( 96)(OUT) \

  350. 	MOVQ    (40)(SP), R13;  ADCQ    $0, R13;       MOVQ    R13, (104)(OUT) \

  351. 	MOVQ    (48)(SP), R14;  ADCQ    $0, R14;       MOVQ    R14, (112)(OUT) \

  352. 	MOVQ    (56)(SP), R15;  ADCQ    $0, R15;       MOVQ    R15, (120)(OUT)

  353. 

  354. // Template for calculating the Montgomery reduction algorithm described in

  355. // section 5.2.3 of https://eprint.iacr.org/2017/1015.pdf. Template must be

  356. // customized with schoolbook multiplicaton for 128 x 320-bit number.

  357. // This macro reuses memory of IN value and *changes* it.

  358. // Input:

  359. //    * IN: 1024-bit number to be reduced

  360. //    * MULS: either MULS_128x320_MULX or MULS_128x320_MULXADX

  361. // Output: OUT 512-bit

  362. #define REDC(OUT, IN, MULS) \

  363. 	MULS(0(IN), ·p503p1, R8, R9, R10, R11, R12, R13, R14, BX, CX, R15) \

  364. 	XORQ    R15, R15        \

  365. 	ADDQ    (24)(IN), R8    \

  366. 	ADCQ    (32)(IN), R9    \

  367. 	ADCQ    (40)(IN), R10   \

  368. 	ADCQ    (48)(IN), R11   \

  369. 	ADCQ    (56)(IN), R12   \

  370. 	ADCQ    (64)(IN), R13   \

  371. 	ADCQ    (72)(IN), R14   \

  372. 	ADCQ    (80)(IN), R15   \

  373. 	MOVQ    R8, (24)(IN)    \

  374. 	MOVQ    R9, (32)(IN)    \

  375. 	MOVQ    R10, (40)(IN)   \

  376. 	MOVQ    R11, (48)(IN)   \

  377. 	MOVQ    R12, (56)(IN)   \

  378. 	MOVQ    R13, (64)(IN)   \

  379. 	MOVQ    R14, (72)(IN)   \

  380. 	MOVQ    R15, (80)(IN)   \

  381. 	MOVQ    (88)(IN), R8    \

  382. 	MOVQ    (96)(IN), R9    \

  383. 	MOVQ    (104)(IN), R10  \

  384. 	MOVQ    (112)(IN), R11  \

  385. 	MOVQ    (120)(IN), R12  \

  386. 	ADCQ    $0, R8          \

  387. 	ADCQ    $0, R9          \

  388. 	ADCQ    $0, R10         \

  389. 	ADCQ    $0, R11         \

  390. 	ADCQ    $0, R12         \

  391. 	MOVQ    R8, (88)(IN)    \

  392. 	MOVQ    R9, (96)(IN)    \

  393. 	MOVQ    R10, (104)(IN)  \

  394. 	MOVQ    R11, (112)(IN)  \

  395. 	MOVQ    R12, (120)(IN)  \

  396. 	\

  397. 	MULS(16(IN), ·p503p1, R8, R9, R10, R11, R12, R13, R14, BX, CX, R15)    \

  398. 	XORQ    R15, R15        \

  399. 	ADDQ    (40)(IN), R8    \

  400. 	ADCQ    (48)(IN), R9    \

  401. 	ADCQ    (56)(IN), R10   \

  402. 	ADCQ    (64)(IN), R11   \

  403. 	ADCQ    (72)(IN), R12   \

  404. 	ADCQ    (80)(IN), R13   \

  405. 	ADCQ    (88)(IN), R14   \

  406. 	ADCQ    (96)(IN), R15   \

  407. 	MOVQ    R8, (40)(IN)    \

  408. 	MOVQ    R9, (48)(IN)    \

  409. 	MOVQ    R10, (56)(IN)   \

  410. 	MOVQ    R11, (64)(IN)   \

  411. 	MOVQ    R12, (72)(IN)   \

  412. 	MOVQ    R13, (80)(IN)   \

  413. 	MOVQ    R14, (88)(IN)   \

  414. 	MOVQ    R15, (96)(IN)   \

  415. 	MOVQ    (104)(IN), R8   \

  416. 	MOVQ    (112)(IN), R9   \

  417. 	MOVQ    (120)(IN), R10  \

  418. 	ADCQ    $0, R8          \

  419. 	ADCQ    $0, R9          \

  420. 	ADCQ    $0, R10         \

  421. 	MOVQ    R8, (104)(IN)   \

  422. 	MOVQ    R9, (112)(IN)   \

  423. 	MOVQ    R10, (120)(IN)  \

  424. 	\

  425. 	MULS(32(IN), ·p503p1, R8, R9, R10, R11, R12, R13, R14, BX, CX, R15)    \

  426. 	XORQ    R15, R15        \

  427. 	XORQ    BX, BX          \

  428. 	ADDQ    ( 56)(IN), R8   \

  429. 	ADCQ    ( 64)(IN), R9   \

  430. 	ADCQ    ( 72)(IN), R10  \

  431. 	ADCQ    ( 80)(IN), R11  \

  432. 	ADCQ    ( 88)(IN), R12  \

  433. 	ADCQ    ( 96)(IN), R13  \

  434. 	ADCQ    (104)(IN), R14  \

  435. 	ADCQ    (112)(IN), R15  \

  436. 	ADCQ    (120)(IN), BX   \

  437. 	MOVQ    R8,  ( 56)(IN)  \

  438. 	MOVQ    R10, ( 72)(IN)  \

  439. 	MOVQ    R11, ( 80)(IN)  \

  440. 	MOVQ    R12, ( 88)(IN)  \

  441. 	MOVQ    R13, ( 96)(IN)  \

  442. 	MOVQ    R14, (104)(IN)  \

  443. 	MOVQ    R15, (112)(IN)  \

  444. 	MOVQ    BX,  (120)(IN)  \

  445. 	MOVQ    R9,  (  0)(OUT) \ // Result: OUT[0]

  446. 	\

  447. 	MULS(48(IN), ·p503p1, R8, R9, R10, R11, R12, R13, R14, BX, CX, R15)    \

  448. 	ADDQ    ( 72)(IN), R8   \

  449. 	ADCQ    ( 80)(IN), R9   \

  450. 	ADCQ    ( 88)(IN), R10  \

  451. 	ADCQ    ( 96)(IN), R11  \

  452. 	ADCQ    (104)(IN), R12  \

  453. 	ADCQ    (112)(IN), R13  \

  454. 	ADCQ    (120)(IN), R14  \

  455. 	MOVQ    R8,  ( 8)(OUT)  \ // Result: OUT[1]

  456. 	MOVQ    R9,  (16)(OUT)  \ // Result: OUT[2]

  457. 	MOVQ    R10, (24)(OUT)  \ // Result: OUT[3]

  458. 	MOVQ    R11, (32)(OUT)  \ // Result: OUT[4]

  459. 	MOVQ    R12, (40)(OUT)  \ // Result: OUT[5]

  460. 	MOVQ    R13, (48)(OUT)  \ // Result: OUT[6] and OUT[7]

  461. 	MOVQ    R14, (56)(OUT)

  462. 

  463. TEXT ·fp503StrongReduce(SB), NOSPLIT, $0-8

  464. 	MOVQ	x+0(FP), REG_P1

  465. 

  466. 	// Zero AX for later use:

  467. 	XORQ	AX, AX

  468. 

  469. 	// Load p into registers:

  470. 	MOVQ	P503_0, R8

  471. 	// P503_{1,2} = P503_0, so reuse R8

  472. 	MOVQ	P503_3, R9

  473. 	MOVQ	P503_4, R10

  474. 	MOVQ	P503_5, R11

  475. 	MOVQ	P503_6, R12

  476. 	MOVQ	P503_7, R13

  477. 

  478. 	// Set x <- x - p

  479. 	SUBQ	R8,  ( 0)(REG_P1)

  480. 	SBBQ	R8,  ( 8)(REG_P1)

  481. 	SBBQ	R8,  (16)(REG_P1)

  482. 	SBBQ	R9,  (24)(REG_P1)

  483. 	SBBQ	R10, (32)(REG_P1)

  484. 	SBBQ	R11, (40)(REG_P1)

  485. 	SBBQ	R12, (48)(REG_P1)

  486. 	SBBQ	R13, (56)(REG_P1)

  487. 

  488. 	// Save carry flag indicating x-p < 0 as a mask

  489. 	SBBQ	$0, AX

  490. 

  491. 	// Conditionally add p to x if x-p < 0

  492. 	ANDQ	AX, R8

  493. 	ANDQ	AX, R9

  494. 	ANDQ	AX, R10

  495. 	ANDQ	AX, R11

  496. 	ANDQ	AX, R12

  497. 	ANDQ	AX, R13

  498. 

  499. 	ADDQ	R8, ( 0)(REG_P1)

  500. 	ADCQ	R8, ( 8)(REG_P1)

  501. 	ADCQ	R8, (16)(REG_P1)

  502. 	ADCQ	R9, (24)(REG_P1)

  503. 	ADCQ	R10,(32)(REG_P1)

  504. 	ADCQ	R11,(40)(REG_P1)

  505. 	ADCQ	R12,(48)(REG_P1)

  506. 	ADCQ	R13,(56)(REG_P1)

  507. 

  508. 	RET

  509. 

  510. TEXT ·fp503ConditionalSwap(SB),NOSPLIT,$0-17

  511. 

  512. 	MOVQ	x+0(FP), REG_P1

  513. 	MOVQ	y+8(FP), REG_P2

  514. 	MOVB	choice+16(FP), AL	// AL = 0 or 1

  515. 	MOVBLZX	AL, AX				// AX = 0 or 1

  516. 	NEGQ	AX					// AX = 0x00..00 or 0xff..ff

  517. 

  518. #ifndef CSWAP_BLOCK

  519. #define CSWAP_BLOCK(idx) 	\

  520. 	MOVQ	(idx*8)(REG_P1), BX	\ // BX = x[idx]

  521. 	MOVQ 	(idx*8)(REG_P2), CX	\ // CX = y[idx]

  522. 	MOVQ	CX, DX				\ // DX = y[idx]

  523. 	XORQ	BX, DX				\ // DX = y[idx] ^ x[idx]

  524. 	ANDQ	AX, DX				\ // DX = (y[idx] ^ x[idx]) & mask

  525. 	XORQ	DX, BX				\ // BX = (y[idx] ^ x[idx]) & mask) ^ x[idx] = x[idx] or y[idx]

  526. 	XORQ	DX, CX				\ // CX = (y[idx] ^ x[idx]) & mask) ^ y[idx] = y[idx] or x[idx]

  527. 	MOVQ	BX, (idx*8)(REG_P1)	\

  528. 	MOVQ	CX, (idx*8)(REG_P2)

  529. #endif

  530. 

  531. 	CSWAP_BLOCK(0)

  532. 	CSWAP_BLOCK(1)

  533. 	CSWAP_BLOCK(2)

  534. 	CSWAP_BLOCK(3)

  535. 	CSWAP_BLOCK(4)

  536. 	CSWAP_BLOCK(5)

  537. 	CSWAP_BLOCK(6)

  538. 	CSWAP_BLOCK(7)

  539. 

  540. #ifdef CSWAP_BLOCK

  541. #undef CSWAP_BLOCK

  542. #endif

  543. 

  544. 	RET

  545. 

  546. TEXT ·fp503AddReduced(SB),NOSPLIT,$0-24

  547. 

  548. 	MOVQ	z+0(FP), REG_P3

  549. 	MOVQ	x+8(FP), REG_P1

  550. 	MOVQ	y+16(FP), REG_P2

  551. 

  552. 	// Used later to calculate a mask

  553. 	XORQ    CX, CX

  554. 

  555. 	// [R8-R15]: z = x + y

  556. 	MOVQ	( 0)(REG_P1), R8

  557. 	MOVQ	( 8)(REG_P1), R9

  558. 	MOVQ	(16)(REG_P1), R10

  559. 	MOVQ	(24)(REG_P1), R11

  560. 	MOVQ	(32)(REG_P1), R12

  561. 	MOVQ	(40)(REG_P1), R13

  562. 	MOVQ	(48)(REG_P1), R14

  563. 	MOVQ	(56)(REG_P1), R15

  564. 	ADDQ	( 0)(REG_P2), R8

  565. 	ADCQ	( 8)(REG_P2), R9

  566. 	ADCQ	(16)(REG_P2), R10

  567. 	ADCQ	(24)(REG_P2), R11

  568. 	ADCQ	(32)(REG_P2), R12

  569. 	ADCQ	(40)(REG_P2), R13

  570. 	ADCQ	(48)(REG_P2), R14

  571. 	ADCQ	(56)(REG_P2), R15

  572. 

  573. 	MOVQ    P503X2_0, AX

  574. 	SUBQ    AX, R8

  575. 	MOVQ    P503X2_1, AX

  576. 	SBBQ    AX, R9

  577. 	SBBQ    AX, R10

  578. 	MOVQ    P503X2_3, AX

  579. 	SBBQ    AX, R11

  580. 	MOVQ    P503X2_4, AX

  581. 	SBBQ    AX, R12

  582. 	MOVQ    P503X2_5, AX

  583. 	SBBQ    AX, R13

  584. 	MOVQ    P503X2_6, AX

  585. 	SBBQ    AX, R14

  586. 	MOVQ    P503X2_7, AX

  587. 	SBBQ    AX, R15

  588. 

  589. 	// mask

  590. 	SBBQ    $0, CX

  591. 

  592. 	// move z to REG_P3

  593. 	MOVQ    R8,  ( 0)(REG_P3)

  594. 	MOVQ    R9,  ( 8)(REG_P3)

  595. 	MOVQ    R10, (16)(REG_P3)

  596. 	MOVQ    R11, (24)(REG_P3)

  597. 	MOVQ    R12, (32)(REG_P3)

  598. 	MOVQ    R13, (40)(REG_P3)

  599. 	MOVQ    R14, (48)(REG_P3)

  600. 	MOVQ    R15, (56)(REG_P3)

  601. 

  602. 	// if z<0 add p503x2 back

  603. 	MOVQ    P503X2_0,   R8

  604. 	MOVQ    P503X2_1,   R9

  605. 	MOVQ    P503X2_3,   R10

  606. 	MOVQ    P503X2_4,   R11

  607. 	MOVQ    P503X2_5,   R12

  608. 	MOVQ    P503X2_6,   R13

  609. 	MOVQ    P503X2_7,   R14

  610. 	ANDQ    CX, R8

  611. 	ANDQ    CX, R9

  612. 	ANDQ    CX, R10

  613. 	ANDQ    CX, R11

  614. 	ANDQ    CX, R12

  615. 	ANDQ    CX, R13

  616. 	ANDQ    CX, R14

  617. 	MOVQ    ( 0)(REG_P3), AX; ADDQ    R8,  AX; MOVQ    AX, ( 0)(REG_P3)

  618. 	MOVQ    ( 8)(REG_P3), AX; ADCQ    R9,  AX; MOVQ    AX, ( 8)(REG_P3)

  619. 	MOVQ    (16)(REG_P3), AX; ADCQ    R9,  AX; MOVQ    AX, (16)(REG_P3)

  620. 	MOVQ    (24)(REG_P3), AX; ADCQ    R10, AX; MOVQ    AX, (24)(REG_P3)

  621. 	MOVQ    (32)(REG_P3), AX; ADCQ    R11, AX; MOVQ    AX, (32)(REG_P3)

  622. 	MOVQ    (40)(REG_P3), AX; ADCQ    R12, AX; MOVQ    AX, (40)(REG_P3)

  623. 	MOVQ    (48)(REG_P3), AX; ADCQ    R13, AX; MOVQ    AX, (48)(REG_P3)

  624. 	MOVQ    (56)(REG_P3), AX; ADCQ    R14, AX; MOVQ    AX, (56)(REG_P3)

  625. 	RET

  626. 

  627. TEXT ·fp503SubReduced(SB), NOSPLIT, $0-24

  628. 

  629. 	MOVQ    z+0(FP), REG_P3

  630. 	MOVQ    x+8(FP), REG_P1

  631. 	MOVQ    y+16(FP), REG_P2

  632. 

  633. 	// Used later to calculate a mask

  634. 	XORQ    CX, CX

  635. 

  636. 	MOVQ    ( 0)(REG_P1), R8

  637. 	MOVQ    ( 8)(REG_P1), R9

  638. 	MOVQ    (16)(REG_P1), R10

  639. 	MOVQ    (24)(REG_P1), R11

  640. 	MOVQ    (32)(REG_P1), R12

  641. 	MOVQ    (40)(REG_P1), R13

  642. 	MOVQ    (48)(REG_P1), R14

  643. 	MOVQ    (56)(REG_P1), R15

  644. 

  645. 	SUBQ    ( 0)(REG_P2), R8

  646. 	SBBQ    ( 8)(REG_P2), R9

  647. 	SBBQ    (16)(REG_P2), R10

  648. 	SBBQ    (24)(REG_P2), R11

  649. 	SBBQ    (32)(REG_P2), R12

  650. 	SBBQ    (40)(REG_P2), R13

  651. 	SBBQ    (48)(REG_P2), R14

  652. 	SBBQ    (56)(REG_P2), R15

  653. 

  654. 	// mask

  655. 	SBBQ    $0, CX

  656. 

  657. 	// store x-y in REG_P3

  658. 	MOVQ    R8,  ( 0)(REG_P3)

  659. 	MOVQ    R9,  ( 8)(REG_P3)

  660. 	MOVQ    R10, (16)(REG_P3)

  661. 	MOVQ    R11, (24)(REG_P3)

  662. 	MOVQ    R12, (32)(REG_P3)

  663. 	MOVQ    R13, (40)(REG_P3)

  664. 	MOVQ    R14, (48)(REG_P3)

  665. 	MOVQ    R15, (56)(REG_P3)

  666. 

  667. 	// if z<0 add p503x2 back

  668. 	MOVQ    P503X2_0, R8

  669. 	MOVQ    P503X2_1, R9

  670. 	MOVQ    P503X2_3, R10

  671. 	MOVQ    P503X2_4, R11

  672. 	MOVQ    P503X2_5, R12

  673. 	MOVQ    P503X2_6, R13

  674. 	MOVQ    P503X2_7, R14

  675. 	ANDQ    CX, R8

  676. 	ANDQ    CX, R9

  677. 	ANDQ    CX, R10

  678. 	ANDQ    CX, R11

  679. 	ANDQ    CX, R12

  680. 	ANDQ    CX, R13

  681. 	ANDQ    CX, R14

  682. 	MOVQ    ( 0)(REG_P3), AX; ADDQ    R8,  AX; MOVQ    AX, ( 0)(REG_P3)

  683. 	MOVQ    ( 8)(REG_P3), AX; ADCQ    R9,  AX; MOVQ    AX, ( 8)(REG_P3)

  684. 	MOVQ    (16)(REG_P3), AX; ADCQ    R9,  AX; MOVQ    AX, (16)(REG_P3)

  685. 	MOVQ    (24)(REG_P3), AX; ADCQ    R10, AX; MOVQ    AX, (24)(REG_P3)

  686. 	MOVQ    (32)(REG_P3), AX; ADCQ    R11, AX; MOVQ    AX, (32)(REG_P3)

  687. 	MOVQ    (40)(REG_P3), AX; ADCQ    R12, AX; MOVQ    AX, (40)(REG_P3)

  688. 	MOVQ    (48)(REG_P3), AX; ADCQ    R13, AX; MOVQ    AX, (48)(REG_P3)

  689. 	MOVQ    (56)(REG_P3), AX; ADCQ    R14, AX; MOVQ    AX, (56)(REG_P3)

  690. 

  691. 	RET

  692. 

  693. TEXT ·mulWithMULXADX(SB), NOSPLIT, $104-24

  694. 	// Actual implementation

  695. 	MOVQ    z+ 0(FP), CX

  696. 	MOVQ    x+ 8(FP), REG_P2

  697. 	MOVQ    y+16(FP), REG_P1

  698. 	MUL(CX, REG_P2, REG_P1, MULS256_MULXADX)

  699. 	RET

  700. 

  701. TEXT ·mulWithMULX(SB), NOSPLIT, $104-24

  702. 	// Actual implementation

  703. 	MOVQ    z+ 0(FP), CX

  704. 	MOVQ    x+ 8(FP), REG_P2

  705. 	MOVQ    y+16(FP), REG_P1

  706. 	MUL(CX, REG_P2, REG_P1, MULS256_MULX)

  707. 	RET

  708. 

  709. TEXT ·mul(SB), $96-24

  710. 	// Uses variant of Karatsuba method.

  711. 	//

  712. 	// Here we store the destination in CX instead of in REG_P3 because the

  713. 	// multiplication instructions use DX as an implicit destination

  714. 	// operand: MULQ $REG sets DX:AX <-- AX * $REG.

  715. 

  716. 	// Actual implementation

  717. 	MOVQ	z+0(FP), CX

  718. 	MOVQ	x+8(FP), REG_P1

  719. 	MOVQ	y+16(FP), REG_P2

  720. 

  721. 	// RAX and RDX will be used for a mask (0-borrow)

  722. 	XORQ	AX, AX

  723. 

  724. 	// RCX[0-3]: U1+U0

  725. 	MOVQ	(32)(REG_P1), R8

  726. 	MOVQ	(40)(REG_P1), R9

  727. 	MOVQ	(48)(REG_P1), R10

  728. 	MOVQ	(56)(REG_P1), R11

  729. 	ADDQ	( 0)(REG_P1), R8

  730. 	ADCQ	( 8)(REG_P1), R9

  731. 	ADCQ	(16)(REG_P1), R10

  732. 	ADCQ	(24)(REG_P1), R11

  733. 	MOVQ	R8,  ( 0)(CX)

  734. 	MOVQ	R9,  ( 8)(CX)

  735. 	MOVQ	R10, (16)(CX)

  736. 	MOVQ	R11, (24)(CX)

  737. 

  738. 	SBBQ	$0, AX

  739. 

  740. 	// R12-R15: V1+V0

  741. 	XORQ	DX, DX

  742. 	MOVQ	(32)(REG_P2), R12

  743. 	MOVQ	(40)(REG_P2), R13

  744. 	MOVQ	(48)(REG_P2), R14

  745. 	MOVQ	(56)(REG_P2), R15

  746. 	ADDQ	( 0)(REG_P2), R12

  747. 	ADCQ	( 8)(REG_P2), R13

  748. 	ADCQ	(16)(REG_P2), R14

  749. 	ADCQ	(24)(REG_P2), R15

  750. 

  751. 	SBBQ	$0, DX

  752. 

  753. 	// Store carries on stack

  754. 	MOVQ	AX, (64)(SP)

  755. 	MOVQ	DX, (72)(SP)

  756. 

  757. 	// (SP[0-3],R8,R9,R10,R11) <- (U0+U1)*(V0+V1).

  758. 	// MUL using comba; In comments below U=U0+U1 V=V0+V1

  759. 

  760. 	// U0*V0

  761. 	MOVQ    (CX), AX

  762. 	MULQ    R12

  763. 	MOVQ    AX, (SP)        // C0

  764. 	MOVQ    DX, R8

  765. 

  766. 	// U0*V1

  767. 	XORQ    R9, R9

  768. 	MOVQ    (CX), AX

  769. 	MULQ    R13

  770. 	ADDQ    AX, R8

  771. 	ADCQ    DX, R9

  772. 

  773. 	// U1*V0

  774. 	XORQ    R10, R10

  775. 	MOVQ    (8)(CX), AX

  776. 	MULQ    R12

  777. 	ADDQ    AX, R8

  778. 	MOVQ    R8, (8)(SP)     // C1

  779. 	ADCQ    DX, R9

  780. 	ADCQ    $0, R10

  781. 

  782. 	// U0*V2

  783. 	XORQ    R8, R8

  784. 	MOVQ    (CX), AX

  785. 	MULQ    R14

  786. 	ADDQ    AX, R9

  787. 	ADCQ    DX, R10

  788. 	ADCQ    $0, R8

  789. 

  790. 	// U2*V0

  791. 	MOVQ    (16)(CX), AX

  792. 	MULQ    R12

  793. 	ADDQ    AX, R9

  794. 	ADCQ    DX, R10

  795. 	ADCQ    $0, R8

  796. 

  797. 	// U1*V1

  798. 	MOVQ    (8)(CX), AX

  799. 	MULQ    R13

  800. 	ADDQ    AX, R9

  801. 	MOVQ    R9, (16)(SP)        // C2

  802. 	ADCQ    DX, R10

  803. 	ADCQ    $0, R8

  804. 

  805. 	// U0*V3

  806. 	XORQ    R9, R9

  807. 	MOVQ    (CX), AX

  808. 	MULQ    R15

  809. 	ADDQ    AX, R10

  810. 	ADCQ    DX, R8

  811. 	ADCQ    $0, R9

  812. 

  813. 	// U3*V0

  814. 	MOVQ    (24)(CX), AX

  815. 	MULQ    R12

  816. 	ADDQ    AX, R10

  817. 	ADCQ    DX, R8

  818. 	ADCQ    $0, R9

  819. 

  820. 	// U1*V2

  821. 	MOVQ    (8)(CX), AX

  822. 	MULQ    R14

  823. 	ADDQ    AX, R10

  824. 	ADCQ    DX, R8

  825. 	ADCQ    $0, R9

  826. 

  827. 	// U2*V1

  828. 	MOVQ    (16)(CX), AX

  829. 	MULQ    R13

  830. 	ADDQ    AX, R10

  831. 	MOVQ    R10, (24)(SP)       // C3

  832. 	ADCQ    DX, R8

  833. 	ADCQ    $0, R9

  834. 

  835. 	// U1*V3

  836. 	XORQ    R10, R10

  837. 	MOVQ    (8)(CX), AX

  838. 	MULQ    R15

  839. 	ADDQ    AX, R8

  840. 	ADCQ    DX, R9

  841. 	ADCQ    $0, R10

  842. 

  843. 	// U3*V1

  844. 	MOVQ    (24)(CX), AX

  845. 	MULQ    R13

  846. 	ADDQ    AX, R8

  847. 	ADCQ    DX, R9

  848. 	ADCQ    $0, R10

  849. 

  850. 	// U2*V2

  851. 	MOVQ    (16)(CX), AX

  852. 	MULQ    R14

  853. 	ADDQ    AX, R8

  854. 	MOVQ    R8, (32)(SP)        // C4

  855. 	ADCQ    DX, R9

  856. 	ADCQ    $0, R10

  857. 

  858. 	// U2*V3

  859. 	XORQ    R11, R11

  860. 	MOVQ    (16)(CX), AX

  861. 	MULQ    R15

  862. 	ADDQ    AX, R9

  863. 	ADCQ    DX, R10

  864. 	ADCQ    $0, R11

  865. 

  866. 	// U3*V2

  867. 	MOVQ    (24)(CX), AX

  868. 	MULQ    R14

  869. 	ADDQ    AX, R9              // C5

  870. 	ADCQ    DX, R10

  871. 	ADCQ    $0, R11

  872. 

  873. 	// U3*V3

  874. 	MOVQ    (24)(CX), AX

  875. 	MULQ    R15

  876. 	ADDQ    AX, R10             // C6

  877. 	ADCQ    DX, R11             // C7

  878. 

  879. 	MOVQ    (64)(SP), AX

  880. 	ANDQ    AX, R12

  881. 	ANDQ    AX, R13

  882. 	ANDQ    AX, R14

  883. 	ANDQ    AX, R15

  884. 	ADDQ    R8, R12

  885. 	ADCQ    R9, R13

  886. 	ADCQ    R10, R14

  887. 	ADCQ    R11, R15

  888. 

  889. 	MOVQ    (72)(SP), AX

  890. 	MOVQ    (CX), R8

  891. 	MOVQ    (8)(CX), R9

  892. 	MOVQ    (16)(CX), R10

  893. 	MOVQ    (24)(CX), R11

  894. 	ANDQ    AX, R8

  895. 	ANDQ    AX, R9

  896. 	ANDQ    AX, R10

  897. 	ANDQ    AX, R11

  898. 	ADDQ    R12, R8

  899. 	ADCQ    R13, R9

  900. 	ADCQ    R14, R10

  901. 	ADCQ    R15, R11

  902. 	MOVQ    R8, (32)(SP)

  903. 	MOVQ    R9, (40)(SP)

  904. 	MOVQ    R10, (48)(SP)

  905. 	MOVQ    R11, (56)(SP)

  906. 

  907. 	// CX[0-7] <- AL*BL

  908. 

  909. 	// U0*V0

  910. 	MOVQ    (REG_P1), R11

  911. 	MOVQ    (REG_P2), AX

  912. 	MULQ    R11

  913. 	XORQ    R9, R9

  914. 	MOVQ    AX, (CX)            // C0

  915. 	MOVQ    DX, R8

  916. 

  917. 	// U0*V1

  918. 	MOVQ    (16)(REG_P1), R14

  919. 	MOVQ    (8)(REG_P2), AX

  920. 	MULQ    R11

  921. 	XORQ    R10, R10

  922. 	ADDQ    AX, R8

  923. 	ADCQ    DX, R9

  924. 

  925. 	// U1*V0

  926. 	MOVQ    (8)(REG_P1), R12

  927. 	MOVQ    (REG_P2), AX

  928. 	MULQ    R12

  929. 	ADDQ    AX, R8

  930. 	MOVQ    R8, (8)(CX)         // C1

  931. 	ADCQ    DX, R9

  932. 	ADCQ    $0, R10

  933. 

  934. 	// U0*V2

  935. 	XORQ    R8, R8

  936. 	MOVQ    (16)(REG_P2), AX

  937. 	MULQ    R11

  938. 	ADDQ    AX, R9

  939. 	ADCQ    DX, R10

  940. 	ADCQ    $0, R8

  941. 

  942. 	// U2*V0

  943. 	MOVQ    (REG_P2), R13

  944. 	MOVQ    R14, AX

  945. 	MULQ    R13

  946. 	ADDQ    AX, R9

  947. 	ADCQ    DX, R10

  948. 	ADCQ    $0, R8

  949. 

  950. 	// U1*V1

  951. 	MOVQ    (8)(REG_P2), AX

  952. 	MULQ    R12

  953. 	ADDQ    AX, R9

  954. 	MOVQ    R9, (16)(CX)        // C2

  955. 	ADCQ    DX, R10

  956. 	ADCQ    $0, R8

  957. 

  958. 	// U0*V3

  959. 	XORQ    R9, R9

  960. 	MOVQ    (24)(REG_P2), AX

  961. 	MULQ    R11

  962. 	MOVQ    (24)(REG_P1), R15

  963. 	ADDQ    AX, R10

  964. 	ADCQ    DX, R8

  965. 	ADCQ    $0, R9

  966. 

  967. 	// U3*V1

  968. 	MOVQ    R15, AX

  969. 	MULQ    R13

  970. 	ADDQ    AX, R10

  971. 	ADCQ    DX, R8

  972. 	ADCQ    $0, R9

  973. 

  974. 	// U2*V2

  975. 	MOVQ    (16)(REG_P2), AX

  976. 	MULQ    R12

  977. 	ADDQ    AX, R10

  978. 	ADCQ    DX, R8

  979. 	ADCQ    $0, R9

  980. 

  981. 	// U2*V3

  982. 	MOVQ    (8)(REG_P2), AX

  983. 	MULQ    R14

  984. 	ADDQ    AX, R10

  985. 	MOVQ    R10, (24)(CX)       // C3

  986. 	ADCQ    DX, R8

  987. 	ADCQ    $0, R9

  988. 

  989. 	// U3*V2

  990. 	XORQ    R10, R10

  991. 	MOVQ    (24)(REG_P2), AX

  992. 	MULQ    R12

  993. 	ADDQ    AX, R8

  994. 	ADCQ    DX, R9

  995. 	ADCQ    $0, R10

  996. 

  997. 	// U3*V1

  998. 	MOVQ    (8)(REG_P2), AX

  999. 	MULQ    R15

  1000. 	ADDQ    AX, R8

  1001. 	ADCQ    DX, R9

  1002. 	ADCQ    $0, R10

  1003. 

  1004. 	// U2*V2

  1005. 	MOVQ    (16)(REG_P2), AX

  1006. 	MULQ    R14

  1007. 	ADDQ    AX, R8

  1008. 	MOVQ    R8, (32)(CX)		// C4

  1009. 	ADCQ    DX, R9

  1010. 	ADCQ    $0, R10

  1011. 

  1012. 	// U2*V3

  1013. 	XORQ    R8, R8

  1014. 	MOVQ    (24)(REG_P2), AX

  1015. 	MULQ    R14

  1016. 	ADDQ    AX, R9

  1017. 	ADCQ    DX, R10

  1018. 	ADCQ    $0, R8

  1019. 

  1020. 	// U3*V2

  1021. 	MOVQ    (16)(REG_P2), AX

  1022. 	MULQ    R15

  1023. 	ADDQ    AX, R9

  1024. 	MOVQ    R9, (40)(CX)		// C5

  1025. 	ADCQ    DX, R10

  1026. 	ADCQ    $0, R8

  1027. 

  1028. 	// U3*V3

  1029. 	MOVQ    (24)(REG_P2), AX

  1030. 	MULQ    R15

  1031. 	ADDQ    AX, R10

  1032. 	MOVQ    R10, (48)(CX)		// C6

  1033. 	ADCQ    DX, R8

  1034. 	MOVQ    R8, (56)(CX)		// C7

  1035. 

  1036. 	// CX[8-15] <- U1*V1

  1037. 	MOVQ    (32)(REG_P1), R11

  1038. 	MOVQ    (32)(REG_P2), AX

  1039. 	MULQ    R11

  1040. 	XORQ    R9, R9

  1041. 	MOVQ    AX, (64)(CX)        // C0

  1042. 	MOVQ    DX, R8

  1043. 

  1044. 	MOVQ    (48)(REG_P1), R14

  1045. 	MOVQ    (40)(REG_P2), AX

  1046. 	MULQ    R11

  1047. 	XORQ    R10, R10

  1048. 	ADDQ    AX, R8

  1049. 	ADCQ    DX, R9

  1050. 

  1051. 	MOVQ    (40)(REG_P1), R12

  1052. 	MOVQ    (32)(REG_P2), AX

  1053. 	MULQ    R12

  1054. 	ADDQ    AX, R8

  1055. 	MOVQ    R8, (72)(CX)        // C1

  1056. 	ADCQ    DX, R9

  1057. 	ADCQ    $0, R10

  1058. 

  1059. 	XORQ    R8, R8

  1060. 	MOVQ    (48)(REG_P2), AX

  1061. 	MULQ    R11

  1062. 	ADDQ    AX, R9

  1063. 	ADCQ    DX, R10

  1064. 	ADCQ    $0, R8

  1065. 

  1066. 	MOVQ    (32)(REG_P2), R13

  1067. 	MOVQ    R14, AX

  1068. 	MULQ    R13

  1069. 	ADDQ    AX, R9

  1070. 	ADCQ    DX, R10

  1071. 	ADCQ    $0, R8

  1072. 

  1073. 	MOVQ    (40)(REG_P2), AX

  1074. 	MULQ    R12

  1075. 	ADDQ    AX, R9

  1076. 	MOVQ    R9, (80)(CX)        // C2

  1077. 	ADCQ    DX, R10

  1078. 	ADCQ    $0, R8

  1079. 

  1080. 	XORQ    R9, R9

  1081. 	MOVQ    (56)(REG_P2), AX

  1082. 	MULQ    R11

  1083. 	MOVQ    (56)(REG_P1), R15

  1084. 	ADDQ    AX, R10

  1085. 	ADCQ    DX, R8

  1086. 	ADCQ    $0, R9

  1087. 

  1088. 	MOVQ    R15, AX

  1089. 	MULQ    R13

  1090. 	ADDQ    AX, R10

  1091. 	ADCQ    DX, R8

  1092. 	ADCQ    $0, R9

  1093. 

  1094. 	MOVQ    (48)(REG_P2), AX

  1095. 	MULQ    R12

  1096. 	ADDQ    AX, R10

  1097. 	ADCQ    DX, R8

  1098. 	ADCQ    $0, R9

  1099. 

  1100. 	MOVQ    (40)(REG_P2), AX

  1101. 	MULQ    R14

  1102. 	ADDQ    AX, R10

  1103. 	MOVQ    R10, (88)(CX)       // C3

  1104. 	ADCQ    DX, R8

  1105. 	ADCQ    $0, R9

  1106. 

  1107. 	XORQ    R10, R10

  1108. 	MOVQ    (56)(REG_P2), AX

  1109. 	MULQ    R12

  1110. 	ADDQ    AX, R8

  1111. 	ADCQ    DX, R9

  1112. 	ADCQ    $0, R10

  1113. 

  1114. 	MOVQ    (40)(REG_P2), AX

  1115. 	MULQ    R15

  1116. 	ADDQ    AX, R8

  1117. 	ADCQ    DX, R9

  1118. 	ADCQ    $0, R10

  1119. 

  1120. 	MOVQ    (48)(REG_P2), AX

  1121. 	MULQ    R14

  1122. 	ADDQ    AX, R8

  1123. 	MOVQ    R8, (96)(CX)        // C4

  1124. 	ADCQ    DX, R9

  1125. 	ADCQ    $0, R10

  1126. 

  1127. 	XORQ    R8, R8

  1128. 	MOVQ    (56)(REG_P2), AX

  1129. 	MULQ    R14

  1130. 	ADDQ    AX, R9

  1131. 	ADCQ    DX, R10

  1132. 	ADCQ    $0, R8

  1133. 

  1134. 	MOVQ    (48)(REG_P2), AX

  1135. 	MULQ    R15

  1136. 	ADDQ    AX, R9

  1137. 	MOVQ    R9, (104)(CX)       // C5

  1138. 	ADCQ    DX, R10

  1139. 	ADCQ    $0, R8

  1140. 

  1141. 	MOVQ    (56)(REG_P2), AX

  1142. 	MULQ    R15

  1143. 	ADDQ    AX, R10

  1144. 	MOVQ    R10, (112)(CX)      // C6

  1145. 	ADCQ    DX, R8

  1146. 	MOVQ    R8, (120)(CX)       // C7

  1147. 

  1148. 	// [R8-R15] <- (U0+U1)*(V0+V1) - U1*V1

  1149. 	MOVQ    (SP), R8

  1150. 	SUBQ    (CX), R8

  1151. 	MOVQ    (8)(SP), R9

  1152. 	SBBQ    (8)(CX), R9

  1153. 	MOVQ    (16)(SP), R10

  1154. 	SBBQ    (16)(CX), R10

  1155. 	MOVQ    (24)(SP), R11

  1156. 	SBBQ    (24)(CX), R11

  1157. 	MOVQ    (32)(SP), R12

  1158. 	SBBQ    (32)(CX), R12

  1159. 	MOVQ    (40)(SP), R13

  1160. 	SBBQ    (40)(CX), R13

  1161. 	MOVQ    (48)(SP), R14

  1162. 	SBBQ    (48)(CX), R14

  1163. 	MOVQ    (56)(SP), R15

  1164. 	SBBQ    (56)(CX), R15

  1165. 

  1166. 	// [R8-R15] <- (U0+U1)*(V0+V1) - U1*V0 - U0*U1

  1167. 	MOVQ    ( 64)(CX), AX;	SUBQ    AX, R8

  1168. 	MOVQ    ( 72)(CX), AX;	SBBQ    AX, R9

  1169. 	MOVQ    ( 80)(CX), AX;	SBBQ    AX, R10

  1170. 	MOVQ    ( 88)(CX), AX;	SBBQ    AX, R11

  1171. 	MOVQ    ( 96)(CX), AX;	SBBQ    AX, R12

  1172. 	MOVQ    (104)(CX), DX;	SBBQ    DX, R13

  1173. 	MOVQ    (112)(CX), DI;	SBBQ    DI, R14

  1174. 	MOVQ    (120)(CX), SI;	SBBQ    SI, R15

  1175. 

  1176. 	// Final result

  1177. 	ADDQ    (32)(CX), R8;	MOVQ    R8,  (32)(CX)

  1178. 	ADCQ    (40)(CX), R9;	MOVQ    R9,  (40)(CX)

  1179. 	ADCQ    (48)(CX), R10;	MOVQ    R10, (48)(CX)

  1180. 	ADCQ    (56)(CX), R11;	MOVQ    R11, (56)(CX)

  1181. 	ADCQ    (64)(CX), R12;	MOVQ    R12, (64)(CX)

  1182. 	ADCQ    (72)(CX), R13;	MOVQ    R13, (72)(CX)

  1183. 	ADCQ    (80)(CX), R14;	MOVQ    R14, (80)(CX)

  1184. 	ADCQ    (88)(CX), R15;	MOVQ    R15, (88)(CX)

  1185. 	ADCQ    $0, AX;        	MOVQ    AX,  (96)(CX)

  1186. 	ADCQ    $0, DX;        	MOVQ    DX, (104)(CX)

  1187. 	ADCQ    $0, DI;         MOVQ    DI, (112)(CX)

  1188. 	ADCQ    $0, SI;     	MOVQ    SI, (120)(CX)

  1189. 

  1190. 	RET

  1191. 

  1192. TEXT ·redc(SB), $0-16

  1193. 	MOVQ	z+0(FP), REG_P2

  1194. 	MOVQ	x+8(FP), REG_P1

  1195. 

  1196. 	MOVQ    (REG_P1), R11

  1197. 	MOVQ    P503P1_3, AX

  1198. 	MULQ    R11

  1199. 	XORQ    R8, R8

  1200. 	ADDQ    (24)(REG_P1), AX

  1201. 	MOVQ    AX, (24)(REG_P2)

  1202. 	ADCQ    DX, R8

  1203. 

  1204. 	XORQ    R9, R9

  1205. 	MOVQ    P503P1_4, AX

  1206. 	MULQ    R11

  1207. 	XORQ    R10, R10

  1208. 	ADDQ    AX, R8

  1209. 	ADCQ    DX, R9

  1210. 

  1211. 	MOVQ    (8)(REG_P1), R12

  1212. 	MOVQ    P503P1_3, AX

  1213. 	MULQ    R12

  1214. 	ADDQ    AX, R8

  1215. 	ADCQ    DX, R9

  1216. 	ADCQ    $0, R10

  1217. 	ADDQ    (32)(REG_P1), R8

  1218. 	MOVQ    R8, (32)(REG_P2)       // Z4

  1219. 	ADCQ    $0, R9

  1220. 	ADCQ    $0, R10

  1221. 

  1222. 	XORQ    R8, R8

  1223. 	MOVQ    P503P1_5, AX

  1224. 	MULQ    R11

  1225. 	ADDQ    AX, R9

  1226. 	ADCQ    DX, R10

  1227. 	ADCQ    $0, R8

  1228. 

  1229. 	MOVQ    P503P1_4, AX

  1230. 	MULQ    R12

  1231. 	ADDQ    AX, R9

  1232. 	ADCQ    DX, R10

  1233. 	ADCQ    $0, R8

  1234. 

  1235. 	MOVQ    (16)(REG_P1), R13

  1236. 	MOVQ    P503P1_3, AX

  1237. 	MULQ    R13

  1238. 	ADDQ    AX, R9

  1239. 	ADCQ    DX, R10

  1240. 	ADCQ    $0, R8

  1241. 	ADDQ    (40)(REG_P1), R9

  1242. 	MOVQ    R9, (40)(REG_P2)       // Z5

  1243. 	ADCQ    $0, R10

  1244. 	ADCQ    $0, R8

  1245. 

  1246. 	XORQ    R9, R9

  1247. 	MOVQ    P503P1_6, AX

  1248. 	MULQ    R11

  1249. 	ADDQ    AX, R10

  1250. 	ADCQ    DX, R8

  1251. 	ADCQ    $0, R9

  1252. 

  1253. 	MOVQ    P503P1_5, AX

  1254. 	MULQ    R12

  1255. 	ADDQ    AX, R10

  1256. 	ADCQ    DX, R8

  1257. 	ADCQ    $0, R9

  1258. 

  1259. 	MOVQ    P503P1_4, AX

  1260. 	MULQ    R13

  1261. 	ADDQ    AX, R10

  1262. 	ADCQ    DX, R8

  1263. 	ADCQ    $0, R9

  1264. 

  1265. 	MOVQ    (24)(REG_P2), R14

  1266. 	MOVQ    P503P1_3, AX

  1267. 	MULQ    R14

  1268. 	ADDQ    AX, R10

  1269. 	ADCQ    DX, R8

  1270. 	ADCQ    $0, R9

  1271. 	ADDQ    (48)(REG_P1), R10

  1272. 	MOVQ    R10, (48)(REG_P2)      // Z6

  1273. 	ADCQ    $0, R8

  1274. 	ADCQ    $0, R9

  1275. 

  1276. 	XORQ    R10, R10

  1277. 	MOVQ    P503P1_7, AX

  1278. 	MULQ    R11

  1279. 	ADDQ    AX, R8

  1280. 	ADCQ    DX, R9

  1281. 	ADCQ    $0, R10

  1282. 

  1283. 	MOVQ    P503P1_6, AX

  1284. 	MULQ    R12

  1285. 	ADDQ    AX, R8

  1286. 	ADCQ    DX, R9

  1287. 	ADCQ    $0, R10

  1288. 

  1289. 	MOVQ    P503P1_5, AX

  1290. 	MULQ    R13

  1291. 	ADDQ    AX, R8

  1292. 	ADCQ    DX, R9

  1293. 	ADCQ    $0, R10

  1294. 

  1295. 	MOVQ    P503P1_4, AX

  1296. 	MULQ    R14

  1297. 	ADDQ    AX, R8

  1298. 	ADCQ    DX, R9

  1299. 	ADCQ    $0, R10

  1300. 

  1301. 	MOVQ    (32)(REG_P2), R15

  1302. 	MOVQ    P503P1_3, AX

  1303. 	MULQ    R15

  1304. 	ADDQ    AX, R8

  1305. 	ADCQ    DX, R9

  1306. 	ADCQ    $0, R10

  1307. 	ADDQ    (56)(REG_P1), R8

  1308. 	MOVQ    R8, (56)(REG_P2)       // Z7

  1309. 	ADCQ    $0, R9

  1310. 	ADCQ    $0, R10

  1311. 

  1312. 	XORQ    R8, R8

  1313. 	MOVQ    P503P1_7, AX

  1314. 	MULQ    R12

  1315. 	ADDQ    AX, R9

  1316. 	ADCQ    DX, R10

  1317. 	ADCQ    $0, R8

  1318. 

  1319. 	MOVQ    P503P1_6, AX

  1320. 	MULQ    R13

  1321. 	ADDQ    AX, R9

  1322. 	ADCQ    DX, R10

  1323. 	ADCQ    $0, R8

  1324. 

  1325. 	MOVQ    P503P1_5, AX

  1326. 	MULQ    R14

  1327. 	ADDQ    AX, R9

  1328. 	ADCQ    DX, R10

  1329. 	ADCQ    $0, R8

  1330. 

  1331. 	MOVQ    P503P1_4, AX

  1332. 	MULQ    R15

  1333. 	ADDQ    AX, R9

  1334. 	ADCQ    DX, R10

  1335. 	ADCQ    $0, R8

  1336. 

  1337. 	MOVQ    (40)(REG_P2), CX

  1338. 	MOVQ    P503P1_3, AX

  1339. 	MULQ    CX

  1340. 	ADDQ    AX, R9

  1341. 	ADCQ    DX, R10

  1342. 	ADCQ    $0, R8

  1343. 	ADDQ    (64)(REG_P1), R9

  1344. 	MOVQ    R9, (REG_P2)           // Z0

  1345. 	ADCQ    $0, R10

  1346. 	ADCQ    $0, R8

  1347. 

  1348. 	XORQ    R9, R9

  1349. 	MOVQ    P503P1_7, AX

  1350. 	MULQ    R13

  1351. 	ADDQ    AX, R10

  1352. 	ADCQ    DX, R8

  1353. 	ADCQ    $0, R9

  1354. 

  1355. 	MOVQ    P503P1_6, AX

  1356. 	MULQ    R14

  1357. 	ADDQ    AX, R10

  1358. 	ADCQ    DX, R8

  1359. 	ADCQ    $0, R9

  1360. 

  1361. 	MOVQ    P503P1_5, AX

  1362. 	MULQ    R15

  1363. 	ADDQ    AX, R10

  1364. 	ADCQ    DX, R8

  1365. 	ADCQ    $0, R9

  1366. 

  1367. 	MOVQ    P503P1_4, AX

  1368. 	MULQ    CX

  1369. 	ADDQ    AX, R10

  1370. 	ADCQ    DX, R8

  1371. 	ADCQ    $0, R9

  1372. 

  1373. 	MOVQ    (48)(REG_P2), R13

  1374. 	MOVQ    P503P1_3, AX

  1375. 	MULQ    R13

  1376. 	ADDQ    AX, R10

  1377. 	ADCQ    DX, R8

  1378. 	ADCQ    $0, R9

  1379. 	ADDQ    (72)(REG_P1), R10

  1380. 	MOVQ    R10, (8)(REG_P2)       // Z1

  1381. 	ADCQ    $0, R8

  1382. 	ADCQ    $0, R9

  1383. 

  1384. 	XORQ    R10, R10

  1385. 	MOVQ    P503P1_7, AX

  1386. 	MULQ    R14

  1387. 	ADDQ    AX, R8

  1388. 	ADCQ    DX, R9

  1389. 	ADCQ    $0, R10

  1390. 

  1391. 	MOVQ    P503P1_6, AX

  1392. 	MULQ    R15

  1393. 	ADDQ    AX, R8

  1394. 	ADCQ    DX, R9

  1395. 	ADCQ    $0, R10

  1396. 

  1397. 	MOVQ    P503P1_5, AX

  1398. 	MULQ    CX

  1399. 	ADDQ    AX, R8

  1400. 	ADCQ    DX, R9

  1401. 	ADCQ    $0, R10

  1402. 

  1403. 	MOVQ    P503P1_4, AX

  1404. 	MULQ    R13

  1405. 	ADDQ    AX, R8

  1406. 	ADCQ    DX, R9

  1407. 	ADCQ    $0, R10

  1408. 

  1409. 	MOVQ    (56)(REG_P2), R14

  1410. 	MOVQ    P503P1_3, AX

  1411. 	MULQ    R14

  1412. 	ADDQ    AX, R8

  1413. 	ADCQ    DX, R9

  1414. 	ADCQ    $0, R10

  1415. 	ADDQ    (80)(REG_P1), R8

  1416. 	MOVQ    R8, (16)(REG_P2)       // Z2

  1417. 	ADCQ    $0, R9

  1418. 	ADCQ    $0, R10

  1419. 

  1420. 	XORQ    R8, R8

  1421. 	MOVQ    P503P1_7, AX

  1422. 	MULQ    R15

  1423. 	ADDQ    AX, R9

  1424. 	ADCQ    DX, R10

  1425. 	ADCQ    $0, R8

  1426. 

  1427. 	MOVQ    P503P1_6, AX

  1428. 	MULQ    CX

  1429. 	ADDQ    AX, R9

  1430. 	ADCQ    DX, R10

  1431. 	ADCQ    $0, R8

  1432. 

  1433. 	MOVQ    P503P1_5, AX

  1434. 	MULQ    R13

  1435. 	ADDQ    AX, R9

  1436. 	ADCQ    DX, R10

  1437. 	ADCQ    $0, R8

  1438. 

  1439. 	MOVQ    P503P1_4, AX

  1440. 	MULQ    R14

  1441. 	ADDQ    AX, R9

  1442. 	ADCQ    DX, R10

  1443. 	ADCQ    $0, R8

  1444. 	ADDQ    (88)(REG_P1), R9

  1445. 	MOVQ    R9, (24)(REG_P2)       // Z3

  1446. 	ADCQ    $0, R10

  1447. 	ADCQ    $0, R8

  1448. 

  1449. 	XORQ    R9, R9

  1450. 	MOVQ    P503P1_7, AX

  1451. 	MULQ    CX

  1452. 	ADDQ    AX, R10

  1453. 	ADCQ    DX, R8

  1454. 	ADCQ    $0, R9

  1455. 

  1456. 	MOVQ    P503P1_6, AX

  1457. 	MULQ    R13

  1458. 	ADDQ    AX, R10

  1459. 	ADCQ    DX, R8

  1460. 	ADCQ    $0, R9

  1461. 

  1462. 	MOVQ    P503P1_5, AX

  1463. 	MULQ    R14

  1464. 	ADDQ    AX, R10

  1465. 	ADCQ    DX, R8

  1466. 	ADCQ    $0, R9

  1467. 	ADDQ    (96)(REG_P1), R10

  1468. 	MOVQ    R10, (32)(REG_P2)      // Z4

  1469. 	ADCQ    $0, R8

  1470. 	ADCQ    $0, R9

  1471. 

  1472. 	XORQ    R10, R10

  1473. 	MOVQ    P503P1_7, AX

  1474. 	MULQ    R13

  1475. 	ADDQ    AX, R8

  1476. 	ADCQ    DX, R9

  1477. 	ADCQ    $0, R10

  1478. 

  1479. 	MOVQ    P503P1_6, AX

  1480. 	MULQ    R14

  1481. 	ADDQ    AX, R8

  1482. 	ADCQ    DX, R9

  1483. 	ADCQ    $0, R10

  1484. 	ADDQ    (104)(REG_P1), R8      // Z5

  1485. 	MOVQ    R8, (40)(REG_P2)       // Z5

  1486. 	ADCQ    $0, R9

  1487. 	ADCQ    $0, R10

  1488. 

  1489. 	MOVQ    P503P1_7, AX

  1490. 	MULQ    R14

  1491. 	ADDQ    AX, R9

  1492. 	ADCQ    DX, R10

  1493. 	ADDQ    (112)(REG_P1), R9      // Z6

  1494. 	MOVQ    R9, (48)(REG_P2)       // Z6

  1495. 	ADCQ    $0, R10

  1496. 	ADDQ    (120)(REG_P1), R10     // Z7

  1497. 	MOVQ    R10, (56)(REG_P2)      // Z7

  1498. 

  1499. 	RET

  1500. 

  1501. TEXT ·redcWithMULX(SB), $0-16

  1502. 	MOVQ    z+0(FP), DI

  1503. 	MOVQ    x+8(FP), SI

  1504. 	REDC(DI, SI, MULS_128x320_MULX)

  1505. 	RET

  1506. 

  1507. TEXT ·redcWithMULXADX(SB), $0-16

  1508. 	MOVQ    z+0(FP), DI

  1509. 	MOVQ    x+8(FP), SI

  1510. 	REDC(DI, SI, MULS_128x320_MULXADX)

  1511. 	RET

  1512. 

  1513. TEXT ·fp503AddLazy(SB), NOSPLIT, $0-24

  1514. 

  1515. 	MOVQ z+0(FP), REG_P3

  1516. 	MOVQ x+8(FP), REG_P1

  1517. 	MOVQ y+16(FP), REG_P2

  1518. 

  1519. 	MOVQ	(REG_P1), R8

  1520. 	MOVQ	(8)(REG_P1), R9

  1521. 	MOVQ	(16)(REG_P1), R10

  1522. 	MOVQ	(24)(REG_P1), R11

  1523. 	MOVQ	(32)(REG_P1), R12

  1524. 	MOVQ	(40)(REG_P1), R13

  1525. 	MOVQ	(48)(REG_P1), R14

  1526. 	MOVQ	(56)(REG_P1), R15

  1527. 

  1528. 	ADDQ	(REG_P2), R8

  1529. 	ADCQ	(8)(REG_P2), R9

  1530. 	ADCQ	(16)(REG_P2), R10

  1531. 	ADCQ	(24)(REG_P2), R11

  1532. 	ADCQ	(32)(REG_P2), R12

  1533. 	ADCQ	(40)(REG_P2), R13

  1534. 	ADCQ	(48)(REG_P2), R14

  1535. 	ADCQ	(56)(REG_P2), R15

  1536. 

  1537. 	MOVQ	R8, (REG_P3)

  1538. 	MOVQ	R9, (8)(REG_P3)

  1539. 	MOVQ	R10, (16)(REG_P3)

  1540. 	MOVQ	R11, (24)(REG_P3)

  1541. 	MOVQ	R12, (32)(REG_P3)

  1542. 	MOVQ	R13, (40)(REG_P3)

  1543. 	MOVQ	R14, (48)(REG_P3)

  1544. 	MOVQ	R15, (56)(REG_P3)

  1545. 

  1546. 	RET

  1547. 

  1548. TEXT ·fp503X2AddLazy(SB), NOSPLIT, $0-24

  1549. 

  1550. 	MOVQ	z+0(FP), REG_P3

  1551. 	MOVQ	x+8(FP), REG_P1

  1552. 	MOVQ	y+16(FP), REG_P2

  1553. 

  1554. 	MOVQ	(REG_P1), R8

  1555. 	MOVQ	(8)(REG_P1), R9

  1556. 	MOVQ	(16)(REG_P1), R10

  1557. 	MOVQ	(24)(REG_P1), R11

  1558. 	MOVQ	(32)(REG_P1), R12

  1559. 	MOVQ	(40)(REG_P1), R13

  1560. 	MOVQ	(48)(REG_P1), R14

  1561. 	MOVQ	(56)(REG_P1), R15

  1562. 	MOVQ	(64)(REG_P1), AX

  1563. 	MOVQ	(72)(REG_P1), BX

  1564. 	MOVQ	(80)(REG_P1), CX

  1565. 

  1566. 	ADDQ	(REG_P2), R8

  1567. 	ADCQ	(8)(REG_P2), R9

  1568. 	ADCQ	(16)(REG_P2), R10

  1569. 	ADCQ	(24)(REG_P2), R11

  1570. 	ADCQ	(32)(REG_P2), R12

  1571. 	ADCQ	(40)(REG_P2), R13

  1572. 	ADCQ	(48)(REG_P2), R14

  1573. 	ADCQ	(56)(REG_P2), R15

  1574. 	ADCQ	(64)(REG_P2), AX

  1575. 	ADCQ	(72)(REG_P2), BX

  1576. 	ADCQ	(80)(REG_P2), CX

  1577. 

  1578. 	MOVQ	R8, (REG_P3)

  1579. 	MOVQ	R9, (8)(REG_P3)

  1580. 	MOVQ	R10, (16)(REG_P3)

  1581. 	MOVQ	R11, (24)(REG_P3)

  1582. 	MOVQ	R12, (32)(REG_P3)

  1583. 	MOVQ	R13, (40)(REG_P3)

  1584. 	MOVQ	R14, (48)(REG_P3)

  1585. 	MOVQ	R15, (56)(REG_P3)

  1586. 	MOVQ	AX, (64)(REG_P3)

  1587. 	MOVQ	BX, (72)(REG_P3)

  1588. 	MOVQ	CX, (80)(REG_P3)

  1589. 

  1590. 	MOVQ	(88)(REG_P1), R8

  1591. 	MOVQ	(96)(REG_P1), R9

  1592. 	MOVQ	(104)(REG_P1), R10

  1593. 	MOVQ	(112)(REG_P1), R11

  1594. 	MOVQ	(120)(REG_P1), R12

  1595. 

  1596. 	ADCQ	(88)(REG_P2), R8

  1597. 	ADCQ	(96)(REG_P2), R9

  1598. 	ADCQ	(104)(REG_P2), R10

  1599. 	ADCQ	(112)(REG_P2), R11

  1600. 	ADCQ	(120)(REG_P2), R12

  1601. 

  1602. 	MOVQ	R8, (88)(REG_P3)

  1603. 	MOVQ	R9, (96)(REG_P3)

  1604. 	MOVQ	R10, (104)(REG_P3)

  1605. 	MOVQ	R11, (112)(REG_P3)

  1606. 	MOVQ	R12, (120)(REG_P3)

  1607. 

  1608. 	RET

  1609. 

  1610. TEXT ·fp503X2SubLazy(SB), NOSPLIT, $0-24

  1611. 

  1612. 	MOVQ z+0(FP), REG_P3

  1613. 	MOVQ x+8(FP), REG_P1

  1614. 	MOVQ y+16(FP), REG_P2

  1615. 	// Used later to store result of 0-borrow

  1616. 	XORQ CX, CX

  1617. 

  1618. 	// SUBC for first 11 limbs

  1619. 	MOVQ	(REG_P1), R8

  1620. 	MOVQ	(8)(REG_P1), R9

  1621. 	MOVQ	(16)(REG_P1), R10

  1622. 	MOVQ	(24)(REG_P1), R11

  1623. 	MOVQ	(32)(REG_P1), R12

  1624. 	MOVQ	(40)(REG_P1), R13

  1625. 	MOVQ	(48)(REG_P1), R14

  1626. 	MOVQ	(56)(REG_P1), R15

  1627. 	MOVQ	(64)(REG_P1), AX

  1628. 	MOVQ	(72)(REG_P1), BX

  1629. 

  1630. 	SUBQ	(REG_P2), R8

  1631. 	SBBQ	(8)(REG_P2), R9

  1632. 	SBBQ	(16)(REG_P2), R10

  1633. 	SBBQ	(24)(REG_P2), R11

  1634. 	SBBQ	(32)(REG_P2), R12

  1635. 	SBBQ	(40)(REG_P2), R13

  1636. 	SBBQ	(48)(REG_P2), R14

  1637. 	SBBQ	(56)(REG_P2), R15

  1638. 	SBBQ	(64)(REG_P2), AX

  1639. 	SBBQ	(72)(REG_P2), BX

  1640. 

  1641. 	MOVQ	R8, (REG_P3)

  1642. 	MOVQ	R9, (8)(REG_P3)

  1643. 	MOVQ	R10, (16)(REG_P3)

  1644. 	MOVQ	R11, (24)(REG_P3)

  1645. 	MOVQ	R12, (32)(REG_P3)

  1646. 	MOVQ	R13, (40)(REG_P3)

  1647. 	MOVQ	R14, (48)(REG_P3)

  1648. 	MOVQ	R15, (56)(REG_P3)

  1649. 	MOVQ	AX, (64)(REG_P3)

  1650. 	MOVQ	BX, (72)(REG_P3)

  1651. 

  1652. 	// SUBC for last 5 limbs

  1653. 	MOVQ	(80)(REG_P1), 	R8

  1654. 	MOVQ	(88)(REG_P1), 	R9

  1655. 	MOVQ	(96)(REG_P1), 	R10

  1656. 	MOVQ	(104)(REG_P1), 	R11

  1657. 	MOVQ	(112)(REG_P1), 	R12

  1658. 	MOVQ	(120)(REG_P1), 	R13

  1659. 

  1660. 	SBBQ	(80)(REG_P2), R8

  1661. 	SBBQ	(88)(REG_P2), R9

  1662. 	SBBQ	(96)(REG_P2), R10

  1663. 	SBBQ	(104)(REG_P2), R11

  1664. 	SBBQ	(112)(REG_P2), R12

  1665. 	SBBQ	(120)(REG_P2), R13

  1666. 

  1667. 	MOVQ	R8, (80)(REG_P3)

  1668. 	MOVQ	R9, (88)(REG_P3)

  1669. 	MOVQ	R10, (96)(REG_P3)

  1670. 	MOVQ	R11, (104)(REG_P3)

  1671. 	MOVQ	R12, (112)(REG_P3)

  1672. 	MOVQ	R13, (120)(REG_P3)

  1673. 

  1674. 	// Now the carry flag is 1 if x-y < 0.  If so, add p*2^512.

  1675. 	SBBQ	$0, CX

  1676. 

  1677. 	// Load p into registers:

  1678. 	MOVQ	P503_0, R8

  1679. 	// P503_{1,2} = P503_0, so reuse R8

  1680. 	MOVQ	P503_3, R9

  1681. 	MOVQ	P503_4, R10

  1682. 	MOVQ	P503_5, R11

  1683. 	MOVQ	P503_6, R12

  1684. 	MOVQ	P503_7, R13

  1685. 

  1686. 	ANDQ	CX, R8

  1687. 	ANDQ	CX, R9

  1688. 	ANDQ	CX, R10

  1689. 	ANDQ	CX, R11

  1690. 	ANDQ	CX, R12

  1691. 	ANDQ	CX, R13

  1692. 

  1693. 	MOVQ   (64   )(REG_P3), AX; ADDQ R8,  AX; MOVQ AX, (64   )(REG_P3)

  1694. 	MOVQ   (64+ 8)(REG_P3), AX; ADCQ R8,  AX; MOVQ AX, (64+ 8)(REG_P3)

  1695. 	MOVQ   (64+16)(REG_P3), AX; ADCQ R8,  AX; MOVQ AX, (64+16)(REG_P3)

  1696. 	MOVQ   (64+24)(REG_P3), AX; ADCQ R9,  AX; MOVQ AX, (64+24)(REG_P3)

  1697. 	MOVQ   (64+32)(REG_P3), AX; ADCQ R10, AX; MOVQ AX, (64+32)(REG_P3)

  1698. 	MOVQ   (64+40)(REG_P3), AX; ADCQ R11, AX; MOVQ AX, (64+40)(REG_P3)

  1699. 	MOVQ   (64+48)(REG_P3), AX; ADCQ R12, AX; MOVQ AX, (64+48)(REG_P3)

  1700. 	MOVQ   (64+56)(REG_P3), AX; ADCQ R13, AX; MOVQ AX, (64+56)(REG_P3)

  1701. 

  1702. 	RET