kris
/
SIDH_p751
1
0
Форк 0
Код Проблеми 0 Запити на злиття 0 Релізи 0 Вікі Активність
Ви не можете вибрати більше 25 тем Теми мають розпочинатися з літери або цифри, можуть містити дефіси (-) і не повинні перевищувати 35 символів.
131 Коміти
14 Гілки
653 KiB
 
 
 
Дерево: cf9e40db2b
Гілки Теги
${ item.name }
Створити гілку ${ searchTerm }
з 'cf9e40db2b'
${ noResults }
SIDH_p751/p503/arith_amd64.s

1709 рядки
39 KiB
Неформатований Звинувачення Історія 

  1. // +build amd64,!noasm

  2. 

  3. #include "textflag.h"

  4. 

  5. // p503

  6. #define P503_0     $0xFFFFFFFFFFFFFFFF

  7. #define P503_1     $0xFFFFFFFFFFFFFFFF

  8. #define P503_2     $0xFFFFFFFFFFFFFFFF

  9. #define P503_3     $0xABFFFFFFFFFFFFFF

  10. #define P503_4     $0x13085BDA2211E7A0

  11. #define P503_5     $0x1B9BF6C87B7E7DAF

  12. #define P503_6     $0x6045C6BDDA77A4D0

  13. #define P503_7     $0x004066F541811E1E

  14. 

  15. // p503+1

  16. #define P503P1_3   $0xAC00000000000000

  17. #define P503P1_4   $0x13085BDA2211E7A0

  18. #define P503P1_5   $0x1B9BF6C87B7E7DAF

  19. #define P503P1_6   $0x6045C6BDDA77A4D0

  20. #define P503P1_7   $0x004066F541811E1E

  21. 

  22. // p503x2

  23. #define P503X2_0   $0xFFFFFFFFFFFFFFFE

  24. #define P503X2_1   $0xFFFFFFFFFFFFFFFF

  25. #define P503X2_2   $0xFFFFFFFFFFFFFFFF

  26. #define P503X2_3   $0x57FFFFFFFFFFFFFF

  27. #define P503X2_4   $0x2610B7B44423CF41

  28. #define P503X2_5   $0x3737ED90F6FCFB5E

  29. #define P503X2_6   $0xC08B8D7BB4EF49A0

  30. #define P503X2_7   $0x0080CDEA83023C3C

  31. 

  32. #define REG_P1 DI

  33. #define REG_P2 SI

  34. #define REG_P3 DX

  35. 

  36. // Performs schoolbook multiplication of 2 256-bit numbers. This optimized version

  37. // uses MULX instruction. Macro smashes value in DX.

  38. // Input: I0 and I1.

  39. // Output: O

  40. // All the other arguments are resgisters, used for storing temporary values

  41. #define MULS256_MULX(O, I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9) \

  42. 	MOVQ    I0, DX          \

  43. 	MULXQ   I1, T1, T0      \   // T0:T1 = A0*B0

  44. 	MOVQ    T1, O           \   // O[0]

  45. 	MULXQ   8+I1, T2, T1    \   // T1:T2 = U0*V1

  46. 	ADDQ    T2, T0          \

  47. 	MULXQ   16+I1, T3, T2   \   // T2:T3 = U0*V2

  48. 	ADCQ    T3, T1          \

  49. 	MULXQ   24+I1, T4, T3   \   // T3:T4 = U0*V3

  50. 	ADCQ    T4, T2          \

  51. 	\ // Column U1

  52. 	MOVQ    8+I0, DX        \

  53. 	ADCQ    $0, T3          \

  54. 	MULXQ   0+I1, T4, T5    \   // T5:T4 = U1*V0

  55. 	MULXQ   8+I1, T7, T6    \   // T6:T7 = U1*V1

  56. 	ADDQ    T7, T5          \

  57. 	MULXQ   16+I1, T8, T7   \   // T7:T8 = U1*V2

  58. 	ADCQ    T8, T6          \

  59. 	MULXQ   24+I1, T9, T8   \   // T8:T9 = U1*V3

  60. 	ADCQ    T9, T7          \

  61. 	ADCQ    $0, T8          \

  62. 	ADDQ    T0, T4          \

  63. 	MOVQ    T4, 8+O         \   // O[1]

  64. 	ADCQ    T1, T5          \

  65. 	ADCQ    T2, T6          \

  66. 	ADCQ    T3, T7          \

  67. 	\ // Column U2

  68. 	MOVQ    16+I0, DX       \

  69. 	ADCQ    $0, T8          \

  70. 	MULXQ   0+I1, T0, T1    \   // T1:T0 = U2*V0

  71. 	MULXQ   8+I1, T3, T2    \   // T2:T3 = U2*V1

  72. 	ADDQ    T3, T1          \

  73. 	MULXQ   16+I1, T4, T3   \   // T3:T4 = U2*V2

  74. 	ADCQ    T4, T2          \

  75. 	MULXQ   24+I1, T9, T4   \   // T4:T9 = U2*V3

  76. 	ADCQ    T9, T3          \

  77. 	\ // Column U3

  78. 	MOVQ    24+I0, DX       \

  79. 	ADCQ    $0, T4          \

  80. 	ADDQ    T5, T0          \

  81. 	MOVQ    T0, 16+O        \   // O[2]

  82. 	ADCQ    T6, T1          \

  83. 	ADCQ    T7, T2          \

  84. 	ADCQ    T8, T3          \

  85. 	ADCQ    $0, T4          \

  86. 	MULXQ   0+I1, T0, T5    \   // T5:T0 = U3*V0

  87. 	MULXQ   8+I1, T7, T6    \   // T6:T7 = U3*V1

  88. 	ADDQ    T7, T5          \

  89. 	MULXQ   16+I1, T8, T7   \   // T7:T8 = U3*V2

  90. 	ADCQ    T8, T6          \

  91. 	MULXQ   24+I1, T9, T8   \   // T8:T9 = U3*V3

  92. 	ADCQ    T9, T7          \

  93. 	ADCQ    $0, T8          \

  94. 	\ // Add values in remaining columns

  95. 	ADDQ    T0, T1          \

  96. 	MOVQ    T1, 24+O        \   // O[3]

  97. 	ADCQ    T5, T2          \

  98. 	MOVQ    T2, 32+O        \   // O[4]

  99. 	ADCQ    T6, T3          \

  100. 	MOVQ    T3, 40+O        \   // O[5]

  101. 	ADCQ    T7, T4          \

  102. 	MOVQ    T4, 48+O        \   // O[6]

  103. 	ADCQ    $0, T8          \   // O[7]

  104. 	MOVQ    T8, 56+O

  105. 

  106. // Performs schoolbook multiplication of 2 256-bit numbers. This optimized version

  107. // uses ADOX, ADCX and MULX instructions. Macro smashes values in AX and DX.

  108. // Input: I0 and I1.

  109. // Output: O

  110. // All the other arguments resgisters are used for storing temporary values

  111. #define MULS256_MULXADX(O, I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9) \

  112. 							\   // U0[0]

  113. 	MOVQ     0+I0, DX       \   // MULX requires multiplayer in DX

  114. 							\   // T0:T1 = I1*DX

  115. 	MULXQ    I1, T1, T0     \   // T0:T1 = U0*V0 (low:high)

  116. 	MOVQ     T1, O          \   // O0[0]

  117. 	MULXQ     8+I1, T2, T1  \   // T2:T1 = U0*V1

  118. 	XORQ     AX, AX         \

  119. 	ADOXQ    T2, T0         \

  120. 	MULXQ    16+I1, T3, T2  \   // T2:T3 = U0*V2

  121. 	ADOXQ    T3, T1         \

  122. 	MULXQ    24+I1, T4, T3  \   // T3:T4 = U0*V3

  123. 	ADOXQ    T4, T2         \

  124. 	\  // Column U1

  125. 	MOVQ      8+I0, DX      \

  126. 	MULXQ    I1, T4, T5     \   // T5:T4 = U1*V0

  127. 	ADOXQ    AX, T3         \

  128. 	XORQ     AX, AX         \

  129. 	MULXQ     8+I1, T7, T6  \   // T6:T7 = U1*V1

  130. 	ADOXQ    T0, T4         \

  131. 	MOVQ     T4, 8+O        \   // O[1]

  132. 	ADCXQ    T7, T5         \

  133. 	MULXQ    16+I1, T8, T7  \   // T7:T8 = U1*V2

  134. 	ADCXQ    T8, T6         \

  135. 	ADOXQ    T1, T5 \

  136. 	MULXQ    24+I1, T9, T8  \   // T8:T9 = U1*V3

  137. 	ADCXQ    T9, T7         \

  138. 	ADCXQ    AX, T8         \

  139. 	ADOXQ    T2, T6         \

  140. 	\ // Column U2

  141. 	MOVQ     16+I0, DX      \

  142. 	MULXQ    I1, T0, T1     \   // T1:T0 = U2*V0

  143. 	ADOXQ    T3, T7         \

  144. 	ADOXQ    AX, T8         \

  145. 	XORQ     AX, AX         \

  146. 	MULXQ    8+I1, T3, T2   \   // T2:T3 = U2*V1

  147. 	ADOXQ    T5, T0         \

  148. 	MOVQ     T0, 16+O       \   // O[2]

  149. 	ADCXQ    T3, T1         \

  150. 	MULXQ    16+I1, T4, T3  \   // T3:T4 = U2*V2

  151. 	ADCXQ    T4, T2         \

  152. 	ADOXQ    T6, T1         \

  153. 	MULXQ    24+I1, T9, T4  \   // T9:T4 = U2*V3

  154. 	ADCXQ    T9, T3         \

  155. 	MOVQ     24+I0, DX      \

  156. 	ADCXQ    AX, T4         \

  157. 	\

  158. 	ADOXQ    T7, T2         \

  159. 	ADOXQ    T8, T3         \

  160. 	ADOXQ    AX, T4         \

  161. 	\ // Column U3

  162. 	MULXQ    I1, T0, T5     \   // T5:T0 = U3*B0

  163. 	XORQ     AX, AX         \

  164. 	MULXQ    8+I1, T7, T6   \   // T6:T7 = U3*B1

  165. 	ADCXQ    T7, T5         \

  166. 	ADOXQ    T0, T1         \

  167. 	MULXQ    16+I1, T8, T7  \   // T7:T8 = U3*V2

  168. 	ADCXQ    T8, T6         \

  169. 	ADOXQ    T5, T2         \

  170. 	MULXQ    24+I1, T9, T8  \   // T8:T9 = U3*V3

  171. 	ADCXQ    T9, T7         \

  172. 	ADCXQ    AX, T8         \

  173. 	\

  174. 	ADOXQ   T6, T3          \

  175. 	ADOXQ   T7, T4          \

  176. 	ADOXQ   AX, T8          \

  177. 	MOVQ    T1, 24+O        \   // O[3]

  178. 	MOVQ    T2, 32+O        \   // O[4]

  179. 	MOVQ    T3, 40+O        \   // O[5]

  180. 	MOVQ    T4, 48+O        \   // O[6] and O[7] below

  181. 	MOVQ    T8, 56+O

  182. 

  183. // Template of a macro that performs schoolbook multiplication of 128-bit with 320-bit

  184. // number. It uses MULX instruction This template must be customized with functions

  185. // performing ADD (add1, add2) and ADD-with-carry (adc1, adc2). addX/adcX may or may

  186. // not be instructions that use two independent carry chains.

  187. // Input:

  188. //   * I0 128-bit number

  189. //   * I1 320-bit number

  190. //   * add1, add2: instruction performing integer addition and starting carry chain

  191. //   * adc1, adc2: instruction performing integer addition with carry

  192. // Output: T[0-6] registers

  193. #define MULS_128x320(I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, add1, add2, adc1, adc2) \

  194. 	\ // Column 0

  195. 	MOVQ    I0, DX              \

  196. 	MULXQ   I1+24(SB), T0, T1   \

  197. 	MULXQ   I1+32(SB), T4, T2   \

  198. 	XORQ    AX, AX              \

  199. 	MULXQ   I1+40(SB), T5, T3   \

  200. 	add1    T4, T1              \

  201. 	adc1    T5, T2              \

  202. 	MULXQ   I1+48(SB), T7, T4   \

  203. 	adc1    T7, T3              \

  204. 	MULXQ   I1+56(SB), T6, T5   \

  205. 	adc1    T6, T4              \

  206. 	adc1    AX, T5              \

  207. 	\ // Column 1

  208. 	MOVQ    8+I0, DX            \

  209. 	MULXQ   I1+24(SB), T6, T7   \

  210. 	add2    T6, T1              \

  211. 	adc2    T7, T2              \

  212. 	MULXQ   I1+32(SB), T8, T6   \

  213. 	adc2    T6, T3              \

  214. 	MULXQ   I1+40(SB), T7, T9   \

  215. 	adc2    T9, T4              \

  216. 	MULXQ   I1+48(SB), T9, T6   \

  217. 	adc2    T6, T5              \

  218. 	MULXQ   I1+56(SB), DX, T6   \

  219. 	adc2    AX, T6              \

  220. 	\ // Output

  221. 	XORQ    AX, AX              \

  222. 	add1    T8, T2              \

  223. 	adc1    T7, T3              \

  224. 	adc1    T9, T4              \

  225. 	adc1    DX, T5              \

  226. 	adc1    AX, T6

  227. 

  228. // Multiplies 128-bit with 320-bit integer. Optimized with MULX instruction.

  229. #define MULS_128x320_MULX(I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9) \

  230. 	MULS_128x320(I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, ADDQ, ADDQ, ADCQ, ADCQ)

  231. 

  232. // Multiplies 128-bit with 320-bit integer. Optimized with  MULX, ADOX and ADCX instructions

  233. #define MULS_128x320_MULXADX(I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9) \

  234. 	MULS_128x320(I0, I1, T0, T1, T2, T3, T4, T5, T6, T7, T8, T9, ADOXQ, ADCXQ, ADOXQ, ADCXQ)

  235. 

  236. // Template of a macro performing multiplication of two 512-bit numbers. It uses one

  237. // level of Karatsuba and one level of schoolbook multiplication. Template must be

  238. // customized with macro performing schoolbook multiplication.

  239. // Input:

  240. //  * I0, I1 - two 512-bit numbers

  241. //  * MULS - either MULS256_MULX or MULS256_MULXADX

  242. // Output: OUT - 1024-bit long

  243. #define MUL(OUT, I0, I1, MULS) \

  244. 	\ // R[8-11]: U1+U0

  245. 	XORQ    AX, AX  \

  246. 	MOVQ    ( 0)(I0), R8    \

  247. 	MOVQ    ( 8)(I0), R9    \

  248. 	MOVQ    (16)(I0), R10   \

  249. 	MOVQ    (24)(I0), R11   \

  250. 	ADDQ    (32)(I0), R8    \

  251. 	ADCQ    (40)(I0), R9    \

  252. 	ADCQ    (48)(I0), R10   \

  253. 	ADCQ    (56)(I0), R11   \

  254. 	SBBQ    $0, AX          \ // store mask

  255. 	MOVQ    R8,  ( 0)(SP)   \

  256. 	MOVQ    R9,  ( 8)(SP)   \

  257. 	MOVQ    R10, (16)(SP)   \

  258. 	MOVQ    R11, (24)(SP)   \

  259. 	\

  260. 	\ // R[12-15]: V1+V0

  261. 	XORQ    BX, BX          \

  262. 	MOVQ    ( 0)(I1), R12   \

  263. 	MOVQ    ( 8)(I1), R13   \

  264. 	MOVQ    (16)(I1), R14   \

  265. 	MOVQ    (24)(I1), R15   \

  266. 	ADDQ    (32)(I1), R12   \

  267. 	ADCQ    (40)(I1), R13   \

  268. 	ADCQ    (48)(I1), R14   \

  269. 	ADCQ    (56)(I1), R15   \

  270. 	SBBQ    $0, BX          \ // store mask

  271. 	MOVQ    R12, (32)(SP)   \

  272. 	MOVQ    R13, (40)(SP)   \

  273. 	MOVQ    R14, (48)(SP)   \

  274. 	MOVQ    R15, (56)(SP)   \

  275. 	\ // Prepare mask for U0+U1 (U1+U0 mod 256^4 if U1+U0 sets carry flag, otherwise 0)

  276. 	ANDQ    AX, R12         \

  277. 	ANDQ    AX, R13         \

  278. 	ANDQ    AX, R14         \

  279. 	ANDQ    AX, R15         \

  280. 	\ // Prepare mask for V0+V1 (V1+V0 mod 256^4 if U1+U0 sets carry flag, otherwise 0)

  281. 	ANDQ    BX, R8          \

  282. 	ANDQ    BX, R9          \

  283. 	ANDQ    BX, R10         \

  284. 	ANDQ    BX, R11         \

  285. 	\ // res = masked(U0+U1) + masked(V0 + V1)

  286. 	ADDQ    R12, R8         \

  287. 	ADCQ    R13, R9         \

  288. 	ADCQ    R14, R10        \

  289. 	ADCQ    R15, R11        \

  290. 	\ // SP[64-96] <- res

  291. 	MOVQ     R8, (64)(SP)   \

  292. 	MOVQ     R9, (72)(SP)   \

  293. 	MOVQ    R10, (80)(SP)   \

  294. 	MOVQ    R11, (88)(SP)   \

  295. 	\ // BP will be used for schoolbook multiplication below

  296. 	MOVQ    BP, 96(SP)  \

  297. 	\ // (U1+U0)*(V1+V0)

  298. 	MULS((64)(OUT), 0(SP), 32(SP), R8, R9, R10, R11, R12, R13, R14, R15, BX, BP)    \

  299. 	\ // U0 x V0

  300. 	MULS(0(OUT), 0(I0), 0(I1), R8, R9, R10, R11, R12, R13, R14, R15, BX, BP)    \

  301. 	\ // U1 x V1

  302. 	MULS(0(SP), 32(I0), 32(I1), R8, R9, R10, R11, R12, R13, R14, R15, BX, BP)  \

  303. 	\ // Recover BP

  304. 	MOVQ    96(SP), BP  \

  305. 	\ // Final part of schoolbook multiplication; R[8-11] = (U0+U1) x (V0+V1)

  306. 	MOVQ    (64)(SP), R8    \

  307. 	MOVQ    (72)(SP), R9    \

  308. 	MOVQ    (80)(SP), R10   \

  309. 	MOVQ    (88)(SP), R11   \

  310. 	MOVQ    (96)(OUT), AX   \

  311. 	ADDQ    AX, R8          \

  312. 	MOVQ    (104)(OUT), AX  \

  313. 	ADCQ    AX, R9          \

  314. 	MOVQ    (112)(OUT), AX  \

  315. 	ADCQ    AX, R10         \

  316. 	MOVQ    (120)(OUT), AX  \

  317. 	ADCQ    AX, R11 \

  318. 	\ // R[12-15, 8-11] = (U0+U1) x (V0+V1) - U0xV0

  319. 	MOVQ    (64)(OUT), R12  \

  320. 	MOVQ    (72)(OUT), R13  \

  321. 	MOVQ    (80)(OUT), R14  \

  322. 	MOVQ    (88)(OUT), R15  \

  323. 	SUBQ    ( 0)(OUT), R12  \

  324. 	SBBQ    ( 8)(OUT), R13  \

  325. 	SBBQ    (16)(OUT), R14  \

  326. 	SBBQ    (24)(OUT), R15  \

  327. 	SBBQ    (32)(OUT), R8   \

  328. 	SBBQ    (40)(OUT), R9   \

  329. 	SBBQ    (48)(OUT), R10  \

  330. 	SBBQ    (56)(OUT), R11  \

  331. 	\ // r8-r15 <- (U0+U1) x (V0+V1) - U0xV0 - U1xV1

  332. 	SUBQ    ( 0)(SP), R12   \

  333. 	SBBQ    ( 8)(SP), R13   \

  334. 	SBBQ    (16)(SP), R14   \

  335. 	SBBQ    (24)(SP), R15   \

  336. 	SBBQ    (32)(SP), R8    \

  337. 	SBBQ    (40)(SP), R9    \

  338. 	SBBQ    (48)(SP), R10   \

  339. 	SBBQ    (56)(SP), R11   \

  340. 	\

  341. 	;                       ADDQ   (32)(OUT), R12; MOVQ    R12, ( 32)(OUT) \

  342. 	;                       ADCQ   (40)(OUT), R13; MOVQ    R13, ( 40)(OUT) \

  343. 	;                       ADCQ   (48)(OUT), R14; MOVQ    R14, ( 48)(OUT) \

  344. 	;                       ADCQ   (56)(OUT), R15; MOVQ    R15, ( 56)(OUT) \

  345. 	MOVQ    ( 0)(SP), AX;   ADCQ    AX,  R8;       MOVQ     R8, ( 64)(OUT) \

  346. 	MOVQ    ( 8)(SP), AX;   ADCQ    AX,  R9;       MOVQ     R9, ( 72)(OUT) \

  347. 	MOVQ    (16)(SP), AX;   ADCQ    AX, R10;       MOVQ    R10, ( 80)(OUT) \

  348. 	MOVQ    (24)(SP), AX;   ADCQ    AX, R11;       MOVQ    R11, ( 88)(OUT) \

  349. 	MOVQ    (32)(SP), R12;  ADCQ    $0, R12;       MOVQ    R12, ( 96)(OUT) \

  350. 	MOVQ    (40)(SP), R13;  ADCQ    $0, R13;       MOVQ    R13, (104)(OUT) \

  351. 	MOVQ    (48)(SP), R14;  ADCQ    $0, R14;       MOVQ    R14, (112)(OUT) \

  352. 	MOVQ    (56)(SP), R15;  ADCQ    $0, R15;       MOVQ    R15, (120)(OUT)

  353. 

  354. // Template for calculating the Montgomery reduction algorithm described in

  355. // section 5.2.3 of https://eprint.iacr.org/2017/1015.pdf. Template must be

  356. // customized with schoolbook multiplicaton for 128 x 320-bit number.

  357. // This macro reuses memory of IN value and *changes* it. Smashes registers

  358. // R[8-15], BX, CX

  359. // Input:

  360. //    * IN: 1024-bit number to be reduced

  361. //    * MULS: either MULS_128x320_MULX or MULS_128x320_MULXADX

  362. // Output: OUT 512-bit

  363. #define REDC(OUT, IN, MULS) \

  364. 	MULS(0(IN), ·p503p1, R8, R9, R10, R11, R12, R13, R14, BX, CX, R15) \

  365. 	XORQ    R15, R15        \

  366. 	ADDQ    (24)(IN), R8    \

  367. 	ADCQ    (32)(IN), R9    \

  368. 	ADCQ    (40)(IN), R10   \

  369. 	ADCQ    (48)(IN), R11   \

  370. 	ADCQ    (56)(IN), R12   \

  371. 	ADCQ    (64)(IN), R13   \

  372. 	ADCQ    (72)(IN), R14   \

  373. 	ADCQ    (80)(IN), R15   \

  374. 	MOVQ    R8, (24)(IN)    \

  375. 	MOVQ    R9, (32)(IN)    \

  376. 	MOVQ    R10, (40)(IN)   \

  377. 	MOVQ    R11, (48)(IN)   \

  378. 	MOVQ    R12, (56)(IN)   \

  379. 	MOVQ    R13, (64)(IN)   \

  380. 	MOVQ    R14, (72)(IN)   \

  381. 	MOVQ    R15, (80)(IN)   \

  382. 	MOVQ    (88)(IN), R8    \

  383. 	MOVQ    (96)(IN), R9    \

  384. 	MOVQ    (104)(IN), R10  \

  385. 	MOVQ    (112)(IN), R11  \

  386. 	MOVQ    (120)(IN), R12  \

  387. 	ADCQ    $0, R8          \

  388. 	ADCQ    $0, R9          \

  389. 	ADCQ    $0, R10         \

  390. 	ADCQ    $0, R11         \

  391. 	ADCQ    $0, R12         \

  392. 	MOVQ    R8, (88)(IN)    \

  393. 	MOVQ    R9, (96)(IN)    \

  394. 	MOVQ    R10, (104)(IN)  \

  395. 	MOVQ    R11, (112)(IN)  \

  396. 	MOVQ    R12, (120)(IN)  \

  397. 	\

  398. 	MULS(16(IN), ·p503p1, R8, R9, R10, R11, R12, R13, R14, BX, CX, R15)    \

  399. 	XORQ    R15, R15        \

  400. 	ADDQ    (40)(IN), R8    \

  401. 	ADCQ    (48)(IN), R9    \

  402. 	ADCQ    (56)(IN), R10   \

  403. 	ADCQ    (64)(IN), R11   \

  404. 	ADCQ    (72)(IN), R12   \

  405. 	ADCQ    (80)(IN), R13   \

  406. 	ADCQ    (88)(IN), R14   \

  407. 	ADCQ    (96)(IN), R15   \

  408. 	MOVQ    R8, (40)(IN)    \

  409. 	MOVQ    R9, (48)(IN)    \

  410. 	MOVQ    R10, (56)(IN)   \

  411. 	MOVQ    R11, (64)(IN)   \

  412. 	MOVQ    R12, (72)(IN)   \

  413. 	MOVQ    R13, (80)(IN)   \

  414. 	MOVQ    R14, (88)(IN)   \

  415. 	MOVQ    R15, (96)(IN)   \

  416. 	MOVQ    (104)(IN), R8   \

  417. 	MOVQ    (112)(IN), R9   \

  418. 	MOVQ    (120)(IN), R10  \

  419. 	ADCQ    $0, R8          \

  420. 	ADCQ    $0, R9          \

  421. 	ADCQ    $0, R10         \

  422. 	MOVQ    R8, (104)(IN)   \

  423. 	MOVQ    R9, (112)(IN)   \

  424. 	MOVQ    R10, (120)(IN)  \

  425. 	\

  426. 	MULS(32(IN), ·p503p1, R8, R9, R10, R11, R12, R13, R14, BX, CX, R15)    \

  427. 	XORQ    R15, R15        \

  428. 	XORQ    BX, BX          \

  429. 	ADDQ    ( 56)(IN), R8   \

  430. 	ADCQ    ( 64)(IN), R9   \

  431. 	ADCQ    ( 72)(IN), R10  \

  432. 	ADCQ    ( 80)(IN), R11  \

  433. 	ADCQ    ( 88)(IN), R12  \

  434. 	ADCQ    ( 96)(IN), R13  \

  435. 	ADCQ    (104)(IN), R14  \

  436. 	ADCQ    (112)(IN), R15  \

  437. 	ADCQ    (120)(IN), BX   \

  438. 	MOVQ    R8,  ( 56)(IN)  \

  439. 	MOVQ    R10, ( 72)(IN)  \

  440. 	MOVQ    R11, ( 80)(IN)  \

  441. 	MOVQ    R12, ( 88)(IN)  \

  442. 	MOVQ    R13, ( 96)(IN)  \

  443. 	MOVQ    R14, (104)(IN)  \

  444. 	MOVQ    R15, (112)(IN)  \

  445. 	MOVQ    BX,  (120)(IN)  \

  446. 	MOVQ    R9,  (  0)(OUT) \ // Result: OUT[0]

  447. 	\

  448. 	MULS(48(IN), ·p503p1, R8, R9, R10, R11, R12, R13, R14, BX, CX, R15)    \

  449. 	ADDQ    ( 72)(IN), R8   \

  450. 	ADCQ    ( 80)(IN), R9   \

  451. 	ADCQ    ( 88)(IN), R10  \

  452. 	ADCQ    ( 96)(IN), R11  \

  453. 	ADCQ    (104)(IN), R12  \

  454. 	ADCQ    (112)(IN), R13  \

  455. 	ADCQ    (120)(IN), R14  \

  456. 	MOVQ    R8,  ( 8)(OUT)  \ // Result: OUT[1]

  457. 	MOVQ    R9,  (16)(OUT)  \ // Result: OUT[2]

  458. 	MOVQ    R10, (24)(OUT)  \ // Result: OUT[3]

  459. 	MOVQ    R11, (32)(OUT)  \ // Result: OUT[4]

  460. 	MOVQ    R12, (40)(OUT)  \ // Result: OUT[5]

  461. 	MOVQ    R13, (48)(OUT)  \ // Result: OUT[6] and OUT[7]

  462. 	MOVQ    R14, (56)(OUT)

  463. 

  464. TEXT ·fp503StrongReduce(SB), NOSPLIT, $0-8

  465. 	MOVQ	x+0(FP), REG_P1

  466. 

  467. 	// Zero AX for later use:

  468. 	XORQ	AX, AX

  469. 

  470. 	// Load p into registers:

  471. 	MOVQ	P503_0, R8

  472. 	// P503_{1,2} = P503_0, so reuse R8

  473. 	MOVQ	P503_3, R9

  474. 	MOVQ	P503_4, R10

  475. 	MOVQ	P503_5, R11

  476. 	MOVQ	P503_6, R12

  477. 	MOVQ	P503_7, R13

  478. 

  479. 	// Set x <- x - p

  480. 	SUBQ	R8,  ( 0)(REG_P1)

  481. 	SBBQ	R8,  ( 8)(REG_P1)

  482. 	SBBQ	R8,  (16)(REG_P1)

  483. 	SBBQ	R9,  (24)(REG_P1)

  484. 	SBBQ	R10, (32)(REG_P1)

  485. 	SBBQ	R11, (40)(REG_P1)

  486. 	SBBQ	R12, (48)(REG_P1)

  487. 	SBBQ	R13, (56)(REG_P1)

  488. 

  489. 	// Save carry flag indicating x-p < 0 as a mask

  490. 	SBBQ	$0, AX

  491. 

  492. 	// Conditionally add p to x if x-p < 0

  493. 	ANDQ	AX, R8

  494. 	ANDQ	AX, R9

  495. 	ANDQ	AX, R10

  496. 	ANDQ	AX, R11

  497. 	ANDQ	AX, R12

  498. 	ANDQ	AX, R13

  499. 

  500. 	ADDQ	R8, ( 0)(REG_P1)

  501. 	ADCQ	R8, ( 8)(REG_P1)

  502. 	ADCQ	R8, (16)(REG_P1)

  503. 	ADCQ	R9, (24)(REG_P1)

  504. 	ADCQ	R10,(32)(REG_P1)

  505. 	ADCQ	R11,(40)(REG_P1)

  506. 	ADCQ	R12,(48)(REG_P1)

  507. 	ADCQ	R13,(56)(REG_P1)

  508. 

  509. 	RET

  510. 

  511. TEXT ·fp503ConditionalSwap(SB),NOSPLIT,$0-17

  512. 

  513. 	MOVQ	x+0(FP), REG_P1

  514. 	MOVQ	y+8(FP), REG_P2

  515. 	MOVB	choice+16(FP), AL	// AL = 0 or 1

  516. 	MOVBLZX	AL, AX				// AX = 0 or 1

  517. 	NEGQ	AX					// AX = 0x00..00 or 0xff..ff

  518. 

  519. #ifndef CSWAP_BLOCK

  520. #define CSWAP_BLOCK(idx) 	\

  521. 	MOVQ	(idx*8)(REG_P1), BX	\ // BX = x[idx]

  522. 	MOVQ 	(idx*8)(REG_P2), CX	\ // CX = y[idx]

  523. 	MOVQ	CX, DX				\ // DX = y[idx]

  524. 	XORQ	BX, DX				\ // DX = y[idx] ^ x[idx]

  525. 	ANDQ	AX, DX				\ // DX = (y[idx] ^ x[idx]) & mask

  526. 	XORQ	DX, BX				\ // BX = (y[idx] ^ x[idx]) & mask) ^ x[idx] = x[idx] or y[idx]

  527. 	XORQ	DX, CX				\ // CX = (y[idx] ^ x[idx]) & mask) ^ y[idx] = y[idx] or x[idx]

  528. 	MOVQ	BX, (idx*8)(REG_P1)	\

  529. 	MOVQ	CX, (idx*8)(REG_P2)

  530. #endif

  531. 

  532. 	CSWAP_BLOCK(0)

  533. 	CSWAP_BLOCK(1)

  534. 	CSWAP_BLOCK(2)

  535. 	CSWAP_BLOCK(3)

  536. 	CSWAP_BLOCK(4)

  537. 	CSWAP_BLOCK(5)

  538. 	CSWAP_BLOCK(6)

  539. 	CSWAP_BLOCK(7)

  540. 

  541. #ifdef CSWAP_BLOCK

  542. #undef CSWAP_BLOCK

  543. #endif

  544. 

  545. 	RET

  546. 

  547. TEXT ·fp503AddReduced(SB),NOSPLIT,$0-24

  548. 

  549. 	MOVQ	z+0(FP), REG_P3

  550. 	MOVQ	x+8(FP), REG_P1

  551. 	MOVQ	y+16(FP), REG_P2

  552. 

  553. 	// Used later to calculate a mask

  554. 	XORQ    CX, CX

  555. 

  556. 	// [R8-R15]: z = x + y

  557. 	MOVQ	( 0)(REG_P1), R8

  558. 	MOVQ	( 8)(REG_P1), R9

  559. 	MOVQ	(16)(REG_P1), R10

  560. 	MOVQ	(24)(REG_P1), R11

  561. 	MOVQ	(32)(REG_P1), R12

  562. 	MOVQ	(40)(REG_P1), R13

  563. 	MOVQ	(48)(REG_P1), R14

  564. 	MOVQ	(56)(REG_P1), R15

  565. 	ADDQ	( 0)(REG_P2), R8

  566. 	ADCQ	( 8)(REG_P2), R9

  567. 	ADCQ	(16)(REG_P2), R10

  568. 	ADCQ	(24)(REG_P2), R11

  569. 	ADCQ	(32)(REG_P2), R12

  570. 	ADCQ	(40)(REG_P2), R13

  571. 	ADCQ	(48)(REG_P2), R14

  572. 	ADCQ	(56)(REG_P2), R15

  573. 

  574. 	MOVQ    P503X2_0, AX

  575. 	SUBQ    AX, R8

  576. 	MOVQ    P503X2_1, AX

  577. 	SBBQ    AX, R9

  578. 	SBBQ    AX, R10

  579. 	MOVQ    P503X2_3, AX

  580. 	SBBQ    AX, R11

  581. 	MOVQ    P503X2_4, AX

  582. 	SBBQ    AX, R12

  583. 	MOVQ    P503X2_5, AX

  584. 	SBBQ    AX, R13

  585. 	MOVQ    P503X2_6, AX

  586. 	SBBQ    AX, R14

  587. 	MOVQ    P503X2_7, AX

  588. 	SBBQ    AX, R15

  589. 

  590. 	// mask

  591. 	SBBQ    $0, CX

  592. 

  593. 	// move z to REG_P3

  594. 	MOVQ    R8,  ( 0)(REG_P3)

  595. 	MOVQ    R9,  ( 8)(REG_P3)

  596. 	MOVQ    R10, (16)(REG_P3)

  597. 	MOVQ    R11, (24)(REG_P3)

  598. 	MOVQ    R12, (32)(REG_P3)

  599. 	MOVQ    R13, (40)(REG_P3)

  600. 	MOVQ    R14, (48)(REG_P3)

  601. 	MOVQ    R15, (56)(REG_P3)

  602. 

  603. 	// if z<0 add p503x2 back

  604. 	MOVQ    P503X2_0,   R8

  605. 	MOVQ    P503X2_1,   R9

  606. 	MOVQ    P503X2_3,   R10

  607. 	MOVQ    P503X2_4,   R11

  608. 	MOVQ    P503X2_5,   R12

  609. 	MOVQ    P503X2_6,   R13

  610. 	MOVQ    P503X2_7,   R14

  611. 	ANDQ    CX, R8

  612. 	ANDQ    CX, R9

  613. 	ANDQ    CX, R10

  614. 	ANDQ    CX, R11

  615. 	ANDQ    CX, R12

  616. 	ANDQ    CX, R13

  617. 	ANDQ    CX, R14

  618. 	MOVQ    ( 0)(REG_P3), AX; ADDQ    R8,  AX; MOVQ    AX, ( 0)(REG_P3)

  619. 	MOVQ    ( 8)(REG_P3), AX; ADCQ    R9,  AX; MOVQ    AX, ( 8)(REG_P3)

  620. 	MOVQ    (16)(REG_P3), AX; ADCQ    R9,  AX; MOVQ    AX, (16)(REG_P3)

  621. 	MOVQ    (24)(REG_P3), AX; ADCQ    R10, AX; MOVQ    AX, (24)(REG_P3)

  622. 	MOVQ    (32)(REG_P3), AX; ADCQ    R11, AX; MOVQ    AX, (32)(REG_P3)

  623. 	MOVQ    (40)(REG_P3), AX; ADCQ    R12, AX; MOVQ    AX, (40)(REG_P3)

  624. 	MOVQ    (48)(REG_P3), AX; ADCQ    R13, AX; MOVQ    AX, (48)(REG_P3)

  625. 	MOVQ    (56)(REG_P3), AX; ADCQ    R14, AX; MOVQ    AX, (56)(REG_P3)

  626. 	RET

  627. 

  628. TEXT ·fp503SubReduced(SB), NOSPLIT, $0-24

  629. 

  630. 	MOVQ    z+0(FP), REG_P3

  631. 	MOVQ    x+8(FP), REG_P1

  632. 	MOVQ    y+16(FP), REG_P2

  633. 

  634. 	// Used later to calculate a mask

  635. 	XORQ    CX, CX

  636. 

  637. 	MOVQ    ( 0)(REG_P1), R8

  638. 	MOVQ    ( 8)(REG_P1), R9

  639. 	MOVQ    (16)(REG_P1), R10

  640. 	MOVQ    (24)(REG_P1), R11

  641. 	MOVQ    (32)(REG_P1), R12

  642. 	MOVQ    (40)(REG_P1), R13

  643. 	MOVQ    (48)(REG_P1), R14

  644. 	MOVQ    (56)(REG_P1), R15

  645. 

  646. 	SUBQ    ( 0)(REG_P2), R8

  647. 	SBBQ    ( 8)(REG_P2), R9

  648. 	SBBQ    (16)(REG_P2), R10

  649. 	SBBQ    (24)(REG_P2), R11

  650. 	SBBQ    (32)(REG_P2), R12

  651. 	SBBQ    (40)(REG_P2), R13

  652. 	SBBQ    (48)(REG_P2), R14

  653. 	SBBQ    (56)(REG_P2), R15

  654. 

  655. 	// mask

  656. 	SBBQ    $0, CX

  657. 

  658. 	// store x-y in REG_P3

  659. 	MOVQ    R8,  ( 0)(REG_P3)

  660. 	MOVQ    R9,  ( 8)(REG_P3)

  661. 	MOVQ    R10, (16)(REG_P3)

  662. 	MOVQ    R11, (24)(REG_P3)

  663. 	MOVQ    R12, (32)(REG_P3)

  664. 	MOVQ    R13, (40)(REG_P3)

  665. 	MOVQ    R14, (48)(REG_P3)

  666. 	MOVQ    R15, (56)(REG_P3)

  667. 

  668. 	// if z<0 add p503x2 back

  669. 	MOVQ    P503X2_0, R8

  670. 	MOVQ    P503X2_1, R9

  671. 	MOVQ    P503X2_3, R10

  672. 	MOVQ    P503X2_4, R11

  673. 	MOVQ    P503X2_5, R12

  674. 	MOVQ    P503X2_6, R13

  675. 	MOVQ    P503X2_7, R14

  676. 	ANDQ    CX, R8

  677. 	ANDQ    CX, R9

  678. 	ANDQ    CX, R10

  679. 	ANDQ    CX, R11

  680. 	ANDQ    CX, R12

  681. 	ANDQ    CX, R13

  682. 	ANDQ    CX, R14

  683. 	MOVQ    ( 0)(REG_P3), AX; ADDQ    R8,  AX; MOVQ    AX, ( 0)(REG_P3)

  684. 	MOVQ    ( 8)(REG_P3), AX; ADCQ    R9,  AX; MOVQ    AX, ( 8)(REG_P3)

  685. 	MOVQ    (16)(REG_P3), AX; ADCQ    R9,  AX; MOVQ    AX, (16)(REG_P3)

  686. 	MOVQ    (24)(REG_P3), AX; ADCQ    R10, AX; MOVQ    AX, (24)(REG_P3)

  687. 	MOVQ    (32)(REG_P3), AX; ADCQ    R11, AX; MOVQ    AX, (32)(REG_P3)

  688. 	MOVQ    (40)(REG_P3), AX; ADCQ    R12, AX; MOVQ    AX, (40)(REG_P3)

  689. 	MOVQ    (48)(REG_P3), AX; ADCQ    R13, AX; MOVQ    AX, (48)(REG_P3)

  690. 	MOVQ    (56)(REG_P3), AX; ADCQ    R14, AX; MOVQ    AX, (56)(REG_P3)

  691. 

  692. 	RET

  693. 

  694. TEXT ·fp503Mul(SB), NOSPLIT, $104-24

  695. 	MOVQ    z+ 0(FP), CX

  696. 	MOVQ    x+ 8(FP), REG_P1

  697. 	MOVQ    y+16(FP), REG_P2

  698. 

  699. 	// Check wether to use optimized implementation

  700. 	CMPB    ·useADXMULX(SB), $1

  701. 	JE      mul_with_mulx_adx

  702. 	CMPB    ·useMULX(SB), $1

  703. 	JE      mul_with_mulx

  704. 

  705. 	// Generic x86 implementation (below) uses variant of Karatsuba method.

  706. 	//

  707. 	// Here we store the destination in CX instead of in REG_P3 because the

  708. 	// multiplication instructions use DX as an implicit destination

  709. 	// operand: MULQ $REG sets DX:AX <-- AX * $REG.

  710. 

  711. 	// RAX and RDX will be used for a mask (0-borrow)

  712. 	XORQ	AX, AX

  713. 

  714. 	// RCX[0-3]: U1+U0

  715. 	MOVQ	(32)(REG_P1), R8

  716. 	MOVQ	(40)(REG_P1), R9

  717. 	MOVQ	(48)(REG_P1), R10

  718. 	MOVQ	(56)(REG_P1), R11

  719. 	ADDQ	( 0)(REG_P1), R8

  720. 	ADCQ	( 8)(REG_P1), R9

  721. 	ADCQ	(16)(REG_P1), R10

  722. 	ADCQ	(24)(REG_P1), R11

  723. 	MOVQ	R8,  ( 0)(CX)

  724. 	MOVQ	R9,  ( 8)(CX)

  725. 	MOVQ	R10, (16)(CX)

  726. 	MOVQ	R11, (24)(CX)

  727. 

  728. 	SBBQ	$0, AX

  729. 

  730. 	// R12-R15: V1+V0

  731. 	XORQ	DX, DX

  732. 	MOVQ	(32)(REG_P2), R12

  733. 	MOVQ	(40)(REG_P2), R13

  734. 	MOVQ	(48)(REG_P2), R14

  735. 	MOVQ	(56)(REG_P2), R15

  736. 	ADDQ	( 0)(REG_P2), R12

  737. 	ADCQ	( 8)(REG_P2), R13

  738. 	ADCQ	(16)(REG_P2), R14

  739. 	ADCQ	(24)(REG_P2), R15

  740. 

  741. 	SBBQ	$0, DX

  742. 

  743. 	// Store carries on stack

  744. 	MOVQ	AX, (64)(SP)

  745. 	MOVQ	DX, (72)(SP)

  746. 

  747. 	// (SP[0-3],R8,R9,R10,R11) <- (U0+U1)*(V0+V1).

  748. 	// MUL using comba; In comments below U=U0+U1 V=V0+V1

  749. 

  750. 	// U0*V0

  751. 	MOVQ    (CX), AX

  752. 	MULQ    R12

  753. 	MOVQ    AX, (SP)        // C0

  754. 	MOVQ    DX, R8

  755. 

  756. 	// U0*V1

  757. 	XORQ    R9, R9

  758. 	MOVQ    (CX), AX

  759. 	MULQ    R13

  760. 	ADDQ    AX, R8

  761. 	ADCQ    DX, R9

  762. 

  763. 	// U1*V0

  764. 	XORQ    R10, R10

  765. 	MOVQ    (8)(CX), AX

  766. 	MULQ    R12

  767. 	ADDQ    AX, R8

  768. 	MOVQ    R8, (8)(SP)     // C1

  769. 	ADCQ    DX, R9

  770. 	ADCQ    $0, R10

  771. 

  772. 	// U0*V2

  773. 	XORQ    R8, R8

  774. 	MOVQ    (CX), AX

  775. 	MULQ    R14

  776. 	ADDQ    AX, R9

  777. 	ADCQ    DX, R10

  778. 	ADCQ    $0, R8

  779. 

  780. 	// U2*V0

  781. 	MOVQ    (16)(CX), AX

  782. 	MULQ    R12

  783. 	ADDQ    AX, R9

  784. 	ADCQ    DX, R10

  785. 	ADCQ    $0, R8

  786. 

  787. 	// U1*V1

  788. 	MOVQ    (8)(CX), AX

  789. 	MULQ    R13

  790. 	ADDQ    AX, R9

  791. 	MOVQ    R9, (16)(SP)        // C2

  792. 	ADCQ    DX, R10

  793. 	ADCQ    $0, R8

  794. 

  795. 	// U0*V3

  796. 	XORQ    R9, R9

  797. 	MOVQ    (CX), AX

  798. 	MULQ    R15

  799. 	ADDQ    AX, R10

  800. 	ADCQ    DX, R8

  801. 	ADCQ    $0, R9

  802. 

  803. 	// U3*V0

  804. 	MOVQ    (24)(CX), AX

  805. 	MULQ    R12

  806. 	ADDQ    AX, R10

  807. 	ADCQ    DX, R8

  808. 	ADCQ    $0, R9

  809. 

  810. 	// U1*V2

  811. 	MOVQ    (8)(CX), AX

  812. 	MULQ    R14

  813. 	ADDQ    AX, R10

  814. 	ADCQ    DX, R8

  815. 	ADCQ    $0, R9

  816. 

  817. 	// U2*V1

  818. 	MOVQ    (16)(CX), AX

  819. 	MULQ    R13

  820. 	ADDQ    AX, R10

  821. 	MOVQ    R10, (24)(SP)       // C3

  822. 	ADCQ    DX, R8

  823. 	ADCQ    $0, R9

  824. 

  825. 	// U1*V3

  826. 	XORQ    R10, R10

  827. 	MOVQ    (8)(CX), AX

  828. 	MULQ    R15

  829. 	ADDQ    AX, R8

  830. 	ADCQ    DX, R9

  831. 	ADCQ    $0, R10

  832. 

  833. 	// U3*V1

  834. 	MOVQ    (24)(CX), AX

  835. 	MULQ    R13

  836. 	ADDQ    AX, R8

  837. 	ADCQ    DX, R9

  838. 	ADCQ    $0, R10

  839. 

  840. 	// U2*V2

  841. 	MOVQ    (16)(CX), AX

  842. 	MULQ    R14

  843. 	ADDQ    AX, R8

  844. 	MOVQ    R8, (32)(SP)        // C4

  845. 	ADCQ    DX, R9

  846. 	ADCQ    $0, R10

  847. 

  848. 	// U2*V3

  849. 	XORQ    R11, R11

  850. 	MOVQ    (16)(CX), AX

  851. 	MULQ    R15

  852. 	ADDQ    AX, R9

  853. 	ADCQ    DX, R10

  854. 	ADCQ    $0, R11

  855. 

  856. 	// U3*V2

  857. 	MOVQ    (24)(CX), AX

  858. 	MULQ    R14

  859. 	ADDQ    AX, R9              // C5

  860. 	ADCQ    DX, R10

  861. 	ADCQ    $0, R11

  862. 

  863. 	// U3*V3

  864. 	MOVQ    (24)(CX), AX

  865. 	MULQ    R15

  866. 	ADDQ    AX, R10             // C6

  867. 	ADCQ    DX, R11             // C7

  868. 

  869. 	MOVQ    (64)(SP), AX

  870. 	ANDQ    AX, R12

  871. 	ANDQ    AX, R13

  872. 	ANDQ    AX, R14

  873. 	ANDQ    AX, R15

  874. 	ADDQ    R8, R12

  875. 	ADCQ    R9, R13

  876. 	ADCQ    R10, R14

  877. 	ADCQ    R11, R15

  878. 

  879. 	MOVQ    (72)(SP), AX

  880. 	MOVQ    (CX), R8

  881. 	MOVQ    (8)(CX), R9

  882. 	MOVQ    (16)(CX), R10

  883. 	MOVQ    (24)(CX), R11

  884. 	ANDQ    AX, R8

  885. 	ANDQ    AX, R9

  886. 	ANDQ    AX, R10

  887. 	ANDQ    AX, R11

  888. 	ADDQ    R12, R8

  889. 	ADCQ    R13, R9

  890. 	ADCQ    R14, R10

  891. 	ADCQ    R15, R11

  892. 	MOVQ    R8, (32)(SP)

  893. 	MOVQ    R9, (40)(SP)

  894. 	MOVQ    R10, (48)(SP)

  895. 	MOVQ    R11, (56)(SP)

  896. 

  897. 	// CX[0-7] <- AL*BL

  898. 

  899. 	// U0*V0

  900. 	MOVQ    (REG_P1), R11

  901. 	MOVQ    (REG_P2), AX

  902. 	MULQ    R11

  903. 	XORQ    R9, R9

  904. 	MOVQ    AX, (CX)            // C0

  905. 	MOVQ    DX, R8

  906. 

  907. 	// U0*V1

  908. 	MOVQ    (16)(REG_P1), R14

  909. 	MOVQ    (8)(REG_P2), AX

  910. 	MULQ    R11

  911. 	XORQ    R10, R10

  912. 	ADDQ    AX, R8

  913. 	ADCQ    DX, R9

  914. 

  915. 	// U1*V0

  916. 	MOVQ    (8)(REG_P1), R12

  917. 	MOVQ    (REG_P2), AX

  918. 	MULQ    R12

  919. 	ADDQ    AX, R8

  920. 	MOVQ    R8, (8)(CX)         // C1

  921. 	ADCQ    DX, R9

  922. 	ADCQ    $0, R10

  923. 

  924. 	// U0*V2

  925. 	XORQ    R8, R8

  926. 	MOVQ    (16)(REG_P2), AX

  927. 	MULQ    R11

  928. 	ADDQ    AX, R9

  929. 	ADCQ    DX, R10

  930. 	ADCQ    $0, R8

  931. 

  932. 	// U2*V0

  933. 	MOVQ    (REG_P2), R13

  934. 	MOVQ    R14, AX

  935. 	MULQ    R13

  936. 	ADDQ    AX, R9

  937. 	ADCQ    DX, R10

  938. 	ADCQ    $0, R8

  939. 

  940. 	// U1*V1

  941. 	MOVQ    (8)(REG_P2), AX

  942. 	MULQ    R12

  943. 	ADDQ    AX, R9

  944. 	MOVQ    R9, (16)(CX)        // C2

  945. 	ADCQ    DX, R10

  946. 	ADCQ    $0, R8

  947. 

  948. 	// U0*V3

  949. 	XORQ    R9, R9

  950. 	MOVQ    (24)(REG_P2), AX

  951. 	MULQ    R11

  952. 	MOVQ    (24)(REG_P1), R15

  953. 	ADDQ    AX, R10

  954. 	ADCQ    DX, R8

  955. 	ADCQ    $0, R9

  956. 

  957. 	// U3*V1

  958. 	MOVQ    R15, AX

  959. 	MULQ    R13

  960. 	ADDQ    AX, R10

  961. 	ADCQ    DX, R8

  962. 	ADCQ    $0, R9

  963. 

  964. 	// U2*V2

  965. 	MOVQ    (16)(REG_P2), AX

  966. 	MULQ    R12

  967. 	ADDQ    AX, R10

  968. 	ADCQ    DX, R8

  969. 	ADCQ    $0, R9

  970. 

  971. 	// U2*V3

  972. 	MOVQ    (8)(REG_P2), AX

  973. 	MULQ    R14

  974. 	ADDQ    AX, R10

  975. 	MOVQ    R10, (24)(CX)       // C3

  976. 	ADCQ    DX, R8

  977. 	ADCQ    $0, R9

  978. 

  979. 	// U3*V2

  980. 	XORQ    R10, R10

  981. 	MOVQ    (24)(REG_P2), AX

  982. 	MULQ    R12

  983. 	ADDQ    AX, R8

  984. 	ADCQ    DX, R9

  985. 	ADCQ    $0, R10

  986. 

  987. 	// U3*V1

  988. 	MOVQ    (8)(REG_P2), AX

  989. 	MULQ    R15

  990. 	ADDQ    AX, R8

  991. 	ADCQ    DX, R9

  992. 	ADCQ    $0, R10

  993. 

  994. 	// U2*V2

  995. 	MOVQ    (16)(REG_P2), AX

  996. 	MULQ    R14

  997. 	ADDQ    AX, R8

  998. 	MOVQ    R8, (32)(CX)		// C4

  999. 	ADCQ    DX, R9

  1000. 	ADCQ    $0, R10

  1001. 

  1002. 	// U2*V3

  1003. 	XORQ    R8, R8

  1004. 	MOVQ    (24)(REG_P2), AX

  1005. 	MULQ    R14

  1006. 	ADDQ    AX, R9

  1007. 	ADCQ    DX, R10

  1008. 	ADCQ    $0, R8

  1009. 

  1010. 	// U3*V2

  1011. 	MOVQ    (16)(REG_P2), AX

  1012. 	MULQ    R15

  1013. 	ADDQ    AX, R9

  1014. 	MOVQ    R9, (40)(CX)		// C5

  1015. 	ADCQ    DX, R10

  1016. 	ADCQ    $0, R8

  1017. 

  1018. 	// U3*V3

  1019. 	MOVQ    (24)(REG_P2), AX

  1020. 	MULQ    R15

  1021. 	ADDQ    AX, R10

  1022. 	MOVQ    R10, (48)(CX)		// C6

  1023. 	ADCQ    DX, R8

  1024. 	MOVQ    R8, (56)(CX)		// C7

  1025. 

  1026. 	// CX[8-15] <- U1*V1

  1027. 	MOVQ    (32)(REG_P1), R11

  1028. 	MOVQ    (32)(REG_P2), AX

  1029. 	MULQ    R11

  1030. 	XORQ    R9, R9

  1031. 	MOVQ    AX, (64)(CX)        // C0

  1032. 	MOVQ    DX, R8

  1033. 

  1034. 	MOVQ    (48)(REG_P1), R14

  1035. 	MOVQ    (40)(REG_P2), AX

  1036. 	MULQ    R11

  1037. 	XORQ    R10, R10

  1038. 	ADDQ    AX, R8

  1039. 	ADCQ    DX, R9

  1040. 

  1041. 	MOVQ    (40)(REG_P1), R12

  1042. 	MOVQ    (32)(REG_P2), AX

  1043. 	MULQ    R12

  1044. 	ADDQ    AX, R8

  1045. 	MOVQ    R8, (72)(CX)        // C1

  1046. 	ADCQ    DX, R9

  1047. 	ADCQ    $0, R10

  1048. 

  1049. 	XORQ    R8, R8

  1050. 	MOVQ    (48)(REG_P2), AX

  1051. 	MULQ    R11

  1052. 	ADDQ    AX, R9

  1053. 	ADCQ    DX, R10

  1054. 	ADCQ    $0, R8

  1055. 

  1056. 	MOVQ    (32)(REG_P2), R13

  1057. 	MOVQ    R14, AX

  1058. 	MULQ    R13

  1059. 	ADDQ    AX, R9

  1060. 	ADCQ    DX, R10

  1061. 	ADCQ    $0, R8

  1062. 

  1063. 	MOVQ    (40)(REG_P2), AX

  1064. 	MULQ    R12

  1065. 	ADDQ    AX, R9

  1066. 	MOVQ    R9, (80)(CX)        // C2

  1067. 	ADCQ    DX, R10

  1068. 	ADCQ    $0, R8

  1069. 

  1070. 	XORQ    R9, R9

  1071. 	MOVQ    (56)(REG_P2), AX

  1072. 	MULQ    R11

  1073. 	MOVQ    (56)(REG_P1), R15

  1074. 	ADDQ    AX, R10

  1075. 	ADCQ    DX, R8

  1076. 	ADCQ    $0, R9

  1077. 

  1078. 	MOVQ    R15, AX

  1079. 	MULQ    R13

  1080. 	ADDQ    AX, R10

  1081. 	ADCQ    DX, R8

  1082. 	ADCQ    $0, R9

  1083. 

  1084. 	MOVQ    (48)(REG_P2), AX

  1085. 	MULQ    R12

  1086. 	ADDQ    AX, R10

  1087. 	ADCQ    DX, R8

  1088. 	ADCQ    $0, R9

  1089. 

  1090. 	MOVQ    (40)(REG_P2), AX

  1091. 	MULQ    R14

  1092. 	ADDQ    AX, R10

  1093. 	MOVQ    R10, (88)(CX)       // C3

  1094. 	ADCQ    DX, R8

  1095. 	ADCQ    $0, R9

  1096. 

  1097. 	XORQ    R10, R10

  1098. 	MOVQ    (56)(REG_P2), AX

  1099. 	MULQ    R12

  1100. 	ADDQ    AX, R8

  1101. 	ADCQ    DX, R9

  1102. 	ADCQ    $0, R10

  1103. 

  1104. 	MOVQ    (40)(REG_P2), AX

  1105. 	MULQ    R15

  1106. 	ADDQ    AX, R8

  1107. 	ADCQ    DX, R9

  1108. 	ADCQ    $0, R10

  1109. 

  1110. 	MOVQ    (48)(REG_P2), AX

  1111. 	MULQ    R14

  1112. 	ADDQ    AX, R8

  1113. 	MOVQ    R8, (96)(CX)        // C4

  1114. 	ADCQ    DX, R9

  1115. 	ADCQ    $0, R10

  1116. 

  1117. 	XORQ    R8, R8

  1118. 	MOVQ    (56)(REG_P2), AX

  1119. 	MULQ    R14

  1120. 	ADDQ    AX, R9

  1121. 	ADCQ    DX, R10

  1122. 	ADCQ    $0, R8

  1123. 

  1124. 	MOVQ    (48)(REG_P2), AX

  1125. 	MULQ    R15

  1126. 	ADDQ    AX, R9

  1127. 	MOVQ    R9, (104)(CX)       // C5

  1128. 	ADCQ    DX, R10

  1129. 	ADCQ    $0, R8

  1130. 

  1131. 	MOVQ    (56)(REG_P2), AX

  1132. 	MULQ    R15

  1133. 	ADDQ    AX, R10

  1134. 	MOVQ    R10, (112)(CX)      // C6

  1135. 	ADCQ    DX, R8

  1136. 	MOVQ    R8, (120)(CX)       // C7

  1137. 

  1138. 	// [R8-R15] <- (U0+U1)*(V0+V1) - U1*V1

  1139. 	MOVQ    (SP), R8

  1140. 	SUBQ    (CX), R8

  1141. 	MOVQ    (8)(SP), R9

  1142. 	SBBQ    (8)(CX), R9

  1143. 	MOVQ    (16)(SP), R10

  1144. 	SBBQ    (16)(CX), R10

  1145. 	MOVQ    (24)(SP), R11

  1146. 	SBBQ    (24)(CX), R11

  1147. 	MOVQ    (32)(SP), R12

  1148. 	SBBQ    (32)(CX), R12

  1149. 	MOVQ    (40)(SP), R13

  1150. 	SBBQ    (40)(CX), R13

  1151. 	MOVQ    (48)(SP), R14

  1152. 	SBBQ    (48)(CX), R14

  1153. 	MOVQ    (56)(SP), R15

  1154. 	SBBQ    (56)(CX), R15

  1155. 

  1156. 	// [R8-R15] <- (U0+U1)*(V0+V1) - U1*V0 - U0*U1

  1157. 	MOVQ    ( 64)(CX), AX;	SUBQ    AX, R8

  1158. 	MOVQ    ( 72)(CX), AX;	SBBQ    AX, R9

  1159. 	MOVQ    ( 80)(CX), AX;	SBBQ    AX, R10

  1160. 	MOVQ    ( 88)(CX), AX;	SBBQ    AX, R11

  1161. 	MOVQ    ( 96)(CX), AX;	SBBQ    AX, R12

  1162. 	MOVQ    (104)(CX), DX;	SBBQ    DX, R13

  1163. 	MOVQ    (112)(CX), DI;	SBBQ    DI, R14

  1164. 	MOVQ    (120)(CX), SI;	SBBQ    SI, R15

  1165. 

  1166. 	// Final result

  1167. 	ADDQ    (32)(CX), R8;	MOVQ    R8,  (32)(CX)

  1168. 	ADCQ    (40)(CX), R9;	MOVQ    R9,  (40)(CX)

  1169. 	ADCQ    (48)(CX), R10;	MOVQ    R10, (48)(CX)

  1170. 	ADCQ    (56)(CX), R11;	MOVQ    R11, (56)(CX)

  1171. 	ADCQ    (64)(CX), R12;	MOVQ    R12, (64)(CX)

  1172. 	ADCQ    (72)(CX), R13;	MOVQ    R13, (72)(CX)

  1173. 	ADCQ    (80)(CX), R14;	MOVQ    R14, (80)(CX)

  1174. 	ADCQ    (88)(CX), R15;	MOVQ    R15, (88)(CX)

  1175. 	ADCQ    $0, AX;        	MOVQ    AX,  (96)(CX)

  1176. 	ADCQ    $0, DX;        	MOVQ    DX, (104)(CX)

  1177. 	ADCQ    $0, DI;         MOVQ    DI, (112)(CX)

  1178. 	ADCQ    $0, SI;     	MOVQ    SI, (120)(CX)

  1179. 	RET

  1180. 

  1181. mul_with_mulx_adx:

  1182. 	// Mul implementation for CPUs supporting two independent carry chain

  1183. 	// (ADOX/ADCX) instructions and carry-less MULX multiplier

  1184. 	MUL(CX, REG_P1, REG_P2, MULS256_MULXADX)

  1185. 	RET

  1186. 

  1187. mul_with_mulx:

  1188. 	// Mul implementation for CPUs supporting carry-less MULX multiplier.

  1189. 	MUL(CX, REG_P1, REG_P2, MULS256_MULX)

  1190. 	RET

  1191. 

  1192. TEXT ·fp503MontgomeryReduce(SB), $0-16

  1193. 	MOVQ    z+0(FP), REG_P2

  1194. 	MOVQ    x+8(FP), REG_P1

  1195. 

  1196. 	// Check wether to use optimized implementation

  1197. 	CMPB    ·useADXMULX(SB), $1

  1198. 	JE      redc_with_mulx_adx

  1199. 	CMPB    ·useMULX(SB), $1

  1200. 	JE      redc_with_mulx

  1201. 

  1202. 	MOVQ    (REG_P1), R11

  1203. 	MOVQ    P503P1_3, AX

  1204. 	MULQ    R11

  1205. 	XORQ    R8, R8

  1206. 	ADDQ    (24)(REG_P1), AX

  1207. 	MOVQ    AX, (24)(REG_P2)

  1208. 	ADCQ    DX, R8

  1209. 

  1210. 	XORQ    R9, R9

  1211. 	MOVQ    P503P1_4, AX

  1212. 	MULQ    R11

  1213. 	XORQ    R10, R10

  1214. 	ADDQ    AX, R8

  1215. 	ADCQ    DX, R9

  1216. 

  1217. 	MOVQ    (8)(REG_P1), R12

  1218. 	MOVQ    P503P1_3, AX

  1219. 	MULQ    R12

  1220. 	ADDQ    AX, R8

  1221. 	ADCQ    DX, R9

  1222. 	ADCQ    $0, R10

  1223. 	ADDQ    (32)(REG_P1), R8

  1224. 	MOVQ    R8, (32)(REG_P2)       // Z4

  1225. 	ADCQ    $0, R9

  1226. 	ADCQ    $0, R10

  1227. 

  1228. 	XORQ    R8, R8

  1229. 	MOVQ    P503P1_5, AX

  1230. 	MULQ    R11

  1231. 	ADDQ    AX, R9

  1232. 	ADCQ    DX, R10

  1233. 	ADCQ    $0, R8

  1234. 

  1235. 	MOVQ    P503P1_4, AX

  1236. 	MULQ    R12

  1237. 	ADDQ    AX, R9

  1238. 	ADCQ    DX, R10

  1239. 	ADCQ    $0, R8

  1240. 

  1241. 	MOVQ    (16)(REG_P1), R13

  1242. 	MOVQ    P503P1_3, AX

  1243. 	MULQ    R13

  1244. 	ADDQ    AX, R9

  1245. 	ADCQ    DX, R10

  1246. 	ADCQ    $0, R8

  1247. 	ADDQ    (40)(REG_P1), R9

  1248. 	MOVQ    R9, (40)(REG_P2)       // Z5

  1249. 	ADCQ    $0, R10

  1250. 	ADCQ    $0, R8

  1251. 

  1252. 	XORQ    R9, R9

  1253. 	MOVQ    P503P1_6, AX

  1254. 	MULQ    R11

  1255. 	ADDQ    AX, R10

  1256. 	ADCQ    DX, R8

  1257. 	ADCQ    $0, R9

  1258. 

  1259. 	MOVQ    P503P1_5, AX

  1260. 	MULQ    R12

  1261. 	ADDQ    AX, R10

  1262. 	ADCQ    DX, R8

  1263. 	ADCQ    $0, R9

  1264. 

  1265. 	MOVQ    P503P1_4, AX

  1266. 	MULQ    R13

  1267. 	ADDQ    AX, R10

  1268. 	ADCQ    DX, R8

  1269. 	ADCQ    $0, R9

  1270. 

  1271. 	MOVQ    (24)(REG_P2), R14

  1272. 	MOVQ    P503P1_3, AX

  1273. 	MULQ    R14

  1274. 	ADDQ    AX, R10

  1275. 	ADCQ    DX, R8

  1276. 	ADCQ    $0, R9

  1277. 	ADDQ    (48)(REG_P1), R10

  1278. 	MOVQ    R10, (48)(REG_P2)      // Z6

  1279. 	ADCQ    $0, R8

  1280. 	ADCQ    $0, R9

  1281. 

  1282. 	XORQ    R10, R10

  1283. 	MOVQ    P503P1_7, AX

  1284. 	MULQ    R11

  1285. 	ADDQ    AX, R8

  1286. 	ADCQ    DX, R9

  1287. 	ADCQ    $0, R10

  1288. 

  1289. 	MOVQ    P503P1_6, AX

  1290. 	MULQ    R12

  1291. 	ADDQ    AX, R8

  1292. 	ADCQ    DX, R9

  1293. 	ADCQ    $0, R10

  1294. 

  1295. 	MOVQ    P503P1_5, AX

  1296. 	MULQ    R13

  1297. 	ADDQ    AX, R8

  1298. 	ADCQ    DX, R9

  1299. 	ADCQ    $0, R10

  1300. 

  1301. 	MOVQ    P503P1_4, AX

  1302. 	MULQ    R14

  1303. 	ADDQ    AX, R8

  1304. 	ADCQ    DX, R9

  1305. 	ADCQ    $0, R10

  1306. 

  1307. 	MOVQ    (32)(REG_P2), R15

  1308. 	MOVQ    P503P1_3, AX

  1309. 	MULQ    R15

  1310. 	ADDQ    AX, R8

  1311. 	ADCQ    DX, R9

  1312. 	ADCQ    $0, R10

  1313. 	ADDQ    (56)(REG_P1), R8

  1314. 	MOVQ    R8, (56)(REG_P2)       // Z7

  1315. 	ADCQ    $0, R9

  1316. 	ADCQ    $0, R10

  1317. 

  1318. 	XORQ    R8, R8

  1319. 	MOVQ    P503P1_7, AX

  1320. 	MULQ    R12

  1321. 	ADDQ    AX, R9

  1322. 	ADCQ    DX, R10

  1323. 	ADCQ    $0, R8

  1324. 

  1325. 	MOVQ    P503P1_6, AX

  1326. 	MULQ    R13

  1327. 	ADDQ    AX, R9

  1328. 	ADCQ    DX, R10

  1329. 	ADCQ    $0, R8

  1330. 

  1331. 	MOVQ    P503P1_5, AX

  1332. 	MULQ    R14

  1333. 	ADDQ    AX, R9

  1334. 	ADCQ    DX, R10

  1335. 	ADCQ    $0, R8

  1336. 

  1337. 	MOVQ    P503P1_4, AX

  1338. 	MULQ    R15

  1339. 	ADDQ    AX, R9

  1340. 	ADCQ    DX, R10

  1341. 	ADCQ    $0, R8

  1342. 

  1343. 	MOVQ    (40)(REG_P2), CX

  1344. 	MOVQ    P503P1_3, AX

  1345. 	MULQ    CX

  1346. 	ADDQ    AX, R9

  1347. 	ADCQ    DX, R10

  1348. 	ADCQ    $0, R8

  1349. 	ADDQ    (64)(REG_P1), R9

  1350. 	MOVQ    R9, (REG_P2)           // Z0

  1351. 	ADCQ    $0, R10

  1352. 	ADCQ    $0, R8

  1353. 

  1354. 	XORQ    R9, R9

  1355. 	MOVQ    P503P1_7, AX

  1356. 	MULQ    R13

  1357. 	ADDQ    AX, R10

  1358. 	ADCQ    DX, R8

  1359. 	ADCQ    $0, R9

  1360. 

  1361. 	MOVQ    P503P1_6, AX

  1362. 	MULQ    R14

  1363. 	ADDQ    AX, R10

  1364. 	ADCQ    DX, R8

  1365. 	ADCQ    $0, R9

  1366. 

  1367. 	MOVQ    P503P1_5, AX

  1368. 	MULQ    R15

  1369. 	ADDQ    AX, R10

  1370. 	ADCQ    DX, R8

  1371. 	ADCQ    $0, R9

  1372. 

  1373. 	MOVQ    P503P1_4, AX

  1374. 	MULQ    CX

  1375. 	ADDQ    AX, R10

  1376. 	ADCQ    DX, R8

  1377. 	ADCQ    $0, R9

  1378. 

  1379. 	MOVQ    (48)(REG_P2), R13

  1380. 	MOVQ    P503P1_3, AX

  1381. 	MULQ    R13

  1382. 	ADDQ    AX, R10

  1383. 	ADCQ    DX, R8

  1384. 	ADCQ    $0, R9

  1385. 	ADDQ    (72)(REG_P1), R10

  1386. 	MOVQ    R10, (8)(REG_P2)       // Z1

  1387. 	ADCQ    $0, R8

  1388. 	ADCQ    $0, R9

  1389. 

  1390. 	XORQ    R10, R10

  1391. 	MOVQ    P503P1_7, AX

  1392. 	MULQ    R14

  1393. 	ADDQ    AX, R8

  1394. 	ADCQ    DX, R9

  1395. 	ADCQ    $0, R10

  1396. 

  1397. 	MOVQ    P503P1_6, AX

  1398. 	MULQ    R15

  1399. 	ADDQ    AX, R8

  1400. 	ADCQ    DX, R9

  1401. 	ADCQ    $0, R10

  1402. 

  1403. 	MOVQ    P503P1_5, AX

  1404. 	MULQ    CX

  1405. 	ADDQ    AX, R8

  1406. 	ADCQ    DX, R9

  1407. 	ADCQ    $0, R10

  1408. 

  1409. 	MOVQ    P503P1_4, AX

  1410. 	MULQ    R13

  1411. 	ADDQ    AX, R8

  1412. 	ADCQ    DX, R9

  1413. 	ADCQ    $0, R10

  1414. 

  1415. 	MOVQ    (56)(REG_P2), R14

  1416. 	MOVQ    P503P1_3, AX

  1417. 	MULQ    R14

  1418. 	ADDQ    AX, R8

  1419. 	ADCQ    DX, R9

  1420. 	ADCQ    $0, R10

  1421. 	ADDQ    (80)(REG_P1), R8

  1422. 	MOVQ    R8, (16)(REG_P2)       // Z2

  1423. 	ADCQ    $0, R9

  1424. 	ADCQ    $0, R10

  1425. 

  1426. 	XORQ    R8, R8

  1427. 	MOVQ    P503P1_7, AX

  1428. 	MULQ    R15

  1429. 	ADDQ    AX, R9

  1430. 	ADCQ    DX, R10

  1431. 	ADCQ    $0, R8

  1432. 

  1433. 	MOVQ    P503P1_6, AX

  1434. 	MULQ    CX

  1435. 	ADDQ    AX, R9

  1436. 	ADCQ    DX, R10

  1437. 	ADCQ    $0, R8

  1438. 

  1439. 	MOVQ    P503P1_5, AX

  1440. 	MULQ    R13

  1441. 	ADDQ    AX, R9

  1442. 	ADCQ    DX, R10

  1443. 	ADCQ    $0, R8

  1444. 

  1445. 	MOVQ    P503P1_4, AX

  1446. 	MULQ    R14

  1447. 	ADDQ    AX, R9

  1448. 	ADCQ    DX, R10

  1449. 	ADCQ    $0, R8

  1450. 	ADDQ    (88)(REG_P1), R9

  1451. 	MOVQ    R9, (24)(REG_P2)       // Z3

  1452. 	ADCQ    $0, R10

  1453. 	ADCQ    $0, R8

  1454. 

  1455. 	XORQ    R9, R9

  1456. 	MOVQ    P503P1_7, AX

  1457. 	MULQ    CX

  1458. 	ADDQ    AX, R10

  1459. 	ADCQ    DX, R8

  1460. 	ADCQ    $0, R9

  1461. 

  1462. 	MOVQ    P503P1_6, AX

  1463. 	MULQ    R13

  1464. 	ADDQ    AX, R10

  1465. 	ADCQ    DX, R8

  1466. 	ADCQ    $0, R9

  1467. 

  1468. 	MOVQ    P503P1_5, AX

  1469. 	MULQ    R14

  1470. 	ADDQ    AX, R10

  1471. 	ADCQ    DX, R8

  1472. 	ADCQ    $0, R9

  1473. 	ADDQ    (96)(REG_P1), R10

  1474. 	MOVQ    R10, (32)(REG_P2)      // Z4

  1475. 	ADCQ    $0, R8

  1476. 	ADCQ    $0, R9

  1477. 

  1478. 	XORQ    R10, R10

  1479. 	MOVQ    P503P1_7, AX

  1480. 	MULQ    R13

  1481. 	ADDQ    AX, R8

  1482. 	ADCQ    DX, R9

  1483. 	ADCQ    $0, R10

  1484. 

  1485. 	MOVQ    P503P1_6, AX

  1486. 	MULQ    R14

  1487. 	ADDQ    AX, R8

  1488. 	ADCQ    DX, R9

  1489. 	ADCQ    $0, R10

  1490. 	ADDQ    (104)(REG_P1), R8      // Z5

  1491. 	MOVQ    R8, (40)(REG_P2)       // Z5

  1492. 	ADCQ    $0, R9

  1493. 	ADCQ    $0, R10

  1494. 

  1495. 	MOVQ    P503P1_7, AX

  1496. 	MULQ    R14

  1497. 	ADDQ    AX, R9

  1498. 	ADCQ    DX, R10

  1499. 	ADDQ    (112)(REG_P1), R9      // Z6

  1500. 	MOVQ    R9, (48)(REG_P2)       // Z6

  1501. 	ADCQ    $0, R10

  1502. 	ADDQ    (120)(REG_P1), R10     // Z7

  1503. 	MOVQ    R10, (56)(REG_P2)      // Z7

  1504. 	RET

  1505. 

  1506. redc_with_mulx_adx:

  1507. 	// Implementation of the Montgomery reduction for CPUs

  1508. 	// supporting two independent carry chain (ADOX/ADCX)

  1509. 	// instructions and carry-less MULX multiplier

  1510. 	REDC(REG_P2, REG_P1, MULS_128x320_MULXADX)

  1511. 	RET

  1512. 

  1513. redc_with_mulx:

  1514. 	// Implementation of the Montgomery reduction for CPUs

  1515. 	// supporting carry-less MULX multiplier.

  1516. 	REDC(REG_P2, REG_P1, MULS_128x320_MULX)

  1517. 	RET

  1518. 

  1519. TEXT ·fp503AddLazy(SB), NOSPLIT, $0-24

  1520. 

  1521. 	MOVQ z+0(FP), REG_P3

  1522. 	MOVQ x+8(FP), REG_P1

  1523. 	MOVQ y+16(FP), REG_P2

  1524. 

  1525. 	MOVQ	(REG_P1), R8

  1526. 	MOVQ	(8)(REG_P1), R9

  1527. 	MOVQ	(16)(REG_P1), R10

  1528. 	MOVQ	(24)(REG_P1), R11

  1529. 	MOVQ	(32)(REG_P1), R12

  1530. 	MOVQ	(40)(REG_P1), R13

  1531. 	MOVQ	(48)(REG_P1), R14

  1532. 	MOVQ	(56)(REG_P1), R15

  1533. 

  1534. 	ADDQ	(REG_P2), R8

  1535. 	ADCQ	(8)(REG_P2), R9

  1536. 	ADCQ	(16)(REG_P2), R10

  1537. 	ADCQ	(24)(REG_P2), R11

  1538. 	ADCQ	(32)(REG_P2), R12

  1539. 	ADCQ	(40)(REG_P2), R13

  1540. 	ADCQ	(48)(REG_P2), R14

  1541. 	ADCQ	(56)(REG_P2), R15

  1542. 

  1543. 	MOVQ	R8, (REG_P3)

  1544. 	MOVQ	R9, (8)(REG_P3)

  1545. 	MOVQ	R10, (16)(REG_P3)

  1546. 	MOVQ	R11, (24)(REG_P3)

  1547. 	MOVQ	R12, (32)(REG_P3)

  1548. 	MOVQ	R13, (40)(REG_P3)

  1549. 	MOVQ	R14, (48)(REG_P3)

  1550. 	MOVQ	R15, (56)(REG_P3)

  1551. 

  1552. 	RET

  1553. 

  1554. TEXT ·fp503X2AddLazy(SB), NOSPLIT, $0-24

  1555. 

  1556. 	MOVQ	z+0(FP), REG_P3

  1557. 	MOVQ	x+8(FP), REG_P1

  1558. 	MOVQ	y+16(FP), REG_P2

  1559. 

  1560. 	MOVQ	(REG_P1), R8

  1561. 	MOVQ	(8)(REG_P1), R9

  1562. 	MOVQ	(16)(REG_P1), R10

  1563. 	MOVQ	(24)(REG_P1), R11

  1564. 	MOVQ	(32)(REG_P1), R12

  1565. 	MOVQ	(40)(REG_P1), R13

  1566. 	MOVQ	(48)(REG_P1), R14

  1567. 	MOVQ	(56)(REG_P1), R15

  1568. 	MOVQ	(64)(REG_P1), AX

  1569. 	MOVQ	(72)(REG_P1), BX

  1570. 	MOVQ	(80)(REG_P1), CX

  1571. 

  1572. 	ADDQ	(REG_P2), R8

  1573. 	ADCQ	(8)(REG_P2), R9

  1574. 	ADCQ	(16)(REG_P2), R10

  1575. 	ADCQ	(24)(REG_P2), R11

  1576. 	ADCQ	(32)(REG_P2), R12

  1577. 	ADCQ	(40)(REG_P2), R13

  1578. 	ADCQ	(48)(REG_P2), R14

  1579. 	ADCQ	(56)(REG_P2), R15

  1580. 	ADCQ	(64)(REG_P2), AX

  1581. 	ADCQ	(72)(REG_P2), BX

  1582. 	ADCQ	(80)(REG_P2), CX

  1583. 

  1584. 	MOVQ	R8, (REG_P3)

  1585. 	MOVQ	R9, (8)(REG_P3)

  1586. 	MOVQ	R10, (16)(REG_P3)

  1587. 	MOVQ	R11, (24)(REG_P3)

  1588. 	MOVQ	R12, (32)(REG_P3)

  1589. 	MOVQ	R13, (40)(REG_P3)

  1590. 	MOVQ	R14, (48)(REG_P3)

  1591. 	MOVQ	R15, (56)(REG_P3)

  1592. 	MOVQ	AX, (64)(REG_P3)

  1593. 	MOVQ	BX, (72)(REG_P3)

  1594. 	MOVQ	CX, (80)(REG_P3)

  1595. 

  1596. 	MOVQ	(88)(REG_P1), R8

  1597. 	MOVQ	(96)(REG_P1), R9

  1598. 	MOVQ	(104)(REG_P1), R10

  1599. 	MOVQ	(112)(REG_P1), R11

  1600. 	MOVQ	(120)(REG_P1), R12

  1601. 

  1602. 	ADCQ	(88)(REG_P2), R8

  1603. 	ADCQ	(96)(REG_P2), R9

  1604. 	ADCQ	(104)(REG_P2), R10

  1605. 	ADCQ	(112)(REG_P2), R11

  1606. 	ADCQ	(120)(REG_P2), R12

  1607. 

  1608. 	MOVQ	R8, (88)(REG_P3)

  1609. 	MOVQ	R9, (96)(REG_P3)

  1610. 	MOVQ	R10, (104)(REG_P3)

  1611. 	MOVQ	R11, (112)(REG_P3)

  1612. 	MOVQ	R12, (120)(REG_P3)

  1613. 

  1614. 	RET

  1615. 

  1616. TEXT ·fp503X2SubLazy(SB), NOSPLIT, $0-24

  1617. 

  1618. 	MOVQ z+0(FP), REG_P3

  1619. 	MOVQ x+8(FP), REG_P1

  1620. 	MOVQ y+16(FP), REG_P2

  1621. 	// Used later to store result of 0-borrow

  1622. 	XORQ CX, CX

  1623. 

  1624. 	// SUBC for first 11 limbs

  1625. 	MOVQ	(REG_P1), R8

  1626. 	MOVQ	(8)(REG_P1), R9

  1627. 	MOVQ	(16)(REG_P1), R10

  1628. 	MOVQ	(24)(REG_P1), R11

  1629. 	MOVQ	(32)(REG_P1), R12

  1630. 	MOVQ	(40)(REG_P1), R13

  1631. 	MOVQ	(48)(REG_P1), R14

  1632. 	MOVQ	(56)(REG_P1), R15

  1633. 	MOVQ	(64)(REG_P1), AX

  1634. 	MOVQ	(72)(REG_P1), BX

  1635. 

  1636. 	SUBQ	(REG_P2), R8

  1637. 	SBBQ	(8)(REG_P2), R9

  1638. 	SBBQ	(16)(REG_P2), R10

  1639. 	SBBQ	(24)(REG_P2), R11

  1640. 	SBBQ	(32)(REG_P2), R12

  1641. 	SBBQ	(40)(REG_P2), R13

  1642. 	SBBQ	(48)(REG_P2), R14

  1643. 	SBBQ	(56)(REG_P2), R15

  1644. 	SBBQ	(64)(REG_P2), AX

  1645. 	SBBQ	(72)(REG_P2), BX

  1646. 

  1647. 	MOVQ	R8, (REG_P3)

  1648. 	MOVQ	R9, (8)(REG_P3)

  1649. 	MOVQ	R10, (16)(REG_P3)

  1650. 	MOVQ	R11, (24)(REG_P3)

  1651. 	MOVQ	R12, (32)(REG_P3)

  1652. 	MOVQ	R13, (40)(REG_P3)

  1653. 	MOVQ	R14, (48)(REG_P3)

  1654. 	MOVQ	R15, (56)(REG_P3)

  1655. 	MOVQ	AX, (64)(REG_P3)

  1656. 	MOVQ	BX, (72)(REG_P3)

  1657. 

  1658. 	// SUBC for last 5 limbs

  1659. 	MOVQ	(80)(REG_P1), 	R8

  1660. 	MOVQ	(88)(REG_P1), 	R9

  1661. 	MOVQ	(96)(REG_P1), 	R10

  1662. 	MOVQ	(104)(REG_P1), 	R11

  1663. 	MOVQ	(112)(REG_P1), 	R12

  1664. 	MOVQ	(120)(REG_P1), 	R13

  1665. 

  1666. 	SBBQ	(80)(REG_P2), R8

  1667. 	SBBQ	(88)(REG_P2), R9

  1668. 	SBBQ	(96)(REG_P2), R10

  1669. 	SBBQ	(104)(REG_P2), R11

  1670. 	SBBQ	(112)(REG_P2), R12

  1671. 	SBBQ	(120)(REG_P2), R13

  1672. 

  1673. 	MOVQ	R8, (80)(REG_P3)

  1674. 	MOVQ	R9, (88)(REG_P3)

  1675. 	MOVQ	R10, (96)(REG_P3)

  1676. 	MOVQ	R11, (104)(REG_P3)

  1677. 	MOVQ	R12, (112)(REG_P3)

  1678. 	MOVQ	R13, (120)(REG_P3)

  1679. 

  1680. 	// Now the carry flag is 1 if x-y < 0.  If so, add p*2^512.

  1681. 	SBBQ	$0, CX

  1682. 

  1683. 	// Load p into registers:

  1684. 	MOVQ	P503_0, R8

  1685. 	// P503_{1,2} = P503_0, so reuse R8

  1686. 	MOVQ	P503_3, R9

  1687. 	MOVQ	P503_4, R10

  1688. 	MOVQ	P503_5, R11

  1689. 	MOVQ	P503_6, R12

  1690. 	MOVQ	P503_7, R13

  1691. 

  1692. 	ANDQ	CX, R8

  1693. 	ANDQ	CX, R9

  1694. 	ANDQ	CX, R10

  1695. 	ANDQ	CX, R11

  1696. 	ANDQ	CX, R12

  1697. 	ANDQ	CX, R13

  1698. 

  1699. 	MOVQ   (64   )(REG_P3), AX; ADDQ R8,  AX; MOVQ AX, (64   )(REG_P3)

  1700. 	MOVQ   (64+ 8)(REG_P3), AX; ADCQ R8,  AX; MOVQ AX, (64+ 8)(REG_P3)

  1701. 	MOVQ   (64+16)(REG_P3), AX; ADCQ R8,  AX; MOVQ AX, (64+16)(REG_P3)

  1702. 	MOVQ   (64+24)(REG_P3), AX; ADCQ R9,  AX; MOVQ AX, (64+24)(REG_P3)

  1703. 	MOVQ   (64+32)(REG_P3), AX; ADCQ R10, AX; MOVQ AX, (64+32)(REG_P3)

  1704. 	MOVQ   (64+40)(REG_P3), AX; ADCQ R11, AX; MOVQ AX, (64+40)(REG_P3)

  1705. 	MOVQ   (64+48)(REG_P3), AX; ADCQ R12, AX; MOVQ AX, (64+48)(REG_P3)

  1706. 	MOVQ   (64+56)(REG_P3), AX; ADCQ R13, AX; MOVQ AX, (64+56)(REG_P3)

  1707. 

  1708. 	RET