kris
/
sike_ifma
1
0
Vork 0
Code Kwesties 0 Pull-aanvragen 0 Publicaties 0 Wiki Activiteit
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
1 Commit
1 Branch
116 KiB
 
 
 
Branch: master
Branches Labels
${ item.name }
Maak branch ${ searchTerm }
van 'master'
${ noResults }
sike_ifma/fp2_751_ifma.S

917 regels
24 KiB
Ruw Permalink Blame Geschiedenis 

  1. 

  2. #if defined(__APPLE__)

  3. /* OS X's C ABI prefixes functions with underscore. */

  4. #define C_ABI(x) _ ## x

  5. #define HIDDEN .private_extern

  6. #else

  7. #define C_ABI(x) x

  8. #define HIDDEN .hidden

  9. #endif

  10. 

  11. .p2align 6

  12. .LpermMask0:

  13. .word 0,1,2,3,   3,4,5,6,   6,7,8,9,    9,10,11,12,    13,14,15,16,   16,17,18,19,   19,20,21,22,    22,23,24,25

  14. .LshiftMask0:

  15. .quad 0,4,8,12,0,4,8,12

  16. .LandMask:

  17. .quad 0xfffffffffffff

  18. 

  19. .p2align 6

  20. .Lpoly:

  21. .quad 0x000fffffffffffff, 0x000fffffffffffff, 0x000fffffffffffff, 0x000fffffffffffff

  22. .quad 0x000fffffffffffff, 0x000fffffffffffff, 0x000fffffffffffff, 0x00049f878a8eeaff

  23. .quad 0x0007cc76e3ec9685, 0x00076da959b1a13f, 0x00084e9867d6ebe8, 0x000b5045cb257480

  24. .quad 0x000f97badc668562, 0x00041f71c0e12909, 0x00000000006fe5d5, 0

  25. 

  26. .LpolyX:

  27. .quad 0x0fffffffffffff00, 0x0fffffffffffff00, 0x0fffffffffffff00, 0x0fffffffffffff00

  28. .quad 0x0fffffffffffff00, 0x0fffffffffffff00, 0x0fffffffffffff00, 0x049f878a8eeaff00

  29. .quad 0x07cc76e3ec968500, 0x076da959b1a13f00, 0x084e9867d6ebe800, 0x0b5045cb25748000

  30. .quad 0x0f97badc66856200, 0x041f71c0e1290900, 0x000000006fe5d500, 0

  31. 

  32. #define felemR %rdi

  33. #define felemA %rsi

  34. #define felemB %rdx

  35. 

  36. #define itr %r10

  37. 

  38. #define M0 %zmm0

  39. #define M1 %zmm1

  40. #define ZERO %zmm2

  41. #define AND_MASK %zmm3

  42. 

  43. #define A0a %zmm4

  44. #define A0b %zmm5

  45. 

  46. #define A1a %zmm6

  47. #define A1b %zmm7

  48. 

  49. #define ACC0a %zmm8

  50. #define ACC0b %zmm9

  51. #define ACC1a %zmm10

  52. #define ACC1b %zmm11

  53. #define ACC2a %zmm12

  54. #define ACC2b %zmm13

  55. #define ACC3a %zmm14

  56. #define ACC3b %zmm15

  57. 

  58. #define B0curr %zmm16

  59. #define B0prev %zmm17

  60. #define B1curr %zmm18

  61. #define B1prev %zmm19

  62. 

  63. #define Y0curr %zmm20

  64. #define Y0prev %zmm21

  65. #define Y1curr %zmm22

  66. #define Y1prev %zmm23

  67. #define Y2curr %zmm24

  68. #define Y2prev %zmm25

  69. #define Y3curr %zmm26

  70. #define Y3prev %zmm27

  71. 

  72. #define T0 %zmm28

  73. #define T1 %zmm29

  74. #define T2 %zmm30

  75. #define T3 %zmm31

  76. 

  77. ###############################################################################

  78. .globl C_ABI(fp2_mul_ifma)

  79. .p2align 6

  80. C_ABI(fp2_mul_ifma):

  81. 

  82.     mov $1, %eax

  83.     kmovw %eax, %k1

  84.     mov $0x7f, %eax

  85.     kmovw %eax, %k5

  86. 

  87.     vpbroadcastq .LandMask(%rip), AND_MASK

  88.     vpxorq ZERO, ZERO, ZERO

  89. 

  90.     vmovdqu64 64*0(felemA), A0a

  91.     vmovdqu64 64*1(felemA), A0b{%k5}{z}

  92.     vmovdqu64 15*8 + 64*0(felemA), A1a

  93.     vmovdqu64 15*8 + 64*1(felemA), A1b{%k5}{z}

  94. 

  95.     # Load the modulus

  96.     vmovdqa64 64*0 + .Lpoly(%rip), M0

  97.     vmovdqa64 64*1 + .Lpoly(%rip), M1

  98. 

  99.     # Prepare the accumulators

  100.     vpxorq ACC0a, ACC0a, ACC0a

  101.     vpxorq ACC0b, ACC0b, ACC0b

  102.     vpxorq ACC1a, ACC1a, ACC1a

  103.     vpxorq ACC1b, ACC1b, ACC1b

  104.     vpxorq ACC2a, ACC2a, ACC2a

  105.     vpxorq ACC2b, ACC2b, ACC2b

  106.     vpxorq ACC3a, ACC3a, ACC3a

  107.     vpxorq ACC3b, ACC3b, ACC3b

  108.     vpxorq T0, T0, T0

  109.     vpxorq T1, T1, T1

  110.     vpxorq T2, T2, T2

  111.     vpxorq T3, T3, T3

  112. 

  113.     # First iteration

  114.     vpbroadcastq (felemB), B0curr

  115.     vpbroadcastq 15*8(felemB), B1curr

  116.     lea 8(felemB), felemB

  117. 

  118.     vpmadd52luq B0curr, A0a, ACC0a

  119.     vpmadd52luq B0curr, A0b, ACC0b

  120.     vpmadd52luq B1curr, A1a, ACC1a

  121.     vpmadd52luq B1curr, A1b, ACC1b

  122.     vpmadd52luq B0curr, A1a, ACC2a

  123.     vpmadd52luq B0curr, A1b, ACC2b

  124.     vpmadd52luq B1curr, A0a, ACC3a

  125.     vpmadd52luq B1curr, A0b, ACC3b

  126. 

  127.     vpermq ACC0a, ZERO, Y0curr

  128.     vpermq ACC1a, ZERO, Y1curr

  129.     vpermq ACC2a, ZERO, Y2curr

  130.     vpermq ACC3a, ZERO, Y3curr

  131. 

  132.     vpmadd52luq Y0curr, M0, ACC0a

  133.     vpmadd52luq Y0curr, M1, ACC0b

  134.     vpmadd52luq Y1curr, M0, ACC1a

  135.     vpmadd52luq Y1curr, M1, ACC1b

  136.     vpmadd52luq Y2curr, M0, ACC2a

  137.     vpmadd52luq Y2curr, M1, ACC2b

  138.     vpmadd52luq Y3curr, M0, ACC3a

  139.     vpmadd52luq Y3curr, M1, ACC3b

  140. 

  141.     vpsrlq $52, ACC0a, T0{%k1}{z}

  142.     vpsrlq $52, ACC1a, T1{%k1}{z}

  143.     vpsrlq $52, ACC2a, T2{%k1}{z}

  144.     vpsrlq $52, ACC3a, T3{%k1}{z}

  145. 

  146.     mov $14, itr

  147. 

  148. 1:

  149.         # Shift the ACC in zmms right by a word

  150.         valignq $1, ACC0a, ACC0b, ACC0a

  151.         valignq $1, ACC0b, ZERO, ACC0b

  152.         valignq $1, ACC1a, ACC1b, ACC1a

  153.         valignq $1, ACC1b, ZERO, ACC1b

  154.         valignq $1, ACC2a, ACC2b, ACC2a

  155.         valignq $1, ACC2b, ZERO, ACC2b

  156.         valignq $1, ACC3a, ACC3b, ACC3a

  157.         valignq $1, ACC3b, ZERO, ACC3b

  158. 

  159.         vmovdqa64 B0curr, B0prev

  160.         vmovdqa64 B1curr, B1prev

  161.         vmovdqa64 Y0curr, Y0prev

  162.         vmovdqa64 Y1curr, Y1prev

  163.         vmovdqa64 Y2curr, Y2prev

  164.         vmovdqa64 Y3curr, Y3prev

  165. 

  166.         vpbroadcastq (felemB), B0curr

  167.         vpbroadcastq 15*8(felemB), B1curr

  168.         lea 8(felemB), felemB

  169. 

  170.         # High multiplications

  171.         vpmadd52huq B0prev, A0a, ACC0a # ACC0 = A0 * B0

  172.         vpmadd52huq B0prev, A0b, ACC0b

  173.         vpmadd52huq B1prev, A1a, ACC1a # ACC1 = A1 * B1

  174.         vpmadd52huq B1prev, A1b, ACC1b

  175.         vpmadd52huq B0prev, A1a, ACC2a # ACC2 = A1 * B0

  176.         vpmadd52huq B0prev, A1b, ACC2b

  177.         vpmadd52huq B1prev, A0a, ACC3a # ACC3 = A0 * B1

  178.         vpmadd52huq B1prev, A0b, ACC3b

  179. 

  180.         vpmadd52huq Y0prev, M0, ACC0a

  181.         vpmadd52huq Y0prev, M1, ACC0b

  182.         vpmadd52huq Y1prev, M0, ACC1a

  183.         vpmadd52huq Y1prev, M1, ACC1b

  184.         vpmadd52huq Y2prev, M0, ACC2a

  185.         vpmadd52huq Y2prev, M1, ACC2b

  186.         vpmadd52huq Y3prev, M0, ACC3a

  187.         vpmadd52huq Y3prev, M1, ACC3b

  188.         # Low multiplications

  189.         vpmadd52luq B0curr, A0a, ACC0a

  190.         vpmadd52luq B0curr, A0b, ACC0b

  191.         vpmadd52luq B1curr, A1a, ACC1a

  192.         vpmadd52luq B1curr, A1b, ACC1b

  193.         vpmadd52luq B0curr, A1a, ACC2a

  194.         vpmadd52luq B0curr, A1b, ACC2b

  195.         vpmadd52luq B1curr, A0a, ACC3a

  196.         vpmadd52luq B1curr, A0b, ACC3b

  197. 

  198.         vpaddq T0, ACC0a, ACC0a

  199.         vpaddq T1, ACC1a, ACC1a

  200.         vpaddq T2, ACC2a, ACC2a

  201.         vpaddq T3, ACC3a, ACC3a

  202.         vpermq ACC0a, ZERO, Y0curr

  203.         vpermq ACC1a, ZERO, Y1curr

  204.         vpermq ACC2a, ZERO, Y2curr

  205.         vpermq ACC3a, ZERO, Y3curr

  206. 

  207.         vpmadd52luq Y0curr, M0, ACC0a

  208.         vpmadd52luq Y0curr, M1, ACC0b

  209.         vpmadd52luq Y1curr, M0, ACC1a

  210.         vpmadd52luq Y1curr, M1, ACC1b

  211.         vpmadd52luq Y2curr, M0, ACC2a

  212.         vpmadd52luq Y2curr, M1, ACC2b

  213.         vpmadd52luq Y3curr, M0, ACC3a

  214.         vpmadd52luq Y3curr, M1, ACC3b

  215. 

  216.         vpsrlq $52, ACC0a, T0{%k1}{z}

  217.         vpsrlq $52, ACC1a, T1{%k1}{z}

  218.         vpsrlq $52, ACC2a, T2{%k1}{z}

  219.         vpsrlq $52, ACC3a, T3{%k1}{z}

  220. 

  221.         dec itr

  222.     jne 1b

  223.     valignq $1, ACC0a, ACC0b, ACC0a

  224.     valignq $1, ACC0b, ZERO, ACC0b

  225.     valignq $1, ACC1a, ACC1b, ACC1a

  226.     valignq $1, ACC1b, ZERO, ACC1b

  227.     valignq $1, ACC2a, ACC2b, ACC2a

  228.     valignq $1, ACC2b, ZERO, ACC2b

  229.     valignq $1, ACC3a, ACC3b, ACC3a

  230.     valignq $1, ACC3b, ZERO, ACC3b

  231.     vpaddq T0, ACC0a, ACC0a

  232.     vpaddq T1, ACC1a, ACC1a

  233.     vpaddq T2, ACC2a, ACC2a

  234.     vpaddq T3, ACC3a, ACC3a

  235. 

  236.     # The last high multiplications

  237.     vpmadd52huq B0curr, A0a, ACC0a

  238.     vpmadd52huq B0curr, A0b, ACC0b

  239.     vpmadd52huq B1curr, A1a, ACC1a

  240.     vpmadd52huq B1curr, A1b, ACC1b

  241.     vpmadd52huq B0curr, A1a, ACC2a

  242.     vpmadd52huq B0curr, A1b, ACC2b

  243.     vpmadd52huq B1curr, A0a, ACC3a

  244.     vpmadd52huq B1curr, A0b, ACC3b

  245. 

  246.     vpmadd52huq Y0curr, M0, ACC0a

  247.     vpmadd52huq Y0curr, M1, ACC0b

  248.     vpmadd52huq Y1curr, M0, ACC1a

  249.     vpmadd52huq Y1curr, M1, ACC1b

  250.     vpmadd52huq Y2curr, M0, ACC2a

  251.     vpmadd52huq Y2curr, M1, ACC2b

  252.     vpmadd52huq Y3curr, M0, ACC3a

  253.     vpmadd52huq Y3curr, M1, ACC3b

  254. 

  255.     # C0 = A0*B0 - A1*B1

  256.     # C1 = A0*B1 + A1*B0

  257.     vpaddq  64*0 + .LpolyX(%rip), ACC0a, ACC0a

  258.     vpaddq  64*1 + .LpolyX(%rip), ACC0b, ACC0b

  259. 

  260.     vpaddq  ACC3a, ACC2a, ACC2a

  261.     vpaddq  ACC3b, ACC2b, ACC2b

  262. 

  263.     vpsubq  ACC1a, ACC0a, ACC0a

  264.     vpsubq  ACC1b, ACC0b, ACC0b

  265.     # Now 'normalize' the acc to 52 bit words

  266.     vpsrlq  $52, ACC0a, A0a

  267.     vpsrlq  $52, ACC0b, A0b

  268. 

  269.     vpsrlq  $52, ACC2a, A1a

  270.     vpsrlq  $52, ACC2b, A1b

  271. 

  272.     vpandq  AND_MASK, ACC0a, ACC0a

  273.     vpandq  AND_MASK, ACC0b, ACC0b

  274.     vpandq  AND_MASK, ACC2a, ACC2a

  275.     vpandq  AND_MASK, ACC2b, ACC2b

  276. 

  277.     valignq $7, A0a, A0b, A0b

  278.     valignq $7, ZERO, A0a, A0a

  279.     valignq $7, A1a, A1b, A1b

  280.     valignq $7, ZERO, A1a, A1a

  281. 

  282.     vpaddq  A0a, ACC0a, ACC0a

  283.     vpaddq  A0b, ACC0b, ACC0b

  284.     vpaddq  A1a, ACC2a, ACC2a

  285.     vpaddq  A1b, ACC2b, ACC2b

  286. 

  287.     vpcmpuq $1, A0a, ACC0a, %k1

  288.     vpcmpuq $1, A0b, ACC0b, %k2

  289.     vpcmpuq $0, AND_MASK, ACC0a, %k3

  290.     vpcmpuq $0, AND_MASK, ACC0b, %k4

  291. 

  292.     kmovb %k1, %eax

  293.     kmovb %k2, %ecx

  294.     kmovb %k3, %r8d

  295.     kmovb %k4, %r9d

  296. 

  297.     add %al, %al

  298.     adc %cl, %cl

  299. 

  300.     add %r8b, %al

  301.     adc %r9b, %cl

  302. 

  303.     xor %r8b, %al

  304.     xor %r9b, %cl

  305. 

  306.     kmovb %eax, %k1

  307.     kmovb %ecx, %k2

  308. 

  309.     vpsubq AND_MASK, ACC0a, ACC0a{%k1}

  310.     vpsubq AND_MASK, ACC0b, ACC0b{%k2}

  311.     vpandq AND_MASK, ACC0a, ACC0a

  312.     vpandq AND_MASK, ACC0b, ACC0b

  313. 

  314.     vpcmpuq $1, A1a, ACC2a, %k1

  315.     vpcmpuq $1, A1b, ACC2b, %k2

  316.     vpcmpuq $0, AND_MASK, ACC2a, %k3

  317.     vpcmpuq $0, AND_MASK, ACC2b, %k4

  318. 

  319.     kmovb %k1, %eax

  320.     kmovb %k2, %ecx

  321.     kmovb %k3, %r8d

  322.     kmovb %k4, %r9d

  323. 

  324.     add %al, %al

  325.     adc %cl, %cl

  326.     add %r8b, %al

  327.     adc %r9b, %cl

  328.     xor %r8b, %al

  329.     xor %r9b, %cl

  330.     kmovb %eax, %k1

  331.     kmovb %ecx, %k2

  332. 

  333.     vpsubq AND_MASK, ACC2a, ACC2a{%k1}

  334.     vpsubq AND_MASK, ACC2b, ACC2b{%k2}

  335.     vpandq AND_MASK, ACC2a, ACC2a

  336.     vpandq AND_MASK, ACC2b, ACC2b

  337. 

  338.     mov $0x7f, %eax

  339.     kmovw %eax, %k1

  340. 

  341.     vmovdqu64 ACC0a, 64*0(felemR)

  342.     vmovdqu64 ACC0b, 64*1(felemR){%k5}

  343.     vmovdqu64 ACC2a, 15*8 + 64*0(felemR)

  344.     vmovdqu64 ACC2b, 15*8 + 64*1(felemR){%k5}

  345.     ret

  346. 

  347. ###############################################################################

  348. 

  349. #define ST0 ACC3a

  350. #define ST1 ACC3b

  351. #define ST2 Y3curr

  352. 

  353. .globl C_ABI(fp2_sqr_ifma)

  354. .p2align 6

  355. C_ABI(fp2_sqr_ifma):

  356. 

  357.     mov $1, %eax

  358.     kmovw %eax, %k1

  359.     mov $0x7f, %eax

  360.     kmovw %eax, %k2

  361. 

  362.     vpbroadcastq .LandMask(%rip), AND_MASK

  363.     vpxorq ZERO, ZERO, ZERO

  364. 

  365.     vmovdqu64 64*0(felemA), A0a

  366.     vmovdqu64 64*1(felemA), A0b{%k2}{z}

  367.     vmovdqu64 15*8 + 64*0(felemA), A1a

  368.     vmovdqu64 15*8 + 64*1(felemA), A1b{%k2}{z}

  369. 

  370.     # Load the modulus

  371.     vmovdqa64 64*0 + .Lpoly(%rip), M0

  372.     vmovdqa64 64*1 + .Lpoly(%rip), M1

  373. 

  374.     # Prepare the accumulators

  375.     vpxorq ACC0a, ACC0a, ACC0a

  376.     vpxorq ACC0b, ACC0b, ACC0b

  377.     vpxorq ACC1a, ACC1a, ACC1a

  378.     vpxorq ACC1b, ACC1b, ACC1b

  379.     vpxorq ACC2a, ACC2a, ACC2a

  380.     vpxorq ACC2b, ACC2b, ACC2b

  381.     vpxorq T0, T0, T0

  382.     vpxorq T1, T1, T1

  383.     vpxorq T2, T2, T2

  384. 

  385.     # First iteration

  386.     vpbroadcastq (felemA), B0curr

  387.     vpbroadcastq 15*8(felemA), B1curr

  388.     lea 8(felemA), felemA

  389. 

  390.     vpmadd52luq B0curr, A0a, ACC0a

  391.     vpmadd52luq B0curr, A0b, ACC0b

  392.     vpmadd52luq B1curr, A1a, ACC1a

  393.     vpmadd52luq B1curr, A1b, ACC1b

  394.     vpmadd52luq B0curr, A1a, ACC2a

  395.     vpmadd52luq B0curr, A1b, ACC2b

  396. 

  397.     vpermq ACC0a, ZERO, Y0curr

  398.     vpermq ACC1a, ZERO, Y1curr

  399.     vpermq ACC2a, ZERO, Y2curr

  400. 

  401.     vpmadd52luq Y0curr, M0, ACC0a

  402.     vpmadd52luq Y0curr, M1, ACC0b

  403.     vpmadd52luq Y1curr, M0, ACC1a

  404.     vpmadd52luq Y1curr, M1, ACC1b

  405.     vpmadd52luq Y2curr, M0, ACC2a

  406.     vpmadd52luq Y2curr, M1, ACC2b

  407. 

  408.     vpsrlq $52, ACC0a, T0{%k1}{z}

  409.     vpsrlq $52, ACC1a, T1{%k1}{z}

  410.     vpsrlq $52, ACC2a, T2{%k1}{z}

  411. 

  412.     mov $14, itr

  413. 

  414. 1:

  415.         # Shift the ACC in zmms right by a word

  416.         valignq $1, ACC0a, ACC0b, ACC0a

  417.         valignq $1, ACC0b, ZERO, ACC0b

  418.         valignq $1, ACC1a, ACC1b, ACC1a

  419.         valignq $1, ACC1b, ZERO, ACC1b

  420.         valignq $1, ACC2a, ACC2b, ACC2a

  421.         valignq $1, ACC2b, ZERO, ACC2b

  422. 

  423.         vpxorq ST0, ST0, ST0

  424.         vpxorq ST1, ST1, ST1

  425.         vpxorq ST2, ST2, ST2

  426. 

  427.         vmovdqa64 B0curr, B0prev

  428.         vmovdqa64 B1curr, B1prev

  429.         vmovdqa64 Y0curr, Y0prev

  430.         vmovdqa64 Y1curr, Y1prev

  431.         vmovdqa64 Y2curr, Y2prev

  432. 

  433.         vpbroadcastq (felemA), B0curr

  434.         vpbroadcastq 15*8(felemA), B1curr

  435.         lea 8(felemA), felemA

  436. 

  437.         # High multiplications

  438.         vpmadd52huq B0prev, A0a, ACC0a # ACC0 = A0 * B0

  439.         vpmadd52huq B1prev, A1a, ACC1a # ACC1 = A1 * B1

  440.         vpmadd52huq B0prev, A1a, ACC2a # ACC2 = A1 * B0

  441.         vpmadd52huq B0prev, A0b, ACC0b

  442.         vpmadd52huq B1prev, A1b, ACC1b

  443.         vpmadd52huq B0prev, A1b, ACC2b

  444.         # We really want to have 8 independent vpmadd instructions in the pipe

  445.         vpmadd52huq Y0prev, M0, T0

  446.         vpmadd52huq Y1prev, M0, T1

  447.         vpmadd52huq Y2prev, M0, T2

  448. 

  449.         vpmadd52huq Y0prev, M1, ACC0b

  450.         vpmadd52huq Y1prev, M1, ACC1b

  451.         vpmadd52huq Y2prev, M1, ACC2b

  452.         # Low multiplications

  453.         vpmadd52luq B0curr, A0a, ACC0a

  454.         vpmadd52luq B1curr, A1a, ACC1a

  455.         vpmadd52luq B0curr, A1a, ACC2a

  456.  

  457.         vpmadd52luq B0curr, A0b, ST0

  458.         vpmadd52luq B1curr, A1b, ST1

  459.         vpmadd52luq B0curr, A1b, ST2

  460. 

  461.         vpaddq T0, ACC0a, ACC0a

  462.         vpaddq T1, ACC1a, ACC1a

  463.         vpaddq T2, ACC2a, ACC2a

  464.         vpermq ACC0a, ZERO, Y0curr

  465.         vpermq ACC1a, ZERO, Y1curr

  466.         vpermq ACC2a, ZERO, Y2curr

  467.         vpaddq ST0, ACC0b, ACC0b

  468.         vpaddq ST1, ACC1b, ACC1b

  469.         vpaddq ST2, ACC2b, ACC2b

  470. 

  471.         vpmadd52luq Y0curr, M0, ACC0a

  472.         vpmadd52luq Y0curr, M1, ACC0b

  473.         vpmadd52luq Y1curr, M0, ACC1a

  474.         vpmadd52luq Y1curr, M1, ACC1b

  475.         vpmadd52luq Y2curr, M0, ACC2a

  476.         vpmadd52luq Y2curr, M1, ACC2b

  477. 

  478.         vpsrlq $52, ACC0a, T0{%k1}{z}

  479.         vpsrlq $52, ACC1a, T1{%k1}{z}

  480.         vpsrlq $52, ACC2a, T2{%k1}{z}

  481. 

  482.         dec itr

  483.     jne 1b

  484.     valignq $1, ACC0a, ACC0b, ACC0a

  485.     valignq $1, ACC0b, ZERO, ACC0b

  486.     valignq $1, ACC1a, ACC1b, ACC1a

  487.     valignq $1, ACC1b, ZERO, ACC1b

  488.     valignq $1, ACC2a, ACC2b, ACC2a

  489.     valignq $1, ACC2b, ZERO, ACC2b

  490.     vpaddq T0, ACC0a, ACC0a

  491.     vpaddq T1, ACC1a, ACC1a

  492.     vpaddq T2, ACC2a, ACC2a

  493. 

  494.     # The last high multiplications

  495.     vpmadd52huq B0curr, A0a, ACC0a

  496.     vpmadd52huq B0curr, A0b, ACC0b

  497.     vpmadd52huq B1curr, A1a, ACC1a

  498.     vpmadd52huq B1curr, A1b, ACC1b

  499.     vpmadd52huq B0curr, A1a, ACC2a

  500.     vpmadd52huq B0curr, A1b, ACC2b

  501. 

  502.     vpmadd52huq Y0curr, M0, ACC0a

  503.     vpmadd52huq Y0curr, M1, ACC0b

  504.     vpmadd52huq Y1curr, M0, ACC1a

  505.     vpmadd52huq Y1curr, M1, ACC1b

  506.     vpmadd52huq Y2curr, M0, ACC2a

  507.     vpmadd52huq Y2curr, M1, ACC2b

  508. 

  509.     # C0 = A0*B0 - A1*B1

  510.     # C1 = A0*B1 + A1*B0

  511.     vpaddq  64*0 + .LpolyX(%rip), ACC0a, ACC0a

  512.     vpaddq  64*1 + .LpolyX(%rip), ACC0b, ACC0b

  513. 

  514.     vpaddq  ACC2a, ACC2a, ACC2a

  515.     vpaddq  ACC2b, ACC2b, ACC2b

  516. 

  517.     vpsubq  ACC1a, ACC0a, ACC0a

  518.     vpsubq  ACC1b, ACC0b, ACC0b

  519. 

  520.     # Now 'normalize' the acc to 52 bit words

  521.     vpsrlq  $52, ACC0a, A0a

  522.     vpsrlq  $52, ACC0b, A0b

  523.     vpsrlq  $52, ACC2a, A1a

  524.     vpsrlq  $52, ACC2b, A1b

  525. 

  526.     vpandq  AND_MASK, ACC0a, ACC0a

  527.     vpandq  AND_MASK, ACC0b, ACC0b

  528.     vpandq  AND_MASK, ACC2a, ACC2a

  529.     vpandq  AND_MASK, ACC2b, ACC2b

  530. 

  531.     valignq $7, A0a, A0b, A0b

  532.     valignq $7, ZERO, A0a, A0a

  533.     valignq $7, A1a, A1b, A1b

  534.     valignq $7, ZERO, A1a, A1a

  535. 

  536.     vpaddq  A0a, ACC0a, ACC0a

  537.     vpaddq  A0b, ACC0b, ACC0b

  538.     vpaddq  A1a, ACC2a, ACC2a

  539.     vpaddq  A1b, ACC2b, ACC2b

  540. 

  541.     vpcmpuq $1, A0a, ACC0a, %k1

  542.     vpcmpuq $1, A0b, ACC0b, %k2

  543.     vpcmpuq $0, AND_MASK, ACC0a, %k3

  544.     vpcmpuq $0, AND_MASK, ACC0b, %k4

  545. 

  546.     kmovb %k1, %eax

  547.     kmovb %k2, %ecx

  548.     kmovb %k3, %r8d

  549.     kmovb %k4, %r9d

  550. 

  551.     add %al, %al

  552.     adc %cl, %cl

  553.     add %r8b, %al

  554.     adc %r9b, %cl

  555.     xor %r8b, %al

  556.     xor %r9b, %cl

  557.     kmovb %eax, %k1

  558.     kmovb %ecx, %k2

  559. 

  560.     vpsubq AND_MASK, ACC0a, ACC0a{%k1}

  561.     vpsubq AND_MASK, ACC0b, ACC0b{%k2}

  562.     vpandq AND_MASK, ACC0a, ACC0a

  563.     vpandq AND_MASK, ACC0b, ACC0b

  564. 

  565.     vpcmpuq $1, A1a, ACC2a, %k1

  566.     vpcmpuq $1, A1b, ACC2b, %k2

  567.     vpcmpuq $0, AND_MASK, ACC2a, %k3

  568.     vpcmpuq $0, AND_MASK, ACC2b, %k4

  569. 

  570.     kmovb %k1, %eax

  571.     kmovb %k2, %ecx

  572.     kmovb %k3, %r8d

  573.     kmovb %k4, %r9d

  574. 

  575.     add %al, %al

  576.     adc %cl, %cl

  577.     add %r8b, %al

  578.     adc %r9b, %cl

  579.     xor %r8b, %al

  580.     xor %r9b, %cl

  581.     kmovb %eax, %k1

  582.     kmovb %ecx, %k2

  583. 

  584.     vpsubq AND_MASK, ACC2a, ACC2a{%k1}

  585.     vpsubq AND_MASK, ACC2b, ACC2b{%k2}

  586.     vpandq AND_MASK, ACC2a, ACC2a

  587.     vpandq AND_MASK, ACC2b, ACC2b

  588. 

  589.     mov $0x7f, %eax

  590.     kmovw %eax, %k1

  591. 

  592.     vmovdqu64 ACC0a, 64*0(felemR)

  593.     vmovdqu64 ACC0b, 64*1(felemR){%k1}

  594.     vmovdqu64 ACC2a, 15*8 + 64*0(felemR)

  595.     vmovdqu64 ACC2b, 15*8 + 64*1(felemR){%k1}

  596.     ret

  597. 

  598. ###############################################################################

  599. .globl C_ABI(fp2_sub)

  600. .p2align 6

  601. C_ABI(fp2_sub):

  602. 

  603.     mov $1, %eax

  604.     kmovw %eax, %k1

  605.     mov $0x7f, %eax

  606.     kmovw %eax, %k2

  607.     

  608.     vmovdqu64 64*0(felemA), ACC0a

  609.     vmovdqu64 64*1(felemA), ACC0b{%k2}{z}

  610.     vmovdqu64 15*8 + 64*0(felemA), ACC1a

  611.     vmovdqu64 15*8 + 64*1(felemA), ACC1b{%k2}{z}

  612. 

  613.     vmovdqu64 64*0(felemB), ACC2a

  614.     vmovdqu64 64*1(felemB), ACC2b{%k2}{z}

  615.     vmovdqu64 15*8 + 64*0(felemB), ACC3a

  616.     vmovdqu64 15*8 + 64*1(felemB), ACC3b{%k2}{z}

  617. 

  618.     vpaddq 64*0 + .LpolyX(%rip), ACC0a, ACC0a

  619.     vpaddq 64*1 + .LpolyX(%rip), ACC0b, ACC0b

  620.     vpaddq 64*0 + .LpolyX(%rip), ACC1a, ACC1a

  621.     vpaddq 64*1 + .LpolyX(%rip), ACC1b, ACC1b

  622. 

  623.     vpsubq ACC2a, ACC0a, ACC0a

  624.     vpsubq ACC2b, ACC0b, ACC0b

  625.     vpsubq ACC3a, ACC1a, ACC2a

  626.     vpsubq ACC3b, ACC1b, ACC2b

  627. 

  628.     jmp fp2_normalize

  629. ###############################################################################

  630. .globl C_ABI(fp2_add)

  631. .p2align 6

  632. C_ABI(fp2_add):

  633. 

  634.     mov $1, %eax

  635.     kmovw %eax, %k1

  636.     mov $0x7f, %eax

  637.     kmovw %eax, %k2

  638.     

  639.     vmovdqu64 64*0(felemA), ACC0a

  640.     vmovdqu64 64*1(felemA), ACC0b{%k2}{z}

  641.     vmovdqu64 15*8 + 64*0(felemA), ACC1a

  642.     vmovdqu64 15*8 + 64*1(felemA), ACC1b{%k2}{z}

  643. 

  644.     vmovdqu64 64*0(felemB), ACC2a

  645.     vmovdqu64 64*1(felemB), ACC2b{%k2}{z}

  646.     vmovdqu64 15*8 + 64*0(felemB), ACC3a

  647.     vmovdqu64 15*8 + 64*1(felemB), ACC3b{%k2}{z}

  648. 

  649.     vpaddq ACC2a, ACC0a, ACC0a

  650.     vpaddq ACC2b, ACC0b, ACC0b

  651.     vpaddq ACC3a, ACC1a, ACC2a

  652.     vpaddq ACC3b, ACC1b, ACC2b

  653. 

  654.     // Fallthrough

  655. ###############################################################################

  656. .p2align 6

  657. C_ABI(fp2_normalize):

  658. 

  659.     vpbroadcastq .LandMask(%rip), AND_MASK

  660.     vpxorq ZERO, ZERO, ZERO

  661. 

  662.     # Now 'normalize' the acc to 52 bit words

  663.     vpsrlq  $52, ACC0a, A0a

  664.     vpsrlq  $52, ACC0b, A0b

  665.     vpsrlq  $52, ACC2a, A1a

  666.     vpsrlq  $52, ACC2b, A1b

  667. 

  668.     vpandq  AND_MASK, ACC0a, ACC0a

  669.     vpandq  AND_MASK, ACC0b, ACC0b

  670.     vpandq  AND_MASK, ACC2a, ACC2a

  671.     vpandq  AND_MASK, ACC2b, ACC2b

  672. 

  673.     valignq $7, A0a, A0b, A0b

  674.     valignq $7, ZERO, A0a, A0a

  675.     valignq $7, A1a, A1b, A1b

  676.     valignq $7, ZERO, A1a, A1a

  677. 

  678.     vpaddq  A0a, ACC0a, ACC0a

  679.     vpaddq  A0b, ACC0b, ACC0b

  680.     vpaddq  A1a, ACC2a, ACC2a

  681.     vpaddq  A1b, ACC2b, ACC2b

  682. 

  683.     vpcmpuq $1, A0a, ACC0a, %k1

  684.     vpcmpuq $1, A0b, ACC0b, %k2

  685.     vpcmpuq $0, AND_MASK, ACC0a, %k3

  686.     vpcmpuq $0, AND_MASK, ACC0b, %k4

  687. 

  688.     kmovb %k1, %eax

  689.     kmovb %k2, %ecx

  690.     kmovb %k3, %r8d

  691.     kmovb %k4, %r9d

  692. 

  693.     add %al, %al

  694.     adc %cl, %cl

  695.     add %r8b, %al

  696.     adc %r9b, %cl

  697.     xor %r8b, %al

  698.     xor %r9b, %cl

  699.     kmovb %eax, %k1

  700.     kmovb %ecx, %k2

  701. 

  702.     vpsubq AND_MASK, ACC0a, ACC0a{%k1}

  703.     vpsubq AND_MASK, ACC0b, ACC0b{%k2}

  704.     vpandq AND_MASK, ACC0a, ACC0a

  705.     vpandq AND_MASK, ACC0b, ACC0b

  706. 

  707.     vpcmpuq $1, A1a, ACC2a, %k1

  708.     vpcmpuq $1, A1b, ACC2b, %k2

  709.     vpcmpuq $0, AND_MASK, ACC2a, %k3

  710.     vpcmpuq $0, AND_MASK, ACC2b, %k4

  711. 

  712.     kmovb %k1, %eax

  713.     kmovb %k2, %ecx

  714.     kmovb %k3, %r8d

  715.     kmovb %k4, %r9d

  716. 

  717.     add %al, %al

  718.     adc %cl, %cl

  719.     add %r8b, %al

  720.     adc %r9b, %cl

  721.     xor %r8b, %al

  722.     xor %r9b, %cl

  723.     kmovb %eax, %k1

  724.     kmovb %ecx, %k2

  725. 

  726.     vpsubq AND_MASK, ACC2a, ACC2a{%k1}

  727.     vpsubq AND_MASK, ACC2b, ACC2b{%k2}

  728.     vpandq AND_MASK, ACC2a, ACC2a

  729.     vpandq AND_MASK, ACC2b, ACC2b

  730. 

  731.     mov $0x7f, %eax

  732.     kmovw %eax, %k1

  733. 

  734.     vmovdqu64 ACC0a, 64*0(felemR)

  735.     vmovdqu64 ACC0b, 64*1(felemR){%k1}

  736.     vmovdqu64 ACC2a, 15*8 + 64*0(felemR)

  737.     vmovdqu64 ACC2b, 15*8 + 64*1(felemR){%k1}

  738. 

  739.     ret

  740. 

  741. 

  742. ###############################################################################

  743. #define p1ptr %rdi

  744. #define p2ptr %rsi

  745. #define swap %rdx

  746. .globl C_ABI(fp2_swap)

  747. .p2align 6

  748. C_ABI(fp2_swap):

  749. 

  750.     mov $0x7f, %eax

  751.     kmovw %eax, %k2

  752.     // TODO: get rid of the masks, not needed

  753.     vmovdqu64 64*0(p1ptr), %zmm0

  754.     vmovdqu64 64*1(p1ptr), %zmm1{%k2}{z}

  755.     vmovdqu64 15*8 + 64*0(p1ptr), %zmm2

  756.     vmovdqu64 15*8 + 64*1(p1ptr), %zmm3{%k2}{z}

  757.     vmovdqu64 2*15*8 + 64*0(p1ptr), %zmm4

  758.     vmovdqu64 2*15*8 + 64*1(p1ptr), %zmm5{%k2}{z}

  759.     vmovdqu64 3*15*8 + 64*0(p1ptr), %zmm6

  760.     vmovdqu64 3*15*8 + 64*1(p1ptr), %zmm7{%k2}{z}

  761. 

  762.     vmovdqu64 64*0(p2ptr), %zmm8

  763.     vmovdqu64 64*1(p2ptr), %zmm9{%k2}{z}

  764.     vmovdqu64 15*8 + 64*0(p2ptr), %zmm10

  765.     vmovdqu64 15*8 + 64*1(p2ptr), %zmm11{%k2}{z}

  766.     vmovdqu64 2*15*8 + 64*0(p2ptr), %zmm12

  767.     vmovdqu64 2*15*8 + 64*1(p2ptr), %zmm13{%k2}{z}

  768.     vmovdqu64 3*15*8 + 64*0(p2ptr), %zmm14

  769.     vmovdqu64 3*15*8 + 64*1(p2ptr), %zmm15{%k2}{z}

  770. 

  771.     vpxorq %zmm16, %zmm16, %zmm16

  772.     vpbroadcastq swap, %zmm17

  773.     vpsubq %zmm17, %zmm16, %zmm16

  774. 

  775.     vmovdqa64 %zmm8, %zmm17

  776.     vmovdqa64 %zmm9, %zmm18

  777.     vmovdqa64 %zmm10, %zmm19

  778.     vmovdqa64 %zmm11, %zmm20

  779.     vmovdqa64 %zmm12, %zmm21

  780.     vmovdqa64 %zmm13, %zmm22

  781.     vmovdqa64 %zmm14, %zmm23

  782.     vmovdqa64 %zmm15, %zmm24

  783. 

  784.     vpternlogq $0xd8, %zmm16, %zmm0, %zmm17

  785.     vpternlogq $0xd8, %zmm16, %zmm1, %zmm18

  786.     vpternlogq $0xd8, %zmm16, %zmm2, %zmm19

  787.     vpternlogq $0xd8, %zmm16, %zmm3, %zmm20

  788.     vpternlogq $0xd8, %zmm16, %zmm4, %zmm21

  789.     vpternlogq $0xd8, %zmm16, %zmm5, %zmm22

  790.     vpternlogq $0xd8, %zmm16, %zmm6, %zmm23

  791.     vpternlogq $0xd8, %zmm16, %zmm7, %zmm24

  792. 

  793.     vpternlogq $0xe4, %zmm16, %zmm0, %zmm8

  794.     vpternlogq $0xe4, %zmm16, %zmm1, %zmm9

  795.     vpternlogq $0xe4, %zmm16, %zmm2, %zmm10

  796.     vpternlogq $0xe4, %zmm16, %zmm3, %zmm11

  797.     vpternlogq $0xe4, %zmm16, %zmm4, %zmm12

  798.     vpternlogq $0xe4, %zmm16, %zmm5, %zmm13

  799.     vpternlogq $0xe4, %zmm16, %zmm6, %zmm14

  800.     vpternlogq $0xe4, %zmm16, %zmm7, %zmm15

  801. 

  802. 

  803.     vmovdqu64 %zmm8, 64*0(p1ptr)

  804.     vmovdqu64 %zmm9, 64*1(p1ptr){%k2}

  805.     vmovdqu64 %zmm10, 15*8 + 64*0(p1ptr)

  806.     vmovdqu64 %zmm11, 15*8 + 64*1(p1ptr){%k2}

  807.     vmovdqu64 %zmm12, 2*15*8 + 64*0(p1ptr)

  808.     vmovdqu64 %zmm13, 2*15*8 + 64*1(p1ptr){%k2}

  809.     vmovdqu64 %zmm14, 3*15*8 + 64*0(p1ptr)

  810.     vmovdqu64 %zmm15, 3*15*8 + 64*1(p1ptr){%k2}

  811. 

  812.     vmovdqu64 %zmm17, 64*0(p2ptr)

  813.     vmovdqu64 %zmm18, 64*1(p2ptr){%k2}

  814.     vmovdqu64 %zmm19, 15*8 + 64*0(p2ptr)

  815.     vmovdqu64 %zmm20, 15*8 + 64*1(p2ptr){%k2}

  816.     vmovdqu64 %zmm21, 2*15*8 + 64*0(p2ptr)

  817.     vmovdqu64 %zmm22, 2*15*8 + 64*1(p2ptr){%k2}

  818.     vmovdqu64 %zmm23, 3*15*8 + 64*0(p2ptr)

  819.     vmovdqu64 %zmm24, 3*15*8 + 64*1(p2ptr){%k2}

  820. 

  821.     ret

  822. ###############################################################################

  823. .globl C_ABI(fp_add)

  824. .p2align 6

  825. C_ABI(fp_add):

  826. 

  827.     mov $0x7f, %eax

  828.     kmovw %eax, %k2

  829.     

  830.     vmovdqu64 64*0(felemA), ACC0a

  831.     vmovdqu64 64*1(felemA), ACC0b{%k2}{z}

  832. 

  833.     vmovdqu64 64*0(felemB), ACC2a

  834.     vmovdqu64 64*1(felemB), ACC2b{%k2}{z}

  835. 

  836.     vpaddq ACC2a, ACC0a, ACC0a

  837.     vpaddq ACC2b, ACC0b, ACC0b

  838. 

  839.     // Fallthrough

  840. ###############################################################################

  841. .p2align 6

  842. C_ABI(fp_normalize):

  843. 

  844.     vpbroadcastq .LandMask(%rip), AND_MASK

  845.     vpxorq ZERO, ZERO, ZERO

  846. 

  847.     # Now 'normalize' the acc to 52 bit words

  848.     vpsrlq  $52, ACC0a, A0a

  849.     vpsrlq  $52, ACC0b, A0b

  850. 

  851.     vpandq  AND_MASK, ACC0a, ACC0a

  852.     vpandq  AND_MASK, ACC0b, ACC0b

  853. 

  854.     valignq $7, A0a, A0b, A0b

  855.     valignq $7, ZERO, A0a, A0a

  856. 

  857.     vpaddq  A0a, ACC0a, ACC0a

  858.     vpaddq  A0b, ACC0b, ACC0b

  859. 

  860.     vpcmpuq $1, A0a, ACC0a, %k1

  861.     vpcmpuq $1, A0b, ACC0b, %k2

  862.     vpcmpuq $0, AND_MASK, ACC0a, %k3

  863.     vpcmpuq $0, AND_MASK, ACC0b, %k4

  864. 

  865.     kmovb %k1, %eax

  866.     kmovb %k2, %ecx

  867.     kmovb %k3, %r8d

  868.     kmovb %k4, %r9d

  869. 

  870.     add %al, %al

  871.     adc %cl, %cl

  872. 

  873.     add %r8b, %al

  874.     adc %r9b, %cl

  875. 

  876.     xor %r8b, %al

  877.     xor %r9b, %cl

  878. 

  879.     kmovb %eax, %k1

  880.     kmovb %ecx, %k2

  881. 

  882.     vpsubq AND_MASK, ACC0a, ACC0a{%k1}

  883.     vpsubq AND_MASK, ACC0b, ACC0b{%k2}

  884.     vpandq AND_MASK, ACC0a, ACC0a

  885.     vpandq AND_MASK, ACC0b, ACC0b

  886. 

  887.     mov $0x7f, %eax

  888.     kmovw %eax, %k1

  889. 

  890.     vmovdqu64 ACC0a, 64*0(%rdi)

  891.     vmovdqu64 ACC0b, 64*1(%rdi){%k1}

  892. 

  893.     ret

  894. 

  895. ###############################################################################

  896. .globl C_ABI(fp_sub)

  897. .p2align 6

  898. C_ABI(fp_sub):

  899. 

  900.     mov $0x7f, %eax

  901.     kmovw %eax, %k2

  902.     

  903.     vmovdqu64 64*0(felemA), ACC0a

  904.     vmovdqu64 64*1(felemA), ACC0b{%k2}{z}

  905. 

  906.     vmovdqu64 64*0(felemB), ACC2a

  907.     vmovdqu64 64*1(felemB), ACC2b{%k2}{z}

  908. 

  909.     vpaddq 64*0 + .LpolyX(%rip), ACC0a, ACC0a

  910.     vpaddq 64*1 + .LpolyX(%rip), ACC0b, ACC0b

  911. 

  912.     vpsubq ACC2a, ACC0a, ACC0a

  913.     vpsubq ACC2b, ACC0b, ACC0b

  914. 

  915.     jmp fp_normalize