kris
/
sike_ifma
1
0
Förgrening 0
Kod Ärenden 0 Pull-förfrågningar 0 Släpp 0 Wiki Aktiviteter
Du kan inte välja fler än 25 ämnen Ämnen måste starta med en bokstav eller siffra, kan innehålla bindestreck ('-') och vara max 35 tecken långa.
1 Commit
1 Gren
116 KiB
 
 
 
Gren: master
Grenar Taggar
${ item.name }
Skapa branchen ${ searchTerm }
från 'master'
${ noResults }
sike_ifma/fp2_packed_751_ifma.S

523 rader
16 KiB
Rå Permalänk Blame Historik 

  1. #if defined(__APPLE__)

  2. /* OS X's C ABI prefixes functions with underscore. */

  3. #define C_ABI(x) _ ## x

  4. #define HIDDEN .private_extern

  5. #else

  6. #define C_ABI(x) x

  7. #define HIDDEN .hidden

  8. #endif

  9. 

  10. #define ACC0 %zmm0

  11. #define ACC1 %zmm1

  12. #define ACC2 %zmm2

  13. #define ACC3 %zmm3

  14. #define ACC4 %zmm4

  15. #define ACC5 %zmm5

  16. #define ACC6 %zmm6

  17. #define ACC7 %zmm7

  18. #define ACC8 %zmm8

  19. #define ACC9 %zmm9

  20. #define ACC10 %zmm10

  21. #define ACC11 %zmm11

  22. #define ACC12 %zmm12

  23. #define ACC13 %zmm13

  24. #define ACC14 %zmm14

  25. #define ACC15 %zmm15

  26. 

  27. #define A0 %zmm16

  28. #define A1 %zmm17

  29. #define A2 %zmm18

  30. #define A3 %zmm19

  31. #define A4 %zmm20

  32. #define A5 %zmm21

  33. #define A6 %zmm22

  34. #define A7 %zmm23

  35. #define A8 %zmm24

  36. #define A9 %zmm25

  37. #define A10 %zmm26

  38. #define A11 %zmm27

  39. #define A12 %zmm28

  40. #define A13 %zmm29

  41. #define A14 %zmm30

  42. 

  43. #define B %zmm31

  44. 

  45. #define rptr %rdi

  46. #define aptr %rsi

  47. #define bptr %rdx

  48. 

  49. #define r0ptr %rdi

  50. #define a0ptr %rsi

  51. #define b0ptr %rdx

  52. 

  53. #define r1ptr %rcx

  54. #define a1ptr %r8

  55. #define b1ptr %r9

  56. 

  57. #define hlp %rax

  58. 

  59. .p2align 6

  60. .Lmask:

  61. .Lpoly:

  62. .quad 0x000fffffffffffff, 0x000fffffffffffff, 0x000fffffffffffff, 0x000fffffffffffff 

  63. .quad 0x000fffffffffffff, 0x000fffffffffffff, 0x000fffffffffffff, 0x00049f878a8eeaff

  64. .quad 0x0007cc76e3ec9685, 0x00076da959b1a13f, 0x00084e9867d6ebe8, 0x000b5045cb257480

  65. .quad 0x000f97badc668562, 0x00041f71c0e12909, 0x00000000006fe5d5, 0

  66. 

  67. .LpolyX:

  68. .quad 0x0fffffffffffff00, 0x0fffffffffffff00, 0x0fffffffffffff00, 0x0fffffffffffff00

  69. .quad 0x0fffffffffffff00, 0x0fffffffffffff00, 0x0fffffffffffff00, 0x049f878a8eeaff00

  70. .quad 0x07cc76e3ec968500, 0x076da959b1a13f00, 0x084e9867d6ebe800, 0x0b5045cb25748000

  71. .quad 0x0f97badc66856200, 0x041f71c0e1290900, 0x000000006fe5d500, 0

  72. 

  73. .Lperm0:

  74. .quad 0,1,0,1,2,3,2,3

  75. 

  76. .Lperm1:

  77. .quad 4,5,5,4,6,7,7,6

  78. 

  79. // TODO: avoid transposing every call by keeping data vertical throughout

  80. 

  81. // Performs 8 field multiplications in parallel

  82. .globl C_ABI(fp2_mul_ifma_x2)

  83. C_ABI(fp2_mul_ifma_x2):

  84. 

  85.     push %rbp

  86.     mov %rsp, %rbp

  87.     sub $960, %rsp

  88.     and $-64, %rsp

  89. 

  90.     mov $0x7f, %rax

  91.     kmovq %rax, %k5

  92. 

  93.     // Load a0[0]

  94.     vmovdqu64 0*64(a0ptr), %zmm0

  95.     vmovdqu64 1*64(a0ptr), %zmm1{%k5}{z}

  96.     lea 15*8(a0ptr), a0ptr

  97.     // Load a0[1]

  98.     vmovdqu64 0*64(a0ptr), %zmm2

  99.     vmovdqu64 1*64(a0ptr), %zmm3{%k5}{z}

  100.     // Load b0[0]

  101.     vmovdqu64 0*64(b0ptr), %zmm4

  102.     vmovdqu64 1*64(b0ptr), %zmm5{%k5}{z}

  103.     lea 15*8(b0ptr), b0ptr

  104.     // Load b0[1]

  105.     vmovdqu64 0*64(b0ptr), %zmm6

  106.     vmovdqu64 1*64(b0ptr), %zmm7{%k5}{z}

  107.     // Load a1[0]

  108.     vmovdqu64 0*64(a1ptr), %zmm8

  109.     vmovdqu64 1*64(a1ptr), %zmm9{%k5}{z}

  110.     lea 15*8(a1ptr), a1ptr

  111.     // Load a1[1]

  112.     vmovdqu64 0*64(a1ptr), %zmm10

  113.     vmovdqu64 1*64(a1ptr), %zmm11{%k5}{z}

  114.     // Load b1[0]

  115.     vmovdqu64 0*64(b1ptr), %zmm12

  116.     vmovdqu64 1*64(b1ptr), %zmm13{%k5}{z}

  117.     lea 15*8(b1ptr), b1ptr

  118.     // Load b1[1]

  119.     vmovdqu64 0*64(b1ptr), %zmm14

  120.     vmovdqu64 1*64(b1ptr), %zmm15{%k5}{z}

  121.     // Transpose

  122.     vpunpcklqdq %zmm2, %zmm0, %zmm16 // 0 0 2 2 4 4 6 6

  123.     vpunpckhqdq %zmm2, %zmm0, %zmm17 // 1 1 3 3 5 5 7 7

  124.     vpunpcklqdq %zmm6, %zmm4, %zmm18 // 0 0 2 2 4 4 6 6

  125.     vpunpckhqdq %zmm6, %zmm4, %zmm19 // 1 1 3 3 5 5 7 7

  126.     vpunpcklqdq %zmm10, %zmm8, %zmm20 // 0 0 2 2 4 4 6 6

  127.     vpunpckhqdq %zmm10, %zmm8, %zmm21 // 1 1 3 3 5 5 7 7

  128.     vpunpcklqdq %zmm14, %zmm12, %zmm22 // 0 0 2 2 4 4 6 6

  129.     vpunpckhqdq %zmm14, %zmm12, %zmm23 // 1 1 3 3 5 5 7 7

  130.     

  131.     vpunpcklqdq %zmm3, %zmm1, %zmm24 // 8 8 10 10 12 12 14 14

  132.     vpunpckhqdq %zmm3, %zmm1, %zmm25 // 9 9 11 11 13 13 15 15

  133.     vpunpcklqdq %zmm7, %zmm5, %zmm26 // 8 8 10 10 12 12 14 14

  134.     vpunpckhqdq %zmm7, %zmm5, %zmm27 // 9 9 11 11 13 13 15 15

  135.     vpunpcklqdq %zmm11, %zmm9, %zmm28 // 8 8 10 10 12 12 14 14

  136.     vpunpckhqdq %zmm11, %zmm9, %zmm29 // 9 9 11 11 13 13 15 15

  137.     vpunpcklqdq %zmm15, %zmm13, %zmm30 // 8 8 10 10 12 12 14 14

  138.     vpunpckhqdq %zmm15, %zmm13, %zmm31 // 9 9 11 11 13 13 15 15

  139. 

  140.     vshufi64x2 $0x44, %zmm20, %zmm16, %zmm0 // 0 0 2 2 0 0 2 2

  141.     vshufi64x2 $0x44, %zmm22, %zmm18, %zmm1 // 0 0 2 2 0 0 2 2

  142.     vshufi64x2 $0xee, %zmm20, %zmm16, %zmm2 // 4 4 6 6 4 4 6 6

  143.     vshufi64x2 $0xee, %zmm22, %zmm18, %zmm3 // 4 4 6 6 4 4 6 6

  144. 

  145.     vshufi64x2 $0x44, %zmm21, %zmm17, %zmm4 // 1 1 3 3 1 1 3 3

  146.     vshufi64x2 $0x44, %zmm23, %zmm19, %zmm5 // 1 1 3 3 1 1 3 3

  147.     vshufi64x2 $0xee, %zmm21, %zmm17, %zmm6 // 5 5 7 7 5 5 7 7

  148.     vshufi64x2 $0xee, %zmm23, %zmm19, %zmm7 // 5 5 7 7 5 5 7 7

  149. 

  150.     vshufi64x2 $0x44, %zmm28, %zmm24, %zmm8 // 8 8 10 10 8 8 10 10

  151.     vshufi64x2 $0x44, %zmm30, %zmm26, %zmm9 // 8 8 10 10 8 8 10 10

  152.     vshufi64x2 $0xee, %zmm28, %zmm24, %zmm10 // 12 12 14 14 12 12 14 14

  153.     vshufi64x2 $0xee, %zmm30, %zmm26, %zmm11 // 12 12 14 14 12 12 14 14

  154. 

  155.     vshufi64x2 $0x44, %zmm29, %zmm25, %zmm12 // 9 9 11 11 9 9 11 11

  156.     vshufi64x2 $0x44, %zmm31, %zmm27, %zmm13 // 9 9 11 11 9 9 11 11

  157.     vshufi64x2 $0xee, %zmm29, %zmm25, %zmm14 // 13 13 15 15 13 13 15 15

  158.     vshufi64x2 $0xee, %zmm31, %zmm27, %zmm15 // 13 13 15 15 13 13 15 15

  159. 

  160.     vshufi64x2 $0x88, %zmm1, %zmm0, %zmm16 //0

  161.     vshufi64x2 $0x88, %zmm5, %zmm4, %zmm17 //1

  162.     vshufi64x2 $0xdd, %zmm1, %zmm0, %zmm18 //

  163.     vshufi64x2 $0xdd, %zmm5, %zmm4, %zmm19

  164.     vshufi64x2 $0x88, %zmm3, %zmm2, %zmm20

  165.     vshufi64x2 $0x88, %zmm7, %zmm6, %zmm21

  166.     vshufi64x2 $0xdd, %zmm3, %zmm2, %zmm22

  167.     vshufi64x2 $0xdd, %zmm7, %zmm6, %zmm23

  168.     vshufi64x2 $0x88, %zmm9, %zmm8, %zmm24

  169.     vshufi64x2 $0x88, %zmm13, %zmm12, %zmm25

  170.     vshufi64x2 $0xdd, %zmm9, %zmm8, %zmm26

  171.     vshufi64x2 $0xdd, %zmm13, %zmm12, %zmm27

  172.     vshufi64x2 $0x88, %zmm11, %zmm10, %zmm28

  173.     vshufi64x2 $0x88, %zmm15, %zmm14, %zmm29

  174.     vshufi64x2 $0xdd, %zmm11, %zmm10, %zmm30

  175. 

  176.     vmovdqa64 .Lperm0(%rip), %zmm31

  177.     vpermq %zmm16, %zmm31, %zmm0

  178.     vpermq %zmm17, %zmm31, %zmm1

  179.     vpermq %zmm18, %zmm31, %zmm2

  180.     vpermq %zmm19, %zmm31, %zmm3

  181.     vpermq %zmm20, %zmm31, %zmm4

  182.     vpermq %zmm21, %zmm31, %zmm5

  183.     vpermq %zmm22, %zmm31, %zmm6

  184.     vpermq %zmm23, %zmm31, %zmm7

  185.     vpermq %zmm24, %zmm31, %zmm8

  186.     vpermq %zmm25, %zmm31, %zmm9

  187.     vpermq %zmm26, %zmm31, %zmm10

  188.     vpermq %zmm27, %zmm31, %zmm11

  189.     vpermq %zmm28, %zmm31, %zmm12

  190.     vpermq %zmm29, %zmm31, %zmm13

  191.     vpermq %zmm30, %zmm31, %zmm14

  192. 

  193.     .irp r,0,1,2,3,4,5,6,7,8,9,10,11,12,13,14

  194.     vmovdqu64 %zmm\r, \r*64(%rsp)

  195.     .endr

  196. 

  197.     vmovdqa64 .Lperm1(%rip), %zmm31

  198.     vpermq %zmm16, %zmm31, A0

  199.     vpermq %zmm17, %zmm31, A1

  200.     vpermq %zmm18, %zmm31, A2

  201.     vpermq %zmm19, %zmm31, A3

  202.     vpermq %zmm20, %zmm31, A4

  203.     vpermq %zmm21, %zmm31, A5

  204.     vpermq %zmm22, %zmm31, A6

  205.     vpermq %zmm23, %zmm31, A7

  206.     vpermq %zmm24, %zmm31, A8

  207.     vpermq %zmm25, %zmm31, A9

  208.     vpermq %zmm26, %zmm31, A10

  209.     vpermq %zmm27, %zmm31, A11

  210.     vpermq %zmm28, %zmm31, A12

  211.     vpermq %zmm29, %zmm31, A13

  212.     vpermq %zmm30, %zmm31, A14

  213. 

  214.     lea (%rsp), bptr

  215.     call do_mul_x2

  216. 

  217.     // After parallel multiplication the layout is:

  218.     // A0[0] * B0[0], A0[1] * B0[1], A0[0] * B0[1], A0[1] * B0[0], A1[0] * B1[0], A1[1] * B1[1], A1[0] * B1[1], A1[1] * B1[0]

  219.     // We need to compute:

  220.     // A0[0] * B0[0] - A0[1] * B0[1],  A0[0] * B0[1] + A0[1] * B0[0], A1[0] * B1[0] - A0[1] * B1[1],  A1[0] * B1[1] + A1[1] * B1[0]

  221.     vpsrldq $8, ACC0, A0

  222.     vpsrldq $8, ACC1, A1

  223.     vpsrldq $8, ACC2, A2

  224.     vpsrldq $8, ACC3, A3

  225.     vpsrldq $8, ACC4, A4

  226.     vpsrldq $8, ACC5, A5

  227.     vpsrldq $8, ACC6, A6

  228.     vpsrldq $8, ACC7, A7

  229.     vpsrldq $8, ACC8, A8

  230.     vpsrldq $8, ACC9, A9

  231.     vpsrldq $8, ACC10, A10

  232.     vpsrldq $8, ACC11, A11

  233.     vpsrldq $8, ACC12, A12

  234.     vpsrldq $8, ACC13, A13

  235.     vpsrldq $8, ACC14, A14

  236. 

  237.     mov $0x44, hlp

  238.     kmovq hlp, %k7

  239. 

  240.     vpaddq A0, ACC0, ACC0{%k7}

  241.     vpaddq A1, ACC1, ACC1{%k7}

  242.     vpaddq A2, ACC2, ACC2{%k7}

  243.     vpaddq A3, ACC3, ACC3{%k7}

  244.     vpaddq A4, ACC4, ACC4{%k7}

  245.     vpaddq A5, ACC5, ACC5{%k7}

  246.     vpaddq A6, ACC6, ACC6{%k7}

  247.     vpaddq A7, ACC7, ACC7{%k7}

  248.     vpaddq A8, ACC8, ACC8{%k7}

  249.     vpaddq A9, ACC9, ACC9{%k7}

  250.     vpaddq A10, ACC10, ACC10{%k7}

  251.     vpaddq A11, ACC11, ACC11{%k7}

  252.     vpaddq A12, ACC12, ACC12{%k7}

  253.     vpaddq A13, ACC13, ACC13{%k7}

  254.     vpaddq A14, ACC14, ACC14{%k7}

  255. 

  256.     mov $0x11, hlp

  257.     kmovq hlp, %k7

  258. 

  259.     vpaddq 0*8+.LpolyX(%rip){1to8}, ACC0, ACC0{%k7}

  260.     vpaddq 1*8+.LpolyX(%rip){1to8}, ACC1, ACC1{%k7}

  261.     vpaddq 2*8+.LpolyX(%rip){1to8}, ACC2, ACC2{%k7}

  262.     vpaddq 3*8+.LpolyX(%rip){1to8}, ACC3, ACC3{%k7}

  263.     vpaddq 4*8+.LpolyX(%rip){1to8}, ACC4, ACC4{%k7}

  264.     vpaddq 5*8+.LpolyX(%rip){1to8}, ACC5, ACC5{%k7}

  265.     vpaddq 6*8+.LpolyX(%rip){1to8}, ACC6, ACC6{%k7}

  266.     vpaddq 7*8+.LpolyX(%rip){1to8}, ACC7, ACC7{%k7}

  267.     vpaddq 8*8+.LpolyX(%rip){1to8}, ACC8, ACC8{%k7}

  268.     vpaddq 9*8+.LpolyX(%rip){1to8}, ACC9, ACC9{%k7}

  269.     vpaddq 10*8+.LpolyX(%rip){1to8}, ACC10, ACC10{%k7}

  270.     vpaddq 11*8+.LpolyX(%rip){1to8}, ACC11, ACC11{%k7}

  271.     vpaddq 12*8+.LpolyX(%rip){1to8}, ACC12, ACC12{%k7}

  272.     vpaddq 13*8+.LpolyX(%rip){1to8}, ACC13, ACC13{%k7}

  273.     vpaddq 14*8+.LpolyX(%rip){1to8}, ACC14, ACC14{%k7}

  274. 

  275.     vpsubq A0, ACC0, ACC0{%k7}

  276.     vpsubq A1, ACC1, ACC1{%k7}

  277.     vpsubq A2, ACC2, ACC2{%k7}

  278.     vpsubq A3, ACC3, ACC3{%k7}

  279.     vpsubq A4, ACC4, ACC4{%k7}

  280.     vpsubq A5, ACC5, ACC5{%k7}

  281.     vpsubq A6, ACC6, ACC6{%k7}

  282.     vpsubq A7, ACC7, ACC7{%k7}

  283.     vpsubq A8, ACC8, ACC8{%k7}

  284.     vpsubq A9, ACC9, ACC9{%k7}

  285.     vpsubq A10, ACC10, ACC10{%k7}

  286.     vpsubq A11, ACC11, ACC11{%k7}

  287.     vpsubq A12, ACC12, ACC12{%k7}

  288.     vpsubq A13, ACC13, ACC13{%k7}

  289.     vpsubq A14, ACC14, ACC14{%k7}

  290.     vpsrlq $52, ACC0, B

  291.     vpaddq B, ACC1, ACC1

  292.     vpandq .Lpoly(%rip){1to8}, ACC0, ACC0

  293. 

  294.     vpsrlq $52, ACC1, B

  295.     vpaddq B, ACC2, ACC2

  296.     vpandq .Lpoly(%rip){1to8}, ACC1, ACC1

  297. 

  298.     vpsrlq $52, ACC2, B

  299.     vpaddq B, ACC3, ACC3

  300.     vpandq .Lpoly(%rip){1to8}, ACC2, ACC2

  301. 

  302.     vpsrlq $52, ACC3, B

  303.     vpaddq B, ACC4, ACC4

  304.     vpandq .Lpoly(%rip){1to8}, ACC3, ACC3

  305. 

  306.     vpsrlq $52, ACC4, B

  307.     vpaddq B, ACC5, ACC5

  308.     vpandq .Lpoly(%rip){1to8}, ACC4, ACC4

  309. 

  310.     vpsrlq $52, ACC5, B

  311.     vpaddq B, ACC6, ACC6

  312.     vpandq .Lpoly(%rip){1to8}, ACC5, ACC5

  313. 

  314.     vpsrlq $52, ACC6, B

  315.     vpaddq B, ACC7, ACC7

  316.     vpandq .Lpoly(%rip){1to8}, ACC6, ACC6

  317. 

  318.     vpsrlq $52, ACC7, B

  319.     vpaddq B, ACC8, ACC8

  320.     vpandq .Lpoly(%rip){1to8}, ACC7, ACC7

  321. 

  322.     vpsrlq $52, ACC8, B

  323.     vpaddq B, ACC9, ACC9

  324.     vpandq .Lpoly(%rip){1to8}, ACC8, ACC8

  325. 

  326.     vpsrlq $52, ACC9, B

  327.     vpaddq B, ACC10, ACC10

  328.     vpandq .Lpoly(%rip){1to8}, ACC9, ACC9

  329. 

  330.     vpsrlq $52, ACC10, B

  331.     vpaddq B, ACC11, ACC11

  332.     vpandq .Lpoly(%rip){1to8}, ACC10, ACC10

  333. 

  334.     vpsrlq $52, ACC11, B

  335.     vpaddq B, ACC12, ACC12

  336.     vpandq .Lpoly(%rip){1to8}, ACC11, ACC11

  337. 

  338.     vpsrlq $52, ACC12, B

  339.     vpaddq B, ACC13, ACC13

  340.     vpandq .Lpoly(%rip){1to8}, ACC12, ACC12

  341. 

  342.     vpsrlq $52, ACC13, B

  343.     vpaddq B, ACC14, ACC14

  344.     vpandq .Lpoly(%rip){1to8}, ACC13, ACC13

  345. 

  346.     vpandq .Lpoly(%rip){1to8}, ACC14, ACC14

  347. 

  348.     // Transpose to horizontal

  349.     vpunpcklqdq ACC1, ACC0, ACC0

  350.     vpunpcklqdq ACC3, ACC2, ACC1

  351.     vpunpcklqdq ACC5, ACC4, ACC2

  352.     vpunpcklqdq ACC7, ACC6, ACC3

  353.     vpunpcklqdq ACC9, ACC8, ACC4

  354.     vpunpcklqdq ACC11, ACC10, ACC5

  355.     vpunpcklqdq ACC13, ACC12, ACC6

  356.     vmovdqa64 ACC14, ACC7

  357. 

  358.     vshufi64x2 $0x44, ACC1, ACC0, A0

  359.     vshufi64x2 $0x44, ACC3, ACC2, A1

  360.     vshufi64x2 $0x44, ACC5, ACC4, A2

  361.     vshufi64x2 $0x44, ACC7, ACC6, A3

  362. 

  363.     vshufi64x2 $0xee, ACC1, ACC0, A4

  364.     vshufi64x2 $0xee, ACC3, ACC2, A5

  365.     vshufi64x2 $0xee, ACC5, ACC4, A6

  366.     vshufi64x2 $0xee, ACC7, ACC6, A7

  367. 

  368.     vshufi64x2 $0x88, A1, A0, ACC0

  369.     vshufi64x2 $0x88, A3, A2, ACC1

  370.     vshufi64x2 $0xdd, A1, A0, ACC2

  371.     vshufi64x2 $0xdd, A3, A2, ACC3

  372. 

  373.     vshufi64x2 $0x88, A5, A4, ACC4

  374.     vshufi64x2 $0x88, A7, A6, ACC5

  375.     vshufi64x2 $0xdd, A5, A4, ACC6

  376.     vshufi64x2 $0xdd, A7, A6, ACC7

  377. 

  378.     vmovdqu64 ACC0, 0*64(r0ptr)

  379.     vmovdqu64 ACC1, 1*64(r0ptr){%k5}

  380.     lea 15*8(r0ptr), r0ptr

  381.     vmovdqu64 ACC2, 0*64(r0ptr)

  382.     vmovdqu64 ACC3, 1*64(r0ptr){%k5}

  383. 

  384.     vmovdqu64 ACC4, 0*64(r1ptr)

  385.     vmovdqu64 ACC5, 1*64(r1ptr){%k5}

  386.     lea 15*8(r1ptr), r1ptr

  387.     vmovdqu64 ACC6, 0*64(r1ptr)

  388.     vmovdqu64 ACC7, 1*64(r1ptr){%k5}

  389. 

  390.     mov %rbp, %rsp

  391.     pop %rbp

  392. ret

  393. 

  394. // Performs 8 field multiplications in parallel

  395. .globl C_ABI(amm_751_ifma_x2)

  396. C_ABI(amm_751_ifma_x2):

  397. 

  398.     vmovdqu64 0*64(aptr), A0

  399.     vmovdqu64 1*64(aptr), A1

  400.     vmovdqu64 2*64(aptr), A2

  401.     vmovdqu64 3*64(aptr), A3

  402.     vmovdqu64 4*64(aptr), A4

  403.     vmovdqu64 5*64(aptr), A5

  404.     vmovdqu64 6*64(aptr), A6

  405.     vmovdqu64 7*64(aptr), A7

  406.     vmovdqu64 8*64(aptr), A8

  407.     vmovdqu64 9*64(aptr), A9

  408.     vmovdqu64 10*64(aptr), A10

  409.     vmovdqu64 11*64(aptr), A11

  410.     vmovdqu64 12*64(aptr), A12

  411.     vmovdqu64 13*64(aptr), A13

  412.     vmovdqu64 14*64(aptr), A14

  413. do_mul_x2:

  414.     vpxorq ACC0, ACC0, ACC0

  415.     vpxorq ACC1, ACC1, ACC1

  416.     vpxorq ACC2, ACC2, ACC2

  417.     vpxorq ACC3, ACC3, ACC3

  418.     vpxorq ACC4, ACC4, ACC4

  419.     vpxorq ACC5, ACC5, ACC5

  420.     vpxorq ACC6, ACC6, ACC6

  421.     vpxorq ACC7, ACC7, ACC7

  422.     vpxorq ACC8, ACC8, ACC8

  423.     vpxorq ACC9, ACC9, ACC9

  424.     vpxorq ACC10, ACC10, ACC10    

  425.     vpxorq ACC11, ACC11, ACC11

  426.     vpxorq ACC12, ACC12, ACC12

  427.     vpxorq ACC13, ACC13, ACC13

  428.     vpxorq ACC14, ACC14, ACC14

  429.     vpxorq ACC15, ACC15, ACC15

  430. 

  431.     mov $15, hlp

  432. 

  433. 1:

  434.         vmovdqu64 (bptr), B

  435.         lea 1*64(bptr), bptr

  436.     

  437.         vpmadd52luq A0, B, ACC0

  438.         vpmadd52luq A1, B, ACC1

  439.         vpmadd52luq A2, B, ACC2

  440.         vpmadd52luq A3, B, ACC3

  441.         vpmadd52luq A4, B, ACC4

  442.         vpmadd52luq A5, B, ACC5

  443.         vpmadd52luq A6, B, ACC6

  444.         vpmadd52luq A7, B, ACC7

  445.         vpmadd52luq A8, B, ACC8

  446.         vpmadd52luq A9, B, ACC9

  447.         vpmadd52luq A10, B, ACC10

  448.         vpmadd52luq A11, B, ACC11

  449.         vpmadd52luq A12, B, ACC12

  450.         vpmadd52luq A13, B, ACC13

  451.         vpmadd52luq A14, B, ACC14

  452. 

  453.         vpmadd52huq A0, B, ACC1

  454.         vpmadd52huq A1, B, ACC2

  455.         vpmadd52huq A2, B, ACC3

  456.         vpmadd52huq A3, B, ACC4

  457.         vpmadd52huq A4, B, ACC5

  458.         vpmadd52huq A5, B, ACC6

  459.         vpmadd52huq A6, B, ACC7

  460.         vpmadd52huq A7, B, ACC8

  461.         vpmadd52huq A8, B, ACC9

  462.         vpmadd52huq A9, B, ACC10

  463.         vpmadd52huq A10, B, ACC11

  464.         vpmadd52huq A11, B, ACC12

  465.         vpmadd52huq A12, B, ACC13

  466.         vpmadd52huq A13, B, ACC14

  467.         vpmadd52huq A14, B, ACC15

  468. 

  469.         vmovdqa64 ACC0, B

  470. 

  471.         vpmadd52luq 0*8 + .Lpoly(%rip){1to8}, B, ACC0

  472.         vpsrlq $52, ACC0, ACC0

  473.         vpmadd52luq 1*8 + .Lpoly(%rip){1to8}, B, ACC1

  474.         vpaddq ACC1, ACC0, ACC0

  475.         vpmadd52luq 2*8 + .Lpoly(%rip){1to8}, B, ACC2

  476.         vmovdqa64 ACC2, ACC1

  477.         vpmadd52luq 3*8 + .Lpoly(%rip){1to8}, B, ACC3

  478.         vmovdqa64 ACC3, ACC2

  479.         vpmadd52luq 4*8 + .Lpoly(%rip){1to8}, B, ACC4

  480.         vmovdqa64 ACC4, ACC3

  481.         vpmadd52luq 5*8 + .Lpoly(%rip){1to8}, B, ACC5

  482.         vmovdqa64 ACC5, ACC4

  483.         vpmadd52luq 6*8 + .Lpoly(%rip){1to8}, B, ACC6

  484.         vmovdqa64 ACC6, ACC5

  485.         vpmadd52luq 7*8 + .Lpoly(%rip){1to8}, B, ACC7

  486.         vmovdqa64 ACC7, ACC6

  487.         vpmadd52luq 8*8 + .Lpoly(%rip){1to8}, B, ACC8

  488.         vmovdqa64 ACC8, ACC7

  489.         vpmadd52luq 9*8 + .Lpoly(%rip){1to8}, B, ACC9

  490.         vmovdqa64 ACC9, ACC8

  491.         vpmadd52luq 10*8 + .Lpoly(%rip){1to8}, B, ACC10

  492.         vmovdqa64 ACC10, ACC9

  493.         vpmadd52luq 11*8 + .Lpoly(%rip){1to8}, B, ACC11

  494.         vmovdqa64 ACC11, ACC10

  495.         vpmadd52luq 12*8 + .Lpoly(%rip){1to8}, B, ACC12

  496.         vmovdqa64 ACC12, ACC11

  497.         vpmadd52luq 13*8 + .Lpoly(%rip){1to8}, B, ACC13

  498.         vmovdqa64 ACC13, ACC12

  499.         vpmadd52luq 14*8 + .Lpoly(%rip){1to8}, B, ACC14

  500.         vmovdqa64 ACC14, ACC13

  501.         vmovdqa64 ACC15, ACC14

  502.         vpxorq ACC15, ACC15, ACC15

  503. 

  504.         vpmadd52huq 0*8 + .Lpoly(%rip){1to8}, B, ACC0

  505.         vpmadd52huq 1*8 + .Lpoly(%rip){1to8}, B, ACC1

  506.         vpmadd52huq 2*8 + .Lpoly(%rip){1to8}, B, ACC2

  507.         vpmadd52huq 3*8 + .Lpoly(%rip){1to8}, B, ACC3

  508.         vpmadd52huq 4*8 + .Lpoly(%rip){1to8}, B, ACC4

  509.         vpmadd52huq 5*8 + .Lpoly(%rip){1to8}, B, ACC5

  510.         vpmadd52huq 6*8 + .Lpoly(%rip){1to8}, B, ACC6

  511.         vpmadd52huq 7*8 + .Lpoly(%rip){1to8}, B, ACC7

  512.         vpmadd52huq 8*8 + .Lpoly(%rip){1to8}, B, ACC8

  513.         vpmadd52huq 9*8 + .Lpoly(%rip){1to8}, B, ACC9

  514.         vpmadd52huq 10*8 + .Lpoly(%rip){1to8}, B, ACC10

  515.         vpmadd52huq 11*8 + .Lpoly(%rip){1to8}, B, ACC11

  516.         vpmadd52huq 12*8 + .Lpoly(%rip){1to8}, B, ACC12

  517.         vpmadd52huq 13*8 + .Lpoly(%rip){1to8}, B, ACC13

  518.         vpmadd52huq 14*8 + .Lpoly(%rip){1to8}, B, ACC14

  519. 

  520.         dec hlp

  521.     jnz 1b

  522. 

  523.     ret