kris
/
boringssl
1
0
Rozštěpit 0
Zdrojový kód Úkoly 0 Požadavky na natažení 0 Vydání 0 Wiki Aktivita
Nelze vybrat více než 25 témat Téma musí začínat písmenem nebo číslem, může obsahovat pomlčky („-“) a může být dlouhé až 35 znaků.
1217 Revize
12 Větve
38 MiB
 
 
 
 
 
 
Strom: 9f33fc63c6
Větve Značky
${ item.name }
Vytvořit větev ${ searchTerm }
z „9f33fc63c6“
${ noResults }
boringssl/crypto/poly1305/poly1305_vec.c

893 řádky
27 KiB
Surový Blame Historie 

  1. /* Copyright (c) 2014, Google Inc.

  2.  *

  3.  * Permission to use, copy, modify, and/or distribute this software for any

  4.  * purpose with or without fee is hereby granted, provided that the above

  5.  * copyright notice and this permission notice appear in all copies.

  6.  *

  7.  * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES

  8.  * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF

  9.  * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY

  10.  * SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES

  11.  * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION

  12.  * OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN

  13.  * CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */

  14. 

  15. /* This implementation of poly1305 is by Andrew Moon

  16.  * (https://github.com/floodyberry/poly1305-donna) and released as public

  17.  * domain. It implements SIMD vectorization based on the algorithm described in

  18.  * http://cr.yp.to/papers.html#neoncrypto. Unrolled to 2 powers, i.e. 64 byte

  19.  * block size */

  20. 

  21. #include <openssl/poly1305.h>

  22. 

  23. 

  24. #if !defined(OPENSSL_WINDOWS) && defined(OPENSSL_X86_64)

  25. 

  26. #include <emmintrin.h>

  27. 

  28. #define ALIGN(x) __attribute__((aligned(x)))

  29. /* inline is not a keyword in C89. */

  30. #define INLINE 

  31. #define U8TO64_LE(m) (*(uint64_t *)(m))

  32. #define U8TO32_LE(m) (*(uint32_t *)(m))

  33. #define U64TO8_LE(m, v) (*(uint64_t *)(m)) = v

  34. 

  35. typedef __m128i xmmi;

  36. typedef unsigned __int128 uint128_t;

  37. 

  38. static const uint32_t ALIGN(16) poly1305_x64_sse2_message_mask[4] = {

  39.     (1 << 26) - 1, 0, (1 << 26) - 1, 0};

  40. static const uint32_t ALIGN(16) poly1305_x64_sse2_5[4] = {5, 0, 5, 0};

  41. static const uint32_t ALIGN(16) poly1305_x64_sse2_1shl128[4] = {(1 << 24), 0,

  42.                                                                 (1 << 24), 0};

  43. 

  44. static uint128_t INLINE add128(uint128_t a, uint128_t b) { return a + b; }

  45. 

  46. static uint128_t INLINE add128_64(uint128_t a, uint64_t b) { return a + b; }

  47. 

  48. static uint128_t INLINE mul64x64_128(uint64_t a, uint64_t b) {

  49.   return (uint128_t)a * b;

  50. }

  51. 

  52. static uint64_t INLINE lo128(uint128_t a) { return (uint64_t)a; }

  53. 

  54. static uint64_t INLINE shr128(uint128_t v, const int shift) {

  55.   return (uint64_t)(v >> shift);

  56. }

  57. 

  58. static uint64_t INLINE shr128_pair(uint64_t hi, uint64_t lo, const int shift) {

  59.   return (uint64_t)((((uint128_t)hi << 64) | lo) >> shift);

  60. }

  61. 

  62. typedef struct poly1305_power_t {

  63.   union {

  64.     xmmi v;

  65.     uint64_t u[2];

  66.     uint32_t d[4];

  67.   } R20, R21, R22, R23, R24, S21, S22, S23, S24;

  68. } poly1305_power;

  69. 

  70. typedef struct poly1305_state_internal_t {

  71.   poly1305_power P[2]; /* 288 bytes, top 32 bit halves unused = 144

  72.                           bytes of free storage */

  73.   union {

  74.     xmmi H[5]; /*  80 bytes  */

  75.     uint64_t HH[10];

  76.   };

  77.   /* uint64_t r0,r1,r2;       [24 bytes] */

  78.   /* uint64_t pad0,pad1;      [16 bytes] */

  79.   uint64_t started;        /*   8 bytes  */

  80.   uint64_t leftover;       /*   8 bytes  */

  81.   uint8_t buffer[64];      /*  64 bytes  */

  82. } poly1305_state_internal; /* 448 bytes total + 63 bytes for

  83.                               alignment = 511 bytes raw */

  84. 

  85. static poly1305_state_internal INLINE *poly1305_aligned_state(

  86.     poly1305_state *state) {

  87.   return (poly1305_state_internal *)(((uint64_t)state + 63) & ~63);

  88. }

  89. 

  90. /* copy 0-63 bytes */

  91. static void INLINE

  92. poly1305_block_copy(uint8_t *dst, const uint8_t *src, size_t bytes) {

  93.   size_t offset = src - dst;

  94.   if (bytes & 32) {

  95.     _mm_storeu_si128((xmmi *)(dst + 0),

  96.                      _mm_loadu_si128((xmmi *)(dst + offset + 0)));

  97.     _mm_storeu_si128((xmmi *)(dst + 16),

  98.                      _mm_loadu_si128((xmmi *)(dst + offset + 16)));

  99.     dst += 32;

  100.   }

  101.   if (bytes & 16) {

  102.     _mm_storeu_si128((xmmi *)dst, _mm_loadu_si128((xmmi *)(dst + offset)));

  103.     dst += 16;

  104.   }

  105.   if (bytes & 8) {

  106.     *(uint64_t *)dst = *(uint64_t *)(dst + offset);

  107.     dst += 8;

  108.   }

  109.   if (bytes & 4) {

  110.     *(uint32_t *)dst = *(uint32_t *)(dst + offset);

  111.     dst += 4;

  112.   }

  113.   if (bytes & 2) {

  114.     *(uint16_t *)dst = *(uint16_t *)(dst + offset);

  115.     dst += 2;

  116.   }

  117.   if (bytes & 1) {

  118.     *(uint8_t *)dst = *(uint8_t *)(dst + offset);

  119.   }

  120. }

  121. 

  122. /* zero 0-15 bytes */

  123. static void INLINE poly1305_block_zero(uint8_t *dst, size_t bytes) {

  124.   if (bytes & 8) {

  125.     *(uint64_t *)dst = 0;

  126.     dst += 8;

  127.   }

  128.   if (bytes & 4) {

  129.     *(uint32_t *)dst = 0;

  130.     dst += 4;

  131.   }

  132.   if (bytes & 2) {

  133.     *(uint16_t *)dst = 0;

  134.     dst += 2;

  135.   }

  136.   if (bytes & 1) {

  137.     *(uint8_t *)dst = 0;

  138.   }

  139. }

  140. 

  141. static size_t INLINE poly1305_min(size_t a, size_t b) {

  142.   return (a < b) ? a : b;

  143. }

  144. 

  145. void CRYPTO_poly1305_init(poly1305_state *state, const uint8_t key[32]) {

  146.   poly1305_state_internal *st = poly1305_aligned_state(state);

  147.   poly1305_power *p;

  148.   uint64_t r0, r1, r2;

  149.   uint64_t t0, t1;

  150. 

  151.   /* clamp key */

  152.   t0 = U8TO64_LE(key + 0);

  153.   t1 = U8TO64_LE(key + 8);

  154.   r0 = t0 & 0xffc0fffffff;

  155.   t0 >>= 44;

  156.   t0 |= t1 << 20;

  157.   r1 = t0 & 0xfffffc0ffff;

  158.   t1 >>= 24;

  159.   r2 = t1 & 0x00ffffffc0f;

  160. 

  161.   /* store r in un-used space of st->P[1] */

  162.   p = &st->P[1];

  163.   p->R20.d[1] = (uint32_t)(r0);

  164.   p->R20.d[3] = (uint32_t)(r0 >> 32);

  165.   p->R21.d[1] = (uint32_t)(r1);

  166.   p->R21.d[3] = (uint32_t)(r1 >> 32);

  167.   p->R22.d[1] = (uint32_t)(r2);

  168.   p->R22.d[3] = (uint32_t)(r2 >> 32);

  169. 

  170.   /* store pad */

  171.   p->R23.d[1] = U8TO32_LE(key + 16);

  172.   p->R23.d[3] = U8TO32_LE(key + 20);

  173.   p->R24.d[1] = U8TO32_LE(key + 24);

  174.   p->R24.d[3] = U8TO32_LE(key + 28);

  175. 

  176.   /* H = 0 */

  177.   st->H[0] = _mm_setzero_si128();

  178.   st->H[1] = _mm_setzero_si128();

  179.   st->H[2] = _mm_setzero_si128();

  180.   st->H[3] = _mm_setzero_si128();

  181.   st->H[4] = _mm_setzero_si128();

  182. 

  183.   st->started = 0;

  184.   st->leftover = 0;

  185. }

  186. 

  187. static void poly1305_first_block(poly1305_state_internal *st,

  188.                                  const uint8_t *m) {

  189.   const xmmi MMASK = _mm_load_si128((xmmi *)poly1305_x64_sse2_message_mask);

  190.   const xmmi FIVE = _mm_load_si128((xmmi *)poly1305_x64_sse2_5);

  191.   const xmmi HIBIT = _mm_load_si128((xmmi *)poly1305_x64_sse2_1shl128);

  192.   xmmi T5, T6;

  193.   poly1305_power *p;

  194.   uint128_t d[3];

  195.   uint64_t r0, r1, r2;

  196.   uint64_t r20, r21, r22, s22;

  197.   uint64_t pad0, pad1;

  198.   uint64_t c;

  199.   uint64_t i;

  200. 

  201.   /* pull out stored info */

  202.   p = &st->P[1];

  203. 

  204.   r0 = ((uint64_t)p->R20.d[3] << 32) | (uint64_t)p->R20.d[1];

  205.   r1 = ((uint64_t)p->R21.d[3] << 32) | (uint64_t)p->R21.d[1];

  206.   r2 = ((uint64_t)p->R22.d[3] << 32) | (uint64_t)p->R22.d[1];

  207.   pad0 = ((uint64_t)p->R23.d[3] << 32) | (uint64_t)p->R23.d[1];

  208.   pad1 = ((uint64_t)p->R24.d[3] << 32) | (uint64_t)p->R24.d[1];

  209. 

  210.   /* compute powers r^2,r^4 */

  211.   r20 = r0;

  212.   r21 = r1;

  213.   r22 = r2;

  214.   for (i = 0; i < 2; i++) {

  215.     s22 = r22 * (5 << 2);

  216. 

  217.     d[0] = add128(mul64x64_128(r20, r20), mul64x64_128(r21 * 2, s22));

  218.     d[1] = add128(mul64x64_128(r22, s22), mul64x64_128(r20 * 2, r21));

  219.     d[2] = add128(mul64x64_128(r21, r21), mul64x64_128(r22 * 2, r20));

  220. 

  221.     r20 = lo128(d[0]) & 0xfffffffffff;

  222.     c = shr128(d[0], 44);

  223.     d[1] = add128_64(d[1], c);

  224.     r21 = lo128(d[1]) & 0xfffffffffff;

  225.     c = shr128(d[1], 44);

  226.     d[2] = add128_64(d[2], c);

  227.     r22 = lo128(d[2]) & 0x3ffffffffff;

  228.     c = shr128(d[2], 42);

  229.     r20 += c * 5;

  230.     c = (r20 >> 44);

  231.     r20 = r20 & 0xfffffffffff;

  232.     r21 += c;

  233. 

  234.     p->R20.v = _mm_shuffle_epi32(_mm_cvtsi32_si128((uint32_t)(r20)&0x3ffffff),

  235.                                  _MM_SHUFFLE(1, 0, 1, 0));

  236.     p->R21.v = _mm_shuffle_epi32(

  237.         _mm_cvtsi32_si128((uint32_t)((r20 >> 26) | (r21 << 18)) & 0x3ffffff),

  238.         _MM_SHUFFLE(1, 0, 1, 0));

  239.     p->R22.v =

  240.         _mm_shuffle_epi32(_mm_cvtsi32_si128((uint32_t)((r21 >> 8)) & 0x3ffffff),

  241.                           _MM_SHUFFLE(1, 0, 1, 0));

  242.     p->R23.v = _mm_shuffle_epi32(

  243.         _mm_cvtsi32_si128((uint32_t)((r21 >> 34) | (r22 << 10)) & 0x3ffffff),

  244.         _MM_SHUFFLE(1, 0, 1, 0));

  245.     p->R24.v = _mm_shuffle_epi32(_mm_cvtsi32_si128((uint32_t)((r22 >> 16))),

  246.                                  _MM_SHUFFLE(1, 0, 1, 0));

  247.     p->S21.v = _mm_mul_epu32(p->R21.v, FIVE);

  248.     p->S22.v = _mm_mul_epu32(p->R22.v, FIVE);

  249.     p->S23.v = _mm_mul_epu32(p->R23.v, FIVE);

  250.     p->S24.v = _mm_mul_epu32(p->R24.v, FIVE);

  251.     p--;

  252.   }

  253. 

  254.   /* put saved info back */

  255.   p = &st->P[1];

  256.   p->R20.d[1] = (uint32_t)(r0);

  257.   p->R20.d[3] = (uint32_t)(r0 >> 32);

  258.   p->R21.d[1] = (uint32_t)(r1);

  259.   p->R21.d[3] = (uint32_t)(r1 >> 32);

  260.   p->R22.d[1] = (uint32_t)(r2);

  261.   p->R22.d[3] = (uint32_t)(r2 >> 32);

  262.   p->R23.d[1] = (uint32_t)(pad0);

  263.   p->R23.d[3] = (uint32_t)(pad0 >> 32);

  264.   p->R24.d[1] = (uint32_t)(pad1);

  265.   p->R24.d[3] = (uint32_t)(pad1 >> 32);

  266. 

  267.   /* H = [Mx,My] */

  268.   T5 = _mm_unpacklo_epi64(_mm_loadl_epi64((xmmi *)(m + 0)),

  269.                           _mm_loadl_epi64((xmmi *)(m + 16)));

  270.   T6 = _mm_unpacklo_epi64(_mm_loadl_epi64((xmmi *)(m + 8)),

  271.                           _mm_loadl_epi64((xmmi *)(m + 24)));

  272.   st->H[0] = _mm_and_si128(MMASK, T5);

  273.   st->H[1] = _mm_and_si128(MMASK, _mm_srli_epi64(T5, 26));

  274.   T5 = _mm_or_si128(_mm_srli_epi64(T5, 52), _mm_slli_epi64(T6, 12));

  275.   st->H[2] = _mm_and_si128(MMASK, T5);

  276.   st->H[3] = _mm_and_si128(MMASK, _mm_srli_epi64(T5, 26));

  277.   st->H[4] = _mm_or_si128(_mm_srli_epi64(T6, 40), HIBIT);

  278. }

  279. 

  280. static void poly1305_blocks(poly1305_state_internal *st, const uint8_t *m,

  281.                             size_t bytes) {

  282.   const xmmi MMASK = _mm_load_si128((xmmi *)poly1305_x64_sse2_message_mask);

  283.   const xmmi FIVE = _mm_load_si128((xmmi *)poly1305_x64_sse2_5);

  284.   const xmmi HIBIT = _mm_load_si128((xmmi *)poly1305_x64_sse2_1shl128);

  285. 

  286.   poly1305_power *p;

  287.   xmmi H0, H1, H2, H3, H4;

  288.   xmmi T0, T1, T2, T3, T4, T5, T6;

  289.   xmmi M0, M1, M2, M3, M4;

  290.   xmmi C1, C2;

  291. 

  292.   H0 = st->H[0];

  293.   H1 = st->H[1];

  294.   H2 = st->H[2];

  295.   H3 = st->H[3];

  296.   H4 = st->H[4];

  297. 

  298.   while (bytes >= 64) {

  299.     /* H *= [r^4,r^4] */

  300.     p = &st->P[0];

  301.     T0 = _mm_mul_epu32(H0, p->R20.v);

  302.     T1 = _mm_mul_epu32(H0, p->R21.v);

  303.     T2 = _mm_mul_epu32(H0, p->R22.v);

  304.     T3 = _mm_mul_epu32(H0, p->R23.v);

  305.     T4 = _mm_mul_epu32(H0, p->R24.v);

  306.     T5 = _mm_mul_epu32(H1, p->S24.v);

  307.     T6 = _mm_mul_epu32(H1, p->R20.v);

  308.     T0 = _mm_add_epi64(T0, T5);

  309.     T1 = _mm_add_epi64(T1, T6);

  310.     T5 = _mm_mul_epu32(H2, p->S23.v);

  311.     T6 = _mm_mul_epu32(H2, p->S24.v);

  312.     T0 = _mm_add_epi64(T0, T5);

  313.     T1 = _mm_add_epi64(T1, T6);

  314.     T5 = _mm_mul_epu32(H3, p->S22.v);

  315.     T6 = _mm_mul_epu32(H3, p->S23.v);

  316.     T0 = _mm_add_epi64(T0, T5);

  317.     T1 = _mm_add_epi64(T1, T6);

  318.     T5 = _mm_mul_epu32(H4, p->S21.v);

  319.     T6 = _mm_mul_epu32(H4, p->S22.v);

  320.     T0 = _mm_add_epi64(T0, T5);

  321.     T1 = _mm_add_epi64(T1, T6);

  322.     T5 = _mm_mul_epu32(H1, p->R21.v);

  323.     T6 = _mm_mul_epu32(H1, p->R22.v);

  324.     T2 = _mm_add_epi64(T2, T5);

  325.     T3 = _mm_add_epi64(T3, T6);

  326.     T5 = _mm_mul_epu32(H2, p->R20.v);

  327.     T6 = _mm_mul_epu32(H2, p->R21.v);

  328.     T2 = _mm_add_epi64(T2, T5);

  329.     T3 = _mm_add_epi64(T3, T6);

  330.     T5 = _mm_mul_epu32(H3, p->S24.v);

  331.     T6 = _mm_mul_epu32(H3, p->R20.v);

  332.     T2 = _mm_add_epi64(T2, T5);

  333.     T3 = _mm_add_epi64(T3, T6);

  334.     T5 = _mm_mul_epu32(H4, p->S23.v);

  335.     T6 = _mm_mul_epu32(H4, p->S24.v);

  336.     T2 = _mm_add_epi64(T2, T5);

  337.     T3 = _mm_add_epi64(T3, T6);

  338.     T5 = _mm_mul_epu32(H1, p->R23.v);

  339.     T4 = _mm_add_epi64(T4, T5);

  340.     T5 = _mm_mul_epu32(H2, p->R22.v);

  341.     T4 = _mm_add_epi64(T4, T5);

  342.     T5 = _mm_mul_epu32(H3, p->R21.v);

  343.     T4 = _mm_add_epi64(T4, T5);

  344.     T5 = _mm_mul_epu32(H4, p->R20.v);

  345.     T4 = _mm_add_epi64(T4, T5);

  346. 

  347.     /* H += [Mx,My]*[r^2,r^2] */

  348.     T5 = _mm_unpacklo_epi64(_mm_loadl_epi64((xmmi *)(m + 0)),

  349.                             _mm_loadl_epi64((xmmi *)(m + 16)));

  350.     T6 = _mm_unpacklo_epi64(_mm_loadl_epi64((xmmi *)(m + 8)),

  351.                             _mm_loadl_epi64((xmmi *)(m + 24)));

  352.     M0 = _mm_and_si128(MMASK, T5);

  353.     M1 = _mm_and_si128(MMASK, _mm_srli_epi64(T5, 26));

  354.     T5 = _mm_or_si128(_mm_srli_epi64(T5, 52), _mm_slli_epi64(T6, 12));

  355.     M2 = _mm_and_si128(MMASK, T5);

  356.     M3 = _mm_and_si128(MMASK, _mm_srli_epi64(T5, 26));

  357.     M4 = _mm_or_si128(_mm_srli_epi64(T6, 40), HIBIT);

  358. 

  359.     p = &st->P[1];

  360.     T5 = _mm_mul_epu32(M0, p->R20.v);

  361.     T6 = _mm_mul_epu32(M0, p->R21.v);

  362.     T0 = _mm_add_epi64(T0, T5);

  363.     T1 = _mm_add_epi64(T1, T6);

  364.     T5 = _mm_mul_epu32(M1, p->S24.v);

  365.     T6 = _mm_mul_epu32(M1, p->R20.v);

  366.     T0 = _mm_add_epi64(T0, T5);

  367.     T1 = _mm_add_epi64(T1, T6);

  368.     T5 = _mm_mul_epu32(M2, p->S23.v);

  369.     T6 = _mm_mul_epu32(M2, p->S24.v);

  370.     T0 = _mm_add_epi64(T0, T5);

  371.     T1 = _mm_add_epi64(T1, T6);

  372.     T5 = _mm_mul_epu32(M3, p->S22.v);

  373.     T6 = _mm_mul_epu32(M3, p->S23.v);

  374.     T0 = _mm_add_epi64(T0, T5);

  375.     T1 = _mm_add_epi64(T1, T6);

  376.     T5 = _mm_mul_epu32(M4, p->S21.v);

  377.     T6 = _mm_mul_epu32(M4, p->S22.v);

  378.     T0 = _mm_add_epi64(T0, T5);

  379.     T1 = _mm_add_epi64(T1, T6);

  380.     T5 = _mm_mul_epu32(M0, p->R22.v);

  381.     T6 = _mm_mul_epu32(M0, p->R23.v);

  382.     T2 = _mm_add_epi64(T2, T5);

  383.     T3 = _mm_add_epi64(T3, T6);

  384.     T5 = _mm_mul_epu32(M1, p->R21.v);

  385.     T6 = _mm_mul_epu32(M1, p->R22.v);

  386.     T2 = _mm_add_epi64(T2, T5);

  387.     T3 = _mm_add_epi64(T3, T6);

  388.     T5 = _mm_mul_epu32(M2, p->R20.v);

  389.     T6 = _mm_mul_epu32(M2, p->R21.v);

  390.     T2 = _mm_add_epi64(T2, T5);

  391.     T3 = _mm_add_epi64(T3, T6);

  392.     T5 = _mm_mul_epu32(M3, p->S24.v);

  393.     T6 = _mm_mul_epu32(M3, p->R20.v);

  394.     T2 = _mm_add_epi64(T2, T5);

  395.     T3 = _mm_add_epi64(T3, T6);

  396.     T5 = _mm_mul_epu32(M4, p->S23.v);

  397.     T6 = _mm_mul_epu32(M4, p->S24.v);

  398.     T2 = _mm_add_epi64(T2, T5);

  399.     T3 = _mm_add_epi64(T3, T6);

  400.     T5 = _mm_mul_epu32(M0, p->R24.v);

  401.     T4 = _mm_add_epi64(T4, T5);

  402.     T5 = _mm_mul_epu32(M1, p->R23.v);

  403.     T4 = _mm_add_epi64(T4, T5);

  404.     T5 = _mm_mul_epu32(M2, p->R22.v);

  405.     T4 = _mm_add_epi64(T4, T5);

  406.     T5 = _mm_mul_epu32(M3, p->R21.v);

  407.     T4 = _mm_add_epi64(T4, T5);

  408.     T5 = _mm_mul_epu32(M4, p->R20.v);

  409.     T4 = _mm_add_epi64(T4, T5);

  410. 

  411.     /* H += [Mx,My] */

  412.     T5 = _mm_unpacklo_epi64(_mm_loadl_epi64((xmmi *)(m + 32)),

  413.                             _mm_loadl_epi64((xmmi *)(m + 48)));

  414.     T6 = _mm_unpacklo_epi64(_mm_loadl_epi64((xmmi *)(m + 40)),

  415.                             _mm_loadl_epi64((xmmi *)(m + 56)));

  416.     M0 = _mm_and_si128(MMASK, T5);

  417.     M1 = _mm_and_si128(MMASK, _mm_srli_epi64(T5, 26));

  418.     T5 = _mm_or_si128(_mm_srli_epi64(T5, 52), _mm_slli_epi64(T6, 12));

  419.     M2 = _mm_and_si128(MMASK, T5);

  420.     M3 = _mm_and_si128(MMASK, _mm_srli_epi64(T5, 26));

  421.     M4 = _mm_or_si128(_mm_srli_epi64(T6, 40), HIBIT);

  422. 

  423.     T0 = _mm_add_epi64(T0, M0);

  424.     T1 = _mm_add_epi64(T1, M1);

  425.     T2 = _mm_add_epi64(T2, M2);

  426.     T3 = _mm_add_epi64(T3, M3);

  427.     T4 = _mm_add_epi64(T4, M4);

  428. 

  429.     /* reduce */

  430.     C1 = _mm_srli_epi64(T0, 26);

  431.     C2 = _mm_srli_epi64(T3, 26);

  432.     T0 = _mm_and_si128(T0, MMASK);

  433.     T3 = _mm_and_si128(T3, MMASK);

  434.     T1 = _mm_add_epi64(T1, C1);

  435.     T4 = _mm_add_epi64(T4, C2);

  436.     C1 = _mm_srli_epi64(T1, 26);

  437.     C2 = _mm_srli_epi64(T4, 26);

  438.     T1 = _mm_and_si128(T1, MMASK);

  439.     T4 = _mm_and_si128(T4, MMASK);

  440.     T2 = _mm_add_epi64(T2, C1);

  441.     T0 = _mm_add_epi64(T0, _mm_mul_epu32(C2, FIVE));

  442.     C1 = _mm_srli_epi64(T2, 26);

  443.     C2 = _mm_srli_epi64(T0, 26);

  444.     T2 = _mm_and_si128(T2, MMASK);

  445.     T0 = _mm_and_si128(T0, MMASK);

  446.     T3 = _mm_add_epi64(T3, C1);

  447.     T1 = _mm_add_epi64(T1, C2);

  448.     C1 = _mm_srli_epi64(T3, 26);

  449.     T3 = _mm_and_si128(T3, MMASK);

  450.     T4 = _mm_add_epi64(T4, C1);

  451. 

  452.     /* H = (H*[r^4,r^4] + [Mx,My]*[r^2,r^2] + [Mx,My]) */

  453.     H0 = T0;

  454.     H1 = T1;

  455.     H2 = T2;

  456.     H3 = T3;

  457.     H4 = T4;

  458. 

  459.     m += 64;

  460.     bytes -= 64;

  461.   }

  462. 

  463.   st->H[0] = H0;

  464.   st->H[1] = H1;

  465.   st->H[2] = H2;

  466.   st->H[3] = H3;

  467.   st->H[4] = H4;

  468. }

  469. 

  470. static size_t poly1305_combine(poly1305_state_internal *st, const uint8_t *m,

  471.                                size_t bytes) {

  472.   const xmmi MMASK = _mm_load_si128((xmmi *)poly1305_x64_sse2_message_mask);

  473.   const xmmi HIBIT = _mm_load_si128((xmmi *)poly1305_x64_sse2_1shl128);

  474.   const xmmi FIVE = _mm_load_si128((xmmi *)poly1305_x64_sse2_5);

  475. 

  476.   poly1305_power *p;

  477.   xmmi H0, H1, H2, H3, H4;

  478.   xmmi M0, M1, M2, M3, M4;

  479.   xmmi T0, T1, T2, T3, T4, T5, T6;

  480.   xmmi C1, C2;

  481. 

  482.   uint64_t r0, r1, r2;

  483.   uint64_t t0, t1, t2, t3, t4;

  484.   uint64_t c;

  485.   size_t consumed = 0;

  486. 

  487.   H0 = st->H[0];

  488.   H1 = st->H[1];

  489.   H2 = st->H[2];

  490.   H3 = st->H[3];

  491.   H4 = st->H[4];

  492. 

  493.   /* p = [r^2,r^2] */

  494.   p = &st->P[1];

  495. 

  496.   if (bytes >= 32) {

  497.     /* H *= [r^2,r^2] */

  498.     T0 = _mm_mul_epu32(H0, p->R20.v);

  499.     T1 = _mm_mul_epu32(H0, p->R21.v);

  500.     T2 = _mm_mul_epu32(H0, p->R22.v);

  501.     T3 = _mm_mul_epu32(H0, p->R23.v);

  502.     T4 = _mm_mul_epu32(H0, p->R24.v);

  503.     T5 = _mm_mul_epu32(H1, p->S24.v);

  504.     T6 = _mm_mul_epu32(H1, p->R20.v);

  505.     T0 = _mm_add_epi64(T0, T5);

  506.     T1 = _mm_add_epi64(T1, T6);

  507.     T5 = _mm_mul_epu32(H2, p->S23.v);

  508.     T6 = _mm_mul_epu32(H2, p->S24.v);

  509.     T0 = _mm_add_epi64(T0, T5);

  510.     T1 = _mm_add_epi64(T1, T6);

  511.     T5 = _mm_mul_epu32(H3, p->S22.v);

  512.     T6 = _mm_mul_epu32(H3, p->S23.v);

  513.     T0 = _mm_add_epi64(T0, T5);

  514.     T1 = _mm_add_epi64(T1, T6);

  515.     T5 = _mm_mul_epu32(H4, p->S21.v);

  516.     T6 = _mm_mul_epu32(H4, p->S22.v);

  517.     T0 = _mm_add_epi64(T0, T5);

  518.     T1 = _mm_add_epi64(T1, T6);

  519.     T5 = _mm_mul_epu32(H1, p->R21.v);

  520.     T6 = _mm_mul_epu32(H1, p->R22.v);

  521.     T2 = _mm_add_epi64(T2, T5);

  522.     T3 = _mm_add_epi64(T3, T6);

  523.     T5 = _mm_mul_epu32(H2, p->R20.v);

  524.     T6 = _mm_mul_epu32(H2, p->R21.v);

  525.     T2 = _mm_add_epi64(T2, T5);

  526.     T3 = _mm_add_epi64(T3, T6);

  527.     T5 = _mm_mul_epu32(H3, p->S24.v);

  528.     T6 = _mm_mul_epu32(H3, p->R20.v);

  529.     T2 = _mm_add_epi64(T2, T5);

  530.     T3 = _mm_add_epi64(T3, T6);

  531.     T5 = _mm_mul_epu32(H4, p->S23.v);

  532.     T6 = _mm_mul_epu32(H4, p->S24.v);

  533.     T2 = _mm_add_epi64(T2, T5);

  534.     T3 = _mm_add_epi64(T3, T6);

  535.     T5 = _mm_mul_epu32(H1, p->R23.v);

  536.     T4 = _mm_add_epi64(T4, T5);

  537.     T5 = _mm_mul_epu32(H2, p->R22.v);

  538.     T4 = _mm_add_epi64(T4, T5);

  539.     T5 = _mm_mul_epu32(H3, p->R21.v);

  540.     T4 = _mm_add_epi64(T4, T5);

  541.     T5 = _mm_mul_epu32(H4, p->R20.v);

  542.     T4 = _mm_add_epi64(T4, T5);

  543. 

  544.     /* H += [Mx,My] */

  545.     T5 = _mm_unpacklo_epi64(_mm_loadl_epi64((xmmi *)(m + 0)),

  546.                             _mm_loadl_epi64((xmmi *)(m + 16)));

  547.     T6 = _mm_unpacklo_epi64(_mm_loadl_epi64((xmmi *)(m + 8)),

  548.                             _mm_loadl_epi64((xmmi *)(m + 24)));

  549.     M0 = _mm_and_si128(MMASK, T5);

  550.     M1 = _mm_and_si128(MMASK, _mm_srli_epi64(T5, 26));

  551.     T5 = _mm_or_si128(_mm_srli_epi64(T5, 52), _mm_slli_epi64(T6, 12));

  552.     M2 = _mm_and_si128(MMASK, T5);

  553.     M3 = _mm_and_si128(MMASK, _mm_srli_epi64(T5, 26));

  554.     M4 = _mm_or_si128(_mm_srli_epi64(T6, 40), HIBIT);

  555. 

  556.     T0 = _mm_add_epi64(T0, M0);

  557.     T1 = _mm_add_epi64(T1, M1);

  558.     T2 = _mm_add_epi64(T2, M2);

  559.     T3 = _mm_add_epi64(T3, M3);

  560.     T4 = _mm_add_epi64(T4, M4);

  561. 

  562.     /* reduce */

  563.     C1 = _mm_srli_epi64(T0, 26);

  564.     C2 = _mm_srli_epi64(T3, 26);

  565.     T0 = _mm_and_si128(T0, MMASK);

  566.     T3 = _mm_and_si128(T3, MMASK);

  567.     T1 = _mm_add_epi64(T1, C1);

  568.     T4 = _mm_add_epi64(T4, C2);

  569.     C1 = _mm_srli_epi64(T1, 26);

  570.     C2 = _mm_srli_epi64(T4, 26);

  571.     T1 = _mm_and_si128(T1, MMASK);

  572.     T4 = _mm_and_si128(T4, MMASK);

  573.     T2 = _mm_add_epi64(T2, C1);

  574.     T0 = _mm_add_epi64(T0, _mm_mul_epu32(C2, FIVE));

  575.     C1 = _mm_srli_epi64(T2, 26);

  576.     C2 = _mm_srli_epi64(T0, 26);

  577.     T2 = _mm_and_si128(T2, MMASK);

  578.     T0 = _mm_and_si128(T0, MMASK);

  579.     T3 = _mm_add_epi64(T3, C1);

  580.     T1 = _mm_add_epi64(T1, C2);

  581.     C1 = _mm_srli_epi64(T3, 26);

  582.     T3 = _mm_and_si128(T3, MMASK);

  583.     T4 = _mm_add_epi64(T4, C1);

  584. 

  585.     /* H = (H*[r^2,r^2] + [Mx,My]) */

  586.     H0 = T0;

  587.     H1 = T1;

  588.     H2 = T2;

  589.     H3 = T3;

  590.     H4 = T4;

  591. 

  592.     consumed = 32;

  593.   }

  594. 

  595.   /* finalize, H *= [r^2,r] */

  596.   r0 = ((uint64_t)p->R20.d[3] << 32) | (uint64_t)p->R20.d[1];

  597.   r1 = ((uint64_t)p->R21.d[3] << 32) | (uint64_t)p->R21.d[1];

  598.   r2 = ((uint64_t)p->R22.d[3] << 32) | (uint64_t)p->R22.d[1];

  599. 

  600.   p->R20.d[2] = (uint32_t)(r0)&0x3ffffff;

  601.   p->R21.d[2] = (uint32_t)((r0 >> 26) | (r1 << 18)) & 0x3ffffff;

  602.   p->R22.d[2] = (uint32_t)((r1 >> 8)) & 0x3ffffff;

  603.   p->R23.d[2] = (uint32_t)((r1 >> 34) | (r2 << 10)) & 0x3ffffff;

  604.   p->R24.d[2] = (uint32_t)((r2 >> 16));

  605.   p->S21.d[2] = p->R21.d[2] * 5;

  606.   p->S22.d[2] = p->R22.d[2] * 5;

  607.   p->S23.d[2] = p->R23.d[2] * 5;

  608.   p->S24.d[2] = p->R24.d[2] * 5;

  609. 

  610.   /* H *= [r^2,r] */

  611.   T0 = _mm_mul_epu32(H0, p->R20.v);

  612.   T1 = _mm_mul_epu32(H0, p->R21.v);

  613.   T2 = _mm_mul_epu32(H0, p->R22.v);

  614.   T3 = _mm_mul_epu32(H0, p->R23.v);

  615.   T4 = _mm_mul_epu32(H0, p->R24.v);

  616.   T5 = _mm_mul_epu32(H1, p->S24.v);

  617.   T6 = _mm_mul_epu32(H1, p->R20.v);

  618.   T0 = _mm_add_epi64(T0, T5);

  619.   T1 = _mm_add_epi64(T1, T6);

  620.   T5 = _mm_mul_epu32(H2, p->S23.v);

  621.   T6 = _mm_mul_epu32(H2, p->S24.v);

  622.   T0 = _mm_add_epi64(T0, T5);

  623.   T1 = _mm_add_epi64(T1, T6);

  624.   T5 = _mm_mul_epu32(H3, p->S22.v);

  625.   T6 = _mm_mul_epu32(H3, p->S23.v);

  626.   T0 = _mm_add_epi64(T0, T5);

  627.   T1 = _mm_add_epi64(T1, T6);

  628.   T5 = _mm_mul_epu32(H4, p->S21.v);

  629.   T6 = _mm_mul_epu32(H4, p->S22.v);

  630.   T0 = _mm_add_epi64(T0, T5);

  631.   T1 = _mm_add_epi64(T1, T6);

  632.   T5 = _mm_mul_epu32(H1, p->R21.v);

  633.   T6 = _mm_mul_epu32(H1, p->R22.v);

  634.   T2 = _mm_add_epi64(T2, T5);

  635.   T3 = _mm_add_epi64(T3, T6);

  636.   T5 = _mm_mul_epu32(H2, p->R20.v);

  637.   T6 = _mm_mul_epu32(H2, p->R21.v);

  638.   T2 = _mm_add_epi64(T2, T5);

  639.   T3 = _mm_add_epi64(T3, T6);

  640.   T5 = _mm_mul_epu32(H3, p->S24.v);

  641.   T6 = _mm_mul_epu32(H3, p->R20.v);

  642.   T2 = _mm_add_epi64(T2, T5);

  643.   T3 = _mm_add_epi64(T3, T6);

  644.   T5 = _mm_mul_epu32(H4, p->S23.v);

  645.   T6 = _mm_mul_epu32(H4, p->S24.v);

  646.   T2 = _mm_add_epi64(T2, T5);

  647.   T3 = _mm_add_epi64(T3, T6);

  648.   T5 = _mm_mul_epu32(H1, p->R23.v);

  649.   T4 = _mm_add_epi64(T4, T5);

  650.   T5 = _mm_mul_epu32(H2, p->R22.v);

  651.   T4 = _mm_add_epi64(T4, T5);

  652.   T5 = _mm_mul_epu32(H3, p->R21.v);

  653.   T4 = _mm_add_epi64(T4, T5);

  654.   T5 = _mm_mul_epu32(H4, p->R20.v);

  655.   T4 = _mm_add_epi64(T4, T5);

  656. 

  657.   C1 = _mm_srli_epi64(T0, 26);

  658.   C2 = _mm_srli_epi64(T3, 26);

  659.   T0 = _mm_and_si128(T0, MMASK);

  660.   T3 = _mm_and_si128(T3, MMASK);

  661.   T1 = _mm_add_epi64(T1, C1);

  662.   T4 = _mm_add_epi64(T4, C2);

  663.   C1 = _mm_srli_epi64(T1, 26);

  664.   C2 = _mm_srli_epi64(T4, 26);

  665.   T1 = _mm_and_si128(T1, MMASK);

  666.   T4 = _mm_and_si128(T4, MMASK);

  667.   T2 = _mm_add_epi64(T2, C1);

  668.   T0 = _mm_add_epi64(T0, _mm_mul_epu32(C2, FIVE));

  669.   C1 = _mm_srli_epi64(T2, 26);

  670.   C2 = _mm_srli_epi64(T0, 26);

  671.   T2 = _mm_and_si128(T2, MMASK);

  672.   T0 = _mm_and_si128(T0, MMASK);

  673.   T3 = _mm_add_epi64(T3, C1);

  674.   T1 = _mm_add_epi64(T1, C2);

  675.   C1 = _mm_srli_epi64(T3, 26);

  676.   T3 = _mm_and_si128(T3, MMASK);

  677.   T4 = _mm_add_epi64(T4, C1);

  678. 

  679.   /* H = H[0]+H[1] */

  680.   H0 = _mm_add_epi64(T0, _mm_srli_si128(T0, 8));

  681.   H1 = _mm_add_epi64(T1, _mm_srli_si128(T1, 8));

  682.   H2 = _mm_add_epi64(T2, _mm_srli_si128(T2, 8));

  683.   H3 = _mm_add_epi64(T3, _mm_srli_si128(T3, 8));

  684.   H4 = _mm_add_epi64(T4, _mm_srli_si128(T4, 8));

  685. 

  686.   t0 = _mm_cvtsi128_si32(H0);

  687.   c = (t0 >> 26);

  688.   t0 &= 0x3ffffff;

  689.   t1 = _mm_cvtsi128_si32(H1) + c;

  690.   c = (t1 >> 26);

  691.   t1 &= 0x3ffffff;

  692.   t2 = _mm_cvtsi128_si32(H2) + c;

  693.   c = (t2 >> 26);

  694.   t2 &= 0x3ffffff;

  695.   t3 = _mm_cvtsi128_si32(H3) + c;

  696.   c = (t3 >> 26);

  697.   t3 &= 0x3ffffff;

  698.   t4 = _mm_cvtsi128_si32(H4) + c;

  699.   c = (t4 >> 26);

  700.   t4 &= 0x3ffffff;

  701.   t0 = t0 + (c * 5);

  702.   c = (t0 >> 26);

  703.   t0 &= 0x3ffffff;

  704.   t1 = t1 + c;

  705. 

  706.   st->HH[0] = ((t0) | (t1 << 26)) & 0xfffffffffffull;

  707.   st->HH[1] = ((t1 >> 18) | (t2 << 8) | (t3 << 34)) & 0xfffffffffffull;

  708.   st->HH[2] = ((t3 >> 10) | (t4 << 16)) & 0x3ffffffffffull;

  709. 

  710.   return consumed;

  711. }

  712. 

  713. void CRYPTO_poly1305_update(poly1305_state *state, const uint8_t *m,

  714.                             size_t bytes) {

  715.   poly1305_state_internal *st = poly1305_aligned_state(state);

  716.   size_t want;

  717. 

  718.   /* need at least 32 initial bytes to start the accelerated branch */

  719.   if (!st->started) {

  720.     if ((st->leftover == 0) && (bytes > 32)) {

  721.       poly1305_first_block(st, m);

  722.       m += 32;

  723.       bytes -= 32;

  724.     } else {

  725.       want = poly1305_min(32 - st->leftover, bytes);

  726.       poly1305_block_copy(st->buffer + st->leftover, m, want);

  727.       bytes -= want;

  728.       m += want;

  729.       st->leftover += want;

  730.       if ((st->leftover < 32) || (bytes == 0)) {

  731.         return;

  732.       }

  733.       poly1305_first_block(st, st->buffer);

  734.       st->leftover = 0;

  735.     }

  736.     st->started = 1;

  737.   }

  738. 

  739.   /* handle leftover */

  740.   if (st->leftover) {

  741.     want = poly1305_min(64 - st->leftover, bytes);

  742.     poly1305_block_copy(st->buffer + st->leftover, m, want);

  743.     bytes -= want;

  744.     m += want;

  745.     st->leftover += want;

  746.     if (st->leftover < 64) {

  747.       return;

  748.     }

  749.     poly1305_blocks(st, st->buffer, 64);

  750.     st->leftover = 0;

  751.   }

  752. 

  753.   /* process 64 byte blocks */

  754.   if (bytes >= 64) {

  755.     want = (bytes & ~63);

  756.     poly1305_blocks(st, m, want);

  757.     m += want;

  758.     bytes -= want;

  759.   }

  760. 

  761.   if (bytes) {

  762.     poly1305_block_copy(st->buffer + st->leftover, m, bytes);

  763.     st->leftover += bytes;

  764.   }

  765. }

  766. 

  767. void CRYPTO_poly1305_finish(poly1305_state *state, uint8_t mac[16]) {

  768.   poly1305_state_internal *st = poly1305_aligned_state(state);

  769.   size_t leftover = st->leftover;

  770.   uint8_t *m = st->buffer;

  771.   uint128_t d[3];

  772.   uint64_t h0, h1, h2;

  773.   uint64_t t0, t1;

  774.   uint64_t g0, g1, g2, c, nc;

  775.   uint64_t r0, r1, r2, s1, s2;

  776.   poly1305_power *p;

  777. 

  778.   if (st->started) {

  779.     size_t consumed = poly1305_combine(st, m, leftover);

  780.     leftover -= consumed;

  781.     m += consumed;

  782.   }

  783. 

  784.   /* st->HH will either be 0 or have the combined result */

  785.   h0 = st->HH[0];

  786.   h1 = st->HH[1];

  787.   h2 = st->HH[2];

  788. 

  789.   p = &st->P[1];

  790.   r0 = ((uint64_t)p->R20.d[3] << 32) | (uint64_t)p->R20.d[1];

  791.   r1 = ((uint64_t)p->R21.d[3] << 32) | (uint64_t)p->R21.d[1];

  792.   r2 = ((uint64_t)p->R22.d[3] << 32) | (uint64_t)p->R22.d[1];

  793.   s1 = r1 * (5 << 2);

  794.   s2 = r2 * (5 << 2);

  795. 

  796.   if (leftover < 16) {

  797.     goto poly1305_donna_atmost15bytes;

  798.   }

  799. 

  800. poly1305_donna_atleast16bytes:

  801.   t0 = U8TO64_LE(m + 0);

  802.   t1 = U8TO64_LE(m + 8);

  803.   h0 += t0 & 0xfffffffffff;

  804.   t0 = shr128_pair(t1, t0, 44);

  805.   h1 += t0 & 0xfffffffffff;

  806.   h2 += (t1 >> 24) | ((uint64_t)1 << 40);

  807. 

  808. poly1305_donna_mul:

  809.   d[0] = add128(add128(mul64x64_128(h0, r0), mul64x64_128(h1, s2)),

  810.                 mul64x64_128(h2, s1));

  811.   d[1] = add128(add128(mul64x64_128(h0, r1), mul64x64_128(h1, r0)),

  812.                 mul64x64_128(h2, s2));

  813.   d[2] = add128(add128(mul64x64_128(h0, r2), mul64x64_128(h1, r1)),

  814.                 mul64x64_128(h2, r0));

  815.   h0 = lo128(d[0]) & 0xfffffffffff;

  816.   c = shr128(d[0], 44);

  817.   d[1] = add128_64(d[1], c);

  818.   h1 = lo128(d[1]) & 0xfffffffffff;

  819.   c = shr128(d[1], 44);

  820.   d[2] = add128_64(d[2], c);

  821.   h2 = lo128(d[2]) & 0x3ffffffffff;

  822.   c = shr128(d[2], 42);

  823.   h0 += c * 5;

  824. 

  825.   m += 16;

  826.   leftover -= 16;

  827.   if (leftover >= 16) {

  828.     goto poly1305_donna_atleast16bytes;

  829.   }

  830. 

  831. /* final bytes */

  832. poly1305_donna_atmost15bytes:

  833.   if (!leftover) {

  834.     goto poly1305_donna_finish;

  835.   }

  836. 

  837.   m[leftover++] = 1;

  838.   poly1305_block_zero(m + leftover, 16 - leftover);

  839.   leftover = 16;

  840. 

  841.   t0 = U8TO64_LE(m + 0);

  842.   t1 = U8TO64_LE(m + 8);

  843.   h0 += t0 & 0xfffffffffff;

  844.   t0 = shr128_pair(t1, t0, 44);

  845.   h1 += t0 & 0xfffffffffff;

  846.   h2 += (t1 >> 24);

  847. 

  848.   goto poly1305_donna_mul;

  849. 

  850. poly1305_donna_finish:

  851.   c = (h0 >> 44);

  852.   h0 &= 0xfffffffffff;

  853.   h1 += c;

  854.   c = (h1 >> 44);

  855.   h1 &= 0xfffffffffff;

  856.   h2 += c;

  857.   c = (h2 >> 42);

  858.   h2 &= 0x3ffffffffff;

  859.   h0 += c * 5;

  860. 

  861.   g0 = h0 + 5;

  862.   c = (g0 >> 44);

  863.   g0 &= 0xfffffffffff;

  864.   g1 = h1 + c;

  865.   c = (g1 >> 44);

  866.   g1 &= 0xfffffffffff;

  867.   g2 = h2 + c - ((uint64_t)1 << 42);

  868. 

  869.   c = (g2 >> 63) - 1;

  870.   nc = ~c;

  871.   h0 = (h0 & nc) | (g0 & c);

  872.   h1 = (h1 & nc) | (g1 & c);

  873.   h2 = (h2 & nc) | (g2 & c);

  874. 

  875.   /* pad */

  876.   t0 = ((uint64_t)p->R23.d[3] << 32) | (uint64_t)p->R23.d[1];

  877.   t1 = ((uint64_t)p->R24.d[3] << 32) | (uint64_t)p->R24.d[1];

  878.   h0 += (t0 & 0xfffffffffff);

  879.   c = (h0 >> 44);

  880.   h0 &= 0xfffffffffff;

  881.   t0 = shr128_pair(t1, t0, 44);

  882.   h1 += (t0 & 0xfffffffffff) + c;

  883.   c = (h1 >> 44);

  884.   h1 &= 0xfffffffffff;

  885.   t1 = (t1 >> 24);

  886.   h2 += (t1)+c;

  887. 

  888.   U64TO8_LE(mac + 0, ((h0) | (h1 << 44)));

  889.   U64TO8_LE(mac + 8, ((h1 >> 20) | (h2 << 24)));

  890. }

  891. 

  892. #endif  /* !OPENSSL_WINDOWS && OPENSSL_X86_64 */