kris
/
pqcrypto
mirror of https://github.com/henrydcase/pqc.git
1
1
Fork 0
Code Issues 1 Releases 2 Wiki Activity
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
834 Commits
9 Branches
76 MiB
 
 
 
Tree: cf5107b69f
Branches Tags
${ item.name }
Create branch ${ searchTerm }
from 'cf5107b69f'
${ noResults }
pqcrypto/crypto_kem/kyber768-90s/avx2/rejsample.c

389 lines
15 KiB
Raw Blame History 

  1. #include "consts.h"

  2. #include "params.h"

  3. #include "rejsample.h"

  4. 

  5. #include <immintrin.h>

  6. #include <stdint.h>

  7. 

  8. 

  9. static const uint8_t idx[256][8] = {

  10.     { 0,  0,  0,  0,  0,  0,  0,  0},

  11.     { 0,  0,  0,  0,  0,  0,  0,  0},

  12.     { 2,  0,  0,  0,  0,  0,  0,  0},

  13.     { 0,  2,  0,  0,  0,  0,  0,  0},

  14.     { 4,  0,  0,  0,  0,  0,  0,  0},

  15.     { 0,  4,  0,  0,  0,  0,  0,  0},

  16.     { 2,  4,  0,  0,  0,  0,  0,  0},

  17.     { 0,  2,  4,  0,  0,  0,  0,  0},

  18.     { 6,  0,  0,  0,  0,  0,  0,  0},

  19.     { 0,  6,  0,  0,  0,  0,  0,  0},

  20.     { 2,  6,  0,  0,  0,  0,  0,  0},

  21.     { 0,  2,  6,  0,  0,  0,  0,  0},

  22.     { 4,  6,  0,  0,  0,  0,  0,  0},

  23.     { 0,  4,  6,  0,  0,  0,  0,  0},

  24.     { 2,  4,  6,  0,  0,  0,  0,  0},

  25.     { 0,  2,  4,  6,  0,  0,  0,  0},

  26.     { 8,  0,  0,  0,  0,  0,  0,  0},

  27.     { 0,  8,  0,  0,  0,  0,  0,  0},

  28.     { 2,  8,  0,  0,  0,  0,  0,  0},

  29.     { 0,  2,  8,  0,  0,  0,  0,  0},

  30.     { 4,  8,  0,  0,  0,  0,  0,  0},

  31.     { 0,  4,  8,  0,  0,  0,  0,  0},

  32.     { 2,  4,  8,  0,  0,  0,  0,  0},

  33.     { 0,  2,  4,  8,  0,  0,  0,  0},

  34.     { 6,  8,  0,  0,  0,  0,  0,  0},

  35.     { 0,  6,  8,  0,  0,  0,  0,  0},

  36.     { 2,  6,  8,  0,  0,  0,  0,  0},

  37.     { 0,  2,  6,  8,  0,  0,  0,  0},

  38.     { 4,  6,  8,  0,  0,  0,  0,  0},

  39.     { 0,  4,  6,  8,  0,  0,  0,  0},

  40.     { 2,  4,  6,  8,  0,  0,  0,  0},

  41.     { 0,  2,  4,  6,  8,  0,  0,  0},

  42.     {10,  0,  0,  0,  0,  0,  0,  0},

  43.     { 0, 10,  0,  0,  0,  0,  0,  0},

  44.     { 2, 10,  0,  0,  0,  0,  0,  0},

  45.     { 0,  2, 10,  0,  0,  0,  0,  0},

  46.     { 4, 10,  0,  0,  0,  0,  0,  0},

  47.     { 0,  4, 10,  0,  0,  0,  0,  0},

  48.     { 2,  4, 10,  0,  0,  0,  0,  0},

  49.     { 0,  2,  4, 10,  0,  0,  0,  0},

  50.     { 6, 10,  0,  0,  0,  0,  0,  0},

  51.     { 0,  6, 10,  0,  0,  0,  0,  0},

  52.     { 2,  6, 10,  0,  0,  0,  0,  0},

  53.     { 0,  2,  6, 10,  0,  0,  0,  0},

  54.     { 4,  6, 10,  0,  0,  0,  0,  0},

  55.     { 0,  4,  6, 10,  0,  0,  0,  0},

  56.     { 2,  4,  6, 10,  0,  0,  0,  0},

  57.     { 0,  2,  4,  6, 10,  0,  0,  0},

  58.     { 8, 10,  0,  0,  0,  0,  0,  0},

  59.     { 0,  8, 10,  0,  0,  0,  0,  0},

  60.     { 2,  8, 10,  0,  0,  0,  0,  0},

  61.     { 0,  2,  8, 10,  0,  0,  0,  0},

  62.     { 4,  8, 10,  0,  0,  0,  0,  0},

  63.     { 0,  4,  8, 10,  0,  0,  0,  0},

  64.     { 2,  4,  8, 10,  0,  0,  0,  0},

  65.     { 0,  2,  4,  8, 10,  0,  0,  0},

  66.     { 6,  8, 10,  0,  0,  0,  0,  0},

  67.     { 0,  6,  8, 10,  0,  0,  0,  0},

  68.     { 2,  6,  8, 10,  0,  0,  0,  0},

  69.     { 0,  2,  6,  8, 10,  0,  0,  0},

  70.     { 4,  6,  8, 10,  0,  0,  0,  0},

  71.     { 0,  4,  6,  8, 10,  0,  0,  0},

  72.     { 2,  4,  6,  8, 10,  0,  0,  0},

  73.     { 0,  2,  4,  6,  8, 10,  0,  0},

  74.     {12,  0,  0,  0,  0,  0,  0,  0},

  75.     { 0, 12,  0,  0,  0,  0,  0,  0},

  76.     { 2, 12,  0,  0,  0,  0,  0,  0},

  77.     { 0,  2, 12,  0,  0,  0,  0,  0},

  78.     { 4, 12,  0,  0,  0,  0,  0,  0},

  79.     { 0,  4, 12,  0,  0,  0,  0,  0},

  80.     { 2,  4, 12,  0,  0,  0,  0,  0},

  81.     { 0,  2,  4, 12,  0,  0,  0,  0},

  82.     { 6, 12,  0,  0,  0,  0,  0,  0},

  83.     { 0,  6, 12,  0,  0,  0,  0,  0},

  84.     { 2,  6, 12,  0,  0,  0,  0,  0},

  85.     { 0,  2,  6, 12,  0,  0,  0,  0},

  86.     { 4,  6, 12,  0,  0,  0,  0,  0},

  87.     { 0,  4,  6, 12,  0,  0,  0,  0},

  88.     { 2,  4,  6, 12,  0,  0,  0,  0},

  89.     { 0,  2,  4,  6, 12,  0,  0,  0},

  90.     { 8, 12,  0,  0,  0,  0,  0,  0},

  91.     { 0,  8, 12,  0,  0,  0,  0,  0},

  92.     { 2,  8, 12,  0,  0,  0,  0,  0},

  93.     { 0,  2,  8, 12,  0,  0,  0,  0},

  94.     { 4,  8, 12,  0,  0,  0,  0,  0},

  95.     { 0,  4,  8, 12,  0,  0,  0,  0},

  96.     { 2,  4,  8, 12,  0,  0,  0,  0},

  97.     { 0,  2,  4,  8, 12,  0,  0,  0},

  98.     { 6,  8, 12,  0,  0,  0,  0,  0},

  99.     { 0,  6,  8, 12,  0,  0,  0,  0},

  100.     { 2,  6,  8, 12,  0,  0,  0,  0},

  101.     { 0,  2,  6,  8, 12,  0,  0,  0},

  102.     { 4,  6,  8, 12,  0,  0,  0,  0},

  103.     { 0,  4,  6,  8, 12,  0,  0,  0},

  104.     { 2,  4,  6,  8, 12,  0,  0,  0},

  105.     { 0,  2,  4,  6,  8, 12,  0,  0},

  106.     {10, 12,  0,  0,  0,  0,  0,  0},

  107.     { 0, 10, 12,  0,  0,  0,  0,  0},

  108.     { 2, 10, 12,  0,  0,  0,  0,  0},

  109.     { 0,  2, 10, 12,  0,  0,  0,  0},

  110.     { 4, 10, 12,  0,  0,  0,  0,  0},

  111.     { 0,  4, 10, 12,  0,  0,  0,  0},

  112.     { 2,  4, 10, 12,  0,  0,  0,  0},

  113.     { 0,  2,  4, 10, 12,  0,  0,  0},

  114.     { 6, 10, 12,  0,  0,  0,  0,  0},

  115.     { 0,  6, 10, 12,  0,  0,  0,  0},

  116.     { 2,  6, 10, 12,  0,  0,  0,  0},

  117.     { 0,  2,  6, 10, 12,  0,  0,  0},

  118.     { 4,  6, 10, 12,  0,  0,  0,  0},

  119.     { 0,  4,  6, 10, 12,  0,  0,  0},

  120.     { 2,  4,  6, 10, 12,  0,  0,  0},

  121.     { 0,  2,  4,  6, 10, 12,  0,  0},

  122.     { 8, 10, 12,  0,  0,  0,  0,  0},

  123.     { 0,  8, 10, 12,  0,  0,  0,  0},

  124.     { 2,  8, 10, 12,  0,  0,  0,  0},

  125.     { 0,  2,  8, 10, 12,  0,  0,  0},

  126.     { 4,  8, 10, 12,  0,  0,  0,  0},

  127.     { 0,  4,  8, 10, 12,  0,  0,  0},

  128.     { 2,  4,  8, 10, 12,  0,  0,  0},

  129.     { 0,  2,  4,  8, 10, 12,  0,  0},

  130.     { 6,  8, 10, 12,  0,  0,  0,  0},

  131.     { 0,  6,  8, 10, 12,  0,  0,  0},

  132.     { 2,  6,  8, 10, 12,  0,  0,  0},

  133.     { 0,  2,  6,  8, 10, 12,  0,  0},

  134.     { 4,  6,  8, 10, 12,  0,  0,  0},

  135.     { 0,  4,  6,  8, 10, 12,  0,  0},

  136.     { 2,  4,  6,  8, 10, 12,  0,  0},

  137.     { 0,  2,  4,  6,  8, 10, 12,  0},

  138.     {14,  0,  0,  0,  0,  0,  0,  0},

  139.     { 0, 14,  0,  0,  0,  0,  0,  0},

  140.     { 2, 14,  0,  0,  0,  0,  0,  0},

  141.     { 0,  2, 14,  0,  0,  0,  0,  0},

  142.     { 4, 14,  0,  0,  0,  0,  0,  0},

  143.     { 0,  4, 14,  0,  0,  0,  0,  0},

  144.     { 2,  4, 14,  0,  0,  0,  0,  0},

  145.     { 0,  2,  4, 14,  0,  0,  0,  0},

  146.     { 6, 14,  0,  0,  0,  0,  0,  0},

  147.     { 0,  6, 14,  0,  0,  0,  0,  0},

  148.     { 2,  6, 14,  0,  0,  0,  0,  0},

  149.     { 0,  2,  6, 14,  0,  0,  0,  0},

  150.     { 4,  6, 14,  0,  0,  0,  0,  0},

  151.     { 0,  4,  6, 14,  0,  0,  0,  0},

  152.     { 2,  4,  6, 14,  0,  0,  0,  0},

  153.     { 0,  2,  4,  6, 14,  0,  0,  0},

  154.     { 8, 14,  0,  0,  0,  0,  0,  0},

  155.     { 0,  8, 14,  0,  0,  0,  0,  0},

  156.     { 2,  8, 14,  0,  0,  0,  0,  0},

  157.     { 0,  2,  8, 14,  0,  0,  0,  0},

  158.     { 4,  8, 14,  0,  0,  0,  0,  0},

  159.     { 0,  4,  8, 14,  0,  0,  0,  0},

  160.     { 2,  4,  8, 14,  0,  0,  0,  0},

  161.     { 0,  2,  4,  8, 14,  0,  0,  0},

  162.     { 6,  8, 14,  0,  0,  0,  0,  0},

  163.     { 0,  6,  8, 14,  0,  0,  0,  0},

  164.     { 2,  6,  8, 14,  0,  0,  0,  0},

  165.     { 0,  2,  6,  8, 14,  0,  0,  0},

  166.     { 4,  6,  8, 14,  0,  0,  0,  0},

  167.     { 0,  4,  6,  8, 14,  0,  0,  0},

  168.     { 2,  4,  6,  8, 14,  0,  0,  0},

  169.     { 0,  2,  4,  6,  8, 14,  0,  0},

  170.     {10, 14,  0,  0,  0,  0,  0,  0},

  171.     { 0, 10, 14,  0,  0,  0,  0,  0},

  172.     { 2, 10, 14,  0,  0,  0,  0,  0},

  173.     { 0,  2, 10, 14,  0,  0,  0,  0},

  174.     { 4, 10, 14,  0,  0,  0,  0,  0},

  175.     { 0,  4, 10, 14,  0,  0,  0,  0},

  176.     { 2,  4, 10, 14,  0,  0,  0,  0},

  177.     { 0,  2,  4, 10, 14,  0,  0,  0},

  178.     { 6, 10, 14,  0,  0,  0,  0,  0},

  179.     { 0,  6, 10, 14,  0,  0,  0,  0},

  180.     { 2,  6, 10, 14,  0,  0,  0,  0},

  181.     { 0,  2,  6, 10, 14,  0,  0,  0},

  182.     { 4,  6, 10, 14,  0,  0,  0,  0},

  183.     { 0,  4,  6, 10, 14,  0,  0,  0},

  184.     { 2,  4,  6, 10, 14,  0,  0,  0},

  185.     { 0,  2,  4,  6, 10, 14,  0,  0},

  186.     { 8, 10, 14,  0,  0,  0,  0,  0},

  187.     { 0,  8, 10, 14,  0,  0,  0,  0},

  188.     { 2,  8, 10, 14,  0,  0,  0,  0},

  189.     { 0,  2,  8, 10, 14,  0,  0,  0},

  190.     { 4,  8, 10, 14,  0,  0,  0,  0},

  191.     { 0,  4,  8, 10, 14,  0,  0,  0},

  192.     { 2,  4,  8, 10, 14,  0,  0,  0},

  193.     { 0,  2,  4,  8, 10, 14,  0,  0},

  194.     { 6,  8, 10, 14,  0,  0,  0,  0},

  195.     { 0,  6,  8, 10, 14,  0,  0,  0},

  196.     { 2,  6,  8, 10, 14,  0,  0,  0},

  197.     { 0,  2,  6,  8, 10, 14,  0,  0},

  198.     { 4,  6,  8, 10, 14,  0,  0,  0},

  199.     { 0,  4,  6,  8, 10, 14,  0,  0},

  200.     { 2,  4,  6,  8, 10, 14,  0,  0},

  201.     { 0,  2,  4,  6,  8, 10, 14,  0},

  202.     {12, 14,  0,  0,  0,  0,  0,  0},

  203.     { 0, 12, 14,  0,  0,  0,  0,  0},

  204.     { 2, 12, 14,  0,  0,  0,  0,  0},

  205.     { 0,  2, 12, 14,  0,  0,  0,  0},

  206.     { 4, 12, 14,  0,  0,  0,  0,  0},

  207.     { 0,  4, 12, 14,  0,  0,  0,  0},

  208.     { 2,  4, 12, 14,  0,  0,  0,  0},

  209.     { 0,  2,  4, 12, 14,  0,  0,  0},

  210.     { 6, 12, 14,  0,  0,  0,  0,  0},

  211.     { 0,  6, 12, 14,  0,  0,  0,  0},

  212.     { 2,  6, 12, 14,  0,  0,  0,  0},

  213.     { 0,  2,  6, 12, 14,  0,  0,  0},

  214.     { 4,  6, 12, 14,  0,  0,  0,  0},

  215.     { 0,  4,  6, 12, 14,  0,  0,  0},

  216.     { 2,  4,  6, 12, 14,  0,  0,  0},

  217.     { 0,  2,  4,  6, 12, 14,  0,  0},

  218.     { 8, 12, 14,  0,  0,  0,  0,  0},

  219.     { 0,  8, 12, 14,  0,  0,  0,  0},

  220.     { 2,  8, 12, 14,  0,  0,  0,  0},

  221.     { 0,  2,  8, 12, 14,  0,  0,  0},

  222.     { 4,  8, 12, 14,  0,  0,  0,  0},

  223.     { 0,  4,  8, 12, 14,  0,  0,  0},

  224.     { 2,  4,  8, 12, 14,  0,  0,  0},

  225.     { 0,  2,  4,  8, 12, 14,  0,  0},

  226.     { 6,  8, 12, 14,  0,  0,  0,  0},

  227.     { 0,  6,  8, 12, 14,  0,  0,  0},

  228.     { 2,  6,  8, 12, 14,  0,  0,  0},

  229.     { 0,  2,  6,  8, 12, 14,  0,  0},

  230.     { 4,  6,  8, 12, 14,  0,  0,  0},

  231.     { 0,  4,  6,  8, 12, 14,  0,  0},

  232.     { 2,  4,  6,  8, 12, 14,  0,  0},

  233.     { 0,  2,  4,  6,  8, 12, 14,  0},

  234.     {10, 12, 14,  0,  0,  0,  0,  0},

  235.     { 0, 10, 12, 14,  0,  0,  0,  0},

  236.     { 2, 10, 12, 14,  0,  0,  0,  0},

  237.     { 0,  2, 10, 12, 14,  0,  0,  0},

  238.     { 4, 10, 12, 14,  0,  0,  0,  0},

  239.     { 0,  4, 10, 12, 14,  0,  0,  0},

  240.     { 2,  4, 10, 12, 14,  0,  0,  0},

  241.     { 0,  2,  4, 10, 12, 14,  0,  0},

  242.     { 6, 10, 12, 14,  0,  0,  0,  0},

  243.     { 0,  6, 10, 12, 14,  0,  0,  0},

  244.     { 2,  6, 10, 12, 14,  0,  0,  0},

  245.     { 0,  2,  6, 10, 12, 14,  0,  0},

  246.     { 4,  6, 10, 12, 14,  0,  0,  0},

  247.     { 0,  4,  6, 10, 12, 14,  0,  0},

  248.     { 2,  4,  6, 10, 12, 14,  0,  0},

  249.     { 0,  2,  4,  6, 10, 12, 14,  0},

  250.     { 8, 10, 12, 14,  0,  0,  0,  0},

  251.     { 0,  8, 10, 12, 14,  0,  0,  0},

  252.     { 2,  8, 10, 12, 14,  0,  0,  0},

  253.     { 0,  2,  8, 10, 12, 14,  0,  0},

  254.     { 4,  8, 10, 12, 14,  0,  0,  0},

  255.     { 0,  4,  8, 10, 12, 14,  0,  0},

  256.     { 2,  4,  8, 10, 12, 14,  0,  0},

  257.     { 0,  2,  4,  8, 10, 12, 14,  0},

  258.     { 6,  8, 10, 12, 14,  0,  0,  0},

  259.     { 0,  6,  8, 10, 12, 14,  0,  0},

  260.     { 2,  6,  8, 10, 12, 14,  0,  0},

  261.     { 0,  2,  6,  8, 10, 12, 14,  0},

  262.     { 4,  6,  8, 10, 12, 14,  0,  0},

  263.     { 0,  4,  6,  8, 10, 12, 14,  0},

  264.     { 2,  4,  6,  8, 10, 12, 14,  0},

  265.     { 0,  2,  4,  6,  8, 10, 12, 14}

  266. };

  267. 

  268. #define _mm256_cmpge_epu16(a, b)  _mm256_cmpeq_epi16(_mm256_max_epu16(a, b), a)

  269. #define _mm_cmpge_epu16(a, b)  _mm_cmpeq_epi16(_mm_max_epu16(a, b), a)

  270. 

  271. size_t PQCLEAN_KYBER76890S_AVX2_rej_uniform(int16_t *r,

  272.         size_t len,

  273.         const uint8_t *buf,

  274.         size_t buflen) {

  275.     size_t ctr, pos;

  276.     uint16_t val;

  277.     uint32_t good0, good1, good2;

  278.     const __m256i bound  = _mm256_set1_epi16((int16_t)(19 * KYBER_Q - 1)); // -1 to use cheaper >= instead of > comparison

  279.     const __m256i ones   = _mm256_set1_epi8(1);

  280.     const __m256i kyberq = _mm256_load_si256(&PQCLEAN_KYBER76890S_AVX2_16xq.as_vec);

  281.     const __m256i v = _mm256_load_si256(&PQCLEAN_KYBER76890S_AVX2_16xv.as_vec);

  282.     __m256i d0, d1, d2, tmp0, tmp1, tmp2, pi0, pi1, pi2;

  283.     __m128i d, tmp, pilo, pihi;

  284. 

  285.     ctr = pos = 0;

  286.     while (ctr + 48 <= len && pos + 96 <= buflen) {

  287.         d0 = _mm256_loadu_si256((__m256i *)&buf[pos + 0]);

  288.         d1 = _mm256_loadu_si256((__m256i *)&buf[pos + 32]);

  289.         d2 = _mm256_loadu_si256((__m256i *)&buf[pos + 64]);

  290. 

  291.         tmp0 = _mm256_cmpge_epu16(bound, d0);

  292.         tmp1 = _mm256_cmpge_epu16(bound, d1);

  293.         tmp2 = _mm256_cmpge_epu16(bound, d2);

  294.         good0 = (uint32_t)_mm256_movemask_epi8(tmp0);

  295.         good1 = (uint32_t)_mm256_movemask_epi8(tmp1);

  296.         good2 = (uint32_t)_mm256_movemask_epi8(tmp2);

  297.         good0 = _pext_u32(good0, 0x55555555);

  298.         good1 = _pext_u32(good1, 0x55555555);

  299.         good2 = _pext_u32(good2, 0x55555555);

  300. 

  301.         pilo = _mm_loadl_epi64((__m128i *)&idx[good0 & 0xFF]);

  302.         pihi = _mm_loadl_epi64((__m128i *)&idx[(good0 >> 8) & 0xFF]);

  303.         pi0 = _mm256_castsi128_si256(pilo);

  304.         pi0 = _mm256_inserti128_si256(pi0, pihi, 1);

  305. 

  306.         pilo = _mm_loadl_epi64((__m128i *)&idx[good1 & 0xFF]);

  307.         pihi = _mm_loadl_epi64((__m128i *)&idx[(good1 >> 8) & 0xFF]);

  308.         pi1 = _mm256_castsi128_si256(pilo);

  309.         pi1 = _mm256_inserti128_si256(pi1, pihi, 1);

  310. 

  311.         pilo = _mm_loadl_epi64((__m128i *)&idx[good2 & 0xFF]);

  312.         pihi = _mm_loadl_epi64((__m128i *)&idx[(good2 >> 8) & 0xFF]);

  313.         pi2 = _mm256_castsi128_si256(pilo);

  314.         pi2 = _mm256_inserti128_si256(pi2, pihi, 1);

  315. 

  316.         tmp0 = _mm256_add_epi8(pi0, ones);

  317.         tmp1 = _mm256_add_epi8(pi1, ones);

  318.         tmp2 = _mm256_add_epi8(pi2, ones);

  319.         pi0 = _mm256_unpacklo_epi8(pi0, tmp0);

  320.         pi1 = _mm256_unpacklo_epi8(pi1, tmp1);

  321.         pi2 = _mm256_unpacklo_epi8(pi2, tmp2);

  322. 

  323.         d0 = _mm256_shuffle_epi8(d0, pi0);

  324.         d1 = _mm256_shuffle_epi8(d1, pi1);

  325.         d2 = _mm256_shuffle_epi8(d2, pi2);

  326. 

  327.         /* Barrett reduction of (still unsigned) d values */

  328.         tmp0 = _mm256_mulhi_epu16(d0, v);

  329.         tmp1 = _mm256_mulhi_epu16(d1, v);

  330.         tmp2 = _mm256_mulhi_epu16(d2, v);

  331.         tmp0 = _mm256_srli_epi16(tmp0, 10);

  332.         tmp1 = _mm256_srli_epi16(tmp1, 10);

  333.         tmp2 = _mm256_srli_epi16(tmp2, 10);

  334.         tmp0 = _mm256_mullo_epi16(tmp0, kyberq);

  335.         tmp1 = _mm256_mullo_epi16(tmp1, kyberq);

  336.         tmp2 = _mm256_mullo_epi16(tmp2, kyberq);

  337.         d0   = _mm256_sub_epi16(d0, tmp0);

  338.         d1   = _mm256_sub_epi16(d1, tmp1);

  339.         d2   = _mm256_sub_epi16(d2, tmp2);

  340. 

  341.         _mm_storeu_si128((__m128i *)&r[ctr], _mm256_castsi256_si128(d0));

  342.         ctr += (unsigned int)_mm_popcnt_u32(good0 & 0xFF);

  343.         _mm_storeu_si128((__m128i *)&r[ctr], _mm256_extracti128_si256(d0, 1));

  344.         ctr += (unsigned int)_mm_popcnt_u32((good0 >> 8) & 0xFF);

  345.         _mm_storeu_si128((__m128i *)&r[ctr], _mm256_castsi256_si128(d1));

  346.         ctr += (unsigned int)_mm_popcnt_u32(good1 & 0xFF);

  347.         _mm_storeu_si128((__m128i *)&r[ctr], _mm256_extracti128_si256(d1, 1));

  348.         ctr += (unsigned int)_mm_popcnt_u32((good1 >> 8) & 0xFF);

  349.         _mm_storeu_si128((__m128i *)&r[ctr], _mm256_castsi256_si128(d2));

  350.         ctr += (unsigned int)_mm_popcnt_u32(good2 & 0xFF);

  351.         _mm_storeu_si128((__m128i *)&r[ctr], _mm256_extracti128_si256(d2, 1));

  352.         ctr += (unsigned int)_mm_popcnt_u32((good2 >> 8) & 0xFF);

  353.         pos += 96;

  354.     }

  355. 

  356.     while (ctr + 8 <= len && pos + 16 <= buflen) {

  357.         d = _mm_loadu_si128((__m128i *)&buf[pos]);

  358.         tmp = _mm_cmpge_epu16(_mm256_castsi256_si128(bound), d);

  359.         good0 = (uint32_t)_mm_movemask_epi8(tmp);

  360.         good0 = _pext_u32(good0, 0x55555555);

  361.         pilo = _mm_loadl_epi64((__m128i *)&idx[good0]);

  362.         pihi = _mm_add_epi8(pilo, _mm256_castsi256_si128(ones));

  363.         pilo = _mm_unpacklo_epi8(pilo, pihi);

  364.         d = _mm_shuffle_epi8(d, pilo);

  365. 

  366.         /* Barrett reduction */

  367.         tmp = _mm_mulhi_epu16(d, _mm256_castsi256_si128(v));

  368.         tmp = _mm_srli_epi16(tmp, 10);

  369.         tmp = _mm_mullo_epi16(tmp, _mm256_castsi256_si128(kyberq));

  370.         d   = _mm_sub_epi16(d, tmp);

  371. 

  372.         _mm_storeu_si128((__m128i *)&r[ctr], d);

  373.         ctr += (unsigned int)_mm_popcnt_u32(good0);

  374.         pos += 16;

  375.     }

  376. 

  377.     while (ctr < len && pos + 2 <= buflen) {

  378.         val = (uint16_t)(buf[pos] | ((uint16_t)buf[pos + 1] << 8));

  379.         pos += 2;

  380. 

  381.         if (val < 19 * KYBER_Q) {

  382.             val -= ((int32_t)val * 20159 >> 26) * KYBER_Q;

  383.             r[ctr++] = (int16_t)val;

  384.         }

  385.     }

  386. 

  387.     return ctr;

  388. }