From cb7675a9b616eaa7ae64183e4da4bb0fa8df4589 Mon Sep 17 00:00:00 2001 From: "John M. Schanck" Date: Mon, 7 Sep 2020 17:00:59 -0400 Subject: [PATCH] Replace avx2 'reduce' in gf2x --- crypto_kem/hqc-128/avx2/gf2x.c | 73 ++++++----------- crypto_kem/hqc-192/avx2/gf2x.c | 77 +++++++----------- crypto_kem/hqc-256/avx2/gf2x.c | 89 ++++++++------------- crypto_kem/hqc-rmrs-128/avx2/gf2x.c | 73 ++++++----------- crypto_kem/hqc-rmrs-192/avx2/gf2x.c | 85 +++++++------------- crypto_kem/hqc-rmrs-256/avx2/gf2x.c | 89 ++++++++------------- test/duplicate_consistency/hqc-192_avx2.yml | 1 + 7 files changed, 177 insertions(+), 310 deletions(-) diff --git a/crypto_kem/hqc-128/avx2/gf2x.c b/crypto_kem/hqc-128/avx2/gf2x.c index 6a1f9bd1..3338b362 100644 --- a/crypto_kem/hqc-128/avx2/gf2x.c +++ b/crypto_kem/hqc-128/avx2/gf2x.c @@ -14,25 +14,21 @@ #define T_TM3_3W_256 32 #define T_TM3_3W_64 128 -#define VEC_N_ARRAY_SIZE_VEC CEIL_DIVIDE(PARAM_N, 256) /*!< The number of needed vectors to store PARAM_N bits*/ -#define WORD 64 -#define LAST64 (PARAM_N >> 6) -uint64_t a1_times_a2[2 * VEC_N_256_SIZE_64 + 1]; -uint64_t tmp_reduce[VEC_N_ARRAY_SIZE_VEC << 2]; -__m256i *o256 = (__m256i *) tmp_reduce; +#define VEC_N_SIZE_256 CEIL_DIVIDE(PARAM_N, 256) /*!< The number of needed vectors to store PARAM_N bits*/ +__m256i a1_times_a2[2 * VEC_N_SIZE_256 + 1]; uint64_t bloc64[PARAM_OMEGA_R]; // Allocation with the biggest possible weight uint64_t bit64[PARAM_OMEGA_R]; // Allocation with the biggest possible weight static inline void reduce(uint64_t *o, const uint64_t *a); -inline static void karat_mult_1(__m128i *C, __m128i *A, __m128i *B); -inline static void karat_mult_2(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_4(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_8(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_16(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_32(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_1(__m128i *C, __m128i *A, __m128i *B); +static inline void karat_mult_2(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_4(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_8(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_16(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_32(__m256i *C, __m256i *A, __m256i *B); static inline void divByXplus1(__m256i *out, __m256i *in, int size); -static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B); +static void TOOM3Mult(__m256i *Out, const uint64_t *A, const uint64_t *B); @@ -45,33 +41,16 @@ static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B); * @param[in] a Pointer to the polynomial a(x) */ static inline void reduce(uint64_t *o, const uint64_t *a) { - __m256i r256, carry256; - __m256i *a256 = (__m256i *) a; - static const int32_t dec64 = PARAM_N & 0x3f; - static const int32_t d0 = WORD - dec64; - int32_t i, i2; + uint64_t r; + uint64_t carry; - for (i = LAST64 ; i < (PARAM_N >> 5) - 4 ; i += 4) { - r256 = _mm256_lddqu_si256((__m256i const *) (& a[i])); - r256 = _mm256_srli_epi64(r256, dec64); - carry256 = _mm256_lddqu_si256((__m256i const *) (& a[i + 1])); - carry256 = _mm256_slli_epi64(carry256, d0); - r256 ^= carry256; - i2 = (i - LAST64) >> 2; - o256[i2] = a256[i2] ^ r256; + for (uint32_t i = 0 ; i < VEC_N_SIZE_64 ; i++) { + r = a[i + VEC_N_SIZE_64 - 1] >> (PARAM_N & 63); + carry = (uint64_t) (a[i + VEC_N_SIZE_64] << (64 - (PARAM_N & 63))); + o[i] = a[i] ^ r ^ carry; } - r256 = (__m256i) { - a[i], a[i + 1], 0x0UL, 0x0UL - }; - carry256 = _mm256_lddqu_si256((__m256i const *) (& a[i + 1])); - r256 = _mm256_srli_epi64(r256, dec64); - carry256 = _mm256_slli_epi64(carry256, d0); - r256 ^= carry256; - i2 = (i - LAST64) >> 2; - o256[i2] = (a256[i2] ^ r256); - tmp_reduce[LAST64] &= RED_MASK; - memcpy(o, tmp_reduce, VEC_N_SIZE_BYTES); + o[VEC_N_SIZE_64 - 1] &= RED_MASK; } /** @@ -83,7 +62,7 @@ static inline void reduce(uint64_t *o, const uint64_t *a) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_1(__m128i *C, __m128i *A, __m128i *B) { +static inline void karat_mult_1(__m128i *C, __m128i *A, __m128i *B) { __m128i D1[2]; __m128i D0[2], D2[2]; __m128i Al = _mm_loadu_si128(A); @@ -140,7 +119,7 @@ inline static void karat_mult_1(__m128i *C, __m128i *A, __m128i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_2(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_2(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[2], D1[2], D2[2], SAA, SBB; __m128i *A128 = (__m128i *)A, *B128 = (__m128i *)B; @@ -170,7 +149,7 @@ inline static void karat_mult_2(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_4(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_4(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[4], D1[4], D2[4], SAA[2], SBB[2]; karat_mult_2( D0, A, B); @@ -207,7 +186,7 @@ inline static void karat_mult_4(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_8(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_8(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[8], D1[8], D2[8], SAA[4], SBB[4]; karat_mult_4( D0, A, B); @@ -246,7 +225,7 @@ inline static void karat_mult_8(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_16(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_16(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[16], D1[16], D2[16], SAA[8], SBB[8]; karat_mult_8( D0, A, B); @@ -285,7 +264,7 @@ inline static void karat_mult_16(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_32(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_32(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[32], D1[32], D2[32], SAA[16], SBB[16]; karat_mult_16( D0, A, B); @@ -344,7 +323,7 @@ static inline void divByXplus1(__m256i *out, __m256i *in, int size) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { +static void TOOM3Mult(__m256i *Out, const uint64_t *A, const uint64_t *B) { static __m256i U0[T_TM3_3W_256], V0[T_TM3_3W_256], U1[T_TM3_3W_256], V1[T_TM3_3W_256], U2[T_TM3_3W_256], V2[T_TM3_3W_256]; static __m256i W0[2 * (T_TM3_3W_256)], W1[2 * (T_TM3_3W_256)], W2[2 * (T_TM3_3W_256)], W3[2 * (T_TM3_3W_256)], W4[2 * (T_TM3_3W_256)]; static __m256i tmp[2 * (T_TM3_3W_256)]; @@ -533,7 +512,7 @@ static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { } for (int32_t i = 0 ; i < 6 * T_TM3_3W_256 - 2 ; i++) { - uint64_t *out64 = Out + (i << 2); + uint64_t *out64 = ((uint64_t *)Out) + (i << 2); _mm256_storeu_si256((__m256i *)out64, ro256[i]); } } @@ -551,8 +530,8 @@ static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { */ void PQCLEAN_HQC128_AVX2_vect_mul(uint64_t *o, const uint64_t *a1, const uint64_t *a2) { TOOM3Mult(a1_times_a2, a1, a2); - reduce(o, a1_times_a2); + reduce(o, (uint64_t *)a1_times_a2); // clear all - memset(a1_times_a2, 0, (VEC_N_SIZE_64 << 1) * sizeof(uint64_t)); + memset(a1_times_a2, 0, (2 * VEC_N_SIZE_256 + 1) * sizeof(__m256i)); } diff --git a/crypto_kem/hqc-192/avx2/gf2x.c b/crypto_kem/hqc-192/avx2/gf2x.c index eda790ad..eb45382b 100644 --- a/crypto_kem/hqc-192/avx2/gf2x.c +++ b/crypto_kem/hqc-192/avx2/gf2x.c @@ -14,26 +14,22 @@ #define T_TM3_3W_256 64 #define T_TM3_3W_64 256 -#define VEC_N_ARRAY_SIZE_VEC CEIL_DIVIDE(PARAM_N, 256) /*!< The number of needed vectors to store PARAM_N bits*/ -#define WORD 64 -#define LAST64 (PARAM_N >> 6) -uint64_t a1_times_a2[2 * VEC_N_256_SIZE_64 + 1]; -uint64_t tmp_reduce[VEC_N_ARRAY_SIZE_VEC << 2]; -__m256i *o256 = (__m256i *) tmp_reduce; +#define VEC_N_SIZE_256 CEIL_DIVIDE(PARAM_N, 256) /*!< The number of needed vectors to store PARAM_N bits*/ +__m256i a1_times_a2[2 * VEC_N_SIZE_256 + 1]; uint64_t bloc64[PARAM_OMEGA_R]; // Allocation with the biggest possible weight uint64_t bit64[PARAM_OMEGA_R]; // Allocation with the biggest possible weight static inline void reduce(uint64_t *o, const uint64_t *a); -inline static void karat_mult_1(__m128i *C, __m128i *A, __m128i *B); -inline static void karat_mult_2(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_4(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_8(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_16(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_32(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_64(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_1(__m128i *C, __m128i *A, __m128i *B); +static inline void karat_mult_2(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_4(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_8(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_16(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_32(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_64(__m256i *C, __m256i *A, __m256i *B); static inline void divByXplus1(__m256i *out, __m256i *in, int size); -static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B); +static void TOOM3Mult(__m256i *Out, const uint64_t *A, const uint64_t *B); @@ -46,35 +42,18 @@ static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B); * @param[in] a Pointer to the polynomial a(x) */ static inline void reduce(uint64_t *o, const uint64_t *a) { - __m256i r256, carry256; - __m256i *a256 = (__m256i *) a; - static const int32_t dec64 = PARAM_N & 0x3f; - static const int32_t d0 = WORD - dec64; - int32_t i, i2; + uint64_t r; + uint64_t carry; - for (i = LAST64 ; i < (PARAM_N >> 5) - 4 ; i += 4) { - r256 = _mm256_lddqu_si256((__m256i const *) (& a[i])); - r256 = _mm256_srli_epi64(r256, dec64); - carry256 = _mm256_lddqu_si256((__m256i const *) (& a[i + 1])); - carry256 = _mm256_slli_epi64(carry256, d0); - r256 ^= carry256; - i2 = (i - LAST64) >> 2; - o256[i2] = a256[i2] ^ r256; + for (uint32_t i = 0 ; i < VEC_N_SIZE_64 ; i++) { + r = a[i + VEC_N_SIZE_64 - 1] >> (PARAM_N & 63); + carry = (uint64_t) (a[i + VEC_N_SIZE_64] << (64 - (PARAM_N & 63))); + o[i] = a[i] ^ r ^ carry; } - r256 = _mm256_lddqu_si256((__m256i const *) (& a[i])); - carry256 = _mm256_lddqu_si256((__m256i const *) (& a[i + 1])); - r256 = _mm256_srli_epi64(r256, dec64); - carry256 = _mm256_slli_epi64(carry256, d0); - r256 ^= carry256; - i2 = (i - LAST64) >> 2; - o256[i2] = (a256[i2] ^ r256); - tmp_reduce[LAST64] &= RED_MASK; - memcpy(o, tmp_reduce, VEC_N_SIZE_BYTES); + o[VEC_N_SIZE_64 - 1] &= RED_MASK; } - - /** * @brief Compute C(x) = A(x)*B(x) * A(x) and B(x) are stored in 128-bit registers @@ -84,7 +63,7 @@ static inline void reduce(uint64_t *o, const uint64_t *a) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_1(__m128i *C, __m128i *A, __m128i *B) { +static inline void karat_mult_1(__m128i *C, __m128i *A, __m128i *B) { __m128i D1[2]; __m128i D0[2], D2[2]; __m128i Al = _mm_loadu_si128(A); @@ -141,7 +120,7 @@ inline static void karat_mult_1(__m128i *C, __m128i *A, __m128i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_2(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_2(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[2], D1[2], D2[2], SAA, SBB; __m128i *A128 = (__m128i *)A, *B128 = (__m128i *)B; @@ -171,7 +150,7 @@ inline static void karat_mult_2(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_4(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_4(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[4], D1[4], D2[4], SAA[2], SBB[2]; karat_mult_2( D0, A, B); @@ -208,7 +187,7 @@ inline static void karat_mult_4(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_8(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_8(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[8], D1[8], D2[8], SAA[4], SBB[4]; karat_mult_4( D0, A, B); @@ -247,7 +226,7 @@ inline static void karat_mult_8(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_16(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_16(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[16], D1[16], D2[16], SAA[8], SBB[8]; karat_mult_8( D0, A, B); @@ -286,7 +265,7 @@ inline static void karat_mult_16(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_32(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_32(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[32], D1[32], D2[32], SAA[16], SBB[16]; karat_mult_16( D0, A, B); @@ -325,7 +304,7 @@ inline static void karat_mult_32(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_64(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_64(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[64], D1[64], D2[64], SAA[32], SBB[32]; karat_mult_32( D0, A, B); @@ -384,7 +363,7 @@ static inline void divByXplus1(__m256i *out, __m256i *in, int size) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { +static void TOOM3Mult(__m256i *Out, const uint64_t *A, const uint64_t *B) { static __m256i U0[T_TM3_3W_256], V0[T_TM3_3W_256], U1[T_TM3_3W_256], V1[T_TM3_3W_256], U2[T_TM3_3W_256], V2[T_TM3_3W_256]; static __m256i W0[2 * (T_TM3_3W_256)], W1[2 * (T_TM3_3W_256)], W2[2 * (T_TM3_3W_256)], W3[2 * (T_TM3_3W_256)], W4[2 * (T_TM3_3W_256)]; static __m256i tmp[2 * (T_TM3_3W_256)]; @@ -572,7 +551,7 @@ static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { } for (int32_t i = 0 ; i < 6 * T_TM3_3W_256 - 2 ; i++) { - uint64_t *out64 = Out + (i << 2); + uint64_t *out64 = ((uint64_t *)Out) + (i << 2); _mm256_storeu_si256((__m256i *)out64, ro256[i]); } } @@ -591,8 +570,8 @@ static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { */ void PQCLEAN_HQC192_AVX2_vect_mul(uint64_t *o, const uint64_t *a1, const uint64_t *a2) { TOOM3Mult(a1_times_a2, a1, a2); - reduce(o, a1_times_a2); + reduce(o, (uint64_t *)a1_times_a2); // clear all - memset(a1_times_a2, 0, (VEC_N_SIZE_64 << 1) * sizeof(uint64_t)); + memset(a1_times_a2, 0, (2 * VEC_N_SIZE_256 + 1) * sizeof(__m256i)); } diff --git a/crypto_kem/hqc-256/avx2/gf2x.c b/crypto_kem/hqc-256/avx2/gf2x.c index abd11014..a86dd4a8 100644 --- a/crypto_kem/hqc-256/avx2/gf2x.c +++ b/crypto_kem/hqc-256/avx2/gf2x.c @@ -18,27 +18,23 @@ #define T_TM3R_3W_256 93 #define T_TM3R_3W_64 (T_TM3R_3W_256<<2) -#define VEC_N_ARRAY_SIZE_VEC CEIL_DIVIDE(PARAM_N, 256) /*!< The number of needed vectors to store PARAM_N bits*/ -#define WORD 64 -#define LAST64 (PARAM_N >> 6) -uint64_t a1_times_a2[2 * VEC_N_256_SIZE_64 + 1]; -uint64_t tmp_reduce[VEC_N_ARRAY_SIZE_VEC << 2]; -__m256i *o256 = (__m256i *) tmp_reduce; +#define VEC_N_SIZE_256 CEIL_DIVIDE(PARAM_N, 256) /*!< The number of needed vectors to store PARAM_N bits*/ +__m256i a1_times_a2[2 * VEC_N_SIZE_256 + 1]; uint64_t bloc64[PARAM_OMEGA_R]; // Allocation with the biggest possible weight uint64_t bit64[PARAM_OMEGA_R]; // Allocation with the biggest possible weight static inline void reduce(uint64_t *o, const uint64_t *a); -inline static void karat_mult_1(__m128i *C, __m128i *A, __m128i *B); -inline static void karat_mult_2(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_4(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_8(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_16(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_32(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_1(__m128i *C, __m128i *A, __m128i *B); +static inline void karat_mult_2(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_4(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_8(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_16(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_32(__m256i *C, __m256i *A, __m256i *B); static inline void divByXplus1(__m256i *out, __m256i *in, int size); static inline void divByXplus1_256(__m256i *out, __m256i *in, int size); -static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B); -static void TOOM3RecMult(uint64_t *Out, const uint64_t *A, const uint64_t *B); +static void TOOM3Mult(__m256i *Out, const uint64_t *A, const uint64_t *B); +static void TOOM3RecMult(__m256i *Out, const uint64_t *A, const uint64_t *B); @@ -51,35 +47,18 @@ static void TOOM3RecMult(uint64_t *Out, const uint64_t *A, const uint64_t *B); * @param[in] a Pointer to the polynomial a(x) */ static inline void reduce(uint64_t *o, const uint64_t *a) { - __m256i r256, carry256; - __m256i *a256 = (__m256i *) a; - static const int32_t dec64 = PARAM_N & 0x3f; - static const int32_t d0 = WORD - dec64; - int32_t i, i2; + uint64_t r; + uint64_t carry; - for (i = LAST64 ; i < (PARAM_N >> 5) - 4 ; i += 4) { - r256 = _mm256_lddqu_si256((__m256i const *) (& a[i])); - r256 = _mm256_srli_epi64(r256, dec64); - carry256 = _mm256_lddqu_si256((__m256i const *) (& a[i + 1])); - carry256 = _mm256_slli_epi64(carry256, d0); - r256 ^= carry256; - i2 = (i - LAST64) >> 2; - o256[i2] = a256[i2] ^ r256; + for (uint32_t i = 0 ; i < VEC_N_SIZE_64 ; i++) { + r = a[i + VEC_N_SIZE_64 - 1] >> (PARAM_N & 63); + carry = (uint64_t) (a[i + VEC_N_SIZE_64] << (64 - (PARAM_N & 63))); + o[i] = a[i] ^ r ^ carry; } - r256 = _mm256_lddqu_si256((__m256i const *)(& a[i])); - carry256 = _mm256_lddqu_si256((__m256i const *)(& a[i + 1])); - r256 = _mm256_srli_epi64(r256, dec64); - carry256 = _mm256_slli_epi64(carry256, d0); - r256 ^= carry256; - i2 = (i - LAST64) >> 2; - o256[i2] = a256[i2] ^ r256; - tmp_reduce[LAST64] &= RED_MASK; - memcpy(o, tmp_reduce, VEC_N_SIZE_BYTES); + o[VEC_N_SIZE_64 - 1] &= RED_MASK; } - - /** * @brief Compute C(x) = A(x)*B(x) * A(x) and B(x) are stored in 128-bit registers @@ -89,7 +68,7 @@ static inline void reduce(uint64_t *o, const uint64_t *a) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_1(__m128i *C, __m128i *A, __m128i *B) { +static inline void karat_mult_1(__m128i *C, __m128i *A, __m128i *B) { __m128i D1[2]; __m128i D0[2], D2[2]; __m128i Al = _mm_loadu_si128(A); @@ -146,7 +125,7 @@ inline static void karat_mult_1(__m128i *C, __m128i *A, __m128i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_2(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_2(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[2], D1[2], D2[2], SAA, SBB; __m128i *A128 = (__m128i *)A, *B128 = (__m128i *)B; @@ -176,7 +155,7 @@ inline static void karat_mult_2(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_4(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_4(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[4], D1[4], D2[4], SAA[2], SBB[2]; karat_mult_2( D0, A, B); @@ -213,7 +192,7 @@ inline static void karat_mult_4(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_8(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_8(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[8], D1[8], D2[8], SAA[4], SBB[4]; karat_mult_4( D0, A, B); @@ -252,7 +231,7 @@ inline static void karat_mult_8(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_16(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_16(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[16], D1[16], D2[16], SAA[8], SBB[8]; karat_mult_8( D0, A, B); @@ -291,7 +270,7 @@ inline static void karat_mult_16(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_32(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_32(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[32], D1[32], D2[32], SAA[16], SBB[16]; karat_mult_16( D0, A, B); @@ -351,7 +330,7 @@ static inline void divByXplus1(__m256i *out, __m256i *in, int size) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { +static void TOOM3Mult(__m256i *Out, const uint64_t *A, const uint64_t *B) { static __m256i U0[T_TM3_3W_256], V0[T_TM3_3W_256], U1[T_TM3_3W_256], V1[T_TM3_3W_256], U2[T_TM3_3W_256], V2[T_TM3_3W_256]; static __m256i W0[2 * (T_TM3_3W_256)], W1[2 * (T_TM3_3W_256)], W2[2 * (T_TM3_3W_256)], W3[2 * (T_TM3_3W_256)], W4[2 * (T_TM3_3W_256)]; static __m256i tmp[2 * (T_TM3_3W_256)]; @@ -540,7 +519,7 @@ static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { } for (int32_t i = 0 ; i < 6 * T_TM3_3W_256 - 2 ; i++) { - uint64_t *out64 = Out + (i << 2); + uint64_t *out64 = ((uint64_t *)Out) + (i << 2); _mm256_storeu_si256((__m256i *)out64, ro256[i]); } } @@ -572,7 +551,7 @@ static inline void divByXplus1_256(__m256i *out, __m256i *in, int32_t size) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -static void TOOM3RecMult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { +static void TOOM3RecMult(__m256i *Out, const uint64_t *A, const uint64_t *B) { __m256i U0[T_TM3R_3W_256 + 2], V0[T_TM3R_3W_256 + 2], U1[T_TM3R_3W_256 + 2], V1[T_TM3R_3W_256 + 2], U2[T_TM3R_3W_256 + 2], V2[T_TM3R_3W_256 + 2]; __m256i W0[2 * (T_TM3R_3W_256 + 2)], W1[2 * (T_TM3R_3W_256 + 2)], W2[2 * (T_TM3R_3W_256 + 2)], W3[2 * (T_TM3R_3W_256 + 2)], W4[2 * (T_TM3R_3W_256 + 2)]; __m256i tmp[2 * (T_TM3R_3W_256 + 2) + 3]; @@ -617,7 +596,7 @@ static void TOOM3RecMult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { } //W1 = W2 * W3 - TOOM3Mult((uint64_t *) W1, (uint64_t *) W2, (uint64_t *) W3); + TOOM3Mult(W1, (uint64_t *) W2, (uint64_t *) W3); //W0 =(U1 + U2*x)*x ; W4 =(V1 + V2*x)*x (SIZE = T_TM3_3W_256 + 2 !) W0[0] = zero; W4[0] = zero; @@ -645,17 +624,17 @@ static void TOOM3RecMult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { } //W3 = W3 * W2 ; W2 = W0 * W4 - TOOM3Mult((uint64_t *) tmp, (uint64_t *) W3, (uint64_t *) W2); + TOOM3Mult(tmp, (uint64_t *) W3, (uint64_t *) W2); for (int32_t i = 0 ; i < 2 * (T_TM3R_3W_256 + 2) ; i++) { W3[i] = tmp[i]; } - TOOM3Mult((uint64_t *) W2, (uint64_t *) W0, (uint64_t *) W4); + TOOM3Mult(W2, (uint64_t *) W0, (uint64_t *) W4); //W4 = U2 * V2 ; W0 = U0 * V0 - TOOM3Mult((uint64_t *) W4, (uint64_t *) U2, (uint64_t *) V2); - TOOM3Mult((uint64_t *) W0, (uint64_t *) U0, (uint64_t *) V0); + TOOM3Mult(W4, (uint64_t *) U2, (uint64_t *) V2); + TOOM3Mult(W0, (uint64_t *) U0, (uint64_t *) V0); //Interpolation phase //9 add, 1 shift, 1 Smul, 2 Sdiv (2n) @@ -737,7 +716,7 @@ static void TOOM3RecMult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { for (int32_t i = 0 ; i < 6 * T_TM3R_3W_256 - 2 ; i++) { - uint64_t *out64 = Out + (i << 2); + uint64_t *out64 = ((uint64_t *)Out) + (i << 2); _mm256_storeu_si256((__m256i *)out64, ro256[i]); } } @@ -756,8 +735,8 @@ static void TOOM3RecMult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { */ void PQCLEAN_HQC256_AVX2_vect_mul(uint64_t *o, const uint64_t *a1, const uint64_t *a2) { TOOM3RecMult(a1_times_a2, a1, a2); - reduce(o, a1_times_a2); + reduce(o, (uint64_t *)a1_times_a2); // clear all - memset(a1_times_a2, 0, (VEC_N_SIZE_64 << 1) * sizeof(uint64_t)); + memset(a1_times_a2, 0, (2 * VEC_N_SIZE_256 + 1) * sizeof(__m256i)); } diff --git a/crypto_kem/hqc-rmrs-128/avx2/gf2x.c b/crypto_kem/hqc-rmrs-128/avx2/gf2x.c index d718fd39..3a91e1ee 100644 --- a/crypto_kem/hqc-rmrs-128/avx2/gf2x.c +++ b/crypto_kem/hqc-rmrs-128/avx2/gf2x.c @@ -14,25 +14,21 @@ #define T_TM3_3W_256 32 #define T_TM3_3W_64 128 -#define VEC_N_ARRAY_SIZE_VEC CEIL_DIVIDE(PARAM_N, 256) /*!< The number of needed vectors to store PARAM_N bits*/ -#define WORD 64 -#define LAST64 (PARAM_N >> 6) -uint64_t a1_times_a2[2 * VEC_N_256_SIZE_64 + 1]; -uint64_t tmp_reduce[VEC_N_ARRAY_SIZE_VEC << 2]; -__m256i *o256 = (__m256i *) tmp_reduce; +#define VEC_N_SIZE_256 CEIL_DIVIDE(PARAM_N, 256) /*!< The number of needed vectors to store PARAM_N bits*/ +__m256i a1_times_a2[2 * VEC_N_SIZE_256 + 1]; uint64_t bloc64[PARAM_OMEGA_R]; // Allocation with the biggest possible weight uint64_t bit64[PARAM_OMEGA_R]; // Allocation with the biggest possible weight static inline void reduce(uint64_t *o, const uint64_t *a); -inline static void karat_mult_1(__m128i *C, __m128i *A, __m128i *B); -inline static void karat_mult_2(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_4(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_8(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_16(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_32(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_1(__m128i *C, __m128i *A, __m128i *B); +static inline void karat_mult_2(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_4(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_8(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_16(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_32(__m256i *C, __m256i *A, __m256i *B); static inline void divByXplus1(__m256i *out, __m256i *in, int size); -static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B); +static void TOOM3Mult(__m256i *Out, const uint64_t *A, const uint64_t *B); @@ -45,33 +41,16 @@ static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B); * @param[in] a Pointer to the polynomial a(x) */ static inline void reduce(uint64_t *o, const uint64_t *a) { - __m256i r256, carry256; - __m256i *a256 = (__m256i *) a; - static const int32_t dec64 = PARAM_N & 0x3f; - static const int32_t d0 = WORD - dec64; - int32_t i, i2; + uint64_t r; + uint64_t carry; - for (i = LAST64 ; i < (PARAM_N >> 5) - 4 ; i += 4) { - r256 = _mm256_lddqu_si256((__m256i const *) (& a[i])); - r256 = _mm256_srli_epi64(r256, dec64); - carry256 = _mm256_lddqu_si256((__m256i const *) (& a[i + 1])); - carry256 = _mm256_slli_epi64(carry256, d0); - r256 ^= carry256; - i2 = (i - LAST64) >> 2; - o256[i2] = a256[i2] ^ r256; + for (uint32_t i = 0 ; i < VEC_N_SIZE_64 ; i++) { + r = a[i + VEC_N_SIZE_64 - 1] >> (PARAM_N & 63); + carry = (uint64_t) (a[i + VEC_N_SIZE_64] << (64 - (PARAM_N & 63))); + o[i] = a[i] ^ r ^ carry; } - r256 = (__m256i) { - a[i], a[i + 1], 0x0UL, 0x0UL - }; - carry256 = _mm256_lddqu_si256((__m256i const *) (& a[i + 1])); - r256 = _mm256_srli_epi64(r256, dec64); - carry256 = _mm256_slli_epi64(carry256, d0); - r256 ^= carry256; - i2 = (i - LAST64) >> 2; - o256[i2] = (a256[i2] ^ r256); - tmp_reduce[LAST64] &= RED_MASK; - memcpy(o, tmp_reduce, VEC_N_SIZE_BYTES); + o[VEC_N_SIZE_64 - 1] &= RED_MASK; } /** @@ -83,7 +62,7 @@ static inline void reduce(uint64_t *o, const uint64_t *a) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_1(__m128i *C, __m128i *A, __m128i *B) { +static inline void karat_mult_1(__m128i *C, __m128i *A, __m128i *B) { __m128i D1[2]; __m128i D0[2], D2[2]; __m128i Al = _mm_loadu_si128(A); @@ -140,7 +119,7 @@ inline static void karat_mult_1(__m128i *C, __m128i *A, __m128i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_2(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_2(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[2], D1[2], D2[2], SAA, SBB; __m128i *A128 = (__m128i *)A, *B128 = (__m128i *)B; @@ -170,7 +149,7 @@ inline static void karat_mult_2(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_4(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_4(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[4], D1[4], D2[4], SAA[2], SBB[2]; karat_mult_2( D0, A, B); @@ -207,7 +186,7 @@ inline static void karat_mult_4(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_8(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_8(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[8], D1[8], D2[8], SAA[4], SBB[4]; karat_mult_4( D0, A, B); @@ -246,7 +225,7 @@ inline static void karat_mult_8(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_16(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_16(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[16], D1[16], D2[16], SAA[8], SBB[8]; karat_mult_8( D0, A, B); @@ -285,7 +264,7 @@ inline static void karat_mult_16(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_32(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_32(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[32], D1[32], D2[32], SAA[16], SBB[16]; karat_mult_16( D0, A, B); @@ -344,7 +323,7 @@ static inline void divByXplus1(__m256i *out, __m256i *in, int size) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { +static void TOOM3Mult(__m256i *Out, const uint64_t *A, const uint64_t *B) { static __m256i U0[T_TM3_3W_256], V0[T_TM3_3W_256], U1[T_TM3_3W_256], V1[T_TM3_3W_256], U2[T_TM3_3W_256], V2[T_TM3_3W_256]; static __m256i W0[2 * (T_TM3_3W_256)], W1[2 * (T_TM3_3W_256)], W2[2 * (T_TM3_3W_256)], W3[2 * (T_TM3_3W_256)], W4[2 * (T_TM3_3W_256)]; static __m256i tmp[2 * (T_TM3_3W_256)]; @@ -533,7 +512,7 @@ static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { } for (int32_t i = 0 ; i < 6 * T_TM3_3W_256 - 2 ; i++) { - uint64_t *out64 = Out + (i << 2); + uint64_t *out64 = ((uint64_t *)Out) + (i << 2); _mm256_storeu_si256((__m256i *)out64, ro256[i]); } } @@ -551,8 +530,8 @@ static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { */ void PQCLEAN_HQCRMRS128_AVX2_vect_mul(uint64_t *o, const uint64_t *a1, const uint64_t *a2) { TOOM3Mult(a1_times_a2, a1, a2); - reduce(o, a1_times_a2); + reduce(o, (uint64_t *)a1_times_a2); // clear all - memset(a1_times_a2, 0, (VEC_N_SIZE_64 << 1) * sizeof(uint64_t)); + memset(a1_times_a2, 0, (2 * VEC_N_SIZE_256 + 1) * sizeof(__m256i)); } diff --git a/crypto_kem/hqc-rmrs-192/avx2/gf2x.c b/crypto_kem/hqc-rmrs-192/avx2/gf2x.c index 51a32cdd..2cd182f1 100644 --- a/crypto_kem/hqc-rmrs-192/avx2/gf2x.c +++ b/crypto_kem/hqc-rmrs-192/avx2/gf2x.c @@ -14,26 +14,22 @@ #define T_TM3_3W_256 64 #define T_TM3_3W_64 256 -#define VEC_N_ARRAY_SIZE_VEC CEIL_DIVIDE(PARAM_N, 256) /*!< The number of needed vectors to store PARAM_N bits*/ -#define WORD 64 -#define LAST64 (PARAM_N >> 6) -uint64_t a1_times_a2[2 * VEC_N_256_SIZE_64 + 1]; -uint64_t tmp_reduce[VEC_N_ARRAY_SIZE_VEC << 2]; -__m256i *o256 = (__m256i *) tmp_reduce; +#define VEC_N_SIZE_256 CEIL_DIVIDE(PARAM_N, 256) /*!< The number of needed vectors to store PARAM_N bits*/ +__m256i a1_times_a2[2 * VEC_N_SIZE_256 + 1]; uint64_t bloc64[PARAM_OMEGA_R]; // Allocation with the biggest possible weight uint64_t bit64[PARAM_OMEGA_R]; // Allocation with the biggest possible weight static inline void reduce(uint64_t *o, const uint64_t *a); -inline static void karat_mult_1(__m128i *C, __m128i *A, __m128i *B); -inline static void karat_mult_2(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_4(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_8(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_16(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_32(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_64(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_1(__m128i *C, __m128i *A, __m128i *B); +static inline void karat_mult_2(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_4(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_8(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_16(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_32(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_64(__m256i *C, __m256i *A, __m256i *B); static inline void divByXplus1(__m256i *out, __m256i *in, int size); -static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B); +static void TOOM3Mult(__m256i *Out, const uint64_t *A, const uint64_t *B); @@ -46,43 +42,18 @@ static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B); * @param[in] a Pointer to the polynomial a(x) */ static inline void reduce(uint64_t *o, const uint64_t *a) { - __m256i r256, carry256; - __m256i *a256 = (__m256i *) a; - static const int32_t dec64 = PARAM_N & 0x3f; - static const int32_t d0 = WORD - dec64; - int32_t i, i2; + uint64_t r; + uint64_t carry; - for (i = LAST64 ; i < (PARAM_N >> 5) - 4 ; i += 4) { - r256 = _mm256_lddqu_si256((__m256i const *) (& a[i])); - r256 = _mm256_srli_epi64(r256, dec64); - carry256 = _mm256_lddqu_si256((__m256i const *) (& a[i + 1])); - carry256 = _mm256_slli_epi64(carry256, d0); - r256 ^= carry256; - i2 = (i - LAST64) >> 2; - o256[i2] = a256[i2] ^ r256; + for (uint32_t i = 0 ; i < VEC_N_SIZE_64 ; i++) { + r = a[i + VEC_N_SIZE_64 - 1] >> (PARAM_N & 63); + carry = (uint64_t) (a[i + VEC_N_SIZE_64] << (64 - (PARAM_N & 63))); + o[i] = a[i] ^ r ^ carry; } - r256 = _mm256_lddqu_si256((__m256i const *)(& a[i])); - r256 = _mm256_srli_epi64(r256, dec64); - carry256 = _mm256_lddqu_si256((__m256i const *)(& a[i + 1])); - carry256 = _mm256_slli_epi64(carry256, d0); - r256 ^= carry256; - i2 = (i - LAST64) >> 2; - o256[i2] = a256[i2] ^ r256; - i += 4; - r256 = _mm256_lddqu_si256((__m256i const *)(& a[i])); - carry256 = _mm256_lddqu_si256((__m256i const *) (& a[i + 1])); - r256 = _mm256_srli_epi64(r256, dec64); - carry256 = _mm256_slli_epi64(carry256, d0); - r256 ^= carry256; - i2 = (i - LAST64) >> 2; - o256[i2] = (a256[i2] ^ r256); - tmp_reduce[LAST64] &= RED_MASK; - memcpy(o, tmp_reduce, VEC_N_SIZE_BYTES); + o[VEC_N_SIZE_64 - 1] &= RED_MASK; } - - /** * @brief Compute C(x) = A(x)*B(x) * A(x) and B(x) are stored in 128-bit registers @@ -92,7 +63,7 @@ static inline void reduce(uint64_t *o, const uint64_t *a) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_1(__m128i *C, __m128i *A, __m128i *B) { +static inline void karat_mult_1(__m128i *C, __m128i *A, __m128i *B) { __m128i D1[2]; __m128i D0[2], D2[2]; __m128i Al = _mm_loadu_si128(A); @@ -149,7 +120,7 @@ inline static void karat_mult_1(__m128i *C, __m128i *A, __m128i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_2(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_2(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[2], D1[2], D2[2], SAA, SBB; __m128i *A128 = (__m128i *)A, *B128 = (__m128i *)B; @@ -179,7 +150,7 @@ inline static void karat_mult_2(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_4(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_4(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[4], D1[4], D2[4], SAA[2], SBB[2]; karat_mult_2( D0, A, B); @@ -216,7 +187,7 @@ inline static void karat_mult_4(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_8(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_8(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[8], D1[8], D2[8], SAA[4], SBB[4]; karat_mult_4( D0, A, B); @@ -255,7 +226,7 @@ inline static void karat_mult_8(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_16(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_16(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[16], D1[16], D2[16], SAA[8], SBB[8]; karat_mult_8( D0, A, B); @@ -294,7 +265,7 @@ inline static void karat_mult_16(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_32(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_32(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[32], D1[32], D2[32], SAA[16], SBB[16]; karat_mult_16( D0, A, B); @@ -333,7 +304,7 @@ inline static void karat_mult_32(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_64(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_64(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[64], D1[64], D2[64], SAA[32], SBB[32]; karat_mult_32( D0, A, B); @@ -392,7 +363,7 @@ static inline void divByXplus1(__m256i *out, __m256i *in, int size) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { +static void TOOM3Mult(__m256i *Out, const uint64_t *A, const uint64_t *B) { static __m256i U0[T_TM3_3W_256], V0[T_TM3_3W_256], U1[T_TM3_3W_256], V1[T_TM3_3W_256], U2[T_TM3_3W_256], V2[T_TM3_3W_256]; static __m256i W0[2 * (T_TM3_3W_256)], W1[2 * (T_TM3_3W_256)], W2[2 * (T_TM3_3W_256)], W3[2 * (T_TM3_3W_256)], W4[2 * (T_TM3_3W_256)]; static __m256i tmp[2 * (T_TM3_3W_256)]; @@ -580,7 +551,7 @@ static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { } for (int32_t i = 0 ; i < 6 * T_TM3_3W_256 - 2 ; i++) { - uint64_t *out64 = Out + (i << 2); + uint64_t *out64 = ((uint64_t *)Out) + (i << 2); _mm256_storeu_si256((__m256i *)out64, ro256[i]); } } @@ -599,8 +570,8 @@ static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { */ void PQCLEAN_HQCRMRS192_AVX2_vect_mul(uint64_t *o, const uint64_t *a1, const uint64_t *a2) { TOOM3Mult(a1_times_a2, a1, a2); - reduce(o, a1_times_a2); + reduce(o, (uint64_t *)a1_times_a2); // clear all - memset(a1_times_a2, 0, (VEC_N_SIZE_64 << 1) * sizeof(uint64_t)); + memset(a1_times_a2, 0, (2 * VEC_N_SIZE_256 + 1) * sizeof(__m256i)); } diff --git a/crypto_kem/hqc-rmrs-256/avx2/gf2x.c b/crypto_kem/hqc-rmrs-256/avx2/gf2x.c index 40d1e7a5..658cfd50 100644 --- a/crypto_kem/hqc-rmrs-256/avx2/gf2x.c +++ b/crypto_kem/hqc-rmrs-256/avx2/gf2x.c @@ -18,27 +18,23 @@ #define T_TM3R_3W_256 93 #define T_TM3R_3W_64 (T_TM3R_3W_256<<2) -#define VEC_N_ARRAY_SIZE_VEC CEIL_DIVIDE(PARAM_N, 256) /*!< The number of needed vectors to store PARAM_N bits*/ -#define WORD 64 -#define LAST64 (PARAM_N >> 6) -uint64_t a1_times_a2[2 * VEC_N_256_SIZE_64 + 1]; -uint64_t tmp_reduce[VEC_N_ARRAY_SIZE_VEC << 2]; -__m256i *o256 = (__m256i *) tmp_reduce; +#define VEC_N_SIZE_256 CEIL_DIVIDE(PARAM_N, 256) /*!< The number of needed vectors to store PARAM_N bits*/ +__m256i a1_times_a2[2 * VEC_N_SIZE_256 + 1]; uint64_t bloc64[PARAM_OMEGA_R]; // Allocation with the biggest possible weight uint64_t bit64[PARAM_OMEGA_R]; // Allocation with the biggest possible weight static inline void reduce(uint64_t *o, const uint64_t *a); -inline static void karat_mult_1(__m128i *C, __m128i *A, __m128i *B); -inline static void karat_mult_2(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_4(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_8(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_16(__m256i *C, __m256i *A, __m256i *B); -inline static void karat_mult_32(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_1(__m128i *C, __m128i *A, __m128i *B); +static inline void karat_mult_2(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_4(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_8(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_16(__m256i *C, __m256i *A, __m256i *B); +static inline void karat_mult_32(__m256i *C, __m256i *A, __m256i *B); static inline void divByXplus1(__m256i *out, __m256i *in, int size); static inline void divByXplus1_256(__m256i *out, __m256i *in, int size); -static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B); -static void TOOM3RecMult(uint64_t *Out, const uint64_t *A, const uint64_t *B); +static void TOOM3Mult(__m256i *Out, const uint64_t *A, const uint64_t *B); +static void TOOM3RecMult(__m256i *Out, const uint64_t *A, const uint64_t *B); @@ -51,35 +47,18 @@ static void TOOM3RecMult(uint64_t *Out, const uint64_t *A, const uint64_t *B); * @param[in] a Pointer to the polynomial a(x) */ static inline void reduce(uint64_t *o, const uint64_t *a) { - __m256i r256, carry256; - __m256i *a256 = (__m256i *) a; - static const int32_t dec64 = PARAM_N & 0x3f; - static const int32_t d0 = WORD - dec64; - int32_t i, i2; + uint64_t r; + uint64_t carry; - for (i = LAST64 ; i < (PARAM_N >> 5) - 4 ; i += 4) { - r256 = _mm256_lddqu_si256((__m256i const *) (& a[i])); - r256 = _mm256_srli_epi64(r256, dec64); - carry256 = _mm256_lddqu_si256((__m256i const *) (& a[i + 1])); - carry256 = _mm256_slli_epi64(carry256, d0); - r256 ^= carry256; - i2 = (i - LAST64) >> 2; - o256[i2] = a256[i2] ^ r256; + for (uint32_t i = 0 ; i < VEC_N_SIZE_64 ; i++) { + r = a[i + VEC_N_SIZE_64 - 1] >> (PARAM_N & 63); + carry = (uint64_t) (a[i + VEC_N_SIZE_64] << (64 - (PARAM_N & 63))); + o[i] = a[i] ^ r ^ carry; } - r256 = _mm256_lddqu_si256((__m256i const *)(& a[i])); - carry256 = _mm256_lddqu_si256((__m256i const *)(& a[i + 1])); - r256 = _mm256_srli_epi64(r256, dec64); - carry256 = _mm256_slli_epi64(carry256, d0); - r256 ^= carry256; - i2 = (i - LAST64) >> 2; - o256[i2] = a256[i2] ^ r256; - tmp_reduce[LAST64] &= RED_MASK; - memcpy(o, tmp_reduce, VEC_N_SIZE_BYTES); + o[VEC_N_SIZE_64 - 1] &= RED_MASK; } - - /** * @brief Compute C(x) = A(x)*B(x) * A(x) and B(x) are stored in 128-bit registers @@ -89,7 +68,7 @@ static inline void reduce(uint64_t *o, const uint64_t *a) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_1(__m128i *C, __m128i *A, __m128i *B) { +static inline void karat_mult_1(__m128i *C, __m128i *A, __m128i *B) { __m128i D1[2]; __m128i D0[2], D2[2]; __m128i Al = _mm_loadu_si128(A); @@ -146,7 +125,7 @@ inline static void karat_mult_1(__m128i *C, __m128i *A, __m128i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_2(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_2(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[2], D1[2], D2[2], SAA, SBB; __m128i *A128 = (__m128i *)A, *B128 = (__m128i *)B; @@ -176,7 +155,7 @@ inline static void karat_mult_2(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_4(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_4(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[4], D1[4], D2[4], SAA[2], SBB[2]; karat_mult_2( D0, A, B); @@ -213,7 +192,7 @@ inline static void karat_mult_4(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_8(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_8(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[8], D1[8], D2[8], SAA[4], SBB[4]; karat_mult_4( D0, A, B); @@ -252,7 +231,7 @@ inline static void karat_mult_8(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_16(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_16(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[16], D1[16], D2[16], SAA[8], SBB[8]; karat_mult_8( D0, A, B); @@ -291,7 +270,7 @@ inline static void karat_mult_16(__m256i *C, __m256i *A, __m256i *B) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -inline static void karat_mult_32(__m256i *C, __m256i *A, __m256i *B) { +static inline void karat_mult_32(__m256i *C, __m256i *A, __m256i *B) { __m256i D0[32], D1[32], D2[32], SAA[16], SBB[16]; karat_mult_16( D0, A, B); @@ -351,7 +330,7 @@ static inline void divByXplus1(__m256i *out, __m256i *in, int size) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { +static void TOOM3Mult(__m256i *Out, const uint64_t *A, const uint64_t *B) { static __m256i U0[T_TM3_3W_256], V0[T_TM3_3W_256], U1[T_TM3_3W_256], V1[T_TM3_3W_256], U2[T_TM3_3W_256], V2[T_TM3_3W_256]; static __m256i W0[2 * (T_TM3_3W_256)], W1[2 * (T_TM3_3W_256)], W2[2 * (T_TM3_3W_256)], W3[2 * (T_TM3_3W_256)], W4[2 * (T_TM3_3W_256)]; static __m256i tmp[2 * (T_TM3_3W_256)]; @@ -540,7 +519,7 @@ static void TOOM3Mult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { } for (int32_t i = 0 ; i < 6 * T_TM3_3W_256 - 2 ; i++) { - uint64_t *out64 = Out + (i << 2); + uint64_t *out64 = ((uint64_t *)Out) + (i << 2); _mm256_storeu_si256((__m256i *)out64, ro256[i]); } } @@ -572,7 +551,7 @@ static inline void divByXplus1_256(__m256i *out, __m256i *in, int32_t size) { * @param[in] A Pointer to the polynomial A(x) * @param[in] B Pointer to the polynomial B(x) */ -static void TOOM3RecMult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { +static void TOOM3RecMult(__m256i *Out, const uint64_t *A, const uint64_t *B) { __m256i U0[T_TM3R_3W_256 + 2], V0[T_TM3R_3W_256 + 2], U1[T_TM3R_3W_256 + 2], V1[T_TM3R_3W_256 + 2], U2[T_TM3R_3W_256 + 2], V2[T_TM3R_3W_256 + 2]; __m256i W0[2 * (T_TM3R_3W_256 + 2)], W1[2 * (T_TM3R_3W_256 + 2)], W2[2 * (T_TM3R_3W_256 + 2)], W3[2 * (T_TM3R_3W_256 + 2)], W4[2 * (T_TM3R_3W_256 + 2)]; __m256i tmp[2 * (T_TM3R_3W_256 + 2) + 3]; @@ -617,7 +596,7 @@ static void TOOM3RecMult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { } //W1 = W2 * W3 - TOOM3Mult((uint64_t *) W1, (uint64_t *) W2, (uint64_t *) W3); + TOOM3Mult(W1, (uint64_t *) W2, (uint64_t *) W3); //W0 =(U1 + U2*x)*x ; W4 =(V1 + V2*x)*x (SIZE = T_TM3_3W_256 + 2 !) W0[0] = zero; W4[0] = zero; @@ -645,17 +624,17 @@ static void TOOM3RecMult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { } //W3 = W3 * W2 ; W2 = W0 * W4 - TOOM3Mult((uint64_t *) tmp, (uint64_t *) W3, (uint64_t *) W2); + TOOM3Mult(tmp, (uint64_t *) W3, (uint64_t *) W2); for (int32_t i = 0 ; i < 2 * (T_TM3R_3W_256 + 2) ; i++) { W3[i] = tmp[i]; } - TOOM3Mult((uint64_t *) W2, (uint64_t *) W0, (uint64_t *) W4); + TOOM3Mult(W2, (uint64_t *) W0, (uint64_t *) W4); //W4 = U2 * V2 ; W0 = U0 * V0 - TOOM3Mult((uint64_t *) W4, (uint64_t *) U2, (uint64_t *) V2); - TOOM3Mult((uint64_t *) W0, (uint64_t *) U0, (uint64_t *) V0); + TOOM3Mult(W4, (uint64_t *) U2, (uint64_t *) V2); + TOOM3Mult(W0, (uint64_t *) U0, (uint64_t *) V0); //Interpolation phase //9 add, 1 shift, 1 Smul, 2 Sdiv (2n) @@ -737,7 +716,7 @@ static void TOOM3RecMult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { for (int32_t i = 0 ; i < 6 * T_TM3R_3W_256 - 2 ; i++) { - uint64_t *out64 = Out + (i << 2); + uint64_t *out64 = ((uint64_t *)Out) + (i << 2); _mm256_storeu_si256((__m256i *)out64, ro256[i]); } } @@ -756,8 +735,8 @@ static void TOOM3RecMult(uint64_t *Out, const uint64_t *A, const uint64_t *B) { */ void PQCLEAN_HQCRMRS256_AVX2_vect_mul(uint64_t *o, const uint64_t *a1, const uint64_t *a2) { TOOM3RecMult(a1_times_a2, a1, a2); - reduce(o, a1_times_a2); + reduce(o, (uint64_t *)a1_times_a2); // clear all - memset(a1_times_a2, 0, (VEC_N_SIZE_64 << 1) * sizeof(uint64_t)); + memset(a1_times_a2, 0, (2 * VEC_N_SIZE_256 + 1) * sizeof(__m256i)); } diff --git a/test/duplicate_consistency/hqc-192_avx2.yml b/test/duplicate_consistency/hqc-192_avx2.yml index 6c2b5158..8b8681e7 100644 --- a/test/duplicate_consistency/hqc-192_avx2.yml +++ b/test/duplicate_consistency/hqc-192_avx2.yml @@ -68,6 +68,7 @@ consistency_checks: - hqc.h - parsing.h - vector.h + - gf2x.c - hqc.c - kem.c - parsing.c