762e1d039c
Patch from https://mta.openssl.org/pipermail/openssl-dev/2016-March/005625.html. Upstream has yet to make a decision on aliasing requirements for their assembly. If they choose to go with the stricter aliasing requirement rather than land this patch, we'll probably want to tweak EVP_AEAD's API guarantees accordingly and then undiverge. In the meantime, import this to avoid a regression on x86 from when we had compiler-vectorized code on GCC platforms. Per our assembly coverage tools and pending multi-CPU-variant tests, we have good coverage here. Unlike Poly1305 (which is currently waiting on yet another upstream bugfix), where there is risk of missed carries everywhere, it is much more difficult to accidentally make a ChaCha20 implementation that fails based on the data passed into it. This restores a sizeable speed improvement on x86. Before: Did 1131000 ChaCha20-Poly1305 (16 bytes) seal operations in 1000205us (1130768.2 ops/sec): 18.1 MB/s Did 161000 ChaCha20-Poly1305 (1350 bytes) seal operations in 1006136us (160018.1 ops/sec): 216.0 MB/s Did 28000 ChaCha20-Poly1305 (8192 bytes) seal operations in 1023264us (27363.4 ops/sec): 224.2 MB/s Did 1166000 ChaCha20-Poly1305-Old (16 bytes) seal operations in 1000447us (1165479.0 ops/sec): 18.6 MB/s Did 160000 ChaCha20-Poly1305-Old (1350 bytes) seal operations in 1004818us (159232.8 ops/sec): 215.0 MB/s Did 30000 ChaCha20-Poly1305-Old (8192 bytes) seal operations in 1016977us (29499.2 ops/sec): 241.7 MB/s After: Did 2208000 ChaCha20-Poly1305 (16 bytes) seal operations in 1000031us (2207931.6 ops/sec): 35.3 MB/s Did 402000 ChaCha20-Poly1305 (1350 bytes) seal operations in 1001717us (401310.9 ops/sec): 541.8 MB/s Did 97000 ChaCha20-Poly1305 (8192 bytes) seal operations in 1005394us (96479.6 ops/sec): 790.4 MB/s Did 2444000 ChaCha20-Poly1305-Old (16 bytes) seal operations in 1000089us (2443782.5 ops/sec): 39.1 MB/s Did 459000 ChaCha20-Poly1305-Old (1350 bytes) seal operations in 1000563us (458741.7 ops/sec): 619.3 MB/s Did 97000 ChaCha20-Poly1305-Old (8192 bytes) seal operations in 1007942us (96235.7 ops/sec): 788.4 MB/s Change-Id: I976da606dae062a776e0cc01229ec03a074035d1 Reviewed-on: https://boringssl-review.googlesource.com/7561 Reviewed-by: Steven Valdez <svaldez@google.com> Reviewed-by: David Benjamin <davidben@google.com>
165 lines
5.1 KiB
C
165 lines
5.1 KiB
C
/* Copyright (c) 2014, Google Inc.
|
|
*
|
|
* Permission to use, copy, modify, and/or distribute this software for any
|
|
* purpose with or without fee is hereby granted, provided that the above
|
|
* copyright notice and this permission notice appear in all copies.
|
|
*
|
|
* THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES
|
|
* WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF
|
|
* MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY
|
|
* SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES
|
|
* WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN ACTION
|
|
* OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF OR IN
|
|
* CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. */
|
|
|
|
/* Adapted from the public domain, estream code by D. Bernstein. */
|
|
|
|
#include <openssl/chacha.h>
|
|
|
|
#include <string.h>
|
|
|
|
#include <openssl/cpu.h>
|
|
|
|
|
|
#define U8TO32_LITTLE(p) \
|
|
(((uint32_t)((p)[0])) | ((uint32_t)((p)[1]) << 8) | \
|
|
((uint32_t)((p)[2]) << 16) | ((uint32_t)((p)[3]) << 24))
|
|
|
|
#if !defined(OPENSSL_NO_ASM) && \
|
|
(defined(OPENSSL_X86) || defined(OPENSSL_X86_64) || \
|
|
defined(OPENSSL_ARM) || defined(OPENSSL_AARCH64))
|
|
|
|
/* ChaCha20_ctr32 is defined in asm/chacha-*.pl. */
|
|
void ChaCha20_ctr32(uint8_t *out, const uint8_t *in, size_t in_len,
|
|
const uint32_t key[8], const uint32_t counter[4]);
|
|
|
|
void CRYPTO_chacha_20(uint8_t *out, const uint8_t *in, size_t in_len,
|
|
const uint8_t key[32], const uint8_t nonce[12],
|
|
uint32_t counter) {
|
|
uint32_t counter_nonce[4];
|
|
counter_nonce[0] = counter;
|
|
counter_nonce[1] = U8TO32_LITTLE(nonce + 0);
|
|
counter_nonce[2] = U8TO32_LITTLE(nonce + 4);
|
|
counter_nonce[3] = U8TO32_LITTLE(nonce + 8);
|
|
|
|
const uint32_t *key_ptr = (const uint32_t *)key;
|
|
#if !defined(OPENSSL_X86) && !defined(OPENSSL_X86_64)
|
|
/* The assembly expects the key to be four-byte aligned. */
|
|
uint32_t key_u32[8];
|
|
if ((((uintptr_t)key) & 3) != 0) {
|
|
key_u32[0] = U8TO32_LITTLE(key + 0);
|
|
key_u32[1] = U8TO32_LITTLE(key + 4);
|
|
key_u32[2] = U8TO32_LITTLE(key + 8);
|
|
key_u32[3] = U8TO32_LITTLE(key + 12);
|
|
key_u32[4] = U8TO32_LITTLE(key + 16);
|
|
key_u32[5] = U8TO32_LITTLE(key + 20);
|
|
key_u32[6] = U8TO32_LITTLE(key + 24);
|
|
key_u32[7] = U8TO32_LITTLE(key + 28);
|
|
|
|
key_ptr = key_u32;
|
|
}
|
|
#endif
|
|
|
|
ChaCha20_ctr32(out, in, in_len, key_ptr, counter_nonce);
|
|
}
|
|
|
|
#else
|
|
|
|
/* sigma contains the ChaCha constants, which happen to be an ASCII string. */
|
|
static const uint8_t sigma[16] = { 'e', 'x', 'p', 'a', 'n', 'd', ' ', '3',
|
|
'2', '-', 'b', 'y', 't', 'e', ' ', 'k' };
|
|
|
|
#define ROTATE(v, n) (((v) << (n)) | ((v) >> (32 - (n))))
|
|
#define XOR(v, w) ((v) ^ (w))
|
|
#define PLUS(x, y) ((x) + (y))
|
|
#define PLUSONE(v) (PLUS((v), 1))
|
|
|
|
#define U32TO8_LITTLE(p, v) \
|
|
{ \
|
|
(p)[0] = (v >> 0) & 0xff; \
|
|
(p)[1] = (v >> 8) & 0xff; \
|
|
(p)[2] = (v >> 16) & 0xff; \
|
|
(p)[3] = (v >> 24) & 0xff; \
|
|
}
|
|
|
|
/* QUARTERROUND updates a, b, c, d with a ChaCha "quarter" round. */
|
|
#define QUARTERROUND(a,b,c,d) \
|
|
x[a] = PLUS(x[a],x[b]); x[d] = ROTATE(XOR(x[d],x[a]),16); \
|
|
x[c] = PLUS(x[c],x[d]); x[b] = ROTATE(XOR(x[b],x[c]),12); \
|
|
x[a] = PLUS(x[a],x[b]); x[d] = ROTATE(XOR(x[d],x[a]), 8); \
|
|
x[c] = PLUS(x[c],x[d]); x[b] = ROTATE(XOR(x[b],x[c]), 7);
|
|
|
|
/* chacha_core performs 20 rounds of ChaCha on the input words in
|
|
* |input| and writes the 64 output bytes to |output|. */
|
|
static void chacha_core(uint8_t output[64], const uint32_t input[16]) {
|
|
uint32_t x[16];
|
|
int i;
|
|
|
|
memcpy(x, input, sizeof(uint32_t) * 16);
|
|
for (i = 20; i > 0; i -= 2) {
|
|
QUARTERROUND(0, 4, 8, 12)
|
|
QUARTERROUND(1, 5, 9, 13)
|
|
QUARTERROUND(2, 6, 10, 14)
|
|
QUARTERROUND(3, 7, 11, 15)
|
|
QUARTERROUND(0, 5, 10, 15)
|
|
QUARTERROUND(1, 6, 11, 12)
|
|
QUARTERROUND(2, 7, 8, 13)
|
|
QUARTERROUND(3, 4, 9, 14)
|
|
}
|
|
|
|
for (i = 0; i < 16; ++i) {
|
|
x[i] = PLUS(x[i], input[i]);
|
|
}
|
|
for (i = 0; i < 16; ++i) {
|
|
U32TO8_LITTLE(output + 4 * i, x[i]);
|
|
}
|
|
}
|
|
|
|
void CRYPTO_chacha_20(uint8_t *out, const uint8_t *in, size_t in_len,
|
|
const uint8_t key[32], const uint8_t nonce[12],
|
|
uint32_t counter) {
|
|
uint32_t input[16];
|
|
uint8_t buf[64];
|
|
size_t todo, i;
|
|
|
|
input[0] = U8TO32_LITTLE(sigma + 0);
|
|
input[1] = U8TO32_LITTLE(sigma + 4);
|
|
input[2] = U8TO32_LITTLE(sigma + 8);
|
|
input[3] = U8TO32_LITTLE(sigma + 12);
|
|
|
|
input[4] = U8TO32_LITTLE(key + 0);
|
|
input[5] = U8TO32_LITTLE(key + 4);
|
|
input[6] = U8TO32_LITTLE(key + 8);
|
|
input[7] = U8TO32_LITTLE(key + 12);
|
|
|
|
input[8] = U8TO32_LITTLE(key + 16);
|
|
input[9] = U8TO32_LITTLE(key + 20);
|
|
input[10] = U8TO32_LITTLE(key + 24);
|
|
input[11] = U8TO32_LITTLE(key + 28);
|
|
|
|
input[12] = counter;
|
|
input[13] = U8TO32_LITTLE(nonce + 0);
|
|
input[14] = U8TO32_LITTLE(nonce + 4);
|
|
input[15] = U8TO32_LITTLE(nonce + 8);
|
|
|
|
while (in_len > 0) {
|
|
todo = sizeof(buf);
|
|
if (in_len < todo) {
|
|
todo = in_len;
|
|
}
|
|
|
|
chacha_core(buf, input);
|
|
for (i = 0; i < todo; i++) {
|
|
out[i] = in[i] ^ buf[i];
|
|
}
|
|
|
|
out += todo;
|
|
in += todo;
|
|
in_len -= todo;
|
|
|
|
input[12]++;
|
|
}
|
|
}
|
|
|
|
#endif
|