From fde02ce2b1912676aa5dbea1a5dfa1189d67c62c Mon Sep 17 00:00:00 2001 From: Douglas Stebila Date: Mon, 18 Feb 2019 10:07:01 -0500 Subject: [PATCH 1/2] Migrate test vector checks to pythonic testing framework --- .travis.yml | 6 +++--- Makefile | 29 +------------------------- test/Makefile | 14 +++++++++---- test/check_tvectors.py | 45 ---------------------------------------- test/test_testvectors.py | 38 +++++++++++++++++++++++++++++++++ 5 files changed, 52 insertions(+), 80 deletions(-) delete mode 100644 test/check_tvectors.py create mode 100644 test/test_testvectors.py diff --git a/.travis.yml b/.travis.yml index 2e9d905e..d6d78217 100644 --- a/.travis.yml +++ b/.travis.yml @@ -47,7 +47,7 @@ matrix: os: linux services: docker env: - - MAKETARGET="run-functest-all run-testvectors-all run-sanitizer-all" + - MAKETARGET="run-functest-all run-sanitizer-all" script: - docker run --rm --privileged multiarch/qemu-user-static:register --reset - docker run --rm -v `pwd`:`pwd` -w `pwd` "pqclean/debian-unstable-ppc" /bin/bash -c "uname -a && @@ -57,7 +57,7 @@ matrix: os: linux services: docker env: - - MAKETARGET="run-functest-all run-testvectors-all" + - MAKETARGET="run-functest-all" script: - docker run --rm --privileged multiarch/qemu-user-static:register --reset - docker run --rm -v `pwd`:`pwd` -w `pwd` "pqclean/debian-buster-arm" /bin/bash -c "uname -a && @@ -67,7 +67,7 @@ matrix: os: linux services: docker env: - - MAKETARGET="run-functest-all run-testvectors-all" + - MAKETARGET="run-functest-all" script: - docker run --rm --privileged multiarch/qemu-user-static:register --reset - docker run --rm -v `pwd`:`pwd` -w `pwd` "pqclean/debian-buster-aarch64" /bin/bash -c "uname -a && diff --git a/Makefile b/Makefile index 0ca441fa..4c77e7f6 100644 --- a/Makefile +++ b/Makefile @@ -56,21 +56,6 @@ bin/sanitizer_$(subst /,_,$(SCHEME)): test/$(dir $(SCHEME))functest.c $(wildcard .PHONY: sanitizer sanitizer: bin/sanitizer_$(subst /,_,$(SCHEME)) -bin/testvectors_$(subst /,_,$(SCHEME)): test/$(dir $(SCHEME))testvectors.c $(wildcard $(SCHEME)/clean/*.c) $(wildcard $(SCHEME)/clean/*.h) | require_scheme - mkdir -p bin - $(CC) $(CFLAGS) \ - -DPQCLEAN_NAMESPACE=$(shell echo PQCLEAN_$(subst -,,$(notdir $(SCHEME))) | tr a-z A-Z) \ - -iquote "./common/" \ - -iquote "$(SCHEME)/clean/" \ - -o bin/testvectors_$(subst /,_,$(SCHEME)) \ - $(COMMON_FILES) \ - common/notrandombytes.c \ - $(SCHEME)/clean/*.c \ - $< - -.PHONY: testvectors -testvectors: bin/testvectors_$(subst /,_,$(SCHEME)) - bin/shared_$(subst /,_,$(SCHEME))_clean.so: $(wildcard $(SCHEME)/clean/*.c) | require_scheme mkdir -p bin $(CC) $(CFLAGS) \ @@ -118,8 +103,6 @@ help: @echo "make functest-all Build functional tests for all schemes" @echo "make run-functest SCHEME=scheme Run functional tests for SCHEME" @echo "make run-functest-all Run all functests" - @echo "make run-testvectors SCHEME=scheme Run testvector checks for SCHEME" - @echo "make run-testvectors-all Run all testvector checks" @echo "make run-sanitizer-all Run address sanitizer for all schemes" @echo "make run-valgrind SCHEME=scheme Run valgrind checks for SCHEME" @echo "make run-valgrind-all Run valgrind checks all schemes" @@ -149,16 +132,6 @@ run-valgrind-all: $(MAKE) run-valgrind SCHEME=$$scheme || exit 1; \ done -.PHONY: run-testvectors -run-testvectors: test/check_tvectors.py | require_scheme - python3 test/check_tvectors.py $(SCHEME) || exit 1; \ - -.PHONY: run-testvectors-all -run-testvectors-all: test/check_tvectors.py - @for scheme in $(ALL_SCHEMES); do \ - python3 test/check_tvectors.py $$scheme || exit 1; \ - done - .PHONY: run-functest-all run-functest-all: functest-all @for functest in $$(find bin/ -maxdepth 1 -name 'functest_*' -not -type d) ; do \ @@ -176,7 +149,7 @@ run-sanitizer-all: sanitizer-all @echo Tests completed .PHONY: test-all -test-all: run-functest-all run-valgrind-all run-sanitizer-all run-testvectors-all +test-all: run-functest-all run-valgrind-all run-sanitizer-all .PHONY: tidy-all tidy-all: diff --git a/test/Makefile b/test/Makefile index 4b4d954e..ab73e429 100644 --- a/test/Makefile +++ b/test/Makefile @@ -9,17 +9,23 @@ SCHEME_DIR="../crypto_$(TYPE)/$(SCHEME)/$(IMPLEMENTATION)" SCHEME_UPPERCASE=$(shell echo $(SCHEME) | tr a-z A-Z | sed 's/-//') COMMON_DIR=../common -COMMON_FILES=$(COMMON_DIR)/randombytes.c $(COMMON_DIR)/fips202.c $(COMMON_DIR)/sha2.c +COMMON_FILES=$(COMMON_DIR)/fips202.c $(COMMON_DIR)/sha2.c DEST_DIR=../bin CFLAGS=-Wall -Wextra -Wpedantic -Werror -std=c99 -I$(COMMON_DIR) $(EXTRAFLAGS) -all: $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION) +all: $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION) $(DEST_DIR)/testvectors_$(SCHEME)_$(IMPLEMENTATION) -$(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION): crypto_$(TYPE)/functest.c $(COMMON_FILES) +$(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION): crypto_$(TYPE)/functest.c $(COMMON_FILES) $(COMMON_DIR)/randombytes.c mkdir -p $(DEST_DIR) cd $(SCHEME_DIR) && make clean && make - $(CC) $(CFLAGS) -DPQCLEAN_NAMESPACE=PQCLEAN_$(SCHEME_UPPERCASE) -I$(SCHEME_DIR) crypto_$(TYPE)/functest.c $(COMMON_FILES) -o $@ -L$(SCHEME_DIR) -l$(SCHEME)_$(IMPLEMENTATION) + $(CC) $(CFLAGS) -DPQCLEAN_NAMESPACE=PQCLEAN_$(SCHEME_UPPERCASE) -I$(SCHEME_DIR) $^ -o $@ -L$(SCHEME_DIR) -l$(SCHEME)_$(IMPLEMENTATION) + +$(DEST_DIR)/testvectors_$(SCHEME)_$(IMPLEMENTATION): crypto_$(TYPE)/testvectors.c $(COMMON_FILES) $(COMMON_DIR)/notrandombytes.c + mkdir -p $(DEST_DIR) + cd $(SCHEME_DIR) && make clean && make + $(CC) $(CFLAGS) -DPQCLEAN_NAMESPACE=PQCLEAN_$(SCHEME_UPPERCASE) -I$(SCHEME_DIR) $^ -o $@ -L$(SCHEME_DIR) -l$(SCHEME)_$(IMPLEMENTATION) clean: $(RM) $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION) + $(RM) $(DEST_DIR)/testvectors_$(SCHEME)_$(IMPLEMENTATION) diff --git a/test/check_tvectors.py b/test/check_tvectors.py deleted file mode 100644 index 54dedb61..00000000 --- a/test/check_tvectors.py +++ /dev/null @@ -1,45 +0,0 @@ -import yaml -import sys -import os -import subprocess -import hashlib - -""" -For a given SCHEME, this script verifies that the hash of the testvectors -produced on this platform matches the one provided in the META file. -""" - -if len(sys.argv) != 2: - print("Provide a scheme name (e.g. crypto_kem/kyber768) as argv[1]") - exit(1) - -SCHEME = sys.argv[1] -SCHEMEFULL = SCHEME.replace('/', '_') # e.g. crypto_kem_kyber768 -SCHEMESHORT = SCHEME.split('/')[1].upper() - - -def get_hash(scheme): - with open("{}/META.yml".format(scheme), 'r', encoding='utf-8') as stream: - meta = yaml.load(stream) - return meta['testvectors-sha256'] - - -expectedTestvectorsHash = get_hash(SCHEME) - -subprocess.run(["make", "testvectors", "SCHEME={}".format(SCHEME)]) -implementations = [ - x for x in os.listdir('bin') - if 'testvectors' in x and SCHEMEFULL in x and '.dSYM' not in x - -] - -for impl in implementations: - testvectors = subprocess.run(["bin/{}".format(impl)], - stdout=subprocess.PIPE) - testvectorsHash = hashlib.sha256(testvectors.stdout).hexdigest() - if testvectorsHash.lower() != expectedTestvectorsHash.lower(): - print("testvectors of {} should be {}, but is {}" - .format(SCHEME, expectedTestvectorsHash, testvectorsHash)) - sys.exit(1) - else: - print("testvectors of {} matched expected hash".format(SCHEME)) diff --git a/test/test_testvectors.py b/test/test_testvectors.py new file mode 100644 index 00000000..42419e85 --- /dev/null +++ b/test/test_testvectors.py @@ -0,0 +1,38 @@ +""" +Checks that (hash of the) test vectors produced on this platform matches +the one provided in the META file for every scheme/implementation. +""" + +import hashlib +import os +import pqclean +import helpers +import subprocess + + +def test_testvectors(): + for scheme in pqclean.Scheme.all_schemes(): + for implementation in scheme.implementations: + yield check_vectors, scheme.name, implementation.name + +def check_vectors(scheme_name, implementation_name): + scheme = pqclean.Scheme.by_name(scheme_name) + implementation = pqclean.Implementation.by_name(scheme_name, implementation_name) + helpers.run_subprocess( + ['make', 'TYPE=' + implementation.scheme.type, 'SCHEME=' + scheme_name, 'IMPLEMENTATION=' + implementation_name], + os.path.join('..', 'test') + ) + out = helpers.run_subprocess( + ['./testvectors_{}_{}'.format(scheme_name, implementation_name)], + os.path.join('..', 'bin'), + ) + assert(scheme.metadata()['testvectors-sha256'].lower() == hashlib.sha256(out.encode('utf-8')).hexdigest().lower()) + + +if __name__ == '__main__': + try: + import nose2 + nose2.main() + except ImportError: + import nose + nose.runmodule() From 0eb589ba2faff1fa03735b414ae179bb6c3e8c59 Mon Sep 17 00:00:00 2001 From: Douglas Stebila Date: Mon, 18 Feb 2019 10:23:08 -0500 Subject: [PATCH 2/2] Minor tweaks --- test/Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/Makefile b/test/Makefile index 694e44b0..50939190 100644 --- a/test/Makefile +++ b/test/Makefile @@ -26,9 +26,9 @@ $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION): rebuild-scheme crypto_$(TYPE)/ $(DEST_DIR)/testvectors_$(SCHEME)_$(IMPLEMENTATION): rebuild-scheme crypto_$(TYPE)/testvectors.c $(COMMON_FILES) $(COMMON_DIR)/notrandombytes.c mkdir -p $(DEST_DIR) - cd $(SCHEME_DIR) && make clean && make $(CC) $(CFLAGS) -DPQCLEAN_NAMESPACE=PQCLEAN_$(SCHEME_UPPERCASE) -I$(SCHEME_DIR) crypto_$(TYPE)/testvectors.c $(COMMON_FILES) $(COMMON_DIR)/notrandombytes.c -o $@ -L$(SCHEME_DIR) -l$(SCHEME)_$(IMPLEMENTATION) +.PHONY: clean clean: $(RM) $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION) $(RM) $(DEST_DIR)/testvectors_$(SCHEME)_$(IMPLEMENTATION)