Migrate test vector checks to pythonic testing framework (#40)

Migrate test vector checks to pythonic testing framework

.travis.yml

@@ -47,7 +47,7 @@ matrix:
       os: linux
       services: docker
       env:
-        - MAKETARGET="run-functest-all run-testvectors-all run-sanitizer-all"
+        - MAKETARGET="run-functest-all run-sanitizer-all"
       script:
         - docker run --rm --privileged multiarch/qemu-user-static:register --reset
         - docker run --rm -v `pwd`:`pwd` -w `pwd` "pqclean/debian-unstable-ppc" /bin/bash -c "uname -a &&
@@ -57,7 +57,7 @@ matrix:
       os: linux
       services: docker
       env:
-        - MAKETARGET="run-functest-all run-testvectors-all"
+        - MAKETARGET="run-functest-all"
       script:
         - docker run --rm --privileged multiarch/qemu-user-static:register --reset
         - docker run --rm -v `pwd`:`pwd` -w `pwd` "pqclean/debian-buster-arm" /bin/bash -c "uname -a &&
@@ -67,7 +67,7 @@ matrix:
       os: linux
       services: docker
       env:
-        - MAKETARGET="run-functest-all run-testvectors-all"
+        - MAKETARGET="run-functest-all"
       script:
         - docker run --rm --privileged multiarch/qemu-user-static:register --reset
         - docker run --rm -v `pwd`:`pwd` -w `pwd` "pqclean/debian-buster-aarch64" /bin/bash -c "uname -a &&

Makefile

@@ -56,21 +56,6 @@ bin/sanitizer_$(subst /,_,$(SCHEME)): test/$(dir $(SCHEME))functest.c $(wildcard
 .PHONY: sanitizer
 sanitizer: bin/sanitizer_$(subst /,_,$(SCHEME))
 
-bin/testvectors_$(subst /,_,$(SCHEME)): test/$(dir $(SCHEME))testvectors.c $(wildcard $(SCHEME)/clean/*.c) $(wildcard $(SCHEME)/clean/*.h) | require_scheme
-	mkdir -p bin
-	$(CC) $(CFLAGS) \
-		-DPQCLEAN_NAMESPACE=$(shell echo PQCLEAN_$(subst -,,$(notdir $(SCHEME))) | tr a-z A-Z) \
-		-iquote "./common/" \
-		-iquote "$(SCHEME)/clean/" \
-		-o bin/testvectors_$(subst /,_,$(SCHEME)) \
-		$(COMMON_FILES) \
-		common/notrandombytes.c \
-		$(SCHEME)/clean/*.c \
-		$<
-
-.PHONY: testvectors
-testvectors: bin/testvectors_$(subst /,_,$(SCHEME))
-
 bin/shared_$(subst /,_,$(SCHEME))_clean.so: $(wildcard $(SCHEME)/clean/*.c) | require_scheme
 	mkdir -p bin
 	$(CC) $(CFLAGS) \
@@ -118,8 +103,6 @@ help:
 	@echo "make functest-all			Build functional tests for all schemes"
 	@echo "make run-functest SCHEME=scheme		Run functional tests for SCHEME"
 	@echo "make run-functest-all			Run all functests"
-	@echo "make run-testvectors SCHEME=scheme	Run testvector checks for SCHEME"
-	@echo "make run-testvectors-all			Run all testvector checks"
 	@echo "make run-sanitizer-all			Run address sanitizer for all schemes"
 	@echo "make run-valgrind SCHEME=scheme		Run valgrind checks for SCHEME"
 	@echo "make run-valgrind-all			Run valgrind checks all schemes"
@@ -149,16 +132,6 @@ run-valgrind-all:
 	    $(MAKE) run-valgrind SCHEME=$$scheme || exit 1; \
 	done
 
-.PHONY: run-testvectors
-run-testvectors: test/check_tvectors.py | require_scheme
-	python3 test/check_tvectors.py $(SCHEME) || exit 1; \
-
-.PHONY: run-testvectors-all
-run-testvectors-all: test/check_tvectors.py
-	@for scheme in $(ALL_SCHEMES); do \
-	    python3 test/check_tvectors.py $$scheme || exit 1; \
-	done
-
 .PHONY: run-functest-all
 run-functest-all: functest-all
 	@for functest in $$(find bin/ -maxdepth 1 -name 'functest_*' -not -type d) ; do \
@@ -176,7 +149,7 @@ run-sanitizer-all: sanitizer-all
 	@echo Tests completed
 
 .PHONY: test-all
-test-all: run-functest-all run-valgrind-all run-sanitizer-all run-testvectors-all
+test-all: run-functest-all run-valgrind-all run-sanitizer-all
 
 .PHONY: tidy-all
 tidy-all:

test/Makefile

@@ -9,20 +9,26 @@ SCHEME_DIR="../crypto_$(TYPE)/$(SCHEME)/$(IMPLEMENTATION)"
 SCHEME_UPPERCASE=$(shell echo $(SCHEME) | tr a-z A-Z | sed 's/-//')
 
 COMMON_DIR=../common
-COMMON_FILES=$(COMMON_DIR)/randombytes.c $(COMMON_DIR)/fips202.c $(COMMON_DIR)/sha2.c
+COMMON_FILES=$(COMMON_DIR)/fips202.c $(COMMON_DIR)/sha2.c
 DEST_DIR=../bin
 
 CFLAGS=-Wall -Wextra -Wpedantic -Werror -std=c99 -I$(COMMON_DIR) $(EXTRAFLAGS)
 
-all: $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION)
+all: $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION) $(DEST_DIR)/testvectors_$(SCHEME)_$(IMPLEMENTATION)
 
 .PHONY: rebuild-scheme
 rebuild-scheme:
 	cd $(SCHEME_DIR) && make
 
-$(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION): rebuild-scheme crypto_$(TYPE)/functest.c $(COMMON_FILES)
+$(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION): rebuild-scheme crypto_$(TYPE)/functest.c $(COMMON_FILES) $(COMMON_DIR)/randombytes.c
 	mkdir -p $(DEST_DIR)
-	$(CC) $(CFLAGS) -DPQCLEAN_NAMESPACE=PQCLEAN_$(SCHEME_UPPERCASE) -I$(SCHEME_DIR) crypto_$(TYPE)/functest.c $(COMMON_FILES) -o $@ -L$(SCHEME_DIR) -l$(SCHEME)_$(IMPLEMENTATION)
+	$(CC) $(CFLAGS) -DPQCLEAN_NAMESPACE=PQCLEAN_$(SCHEME_UPPERCASE) -I$(SCHEME_DIR) crypto_$(TYPE)/testvectors.c $(COMMON_FILES) $(COMMON_DIR)/randombytes.c -o $@ -L$(SCHEME_DIR) -l$(SCHEME)_$(IMPLEMENTATION)
 
+$(DEST_DIR)/testvectors_$(SCHEME)_$(IMPLEMENTATION): rebuild-scheme crypto_$(TYPE)/testvectors.c $(COMMON_FILES) $(COMMON_DIR)/notrandombytes.c
+	mkdir -p $(DEST_DIR)
+	$(CC) $(CFLAGS) -DPQCLEAN_NAMESPACE=PQCLEAN_$(SCHEME_UPPERCASE) -I$(SCHEME_DIR) crypto_$(TYPE)/testvectors.c $(COMMON_FILES) $(COMMON_DIR)/notrandombytes.c -o $@ -L$(SCHEME_DIR) -l$(SCHEME)_$(IMPLEMENTATION)
+
+.PHONY: clean
 clean:
 	$(RM) $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION)
+	$(RM) $(DEST_DIR)/testvectors_$(SCHEME)_$(IMPLEMENTATION)

test/check_tvectors.py

@@ -1,45 +0,0 @@
-import yaml
-import sys
-import os
-import subprocess
-import hashlib
-
-"""
-For a given SCHEME, this script verifies that the hash of the testvectors
-produced on this platform matches the one provided in the META file.
-"""
-
-if len(sys.argv) != 2:
-    print("Provide a scheme name (e.g. crypto_kem/kyber768) as argv[1]")
-    exit(1)
-
-SCHEME = sys.argv[1]
-SCHEMEFULL = SCHEME.replace('/', '_')  # e.g. crypto_kem_kyber768
-SCHEMESHORT = SCHEME.split('/')[1].upper()
-
-
-def get_hash(scheme):
-    with open("{}/META.yml".format(scheme), 'r', encoding='utf-8') as stream:
-        meta = yaml.load(stream)
-        return meta['testvectors-sha256']
-
-
-expectedTestvectorsHash = get_hash(SCHEME)
-
-subprocess.run(["make", "testvectors", "SCHEME={}".format(SCHEME)])
-implementations = [
-    x for x in os.listdir('bin')
-    if 'testvectors' in x and SCHEMEFULL in x and '.dSYM' not in x
-
-]
-
-for impl in implementations:
-    testvectors = subprocess.run(["bin/{}".format(impl)],
-                                 stdout=subprocess.PIPE)
-    testvectorsHash = hashlib.sha256(testvectors.stdout).hexdigest()
-    if testvectorsHash.lower() != expectedTestvectorsHash.lower():
-        print("testvectors of {} should be {}, but is {}"
-              .format(SCHEME, expectedTestvectorsHash, testvectorsHash))
-        sys.exit(1)
-    else:
-        print("testvectors of {} matched expected hash".format(SCHEME))

test/test_testvectors.py

@@ -0,0 +1,38 @@
+"""
+Checks that (hash of the) test vectors produced on this platform matches
+the one provided in the META file for every scheme/implementation.
+"""
+
+import hashlib
+import os
+import pqclean
+import helpers
+import subprocess
+
+
+def test_testvectors():
+    for scheme in pqclean.Scheme.all_schemes():
+        for implementation in scheme.implementations:
+            yield check_vectors, scheme.name, implementation.name
+
+def check_vectors(scheme_name, implementation_name):
+    scheme = pqclean.Scheme.by_name(scheme_name)
+    implementation = pqclean.Implementation.by_name(scheme_name, implementation_name)
+    helpers.run_subprocess(
+        ['make', 'TYPE=' + implementation.scheme.type, 'SCHEME=' + scheme_name, 'IMPLEMENTATION=' + implementation_name],
+        os.path.join('..', 'test')
+    )
+    out = helpers.run_subprocess(
+        ['./testvectors_{}_{}'.format(scheme_name, implementation_name)],
+        os.path.join('..', 'bin'),
+    )
+    assert(scheme.metadata()['testvectors-sha256'].lower() == hashlib.sha256(out.encode('utf-8')).hexdigest().lower())
+
+
+if __name__ == '__main__':
+    try:
+        import nose2
+        nose2.main()
+    except ImportError:
+        import nose
+        nose.runmodule()