From f64a7e0dd47d179321f307d0e62b313bd62ab208 Mon Sep 17 00:00:00 2001 From: Douglas Stebila Date: Wed, 13 Feb 2019 22:25:34 -0500 Subject: [PATCH] Reimplement Python tests using nose framework --- .gitignore | 2 + .travis.yml | 5 +- Makefile | 8 +- crypto_kem/kyber768/clean/Makefile | 6 +- requirements.txt | 1 + test/Makefile | 25 +++++ ...check_testvectors.py => check_tvectors.py} | 0 test/helpers.py | 13 +++ test/pqclean.py | 103 ++++++++++++++++++ test/test_compile_lib.py | 15 +++ test/test_functest.py | 23 ++++ test/test_license.py | 13 +++ 12 files changed, 206 insertions(+), 8 deletions(-) create mode 100644 test/Makefile rename test/{check_testvectors.py => check_tvectors.py} (100%) create mode 100644 test/helpers.py create mode 100644 test/pqclean.py create mode 100644 test/test_compile_lib.py create mode 100644 test/test_functest.py create mode 100644 test/test_license.py diff --git a/.gitignore b/.gitignore index 3794b9c0..ac2d8c88 100644 --- a/.gitignore +++ b/.gitignore @@ -8,3 +8,5 @@ bin/ # Object and library files on Windows *.lib *.obj + +__pycache__ diff --git a/.travis.yml b/.travis.yml index d53c2f2c..0cfd02e3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,6 +14,7 @@ matrix: packages: - python3 - python3-yaml + - python3-nose - valgrind - name: "Linux + Clang" os: linux @@ -25,6 +26,7 @@ matrix: packages: - python3 - python3-yaml + - python3-nose - valgrind - name: "Linux 32-bit GCC" os: linux @@ -35,6 +37,7 @@ matrix: - gcc-multilib - python3 - python3-yaml + - python3-nose - valgrind before_install: - sudo dpkg --add-architecture i386 @@ -74,7 +77,7 @@ matrix: script: - make ${MAKETARGET} - + - cd test && nosetest -v cache: pip diff --git a/Makefile b/Makefile index 791962aa..feb7ad55 100644 --- a/Makefile +++ b/Makefile @@ -159,17 +159,17 @@ run-valgrind-all: done .PHONY: run-testvectors -run-testvectors: test/check_testvectors.py | require_scheme - python3 test/check_testvectors.py $(SCHEME) || exit 1; \ +run-testvectors: test/check_tvectors.py | require_scheme + python3 test/check_tvectors.py $(SCHEME) || exit 1; \ .PHONY: run-symbol-namespace run-symbol-namespace: test/check_symbol_namespace.py | require_scheme python3 test/check_symbol_namespace.py $(SCHEME) || exit 1; \ .PHONY: run-testvectors-all -run-testvectors-all: test/check_testvectors.py +run-testvectors-all: test/check_tvectors.py @for scheme in $(ALL_SCHEMES); do \ - python3 test/check_testvectors.py $$scheme || exit 1; \ + python3 test/check_tvectors.py $$scheme || exit 1; \ done .PHONY: run-symbol-namespace-all diff --git a/crypto_kem/kyber768/clean/Makefile b/crypto_kem/kyber768/clean/Makefile index 30340cc6..a2771e61 100644 --- a/crypto_kem/kyber768/clean/Makefile +++ b/crypto_kem/kyber768/clean/Makefile @@ -8,8 +8,8 @@ CFLAGS=-Wall -Wextra -Wpedantic -Werror -std=c99 -I../../../common $(EXTRAFLAGS) all: $(LIB) $(LIB): $(OBJECTS) - $(AR) -r $@ $(OBJECTS) + $(AR) -r $@ $(OBJECTS) clean: - $(RM) $(OBJECTS) - $(RM) $(LIB) + $(RM) $(OBJECTS) + $(RM) $(LIB) diff --git a/requirements.txt b/requirements.txt index 5500f007..4cf63cf2 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1 +1,2 @@ PyYAML +nose diff --git a/test/Makefile b/test/Makefile new file mode 100644 index 00000000..4b4d954e --- /dev/null +++ b/test/Makefile @@ -0,0 +1,25 @@ +# This Makefile can be used with GNU Make or BSD Make + +# override as desired +TYPE=kem +SCHEME=kyber768 +IMPLEMENTATION=clean + +SCHEME_DIR="../crypto_$(TYPE)/$(SCHEME)/$(IMPLEMENTATION)" +SCHEME_UPPERCASE=$(shell echo $(SCHEME) | tr a-z A-Z | sed 's/-//') + +COMMON_DIR=../common +COMMON_FILES=$(COMMON_DIR)/randombytes.c $(COMMON_DIR)/fips202.c $(COMMON_DIR)/sha2.c +DEST_DIR=../bin + +CFLAGS=-Wall -Wextra -Wpedantic -Werror -std=c99 -I$(COMMON_DIR) $(EXTRAFLAGS) + +all: $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION) + +$(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION): crypto_$(TYPE)/functest.c $(COMMON_FILES) + mkdir -p $(DEST_DIR) + cd $(SCHEME_DIR) && make clean && make + $(CC) $(CFLAGS) -DPQCLEAN_NAMESPACE=PQCLEAN_$(SCHEME_UPPERCASE) -I$(SCHEME_DIR) crypto_$(TYPE)/functest.c $(COMMON_FILES) -o $@ -L$(SCHEME_DIR) -l$(SCHEME)_$(IMPLEMENTATION) + +clean: + $(RM) $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION) diff --git a/test/check_testvectors.py b/test/check_tvectors.py similarity index 100% rename from test/check_testvectors.py rename to test/check_tvectors.py diff --git a/test/helpers.py b/test/helpers.py new file mode 100644 index 00000000..06ccdbbf --- /dev/null +++ b/test/helpers.py @@ -0,0 +1,13 @@ +import subprocess + +def run_subprocess(command, working_dir, expected_returncode = 0): + """Helper function to run a shell command and report success/failure depending on the exit status of the shell command.""" + # Note we need to capture stdout/stderr from the subprocess, then print it, which nose/unittest will then capture and buffer appropriately + result = subprocess.run( + command, + stdout = subprocess.PIPE, + stderr = subprocess.STDOUT, + cwd = working_dir + ) + print(result.stdout.decode('utf-8')) + assert(result.returncode == expected_returncode) diff --git a/test/pqclean.py b/test/pqclean.py new file mode 100644 index 00000000..f52cee25 --- /dev/null +++ b/test/pqclean.py @@ -0,0 +1,103 @@ +import os +import yaml + +class Scheme: + def __init__(self): + self.type = None + self.name = None + self.implementations = [] + + def path(self, base='..'): + return os.path.join(base, 'crypto_' + self.type, self.name) + + @staticmethod + def by_name(scheme_name): + for scheme in Scheme.all_schemes(): + if scheme.name == scheme_name: + return scheme + raise KeyError() + + @staticmethod + def all_schemes(): + schemes = [] + schemes.extend(Scheme.all_schemes_of_type('kem')) + schemes.extend(Scheme.all_schemes_of_type('sign')) + return schemes + + @staticmethod + def all_implementations(): + implementations = dict() + for scheme in Scheme.all_schemes().values(): + implementations.extend(scheme.all_implementations()) + return implementations + + @staticmethod + def all_schemes_of_type(type: str) -> list: + schemes = [] + p = os.path.join('..', 'crypto_' + type) + for d in os.listdir(p): + if os.path.isdir(os.path.join(p, d)): + if type == 'kem': + schemes.append(KEM(d)) + elif type == 'sign': + schemes.append(Signature(d)) + else: + assert('Unknown type') + return schemes + + def metadata(self): + metafile = os.path.join(self.path(), 'META.yml') + try: + with open(metafile, encoding='utf-8') as f: + metadata = yaml.load(f.read()) + return metadata + except Exception as e: + print("Can't open {}: {}".format(metafile, e)) + return None + +class Implementation: + + def __init__(self, scheme, name): + self.scheme = scheme + self.name = name + + def path(self, base='..') -> str: + return os.path.join(self.scheme.path(), self.name) + + @staticmethod + def by_name(scheme_name, implementation_name): + scheme = Scheme.by_name(scheme_name) + for implementation in scheme.implementations: + if implementation.name == implementation_name: + return implementation + raise KeyError() + + @staticmethod + def all_implementations(scheme: Scheme) -> list: + implementations = [] + for d in os.listdir(scheme.path()): + if os.path.isdir(os.path.join(scheme.path(), d)): + implementations.append(Implementation(scheme, d)) + return implementations + +class KEM(Scheme): + + def __init__(self, name: str): + self.type = 'kem' + self.name = name; + self.implementations = Implementation.all_implementations(self) + + @staticmethod + def all_kems() -> list: + return Scheme.all_schemes_of_type('kem') + +class Signature(Scheme): + + def __init__(self, name: str): + self.type = 'sign' + self.name = name; + self.implementations = Implementation.all_implementations(self) + + @staticmethod + def all_sigs(): + return Scheme.all_schemes_of_type('sig') diff --git a/test/test_compile_lib.py b/test/test_compile_lib.py new file mode 100644 index 00000000..f7a67f69 --- /dev/null +++ b/test/test_compile_lib.py @@ -0,0 +1,15 @@ +import os +import pqclean +import helpers + +def test_compile_lib(): + for scheme in pqclean.Scheme.all_schemes(): + for implementation in scheme.implementations: + yield check_compile_lib, scheme.name, implementation.name + +def check_compile_lib(scheme_name, implementation_name): + implementation = pqclean.Implementation.by_name(scheme_name, implementation_name) + helpers.run_subprocess( + ['make'], + implementation.path() + ) diff --git a/test/test_functest.py b/test/test_functest.py new file mode 100644 index 00000000..f10e66fd --- /dev/null +++ b/test/test_functest.py @@ -0,0 +1,23 @@ +import os +import pqclean +import helpers + +def test_functest(): + for scheme in pqclean.Scheme.all_schemes(): + for implementation in scheme.implementations: + yield check_functest, scheme.name, implementation.name + +def check_functest(scheme_name, implementation_name): + implementation = pqclean.Implementation.by_name(scheme_name, implementation_name) + helpers.run_subprocess( + ['make', 'clean', 'TYPE=' + implementation.scheme.type, 'SCHEME=' + scheme_name, 'IMPLEMENTATION=' + implementation_name], + os.path.join('..', 'test') + ) + helpers.run_subprocess( + ['make', 'TYPE=' + implementation.scheme.type, 'SCHEME=' + scheme_name, 'IMPLEMENTATION=' + implementation_name], + os.path.join('..', 'test') + ) + helpers.run_subprocess( + ['./functest_{}_{}'.format(scheme_name, implementation_name)], + os.path.join('..', 'bin'), + ) diff --git a/test/test_license.py b/test/test_license.py new file mode 100644 index 00000000..0260a124 --- /dev/null +++ b/test/test_license.py @@ -0,0 +1,13 @@ +import os +import pqclean + +def test_license(): + for scheme in pqclean.Scheme.all_schemes(): + for implementation in scheme.implementations: + yield check_license, scheme.name, implementation.name + +def check_license(scheme_name, implementation_name): + implementation = pqclean.Implementation.by_name(scheme_name, implementation_name) + p1 = os.path.join(implementation.path(), 'LICENSE') + p2 = os.path.join(implementation.path(), 'LICENSE.txt') + assert(os.path.isfile(p1) or os.path.isfile(p2))