Merge pull request #27 from PQClean/ds-unittest

Python-driven test framework using nose for unit tests

.gitignore

@@ -8,3 +8,5 @@ bin/
 # Object and library files on Windows
 *.lib
 *.obj
+
+__pycache__

.travis.yml

@@ -14,6 +14,7 @@ matrix:
           packages:
             - python3
             - python3-yaml
+            - python3-nose
             - valgrind
     - name: "Linux + Clang"
       os: linux
@@ -25,6 +26,7 @@ matrix:
           packages:
             - python3
             - python3-yaml
+            - python3-nose
             - valgrind
     - name: "Linux 32-bit GCC"
       os: linux
@@ -35,6 +37,7 @@ matrix:
             - gcc-multilib
             - python3
             - python3-yaml
+            - python3-nose
             - valgrind
       before_install:
         - sudo dpkg --add-architecture i386
@@ -74,7 +77,7 @@ matrix:
 
 script:
   - make ${MAKETARGET}
-
+  - cd test && python3 -m nose -v
 
 cache: pip
 

Makefile

@@ -159,17 +159,17 @@ run-valgrind-all:
 	done
 
 .PHONY: run-testvectors
-run-testvectors: test/check_testvectors.py | require_scheme
+run-testvectors: test/check_tvectors.py | require_scheme
-	python3 test/check_testvectors.py $(SCHEME) || exit 1; \
+	python3 test/check_tvectors.py $(SCHEME) || exit 1; \
 
 .PHONY: run-symbol-namespace
 run-symbol-namespace: test/check_symbol_namespace.py | require_scheme
 	python3 test/check_symbol_namespace.py $(SCHEME) || exit 1; \
 
 .PHONY: run-testvectors-all
-run-testvectors-all: test/check_testvectors.py
+run-testvectors-all: test/check_tvectors.py
 	@for scheme in $(ALL_SCHEMES); do \
-	    python3 test/check_testvectors.py $$scheme || exit 1; \
+	    python3 test/check_tvectors.py $$scheme || exit 1; \
 	done
 
 .PHONY: run-symbol-namespace-all

requirements.txt

@@ -1 +1,2 @@
 PyYAML
+nose

test/Makefile

@@ -0,0 +1,25 @@
+# This Makefile can be used with GNU Make or BSD Make
+
+# override as desired
+TYPE=kem
+SCHEME=kyber768
+IMPLEMENTATION=clean
+
+SCHEME_DIR="../crypto_$(TYPE)/$(SCHEME)/$(IMPLEMENTATION)"
+SCHEME_UPPERCASE=$(shell echo $(SCHEME) | tr a-z A-Z | sed 's/-//')
+
+COMMON_DIR=../common
+COMMON_FILES=$(COMMON_DIR)/randombytes.c $(COMMON_DIR)/fips202.c $(COMMON_DIR)/sha2.c
+DEST_DIR=../bin
+
+CFLAGS=-Wall -Wextra -Wpedantic -Werror -std=c99 -I$(COMMON_DIR) $(EXTRAFLAGS)
+
+all: $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION)
+
+$(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION): crypto_$(TYPE)/functest.c $(COMMON_FILES)
+	mkdir -p $(DEST_DIR)
+	cd $(SCHEME_DIR) && make clean && make
+	$(CC) $(CFLAGS) -DPQCLEAN_NAMESPACE=PQCLEAN_$(SCHEME_UPPERCASE) -I$(SCHEME_DIR) crypto_$(TYPE)/functest.c $(COMMON_FILES) -o $@ -L$(SCHEME_DIR) -l$(SCHEME)_$(IMPLEMENTATION)
+
+clean:
+	$(RM) $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION)

test/helpers.py

@@ -0,0 +1,13 @@
+import subprocess
+
+def run_subprocess(command, working_dir, expected_returncode = 0):
+    """Helper function to run a shell command and report success/failure depending on the exit status of the shell command."""
+    # Note we need to capture stdout/stderr from the subprocess, then print it, which nose/unittest will then capture and buffer appropriately
+    result = subprocess.run(
+        command,
+        stdout = subprocess.PIPE,
+        stderr = subprocess.STDOUT,
+        cwd = working_dir
+    )
+    print(result.stdout.decode('utf-8'))
+    assert(result.returncode == expected_returncode)

test/pqclean.py

@@ -0,0 +1,103 @@
+import os
+import yaml
+
+class Scheme:
+    def __init__(self):
+        self.type = None
+        self.name = None
+        self.implementations = []
+
+    def path(self, base='..'):
+        return os.path.join(base, 'crypto_' + self.type, self.name)
+
+    @staticmethod
+    def by_name(scheme_name):
+        for scheme in Scheme.all_schemes():
+            if scheme.name == scheme_name:
+                return scheme
+        raise KeyError()
+
+    @staticmethod
+    def all_schemes():
+        schemes = []
+        schemes.extend(Scheme.all_schemes_of_type('kem'))
+        schemes.extend(Scheme.all_schemes_of_type('sign'))
+        return schemes
+
+    @staticmethod
+    def all_implementations():
+        implementations = dict()
+        for scheme in Scheme.all_schemes().values():
+            implementations.extend(scheme.all_implementations())
+        return implementations
+
+    @staticmethod
+    def all_schemes_of_type(type: str) -> list:
+        schemes = []
+        p = os.path.join('..', 'crypto_' + type)
+        for d in os.listdir(p):
+            if os.path.isdir(os.path.join(p, d)):
+                if type == 'kem':
+                    schemes.append(KEM(d))
+                elif type == 'sign':
+                    schemes.append(Signature(d))
+                else:
+                    assert('Unknown type')
+        return schemes
+
+    def metadata(self):
+        metafile = os.path.join(self.path(), 'META.yml')
+        try:
+            with open(metafile, encoding='utf-8') as f:
+                metadata = yaml.load(f.read())
+                return metadata
+        except Exception as e:
+            print("Can't open {}: {}".format(metafile, e))
+            return None
+
+class Implementation:
+
+    def __init__(self, scheme, name):
+        self.scheme = scheme
+        self.name = name
+
+    def path(self, base='..') -> str:
+        return os.path.join(self.scheme.path(), self.name)
+
+    @staticmethod
+    def by_name(scheme_name, implementation_name):
+        scheme = Scheme.by_name(scheme_name)
+        for implementation in scheme.implementations:
+            if implementation.name == implementation_name:
+                return implementation
+        raise KeyError()
+
+    @staticmethod
+    def all_implementations(scheme: Scheme) -> list:
+        implementations = []
+        for d in os.listdir(scheme.path()):
+            if os.path.isdir(os.path.join(scheme.path(), d)):
+                implementations.append(Implementation(scheme, d))
+        return implementations
+
+class KEM(Scheme):
+
+    def __init__(self, name: str):
+        self.type = 'kem'
+        self.name = name;
+        self.implementations = Implementation.all_implementations(self)
+
+    @staticmethod
+    def all_kems() -> list:
+        return Scheme.all_schemes_of_type('kem')
+
+class Signature(Scheme):
+
+    def __init__(self, name: str):
+        self.type = 'sign'
+        self.name = name;
+        self.implementations = Implementation.all_implementations(self)
+
+    @staticmethod
+    def all_sigs():
+        return Scheme.all_schemes_of_type('sig')

test/test_compile_lib.py

@@ -0,0 +1,19 @@
+"""
+Checks that the archive library can be successfully built for every scheme/implementation.
+"""
+
+import os
+import pqclean
+import helpers
+
+def test_compile_lib():
+    for scheme in pqclean.Scheme.all_schemes():
+        for implementation in scheme.implementations:
+            yield check_compile_lib, scheme.name, implementation.name
+
+def check_compile_lib(scheme_name, implementation_name):
+    implementation = pqclean.Implementation.by_name(scheme_name, implementation_name)
+    helpers.run_subprocess(
+        ['make'],
+        implementation.path()
+    )

test/test_functest.py

@@ -0,0 +1,27 @@
+"""
+Checks that the functional test program (functest) can be successfully built and executed for every scheme/implementation.
+"""
+
+import os
+import pqclean
+import helpers
+
+def test_functest():
+    for scheme in pqclean.Scheme.all_schemes():
+        for implementation in scheme.implementations:
+            yield check_functest, scheme.name, implementation.name
+
+def check_functest(scheme_name, implementation_name):
+    implementation = pqclean.Implementation.by_name(scheme_name, implementation_name)
+    helpers.run_subprocess(
+        ['make', 'clean', 'TYPE=' + implementation.scheme.type, 'SCHEME=' + scheme_name, 'IMPLEMENTATION=' + implementation_name],
+        os.path.join('..', 'test')
+    )
+    helpers.run_subprocess(
+        ['make', 'TYPE=' + implementation.scheme.type, 'SCHEME=' + scheme_name, 'IMPLEMENTATION=' + implementation_name],
+        os.path.join('..', 'test')
+    )
+    helpers.run_subprocess(
+        ['./functest_{}_{}'.format(scheme_name, implementation_name)],
+        os.path.join('..', 'bin'),
+    )

test/test_license.py

@@ -0,0 +1,17 @@
+"""
+Checks that a LICENSE or LICENSE.txt file is present for every implementation of the specified scheme.
+"""
+
+import os
+import pqclean
+
+def test_license():
+    for scheme in pqclean.Scheme.all_schemes():
+        for implementation in scheme.implementations:
+            yield check_license, scheme.name, implementation.name
+
+def check_license(scheme_name, implementation_name):
+    implementation = pqclean.Implementation.by_name(scheme_name, implementation_name)
+    p1 = os.path.join(implementation.path(), 'LICENSE')
+    p2 = os.path.join(implementation.path(), 'LICENSE.txt')
+    assert(os.path.isfile(p1) or os.path.isfile(p2))