diff --git a/test/helpers.py b/test/helpers.py index b97457a5..d8b7afbd 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -7,6 +7,8 @@ import sys import pqclean +import logging + def run_subprocess(command, working_dir='.', env=None, expected_returncode=0): """ @@ -113,7 +115,7 @@ def ensure_available(executable): raise AssertionError("{} not available on CI".format(executable)) -def permit_test(testname, thing, **args): +def permit_test(testname, thing, *args, **kwargs): if 'PQCLEAN_ONLY_TESTS' in os.environ: if not(testname.lower() in os.environ['PQCLEAN_ONLY_TESTS'].lower().split(',')): return False @@ -169,6 +171,7 @@ def permit_test(testname, thing, **args): for diff_line in diff_result.stdout.decode('utf-8').splitlines(): # don't skip test if there are any changes outside schemes if not(diff_line.startswith('crypto_kem')) and not (diff_line.startswith('crypto_sign')): + logging.info("Running all tests as there are changes outside of schemes") return True # do test if the scheme in question has been changed if diff_line.startswith(thing.path(base='')): @@ -177,3 +180,15 @@ def permit_test(testname, thing, **args): return False return True + + +def filtered_test(func): + funcname = func.__name__[len("check_"):] + + @functools.wraps(func) + def wrapper(*args, **kwargs): + if permit_test(funcname, *args, **kwargs): + return func(*args, **kwargs) + else: + raise unittest.SkipTest("Test disabled by filter") + return wrapper diff --git a/test/test_api_h.py b/test/test_api_h.py index 205793f2..dbb78ce7 100644 --- a/test/test_api_h.py +++ b/test/test_api_h.py @@ -5,14 +5,14 @@ import helpers import pqclean -def test_preprocessor(): +def test_api_h(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - if helpers.permit_test('preprocessor', implementation): - yield check_preprocessor, implementation + yield check_api_h, implementation -def check_preprocessor(implementation: pqclean.Implementation): +@helpers.filtered_test +def check_api_h(implementation: pqclean.Implementation): apipath = os.path.join(implementation.path(), 'api.h') errors = [] p = re.compile(r'^\s*#include\s*"') @@ -25,6 +25,7 @@ def check_preprocessor(implementation: pqclean.Implementation): "Prohibited external include in api.h" + "".join(errors) ) + if __name__ == "__main__": try: import nose2 diff --git a/test/test_char.py b/test/test_char.py index 3f8f7296..53a5057f 100644 --- a/test/test_char.py +++ b/test/test_char.py @@ -17,8 +17,7 @@ def test_char(): ) for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - if helpers.permit_test('char', implementation): - yield check_char, implementation + yield check_char, implementation def walk_tree(ast): @@ -30,6 +29,7 @@ def walk_tree(ast): yield from walk_tree(child) # recursively yield prohibited nodes +@helpers.filtered_test @helpers.skip_windows() def check_char(implementation): errors = [] diff --git a/test/test_common.py b/test/test_common.py index e4c89939..41eb3efe 100644 --- a/test/test_common.py +++ b/test/test_common.py @@ -12,7 +12,8 @@ import helpers def test_common(): for d in os.listdir('common'): primitive = re.sub(r"\.c$", "", d) - if helpers.permit_test('common', None): yield check_common, primitive + if helpers.permit_test('common', None): + yield check_common, primitive def check_common(primitive): diff --git a/test/test_compile_lib.py b/test/test_compile_lib.py index c3888543..67ede139 100644 --- a/test/test_compile_lib.py +++ b/test/test_compile_lib.py @@ -10,10 +10,10 @@ import helpers def test_compile_lib(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - if helpers.permit_test('compile_lib', implementation): - yield check_compile_lib, implementation + yield check_compile_lib, implementation +@helpers.filtered_test def check_compile_lib(implementation): helpers.make('clean', working_dir=implementation.path()) helpers.make(working_dir=implementation.path()) diff --git a/test/test_duplicate_consistency.py b/test/test_duplicate_consistency.py index 2f86b74a..61119337 100644 --- a/test/test_duplicate_consistency.py +++ b/test/test_duplicate_consistency.py @@ -8,33 +8,54 @@ import helpers import unittest import yaml -helpers.skip_windows() + +def skipped_test(*args, **kwargs): + raise unittest.SkipTest("Skipped consistency check") + + def test_duplicate_consistency(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - if helpers.permit_test('duplicate_consistency', implementation): - if os.path.isfile(os.path.join('duplicate_consistency', '{}_{}.yml'.format(scheme.name, implementation.name))): - metafile = os.path.join('duplicate_consistency', '{}_{}.yml'.format(implementation.scheme.name, implementation.name)) - with open(metafile, encoding='utf-8') as f: - metadata = yaml.safe_load(f.read()) - for group in metadata['consistency_checks']: - source = pqclean.Implementation.by_name(group['source']['scheme'], group['source']['implementation']) - for file in group['files']: - yield check_duplicate_consistency, implementation, source, file + if not helpers.permit_test('duplicate_consistency', + implementation): + yield skipped_test, implementation + continue + + if os.path.isfile( + os.path.join( + 'duplicate_consistency', + '{}_{}.yml'.format(scheme.name, implementation.name))): + metafile = os.path.join( + 'duplicate_consistency', + '{}_{}.yml'.format(scheme.name, implementation.name)) + with open(metafile, encoding='utf-8') as f: + metadata = yaml.safe_load(f.read()) + for group in metadata['consistency_checks']: + source = pqclean.Implementation.by_name( + group['source']['scheme'], + group['source']['implementation']) + for file in group['files']: + yield (check_duplicate_consistency, implementation, + source, file) + def file_get_contents(filename): with open(filename) as f: return f.read() + +@helpers.skip_windows() def check_duplicate_consistency(implementation, source, file): transformed_src = helpers.run_subprocess( - ['sed', '-e', 's/{}/{}/g'.format(source.namespace_prefix(), implementation.namespace_prefix()), os.path.join(source.path(), file)] + ['sed', '-e', 's/{}/{}/g'.format(source.namespace_prefix(), + implementation.namespace_prefix()), os.path.join(source.path(), file)] ) this_src = file_get_contents(os.path.join(implementation.path(), file)) print(os.path.join(implementation.path(), file)) print(this_src) assert(transformed_src == this_src) + if __name__ == '__main__': try: import nose2 diff --git a/test/test_dynamic_memory.py b/test/test_dynamic_memory.py index 27257006..4f348c96 100644 --- a/test/test_dynamic_memory.py +++ b/test/test_dynamic_memory.py @@ -4,19 +4,17 @@ Checks that no dynamic memory functions are used import pqclean import helpers -import sys -import unittest def test_dynamic_memory(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - if helpers.permit_test('dynamic_memory', implementation): - # Keep this loop outside, to allow multiple assertions - for function in ['malloc', 'free', 'realloc', 'calloc']: - yield (check_dynamic_memory, implementation, function) + # Keep this loop outside, to allow multiple assertions + for function in ['malloc', 'free', 'realloc', 'calloc']: + yield (check_dynamic_memory, implementation, function) +@helpers.filtered_test @helpers.skip_windows() def check_dynamic_memory(implementation, function): # 'make' will take care of not rebuilding existing library files diff --git a/test/test_format.py b/test/test_format.py index a55c78ef..5a4d19c1 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -5,19 +5,20 @@ import pqclean def test_formatting(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - if helpers.permit_test('format', implementation): - yield check_format, implementation + yield check_format, implementation +@helpers.filtered_test def check_format(implementation: pqclean.Implementation): helpers.ensure_available('astyle') cfiles = implementation.cfiles() hfiles = implementation.hfiles() - result = helpers.run_subprocess(['astyle', - '--dry-run', - '--options=../.astylerc', - *cfiles, - *hfiles]) + result = helpers.run_subprocess( + ['astyle', + '--dry-run', + '--options=../.astylerc', + *cfiles, + *hfiles]) assert(not('Formatted' in result)) diff --git a/test/test_functest.py b/test/test_functest.py index e38eed50..ff79cedf 100644 --- a/test/test_functest.py +++ b/test/test_functest.py @@ -14,17 +14,16 @@ import helpers def test_functest(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - if helpers.permit_test('functest', implementation): - yield check_functest, implementation + yield check_functest, implementation def test_functest_sanitizers(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - if helpers.permit_test('functest_sanitizers', implementation): - yield check_functest_sanitizers, implementation + yield check_functest_sanitizers, implementation +@helpers.filtered_test def check_functest(implementation): helpers.make('functest', TYPE=implementation.scheme.type, @@ -41,6 +40,7 @@ def check_functest(implementation): ) +@helpers.filtered_test @helpers.skip_windows() @helpers.slow_test def check_functest_sanitizers(implementation): diff --git a/test/test_license.py b/test/test_license.py index 15c61491..da17f35c 100644 --- a/test/test_license.py +++ b/test/test_license.py @@ -11,10 +11,10 @@ import helpers def test_license(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - if helpers.permit_test('license', implementation): - yield check_license, implementation + yield check_license, implementation +@helpers.filtered_test def check_license(implementation): p1 = os.path.join(implementation.path(), 'LICENSE') p2 = os.path.join(implementation.path(), 'LICENSE.txt') diff --git a/test/test_linter.py b/test/test_linter.py index 64d408cc..00e89580 100644 --- a/test/test_linter.py +++ b/test/test_linter.py @@ -12,10 +12,10 @@ additional_flags = [] def test_clang_tidy(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - if helpers.permit_test('linter', implementation): - yield check_tidy, implementation + yield check_tidy, implementation +@helpers.filtered_test @helpers.skip_windows() def check_tidy(implementation: pqclean.Implementation): helpers.ensure_available('clang-tidy') diff --git a/test/test_makefile_dependencies.py b/test/test_makefile_dependencies.py index 8305745e..011f66a9 100644 --- a/test/test_makefile_dependencies.py +++ b/test/test_makefile_dependencies.py @@ -8,20 +8,30 @@ import pqclean import helpers import glob import datetime +import unittest + + +def _skipped_test(*args, **kwargs): + """Used to indicate skipped tests""" + raise unittest.SkipTest("Skipped makefile dependencies test") def test_makefile_dependencies(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - if helpers.permit_test('makefile_dependencies', implementation): - # initial build - want to have *all* files in place at beginning - helpers.make('clean', working_dir=implementation.path()) - helpers.make(working_dir=implementation.path()) - # test case for each candidate file - cfiles = glob.glob(os.path.join(implementation.path(), '*.c')) - hfiles = glob.glob(os.path.join(implementation.path(), '*.h')) - for file in (cfiles + hfiles): - yield (check_makefile_dependencies, implementation, file) + if not helpers.permit_test( + 'makefile_dependencies', implementation): + yield _skipped_test, implementation + continue + + # initial build - want to have *all* files in place at beginning + helpers.make('clean', working_dir=implementation.path()) + helpers.make(working_dir=implementation.path()) + # test case for each candidate file + cfiles = glob.glob(os.path.join(implementation.path(), '*.c')) + hfiles = glob.glob(os.path.join(implementation.path(), '*.h')) + for file in (cfiles + hfiles): + yield (check_makefile_dependencies, implementation, file) def touch(time, *files): diff --git a/test/test_metadata.py b/test/test_metadata.py index 8c22a980..69ad18e3 100644 --- a/test/test_metadata.py +++ b/test/test_metadata.py @@ -10,10 +10,10 @@ import pqclean def test_metadata(): for scheme in pqclean.Scheme.all_schemes(): - if helpers.permit_test('metadata', scheme): - yield check_metadata, scheme + yield check_metadata, scheme +@helpers.filtered_test def check_metadata(scheme): metadata = scheme.metadata() diff --git a/test/test_metadata_sizes.py b/test/test_metadata_sizes.py index 20915cce..47ef863d 100644 --- a/test/test_metadata_sizes.py +++ b/test/test_metadata_sizes.py @@ -8,10 +8,10 @@ import helpers def test_metadata_sizes(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - if helpers.permit_test('metadata_sizes', implementation): - yield check_metadata_sizes, implementation + yield check_metadata_sizes, implementation +@helpers.filtered_test def check_metadata_sizes(implementation): metadata = implementation.scheme.metadata() impl_meta = next((impl for impl in metadata['implementations'] diff --git a/test/test_nistkat.py b/test/test_nistkat.py index 215a024e..97c8293f 100644 --- a/test/test_nistkat.py +++ b/test/test_nistkat.py @@ -2,7 +2,7 @@ Checks that (hash of the) KATs (in NIST format) produced on this platform matches the one provided in the META file for every scheme/implementation. -Note that this only uses the first test case from the NIST-format KAT files. +Note that this only uses the first test case from the NIST-format KAT files. The appropriate hash can be generated from the original submission's KAT file using the command: cat PQCkemKAT_whatever.rsp | head -n 8 | tail -n 6 | sha256sum @@ -14,17 +14,21 @@ import pqclean import helpers import unittest + def test_nistkat(): for scheme in pqclean.Scheme.all_schemes(): - if scheme.type != 'kem': continue + if scheme.type != 'kem': + continue for implementation in scheme.implementations: - if helpers.permit_test('nistkat', implementation): - yield check_nistkat, implementation + yield check_nistkat, implementation +@helpers.filtered_test def check_nistkat(implementation): if implementation.scheme.name == "kyber768": - raise unittest.SkipTest("Temporarily skip NIST KAT check for kyber768 since it's an outdated implementation") + raise unittest.SkipTest( + "Temporarily skip NIST KAT check for kyber768 since it's " + "an outdated implementation") helpers.make('nistkat', TYPE=implementation.scheme.type, SCHEME=implementation.scheme.name, diff --git a/test/test_no_symlinks.py b/test/test_no_symlinks.py index eb2a0656..93f2937d 100644 --- a/test/test_no_symlinks.py +++ b/test/test_no_symlinks.py @@ -10,10 +10,10 @@ import helpers def test_no_symlinks(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - if helpers.permit_test('no_symlinks', implementation): - yield check_no_symlinks, implementation + yield check_no_symlinks, implementation +@helpers.filtered_test def check_no_symlinks(implementation): for file in os.listdir(implementation.path()): fpath = os.path.join(implementation.path(), file) diff --git a/test/test_preprocessor.py b/test/test_preprocessor.py index 84c4c8f4..043b7a50 100644 --- a/test/test_preprocessor.py +++ b/test/test_preprocessor.py @@ -5,10 +5,10 @@ import helpers def test_preprocessor(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - if helpers.permit_test('preprocessor', implementation): - yield check_preprocessor, implementation + yield check_preprocessor, implementation +@helpers.filtered_test def check_preprocessor(implementation: pqclean.Implementation): cfiles = implementation.cfiles() hfiles = implementation.hfiles() diff --git a/test/test_symbol_namespace.py b/test/test_symbol_namespace.py index 0b9fe933..cefc8af2 100644 --- a/test/test_symbol_namespace.py +++ b/test/test_symbol_namespace.py @@ -12,10 +12,10 @@ import unittest def test_symbol_namespace(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - if helpers.permit_test('symbol_namespace', implementation): - yield check_symbol_namespace, implementation + yield check_symbol_namespace, implementation +@helpers.filtered_test def check_symbol_namespace(implementation): if sys.platform not in ['linux', 'darwin']: raise unittest.SkipTest("Unsupported platform") diff --git a/test/test_testvectors.py b/test/test_testvectors.py index 87c9c137..c4834c31 100644 --- a/test/test_testvectors.py +++ b/test/test_testvectors.py @@ -10,28 +10,27 @@ import helpers def test_testvectors(): + @helpers.filtered_test + def check_testvectors(implementation): + helpers.make('testvectors', + TYPE=implementation.scheme.type, + SCHEME=implementation.scheme.name, + IMPLEMENTATION=implementation.name, + working_dir=os.path.join('..', 'test')) + out = helpers.run_subprocess( + [os.path.join('..', 'bin', 'testvectors_{}_{}{}'.format( + implementation.scheme.name, + implementation.name, + '.exe' if os.name == 'nt' else '' + ))], + os.path.join('..', 'bin'), + ).replace('\r', '') + assert(implementation.scheme.metadata()['testvectors-sha256'].lower() + == hashlib.sha256(out.encode('utf-8')).hexdigest().lower()) + for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - if helpers.permit_test('testvectors', implementation): - yield check_vectors, implementation - - -def check_vectors(implementation): - helpers.make('testvectors', - TYPE=implementation.scheme.type, - SCHEME=implementation.scheme.name, - IMPLEMENTATION=implementation.name, - working_dir=os.path.join('..', 'test')) - out = helpers.run_subprocess( - [os.path.join('..', 'bin', 'testvectors_{}_{}{}'.format( - implementation.scheme.name, - implementation.name, - '.exe' if os.name == 'nt' else '' - ))], - os.path.join('..', 'bin'), - ).replace('\r', '') - assert(implementation.scheme.metadata()['testvectors-sha256'].lower() - == hashlib.sha256(out.encode('utf-8')).hexdigest().lower()) + yield check_testvectors, implementation if __name__ == '__main__': diff --git a/test/test_valgrind.py b/test/test_valgrind.py index 6afba526..815c5f8e 100644 --- a/test/test_valgrind.py +++ b/test/test_valgrind.py @@ -13,11 +13,11 @@ import helpers def test_functest(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - if helpers.permit_test('valgrind', implementation): - yield check_valgrind, implementation + yield check_valgrind, implementation @helpers.slow_test +@helpers.filtered_test def check_valgrind(implementation: pqclean.Implementation): if (platform.machine() not in ('i386', 'x86_64') or platform.system() != 'Linux'):