diff --git a/test/helpers.py b/test/helpers.py index e6c3adc1..966510f1 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -5,6 +5,8 @@ import unittest import shutil import sys +import pqclean + def run_subprocess(command, working_dir='.', env=None, expected_returncode=0): """ @@ -97,3 +99,32 @@ def ensure_available(executable): else "On Windows, make sure to add it to PATH") ) raise AssertionError("{} not available on CI".format(executable)) + + +def permit_test(testname, thing, **args): + if 'PQCLEAN_ONLY_TESTS' in os.environ: + if not(testname.lower() in os.environ['PQCLEAN_ONLY_TESTS'].lower().split(',')): + return False + if 'PQCLEAN_SKIP_TESTS' in os.environ: + if testname.lower() in os.environ['PQCLEAN_SKIP_TESTS'].lower().split(','): + return False + + if isinstance(thing, pqclean.Implementation): + scheme = thing.scheme + else: + scheme = thing + + if 'PQCLEAN_ONLY_TYPES' in os.environ: + if not(scheme.type.lower() in os.environ['PQCLEAN_ONLY_TYPES'].lower().split(',')): + return False + if 'PQCLEAN_SKIP_TYPES' in os.environ: + if scheme.type.lower() in os.environ['PQCLEAN_SKIP_TYPES'].lower().split(','): + return False + if 'PQCLEAN_ONLY_SCHEMES' in os.environ: + if not(scheme.name.lower() in os.environ['PQCLEAN_ONLY_SCHEMES'].lower().split(',')): + return False + if 'PQCLEAN_SKIP_SCHEMES' in os.environ: + if scheme.name.lower() in os.environ['PQCLEAN_SKIP_SCHEMES'].lower().split(','): + return False + + return True diff --git a/test/test_api_h.py b/test/test_api_h.py index 501791f9..5ed2d121 100644 --- a/test/test_api_h.py +++ b/test/test_api_h.py @@ -1,13 +1,15 @@ import os import re +import helpers import pqclean def test_preprocessor(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - yield check_preprocessor, implementation + if helpers.permit_test('preprocessor', implementation): + yield check_preprocessor, implementation def check_preprocessor(implementation: pqclean.Implementation): diff --git a/test/test_char.py b/test/test_char.py index 98f633ed..48aa55c9 100644 --- a/test/test_char.py +++ b/test/test_char.py @@ -17,7 +17,8 @@ def test_char(): ) for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - yield check_char, implementation + if helpers.permit_test('char', implementation): + yield check_char, implementation def walk_tree(ast): diff --git a/test/test_compile_lib.py b/test/test_compile_lib.py index 7151975e..e4b1ae5c 100644 --- a/test/test_compile_lib.py +++ b/test/test_compile_lib.py @@ -10,7 +10,8 @@ import helpers def test_compile_lib(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - yield check_compile_lib, implementation + if helpers.permit_test('compile_lib', implementation): + yield check_compile_lib, implementation def check_compile_lib(implementation): diff --git a/test/test_duplicate_consistency.py b/test/test_duplicate_consistency.py index 52755a3b..041b5661 100644 --- a/test/test_duplicate_consistency.py +++ b/test/test_duplicate_consistency.py @@ -11,8 +11,9 @@ import yaml def test_duplicate_consistency(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - if os.path.isfile(os.path.join('duplicate_consistency', '{}_{}.yml'.format(scheme.name, implementation.name))): - yield check_duplicate_consistency, implementation + if helpers.permit_test('duplicate_consistency', implementation): + if os.path.isfile(os.path.join('duplicate_consistency', '{}_{}.yml'.format(scheme.name, implementation.name))): + yield check_duplicate_consistency, implementation def file_get_contents(filename): with open(filename) as f: diff --git a/test/test_dynamic_memory.py b/test/test_dynamic_memory.py index 8853eaea..27257006 100644 --- a/test/test_dynamic_memory.py +++ b/test/test_dynamic_memory.py @@ -11,9 +11,10 @@ import unittest def test_dynamic_memory(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - # Keep this loop outside, to allow multiple assertions - for function in ['malloc', 'free', 'realloc', 'calloc']: - yield (check_dynamic_memory, implementation, function) + if helpers.permit_test('dynamic_memory', implementation): + # Keep this loop outside, to allow multiple assertions + for function in ['malloc', 'free', 'realloc', 'calloc']: + yield (check_dynamic_memory, implementation, function) @helpers.skip_windows() diff --git a/test/test_format.py b/test/test_format.py index 09cc61fd..427a98a0 100644 --- a/test/test_format.py +++ b/test/test_format.py @@ -1,18 +1,19 @@ +import helpers import pqclean -from helpers import run_subprocess, ensure_available def test_formatting(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - yield check_format, implementation + if helpers.permit_test('format', implementation): + yield check_format, implementation def check_format(implementation: pqclean.Implementation): - ensure_available('astyle') + helpers.ensure_available('astyle') cfiles = implementation.cfiles() hfiles = implementation.hfiles() - run_subprocess(['astyle', + helpers.run_subprocess(['astyle', '--dry-run', '--options=../.astylerc', *cfiles, diff --git a/test/test_functest.py b/test/test_functest.py index 01807e75..d4517947 100644 --- a/test/test_functest.py +++ b/test/test_functest.py @@ -14,13 +14,15 @@ import helpers def test_functest(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - yield check_functest, implementation + if helpers.permit_test('functest', implementation): + yield check_functest, implementation def test_functest_sanitizers(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - yield check_functest_sanitizers, implementation + if helpers.permit_test('functest_sanitizers', implementation): + yield check_functest_sanitizers, implementation def check_functest(implementation): diff --git a/test/test_license.py b/test/test_license.py index 64633a40..15c61491 100644 --- a/test/test_license.py +++ b/test/test_license.py @@ -5,12 +5,14 @@ implementation of the specified scheme. import os import pqclean +import helpers def test_license(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - yield check_license, implementation + if helpers.permit_test('license', implementation): + yield check_license, implementation def check_license(implementation): diff --git a/test/test_linter.py b/test/test_linter.py index 33563f8b..f9bf430e 100644 --- a/test/test_linter.py +++ b/test/test_linter.py @@ -2,20 +2,21 @@ import os from glob import glob import pqclean -from helpers import run_subprocess, ensure_available +import helpers def test_clang_tidy(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - yield check_tidy, implementation + if helpers.permit_test('linter', implementation): + yield check_tidy, implementation def check_tidy(implementation: pqclean.Implementation): - ensure_available('clang-tidy') + helpers.ensure_available('clang-tidy') cfiles = glob(os.path.join(implementation.path(), '*.c')) common_files = glob(os.path.join('..', 'common', '*.c')) - run_subprocess(['clang-tidy', + helpers.run_subprocess(['clang-tidy', '-quiet', '-header-filter=.*', *cfiles, diff --git a/test/test_makefile_dependencies.py b/test/test_makefile_dependencies.py index 86730890..8305745e 100644 --- a/test/test_makefile_dependencies.py +++ b/test/test_makefile_dependencies.py @@ -13,14 +13,15 @@ import datetime def test_makefile_dependencies(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - # initial build - want to have *all* files in place at beginning - helpers.make('clean', working_dir=implementation.path()) - helpers.make(working_dir=implementation.path()) - # test case for each candidate file - cfiles = glob.glob(os.path.join(implementation.path(), '*.c')) - hfiles = glob.glob(os.path.join(implementation.path(), '*.h')) - for file in (cfiles + hfiles): - yield (check_makefile_dependencies, implementation, file) + if helpers.permit_test('makefile_dependencies', implementation): + # initial build - want to have *all* files in place at beginning + helpers.make('clean', working_dir=implementation.path()) + helpers.make(working_dir=implementation.path()) + # test case for each candidate file + cfiles = glob.glob(os.path.join(implementation.path(), '*.c')) + hfiles = glob.glob(os.path.join(implementation.path(), '*.h')) + for file in (cfiles + hfiles): + yield (check_makefile_dependencies, implementation, file) def touch(time, *files): diff --git a/test/test_metadata.py b/test/test_metadata.py index 11556a08..997ce831 100644 --- a/test/test_metadata.py +++ b/test/test_metadata.py @@ -3,13 +3,15 @@ Verify the metadata specified in the META.yml files. """ import copy +import helpers import itertools import pqclean def test_metadata(): for scheme in pqclean.Scheme.all_schemes(): - yield check_metadata, scheme + if helpers.permit_test('metadata', scheme): + yield check_metadata, scheme def check_metadata(scheme): diff --git a/test/test_metadata_sizes.py b/test/test_metadata_sizes.py index 38c77a5f..20915cce 100644 --- a/test/test_metadata_sizes.py +++ b/test/test_metadata_sizes.py @@ -8,7 +8,8 @@ import helpers def test_metadata_sizes(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - yield check_metadata_sizes, implementation + if helpers.permit_test('metadata_sizes', implementation): + yield check_metadata_sizes, implementation def check_metadata_sizes(implementation): diff --git a/test/test_no_symlinks.py b/test/test_no_symlinks.py index 6f8eac6a..eb2a0656 100644 --- a/test/test_no_symlinks.py +++ b/test/test_no_symlinks.py @@ -4,12 +4,14 @@ Checks that no implementation makes use of symbolic links. import os import pqclean +import helpers def test_no_symlinks(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - yield check_no_symlinks, implementation + if helpers.permit_test('no_symlinks', implementation): + yield check_no_symlinks, implementation def check_no_symlinks(implementation): diff --git a/test/test_preprocessor.py b/test/test_preprocessor.py index 64e0b1f4..c332362e 100644 --- a/test/test_preprocessor.py +++ b/test/test_preprocessor.py @@ -2,13 +2,14 @@ import os from glob import glob import pqclean -from helpers import run_subprocess, ensure_available +import helpers def test_preprocessor(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - yield check_preprocessor, implementation + if helpers.permit_test('preprocessor', implementation): + yield check_preprocessor, implementation def check_preprocessor(implementation: pqclean.Implementation): diff --git a/test/test_symbol_namespace.py b/test/test_symbol_namespace.py index 5d6ea9a0..0b9fe933 100644 --- a/test/test_symbol_namespace.py +++ b/test/test_symbol_namespace.py @@ -12,7 +12,8 @@ import unittest def test_symbol_namespace(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - yield check_symbol_namespace, implementation + if helpers.permit_test('symbol_namespace', implementation): + yield check_symbol_namespace, implementation def check_symbol_namespace(implementation): diff --git a/test/test_testvectors.py b/test/test_testvectors.py index 14afb92f..87c9c137 100644 --- a/test/test_testvectors.py +++ b/test/test_testvectors.py @@ -12,7 +12,8 @@ import helpers def test_testvectors(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - yield check_vectors, implementation + if helpers.permit_test('testvectors', implementation): + yield check_vectors, implementation def check_vectors(implementation): diff --git a/test/test_valgrind.py b/test/test_valgrind.py index 219791cd..05341456 100644 --- a/test/test_valgrind.py +++ b/test/test_valgrind.py @@ -13,7 +13,8 @@ import helpers def test_functest(): for scheme in pqclean.Scheme.all_schemes(): for implementation in scheme.implementations: - yield check_valgrind, implementation + if helpers.permit_test('valgrind', implementation): + yield check_valgrind, implementation def check_valgrind(implementation: pqclean.Implementation):