Selectively filter tests (#102)

Selectively filter tests

test/helpers.py

@@ -5,6 +5,8 @@ import unittest
 import shutil
 import sys
 
+import pqclean
+
 
 def run_subprocess(command, working_dir='.', env=None, expected_returncode=0):
     """
@@ -97,3 +99,32 @@ def ensure_available(executable):
                         else "On Windows, make sure to add it to PATH")
                 )
     raise AssertionError("{} not available on CI".format(executable))
+
+
+def permit_test(testname, thing, **args):
+    if 'PQCLEAN_ONLY_TESTS' in os.environ:
+        if not(testname.lower() in os.environ['PQCLEAN_ONLY_TESTS'].lower().split(',')):
+            return False
+    if 'PQCLEAN_SKIP_TESTS' in os.environ:
+        if testname.lower() in os.environ['PQCLEAN_SKIP_TESTS'].lower().split(','):
+            return False
+
+    if isinstance(thing, pqclean.Implementation):
+        scheme = thing.scheme
+    else:
+        scheme = thing
+
+    if 'PQCLEAN_ONLY_TYPES' in os.environ:
+        if not(scheme.type.lower() in os.environ['PQCLEAN_ONLY_TYPES'].lower().split(',')):
+            return False
+    if 'PQCLEAN_SKIP_TYPES' in os.environ:
+        if scheme.type.lower() in os.environ['PQCLEAN_SKIP_TYPES'].lower().split(','):
+            return False
+    if 'PQCLEAN_ONLY_SCHEMES' in os.environ:
+        if not(scheme.name.lower() in os.environ['PQCLEAN_ONLY_SCHEMES'].lower().split(',')):
+            return False
+    if 'PQCLEAN_SKIP_SCHEMES' in os.environ:
+        if scheme.name.lower() in os.environ['PQCLEAN_SKIP_SCHEMES'].lower().split(','):
+            return False
+
+    return True

test/test_api_h.py

@@ -1,13 +1,15 @@
 import os
 import re
 
+import helpers
 import pqclean
 
 
 def test_preprocessor():
     for scheme in pqclean.Scheme.all_schemes():
         for implementation in scheme.implementations:
-            yield check_preprocessor, implementation
+            if helpers.permit_test('preprocessor', implementation): 
+                yield check_preprocessor, implementation
 
 
 def check_preprocessor(implementation: pqclean.Implementation):

test/test_char.py

@@ -17,7 +17,8 @@ def test_char():
         )
     for scheme in pqclean.Scheme.all_schemes():
         for implementation in scheme.implementations:
-            yield check_char, implementation
+            if helpers.permit_test('char', implementation): 
+                yield check_char, implementation
 
 
 def walk_tree(ast):

test/test_compile_lib.py

@@ -10,7 +10,8 @@ import helpers
 def test_compile_lib():
     for scheme in pqclean.Scheme.all_schemes():
         for implementation in scheme.implementations:
-            yield check_compile_lib, implementation
+            if helpers.permit_test('compile_lib', implementation): 
+                yield check_compile_lib, implementation
 
 
 def check_compile_lib(implementation):

test/test_duplicate_consistency.py

@@ -11,8 +11,9 @@ import yaml
 def test_duplicate_consistency():
     for scheme in pqclean.Scheme.all_schemes():
         for implementation in scheme.implementations:
-            if os.path.isfile(os.path.join('duplicate_consistency', '{}_{}.yml'.format(scheme.name, implementation.name))):
-                yield check_duplicate_consistency, implementation
+            if helpers.permit_test('duplicate_consistency', implementation): 
+                if os.path.isfile(os.path.join('duplicate_consistency', '{}_{}.yml'.format(scheme.name, implementation.name))):
+                    yield check_duplicate_consistency, implementation
 
 def file_get_contents(filename):
     with open(filename) as f:

test/test_dynamic_memory.py

@@ -11,9 +11,10 @@ import unittest
 def test_dynamic_memory():
     for scheme in pqclean.Scheme.all_schemes():
         for implementation in scheme.implementations:
-            # Keep this loop outside, to allow multiple assertions
-            for function in ['malloc', 'free', 'realloc', 'calloc']:
-                yield (check_dynamic_memory, implementation, function)
+            if helpers.permit_test('dynamic_memory', implementation): 
+                # Keep this loop outside, to allow multiple assertions
+                for function in ['malloc', 'free', 'realloc', 'calloc']:
+                    yield (check_dynamic_memory, implementation, function)
 
 
 @helpers.skip_windows()

test/test_format.py

@@ -1,18 +1,19 @@
+import helpers
 import pqclean
-from helpers import run_subprocess, ensure_available
 
 
 def test_formatting():
     for scheme in pqclean.Scheme.all_schemes():
         for implementation in scheme.implementations:
-            yield check_format, implementation
+            if helpers.permit_test('format', implementation): 
+                yield check_format, implementation
 
 
 def check_format(implementation: pqclean.Implementation):
-    ensure_available('astyle')
+    helpers.ensure_available('astyle')
     cfiles = implementation.cfiles()
     hfiles = implementation.hfiles()
-    run_subprocess(['astyle',
+    helpers.run_subprocess(['astyle',
                     '--dry-run',
                     '--options=../.astylerc',
                     *cfiles,

test/test_functest.py

@@ -14,13 +14,15 @@ import helpers
 def test_functest():
     for scheme in pqclean.Scheme.all_schemes():
         for implementation in scheme.implementations:
-            yield check_functest, implementation
+            if helpers.permit_test('functest', implementation): 
+                yield check_functest, implementation
 
 
 def test_functest_sanitizers():
     for scheme in pqclean.Scheme.all_schemes():
         for implementation in scheme.implementations:
-            yield check_functest_sanitizers, implementation
+            if helpers.permit_test('functest_sanitizers', implementation): 
+                yield check_functest_sanitizers, implementation
 
 
 def check_functest(implementation):

test/test_license.py

@@ -5,12 +5,14 @@ implementation of the specified scheme.
 
 import os
 import pqclean
+import helpers
 
 
 def test_license():
     for scheme in pqclean.Scheme.all_schemes():
         for implementation in scheme.implementations:
-            yield check_license, implementation
+            if helpers.permit_test('license', implementation): 
+                yield check_license, implementation
 
 
 def check_license(implementation):

test/test_linter.py

@@ -2,20 +2,21 @@ import os
 from glob import glob
 
 import pqclean
-from helpers import run_subprocess, ensure_available
+import helpers
 
 
 def test_clang_tidy():
     for scheme in pqclean.Scheme.all_schemes():
         for implementation in scheme.implementations:
-            yield check_tidy, implementation
+            if helpers.permit_test('linter', implementation):
+                yield check_tidy, implementation
 
 
 def check_tidy(implementation: pqclean.Implementation):
-    ensure_available('clang-tidy')
+    helpers.ensure_available('clang-tidy')
     cfiles = glob(os.path.join(implementation.path(), '*.c'))
     common_files = glob(os.path.join('..', 'common', '*.c'))
-    run_subprocess(['clang-tidy',
+    helpers.run_subprocess(['clang-tidy',
                     '-quiet',
                     '-header-filter=.*',
                     *cfiles,

test/test_makefile_dependencies.py

@@ -13,14 +13,15 @@ import datetime
 def test_makefile_dependencies():
     for scheme in pqclean.Scheme.all_schemes():
         for implementation in scheme.implementations:
-            # initial build - want to have *all* files in place at beginning
-            helpers.make('clean', working_dir=implementation.path())
-            helpers.make(working_dir=implementation.path())
-            # test case for each candidate file
-            cfiles = glob.glob(os.path.join(implementation.path(), '*.c'))
-            hfiles = glob.glob(os.path.join(implementation.path(), '*.h'))
-            for file in (cfiles + hfiles):
-                yield (check_makefile_dependencies, implementation, file)
+            if helpers.permit_test('makefile_dependencies', implementation):
+                # initial build - want to have *all* files in place at beginning
+                helpers.make('clean', working_dir=implementation.path())
+                helpers.make(working_dir=implementation.path())
+                # test case for each candidate file
+                cfiles = glob.glob(os.path.join(implementation.path(), '*.c'))
+                hfiles = glob.glob(os.path.join(implementation.path(), '*.h'))
+                for file in (cfiles + hfiles):
+                    yield (check_makefile_dependencies, implementation, file)
 
 
 def touch(time, *files):

test/test_metadata.py

@@ -3,13 +3,15 @@ Verify the metadata specified in the META.yml files.
 """
 
 import copy
+import helpers
 import itertools
 import pqclean
 
 
 def test_metadata():
     for scheme in pqclean.Scheme.all_schemes():
-        yield check_metadata, scheme
+        if helpers.permit_test('metadata', scheme):
+            yield check_metadata, scheme
 
 
 def check_metadata(scheme):

test/test_metadata_sizes.py

@@ -8,7 +8,8 @@ import helpers
 def test_metadata_sizes():
     for scheme in pqclean.Scheme.all_schemes():
         for implementation in scheme.implementations:
-            yield check_metadata_sizes, implementation
+            if helpers.permit_test('metadata_sizes', implementation):
+                yield check_metadata_sizes, implementation
 
 
 def check_metadata_sizes(implementation):

test/test_no_symlinks.py

@@ -4,12 +4,14 @@ Checks that no implementation makes use of symbolic links.
 
 import os
 import pqclean
+import helpers
 
 
 def test_no_symlinks():
     for scheme in pqclean.Scheme.all_schemes():
         for implementation in scheme.implementations:
-            yield check_no_symlinks, implementation
+            if helpers.permit_test('no_symlinks', implementation):
+                yield check_no_symlinks, implementation
 
 
 def check_no_symlinks(implementation):

test/test_preprocessor.py

@@ -2,13 +2,14 @@ import os
 from glob import glob
 
 import pqclean
-from helpers import run_subprocess, ensure_available
+import helpers
 
 
 def test_preprocessor():
     for scheme in pqclean.Scheme.all_schemes():
         for implementation in scheme.implementations:
-            yield check_preprocessor, implementation
+            if helpers.permit_test('preprocessor', implementation):
+                yield check_preprocessor, implementation
 
 
 def check_preprocessor(implementation: pqclean.Implementation):

test/test_symbol_namespace.py

@@ -12,7 +12,8 @@ import unittest
 def test_symbol_namespace():
     for scheme in pqclean.Scheme.all_schemes():
         for implementation in scheme.implementations:
-            yield check_symbol_namespace, implementation
+            if helpers.permit_test('symbol_namespace', implementation):
+                yield check_symbol_namespace, implementation
 
 
 def check_symbol_namespace(implementation):

test/test_testvectors.py

@@ -12,7 +12,8 @@ import helpers
 def test_testvectors():
     for scheme in pqclean.Scheme.all_schemes():
         for implementation in scheme.implementations:
-            yield check_vectors, implementation
+            if helpers.permit_test('testvectors', implementation):
+                yield check_vectors, implementation
 
 
 def check_vectors(implementation):

test/test_valgrind.py

@@ -13,7 +13,8 @@ import helpers
 def test_functest():
     for scheme in pqclean.Scheme.all_schemes():
         for implementation in scheme.implementations:
-            yield check_valgrind, implementation
+            if helpers.permit_test('valgrind', implementation):
+                yield check_valgrind, implementation
 
 
 def check_valgrind(implementation: pqclean.Implementation):