kris/pqc
1
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Selectively filter tests (#102)

Browse Source 
Selectively filter tests
This commit is contained in:
Douglas Stebila 2019-04-10 19:27:25 -04:00 committed by GitHub
commit f388d6f93e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
18 changed files with 88 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

31
test/helpers.py
View File

@ -5,6 +5,8 @@ import unittest
import shutil
import sys
import pqclean
def run_subprocess(command, working_dir='.', env=None, expected_returncode=0):
"""
@ -97,3 +99,32 @@ def ensure_available(executable):
else "On Windows, make sure to add it to PATH")
)
raise AssertionError("{} not available on CI".format(executable))
def permit_test(testname, thing, **args):
if 'PQCLEAN_ONLY_TESTS' in os.environ:
if not(testname.lower() in os.environ['PQCLEAN_ONLY_TESTS'].lower().split(',')):
return False
if 'PQCLEAN_SKIP_TESTS' in os.environ:
if testname.lower() in os.environ['PQCLEAN_SKIP_TESTS'].lower().split(','):
return False
if isinstance(thing, pqclean.Implementation):
scheme = thing.scheme
else:
scheme = thing
if 'PQCLEAN_ONLY_TYPES' in os.environ:
if not(scheme.type.lower() in os.environ['PQCLEAN_ONLY_TYPES'].lower().split(',')):
return False
if 'PQCLEAN_SKIP_TYPES' in os.environ:
if scheme.type.lower() in os.environ['PQCLEAN_SKIP_TYPES'].lower().split(','):
return False
if 'PQCLEAN_ONLY_SCHEMES' in os.environ:
if not(scheme.name.lower() in os.environ['PQCLEAN_ONLY_SCHEMES'].lower().split(',')):
return False
if 'PQCLEAN_SKIP_SCHEMES' in os.environ:
if scheme.name.lower() in os.environ['PQCLEAN_SKIP_SCHEMES'].lower().split(','):
return False
return True

2
test/test_api_h.py
View File

@ -1,12 +1,14 @@
import os
import re
import helpers
import pqclean
def test_preprocessor():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('preprocessor', implementation):
yield check_preprocessor, implementation

1
test/test_char.py
View File

@ -17,6 +17,7 @@ def test_char():
)
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('char', implementation):
yield check_char, implementation

1
test/test_compile_lib.py
View File

@ -10,6 +10,7 @@ import helpers
def test_compile_lib():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('compile_lib', implementation):
yield check_compile_lib, implementation

1
test/test_duplicate_consistency.py
View File

@ -11,6 +11,7 @@ import yaml
def test_duplicate_consistency():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('duplicate_consistency', implementation):
if os.path.isfile(os.path.join('duplicate_consistency', '{}_{}.yml'.format(scheme.name, implementation.name))):
yield check_duplicate_consistency, implementation

1
test/test_dynamic_memory.py
View File

@ -11,6 +11,7 @@ import unittest
def test_dynamic_memory():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('dynamic_memory', implementation):
# Keep this loop outside, to allow multiple assertions
for function in ['malloc', 'free', 'realloc', 'calloc']:
yield (check_dynamic_memory, implementation, function)

7
test/test_format.py
View File

@ -1,18 +1,19 @@
import helpers
import pqclean
from helpers import run_subprocess, ensure_available
def test_formatting():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('format', implementation):
yield check_format, implementation
def check_format(implementation: pqclean.Implementation):
ensure_available('astyle')
helpers.ensure_available('astyle')
cfiles = implementation.cfiles()
hfiles = implementation.hfiles()
run_subprocess(['astyle',
helpers.run_subprocess(['astyle',
'--dry-run',
'--options=../.astylerc',
*cfiles,

2
test/test_functest.py
View File

@ -14,12 +14,14 @@ import helpers
def test_functest():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('functest', implementation):
yield check_functest, implementation
def test_functest_sanitizers():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('functest_sanitizers', implementation):
yield check_functest_sanitizers, implementation

2
test/test_license.py
View File

@ -5,11 +5,13 @@ implementation of the specified scheme.
import os
import pqclean
import helpers
def test_license():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('license', implementation):
yield check_license, implementation

7
test/test_linter.py
View File

@ -2,20 +2,21 @@ import os
from glob import glob
import pqclean
from helpers import run_subprocess, ensure_available
import helpers
def test_clang_tidy():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('linter', implementation):
yield check_tidy, implementation
def check_tidy(implementation: pqclean.Implementation):
ensure_available('clang-tidy')
helpers.ensure_available('clang-tidy')
cfiles = glob(os.path.join(implementation.path(), '*.c'))
common_files = glob(os.path.join('..', 'common', '*.c'))
run_subprocess(['clang-tidy',
helpers.run_subprocess(['clang-tidy',
'-quiet',
'-header-filter=.*',
*cfiles,

1
test/test_makefile_dependencies.py
View File

@ -13,6 +13,7 @@ import datetime
def test_makefile_dependencies():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('makefile_dependencies', implementation):
# initial build - want to have *all* files in place at beginning
helpers.make('clean', working_dir=implementation.path())
helpers.make(working_dir=implementation.path())

2
test/test_metadata.py
View File

@ -3,12 +3,14 @@ Verify the metadata specified in the META.yml files.
"""
import copy
import helpers
import itertools
import pqclean
def test_metadata():
for scheme in pqclean.Scheme.all_schemes():
if helpers.permit_test('metadata', scheme):
yield check_metadata, scheme

1
test/test_metadata_sizes.py
View File

@ -8,6 +8,7 @@ import helpers
def test_metadata_sizes():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('metadata_sizes', implementation):
yield check_metadata_sizes, implementation

2
test/test_no_symlinks.py
View File

@ -4,11 +4,13 @@ Checks that no implementation makes use of symbolic links.
import os
import pqclean
import helpers
def test_no_symlinks():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('no_symlinks', implementation):
yield check_no_symlinks, implementation

3
test/test_preprocessor.py
View File

@ -2,12 +2,13 @@ import os
from glob import glob
import pqclean
from helpers import run_subprocess, ensure_available
import helpers
def test_preprocessor():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('preprocessor', implementation):
yield check_preprocessor, implementation

1
test/test_symbol_namespace.py
View File

@ -12,6 +12,7 @@ import unittest
def test_symbol_namespace():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('symbol_namespace', implementation):
yield check_symbol_namespace, implementation

1
test/test_testvectors.py
View File

@ -12,6 +12,7 @@ import helpers
def test_testvectors():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('testvectors', implementation):
yield check_vectors, implementation

1
test/test_valgrind.py
View File

@ -13,6 +13,7 @@ import helpers
def test_functest():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('valgrind', implementation):
yield check_valgrind, implementation