kris/pqc
1
1
Fork 0
Code Issues Pull Requests Releases Wiki Activity

Change permit test into a decorator

Browse Source 
This way we can properly raise skiptest
This commit is contained in:
Thom Wiggers 2019-04-18 10:00:08 +02:00
parent 98780c6e8e
commit 65d43df241
No known key found for this signature in database
GPG Key ID: 001BB0A7CE26E363
20 changed files with 138 additions and 88 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

17
test/helpers.py
View File

@ -7,6 +7,8 @@ import sys
import pqclean
import logging
def run_subprocess(command, working_dir='.', env=None, expected_returncode=0):
"""
@ -113,7 +115,7 @@ def ensure_available(executable):
raise AssertionError("{} not available on CI".format(executable))
def permit_test(testname, thing, **args):
def permit_test(testname, thing, *args, **kwargs):
if 'PQCLEAN_ONLY_TESTS' in os.environ:
if not(testname.lower() in os.environ['PQCLEAN_ONLY_TESTS'].lower().split(',')):
return False
@ -169,6 +171,7 @@ def permit_test(testname, thing, **args):
for diff_line in diff_result.stdout.decode('utf-8').splitlines():
# don't skip test if there are any changes outside schemes
if not(diff_line.startswith('crypto_kem')) and not (diff_line.startswith('crypto_sign')):
logging.info("Running all tests as there are changes outside of schemes")
return True
# do test if the scheme in question has been changed
if diff_line.startswith(thing.path(base='')):
@ -177,3 +180,15 @@ def permit_test(testname, thing, **args):
return False
return True
def filtered_test(func):
funcname = func.__name__[len("check_"):]
@functools.wraps(func)
def wrapper(*args, **kwargs):
if permit_test(funcname, *args, **kwargs):
return func(*args, **kwargs)
else:
raise unittest.SkipTest("Test disabled by filter")
return wrapper

9
test/test_api_h.py
View File

@ -5,14 +5,14 @@ import helpers
import pqclean
def test_preprocessor():
def test_api_h():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('preprocessor', implementation):
yield check_preprocessor, implementation
yield check_api_h, implementation
def check_preprocessor(implementation: pqclean.Implementation):
@helpers.filtered_test
def check_api_h(implementation: pqclean.Implementation):
apipath = os.path.join(implementation.path(), 'api.h')
errors = []
p = re.compile(r'^\s*#include\s*"')
@ -25,6 +25,7 @@ def check_preprocessor(implementation: pqclean.Implementation):
"Prohibited external include in api.h" + "".join(errors)
)
if __name__ == "__main__":
try:
import nose2

4
test/test_char.py
View File

@ -17,8 +17,7 @@ def test_char():
)
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('char', implementation):
yield check_char, implementation
yield check_char, implementation
def walk_tree(ast):
@ -30,6 +29,7 @@ def walk_tree(ast):
yield from walk_tree(child) # recursively yield prohibited nodes
@helpers.filtered_test
@helpers.skip_windows()
def check_char(implementation):
errors = []

3
test/test_common.py
View File

@ -12,7 +12,8 @@ import helpers
def test_common():
for d in os.listdir('common'):
primitive = re.sub(r"\.c$", "", d)
if helpers.permit_test('common', None): yield check_common, primitive
if helpers.permit_test('common', None):
yield check_common, primitive
def check_common(primitive):

4
test/test_compile_lib.py
View File

@ -10,10 +10,10 @@ import helpers
def test_compile_lib():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('compile_lib', implementation):
yield check_compile_lib, implementation
yield check_compile_lib, implementation
@helpers.filtered_test
def check_compile_lib(implementation):
helpers.make('clean', working_dir=implementation.path())
helpers.make(working_dir=implementation.path())

43
test/test_duplicate_consistency.py
View File

@ -8,33 +8,54 @@ import helpers
import unittest
import yaml
helpers.skip_windows()
def skipped_test(*args, **kwargs):
raise unittest.SkipTest("Skipped consistency check")
def test_duplicate_consistency():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('duplicate_consistency', implementation):
if os.path.isfile(os.path.join('duplicate_consistency', '{}_{}.yml'.format(scheme.name, implementation.name))):
metafile = os.path.join('duplicate_consistency', '{}_{}.yml'.format(implementation.scheme.name, implementation.name))
with open(metafile, encoding='utf-8') as f:
metadata = yaml.safe_load(f.read())
for group in metadata['consistency_checks']:
source = pqclean.Implementation.by_name(group['source']['scheme'], group['source']['implementation'])
for file in group['files']:
yield check_duplicate_consistency, implementation, source, file
if not helpers.permit_test('duplicate_consistency',
implementation):
yield skipped_test, implementation
continue
if os.path.isfile(
os.path.join(
'duplicate_consistency',
'{}_{}.yml'.format(scheme.name, implementation.name))):
metafile = os.path.join(
'duplicate_consistency',
'{}_{}.yml'.format(scheme.name, implementation.name))
with open(metafile, encoding='utf-8') as f:
metadata = yaml.safe_load(f.read())
for group in metadata['consistency_checks']:
source = pqclean.Implementation.by_name(
group['source']['scheme'],
group['source']['implementation'])
for file in group['files']:
yield (check_duplicate_consistency, implementation,
source, file)
def file_get_contents(filename):
with open(filename) as f:
return f.read()
@helpers.skip_windows()
def check_duplicate_consistency(implementation, source, file):
transformed_src = helpers.run_subprocess(
['sed', '-e', 's/{}/{}/g'.format(source.namespace_prefix(), implementation.namespace_prefix()), os.path.join(source.path(), file)]
['sed', '-e', 's/{}/{}/g'.format(source.namespace_prefix(),
implementation.namespace_prefix()), os.path.join(source.path(), file)]
)
this_src = file_get_contents(os.path.join(implementation.path(), file))
print(os.path.join(implementation.path(), file))
print(this_src)
assert(transformed_src == this_src)
if __name__ == '__main__':
try:
import nose2

10
test/test_dynamic_memory.py
View File

@ -4,19 +4,17 @@ Checks that no dynamic memory functions are used
import pqclean
import helpers
import sys
import unittest
def test_dynamic_memory():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('dynamic_memory', implementation):
# Keep this loop outside, to allow multiple assertions
for function in ['malloc', 'free', 'realloc', 'calloc']:
yield (check_dynamic_memory, implementation, function)
# Keep this loop outside, to allow multiple assertions
for function in ['malloc', 'free', 'realloc', 'calloc']:
yield (check_dynamic_memory, implementation, function)
@helpers.filtered_test
@helpers.skip_windows()
def check_dynamic_memory(implementation, function):
# 'make' will take care of not rebuilding existing library files

15
test/test_format.py
View File

@ -5,19 +5,20 @@ import pqclean
def test_formatting():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('format', implementation):
yield check_format, implementation
yield check_format, implementation
@helpers.filtered_test
def check_format(implementation: pqclean.Implementation):
helpers.ensure_available('astyle')
cfiles = implementation.cfiles()
hfiles = implementation.hfiles()
result = helpers.run_subprocess(['astyle',
'--dry-run',
'--options=../.astylerc',
*cfiles,
*hfiles])
result = helpers.run_subprocess(
['astyle',
'--dry-run',
'--options=../.astylerc',
*cfiles,
*hfiles])
assert(not('Formatted' in result))

8
test/test_functest.py
View File

@ -14,17 +14,16 @@ import helpers
def test_functest():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('functest', implementation):
yield check_functest, implementation
yield check_functest, implementation
def test_functest_sanitizers():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('functest_sanitizers', implementation):
yield check_functest_sanitizers, implementation
yield check_functest_sanitizers, implementation
@helpers.filtered_test
def check_functest(implementation):
helpers.make('functest',
TYPE=implementation.scheme.type,
@ -41,6 +40,7 @@ def check_functest(implementation):
)
@helpers.filtered_test
@helpers.skip_windows()
@helpers.slow_test
def check_functest_sanitizers(implementation):

4
test/test_license.py
View File

@ -11,10 +11,10 @@ import helpers
def test_license():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('license', implementation):
yield check_license, implementation
yield check_license, implementation
@helpers.filtered_test
def check_license(implementation):
p1 = os.path.join(implementation.path(), 'LICENSE')
p2 = os.path.join(implementation.path(), 'LICENSE.txt')

4
test/test_linter.py
View File

@ -12,10 +12,10 @@ additional_flags = []
def test_clang_tidy():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('linter', implementation):
yield check_tidy, implementation
yield check_tidy, implementation
@helpers.filtered_test
@helpers.skip_windows()
def check_tidy(implementation: pqclean.Implementation):
helpers.ensure_available('clang-tidy')

28
test/test_makefile_dependencies.py
View File

@ -8,20 +8,30 @@ import pqclean
import helpers
import glob
import datetime
import unittest
def _skipped_test(*args, **kwargs):
"""Used to indicate skipped tests"""
raise unittest.SkipTest("Skipped makefile dependencies test")
def test_makefile_dependencies():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('makefile_dependencies', implementation):
# initial build - want to have *all* files in place at beginning
helpers.make('clean', working_dir=implementation.path())
helpers.make(working_dir=implementation.path())
# test case for each candidate file
cfiles = glob.glob(os.path.join(implementation.path(), '*.c'))
hfiles = glob.glob(os.path.join(implementation.path(), '*.h'))
for file in (cfiles + hfiles):
yield (check_makefile_dependencies, implementation, file)
if not helpers.permit_test(
'makefile_dependencies', implementation):
yield _skipped_test, implementation
continue
# initial build - want to have *all* files in place at beginning
helpers.make('clean', working_dir=implementation.path())
helpers.make(working_dir=implementation.path())
# test case for each candidate file
cfiles = glob.glob(os.path.join(implementation.path(), '*.c'))
hfiles = glob.glob(os.path.join(implementation.path(), '*.h'))
for file in (cfiles + hfiles):
yield (check_makefile_dependencies, implementation, file)
def touch(time, *files):

4
test/test_metadata.py
View File

@ -10,10 +10,10 @@ import pqclean
def test_metadata():
for scheme in pqclean.Scheme.all_schemes():
if helpers.permit_test('metadata', scheme):
yield check_metadata, scheme
yield check_metadata, scheme
@helpers.filtered_test
def check_metadata(scheme):
metadata = scheme.metadata()

4
test/test_metadata_sizes.py
View File

@ -8,10 +8,10 @@ import helpers
def test_metadata_sizes():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('metadata_sizes', implementation):
yield check_metadata_sizes, implementation
yield check_metadata_sizes, implementation
@helpers.filtered_test
def check_metadata_sizes(implementation):
metadata = implementation.scheme.metadata()
impl_meta = next((impl for impl in metadata['implementations']

14
test/test_nistkat.py
View File

@ -2,7 +2,7 @@
Checks that (hash of the) KATs (in NIST format) produced on this platform matches
the one provided in the META file for every scheme/implementation.
Note that this only uses the first test case from the NIST-format KAT files.
Note that this only uses the first test case from the NIST-format KAT files.
The appropriate hash can be generated from the original submission's KAT file
using the command:
cat PQCkemKAT_whatever.rsp | head -n 8 | tail -n 6 | sha256sum
@ -14,17 +14,21 @@ import pqclean
import helpers
import unittest
def test_nistkat():
for scheme in pqclean.Scheme.all_schemes():
if scheme.type != 'kem': continue
if scheme.type != 'kem':
continue
for implementation in scheme.implementations:
if helpers.permit_test('nistkat', implementation):
yield check_nistkat, implementation
yield check_nistkat, implementation
@helpers.filtered_test
def check_nistkat(implementation):
if implementation.scheme.name == "kyber768":
raise unittest.SkipTest("Temporarily skip NIST KAT check for kyber768 since it's an outdated implementation")
raise unittest.SkipTest(
"Temporarily skip NIST KAT check for kyber768 since it's "
"an outdated implementation")
helpers.make('nistkat',
TYPE=implementation.scheme.type,
SCHEME=implementation.scheme.name,

4
test/test_no_symlinks.py
View File

@ -10,10 +10,10 @@ import helpers
def test_no_symlinks():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('no_symlinks', implementation):
yield check_no_symlinks, implementation
yield check_no_symlinks, implementation
@helpers.filtered_test
def check_no_symlinks(implementation):
for file in os.listdir(implementation.path()):
fpath = os.path.join(implementation.path(), file)

4
test/test_preprocessor.py
View File

@ -5,10 +5,10 @@ import helpers
def test_preprocessor():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('preprocessor', implementation):
yield check_preprocessor, implementation
yield check_preprocessor, implementation
@helpers.filtered_test
def check_preprocessor(implementation: pqclean.Implementation):
cfiles = implementation.cfiles()
hfiles = implementation.hfiles()

4
test/test_symbol_namespace.py
View File

@ -12,10 +12,10 @@ import unittest
def test_symbol_namespace():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('symbol_namespace', implementation):
yield check_symbol_namespace, implementation
yield check_symbol_namespace, implementation
@helpers.filtered_test
def check_symbol_namespace(implementation):
if sys.platform not in ['linux', 'darwin']:
raise unittest.SkipTest("Unsupported platform")

39
test/test_testvectors.py
View File

@ -10,28 +10,27 @@ import helpers
def test_testvectors():
@helpers.filtered_test
def check_testvectors(implementation):
helpers.make('testvectors',
TYPE=implementation.scheme.type,
SCHEME=implementation.scheme.name,
IMPLEMENTATION=implementation.name,
working_dir=os.path.join('..', 'test'))
out = helpers.run_subprocess(
[os.path.join('..', 'bin', 'testvectors_{}_{}{}'.format(
implementation.scheme.name,
implementation.name,
'.exe' if os.name == 'nt' else ''
))],
os.path.join('..', 'bin'),
).replace('\r', '')
assert(implementation.scheme.metadata()['testvectors-sha256'].lower()
== hashlib.sha256(out.encode('utf-8')).hexdigest().lower())
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('testvectors', implementation):
yield check_vectors, implementation
def check_vectors(implementation):
helpers.make('testvectors',
TYPE=implementation.scheme.type,
SCHEME=implementation.scheme.name,
IMPLEMENTATION=implementation.name,
working_dir=os.path.join('..', 'test'))
out = helpers.run_subprocess(
[os.path.join('..', 'bin', 'testvectors_{}_{}{}'.format(
implementation.scheme.name,
implementation.name,
'.exe' if os.name == 'nt' else ''
))],
os.path.join('..', 'bin'),
).replace('\r', '')
assert(implementation.scheme.metadata()['testvectors-sha256'].lower()
== hashlib.sha256(out.encode('utf-8')).hexdigest().lower())
yield check_testvectors, implementation
if __name__ == '__main__':

4
test/test_valgrind.py
View File

@ -13,11 +13,11 @@ import helpers
def test_functest():
for scheme in pqclean.Scheme.all_schemes():
for implementation in scheme.implementations:
if helpers.permit_test('valgrind', implementation):
yield check_valgrind, implementation
yield check_valgrind, implementation
@helpers.slow_test
@helpers.filtered_test
def check_valgrind(implementation: pqclean.Implementation):
if (platform.machine() not in ('i386', 'x86_64') or
platform.system() != 'Linux'):