diff --git a/Makefile b/Makefile index 44d53d92..1abfdec7 100644 --- a/Makefile +++ b/Makefile @@ -1,9 +1,4 @@ -# This -Wall was supported by the European Commission through the ERC Starting Grant 805031 (EPOQUE) -CFLAGS=-Wall -Wextra -Wpedantic -Werror -std=c99 -g $(EXTRAFLAGS) - ALL_SCHEMES=$(filter-out crypto_%.c, $(wildcard crypto_*/*)) -COMMON_FILES = common/fips202.c common/sha2.c -RANDOM_IMPL = common/randombytes.c default: help @@ -14,60 +9,6 @@ ifndef SCHEME $(error The SCHEME variable is not set. Example: SCHEME=crypto_kem/kyber768) endif -bin/functest_$(subst /,_,$(SCHEME)): test/$(dir $(SCHEME))functest.c $(wildcard $(SCHEME)/clean/*.c) $(wildcard $(SCHEME)/clean/*.h) | require_scheme - mkdir -p bin - $(CC) $(CFLAGS) \ - -DPQCLEAN_NAMESPACE=$(shell echo PQCLEAN_$(subst -,,$(notdir $(SCHEME)))_CLEAN | tr a-z A-Z) \ - -iquote "./common/" \ - -iquote "$(SCHEME)/clean/" \ - -o bin/functest_$(subst /,_,$(SCHEME)) \ - $(SCHEME)/clean/*.c \ - $(COMMON_FILES) \ - $(RANDOM_IMPL) \ - $< - -.PHONY: functest -functest: bin/functest_$(subst /,_,$(SCHEME)) - -.PHONY: run-functest -run-functest: bin/functest_$(subst /,_,$(SCHEME)) - ./$< - -.PHONY: run-valgrind -run-valgrind: bin/functest_$(subst /,_,$(SCHEME)) -ifeq ($(shell uname -s),Linux) - valgrind --leak-check=full --error-exitcode=1 $< -else - @echo "Valgrind not supported on this platform." -endif - -bin/sanitizer_$(subst /,_,$(SCHEME)): test/$(dir $(SCHEME))functest.c $(wildcard $(SCHEME)/clean/*.c) $(wildcard $(SCHEME)/clean/*.h) | require_scheme - mkdir -p bin - $(CC) $(CFLAGS) -fsanitize=address,undefined \ - -DPQCLEAN_NAMESPACE=$(shell echo PQCLEAN_$(subst -,,$(notdir $(SCHEME)))_CLEAN | tr a-z A-Z) \ - -iquote "./common/" \ - -iquote "$(SCHEME)/clean/" \ - -o bin/sanitizer_$(subst /,_,$(SCHEME)) \ - $(COMMON_FILES) \ - $(RANDOM_IMPL) \ - $(SCHEME)/clean/*.c \ - $< - -.PHONY: sanitizer -sanitizer: bin/sanitizer_$(subst /,_,$(SCHEME)) - -bin/shared_$(subst /,_,$(SCHEME))_clean.so: $(wildcard $(SCHEME)/clean/*.c) | require_scheme - mkdir -p bin - $(CC) $(CFLAGS) \ - -DPQCLEAN_NAMESPACE=$(shell echo PQCLEAN_$(subst -,,$(notdir $(SCHEME)))_CLEAN | tr a-z A-Z) \ - -nostdlib \ - -shared \ - -fPIC \ - -iquote "./common/" \ - -iquote "$(SCHEME)/clean/" \ - -o $@ \ - $^ - .PHONY: clean clean: rm -rf bin @@ -98,14 +39,6 @@ apply-tidy: # The below should be outlined with ts=8 .PHONY: help help: - @echo "make test-all Run all tests" - @echo "make functest SCHEME=scheme Build functional tests for SCHEME" - @echo "make functest-all Build functional tests for all schemes" - @echo "make run-functest SCHEME=scheme Run functional tests for SCHEME" - @echo "make run-functest-all Run all functests" - @echo "make run-sanitizer-all Run address sanitizer for all schemes" - @echo "make run-valgrind SCHEME=scheme Run valgrind checks for SCHEME" - @echo "make run-valgrind-all Run valgrind checks all schemes" @echo "make clean Clean up the bin/ folder" @echo "make format Automatically formats all the source code" @echo "make tidy SCHEME=scheme Runs the clang-tidy linter against SCHEME" @@ -114,42 +47,6 @@ help: @echo "make apply-tidy-all Tidy up all schemes" @echo "make help Displays this message" -.PHONY: functest-all -functest-all: - @for scheme in $(ALL_SCHEMES); do \ - $(MAKE) functest SCHEME=$$scheme || exit 1; \ - done - -.PHONY: sanitizer-all -sanitizer-all: - @for scheme in $(ALL_SCHEMES); do \ - $(MAKE) sanitizer SCHEME=$$scheme || exit 1; \ - done - -.PHONY: run-valgrind-all -run-valgrind-all: - @for scheme in $(ALL_SCHEMES); do \ - $(MAKE) run-valgrind SCHEME=$$scheme || exit 1; \ - done - -.PHONY: run-functest-all -run-functest-all: functest-all - @for functest in $$(find bin/ -maxdepth 1 -name 'functest_*' -not -type d) ; do \ - echo ./$$functest ; \ - ./$$functest || exit 1 ;\ - done - @echo Tests completed - -.PHONY: run-sanitizer-all -run-sanitizer-all: sanitizer-all - @for sanitizer in $$(find bin/ -maxdepth 1 -name 'sanitizer_*' -not -type d) ; do \ - echo ./$$sanitizer ; \ - ./$$sanitizer || exit 1 ;\ - done - @echo Tests completed - -.PHONY: test-all -test-all: run-functest-all run-valgrind-all run-sanitizer-all .PHONY: tidy-all tidy-all: diff --git a/test/Makefile b/test/Makefile index 31967c2f..5d184dff 100644 --- a/test/Makefile +++ b/test/Makefile @@ -14,6 +14,7 @@ COMMON_FILES=$(COMMON_DIR)/fips202.c $(COMMON_DIR)/sha2.c COMMON_HEADERS=$(COMMON_DIR)/fips202.h $(COMMON_DIR)/randombytes.h $(COMMON_DIR)/sha2.h DEST_DIR=../bin +# This -Wall was supported by the European Commission through the ERC Starting Grant 805031 (EPOQUE) CFLAGS=-Wall -Wextra -Wpedantic -Werror -std=c99 -I$(COMMON_DIR) $(EXTRAFLAGS) all: $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION) $(DEST_DIR)/testvectors_$(SCHEME)_$(IMPLEMENTATION) @@ -22,6 +23,16 @@ all: $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION) $(DEST_DIR)/testvectors_$( build-scheme: cd $(SCHEME_DIR) && $(MAKE) +.PHONY: clean-scheme +clean-scheme: + cd $(SCHEME_DIR) && $(MAKE) clean + +.PHONY: functest +functest: $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION) + +.PHONY: testvectors +testvectors: $(DEST_DIR)/testvectors_$(SCHEME)_$(IMPLEMENTATION) + $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION): build-scheme crypto_$(TYPE)/functest.c $(COMMON_FILES) $(COMMON_DIR)/randombytes.c $(COMMON_HEADERS) mkdir -p $(DEST_DIR) $(CC) $(CFLAGS) -DPQCLEAN_NAMESPACE=PQCLEAN_$(SCHEME_UPPERCASE)_$(IMPLEMENTATION_UPPERCASE) -I$(SCHEME_DIR) crypto_$(TYPE)/functest.c $(COMMON_FILES) $(COMMON_DIR)/notrandombytes.c -o $@ -L$(SCHEME_DIR) -l$(SCHEME)_$(IMPLEMENTATION) diff --git a/test/Makefile.Microsoft_nmake b/test/Makefile.Microsoft_nmake index 60e0682f..c37a727e 100644 --- a/test/Makefile.Microsoft_nmake +++ b/test/Makefile.Microsoft_nmake @@ -2,6 +2,7 @@ # nmake /f Makefile.Microsoft_nmake # override as desired +# vim: set ts=4 sw=4 et: TYPE=kem SCHEME=kyber768 SCHEME_UPPERCASE=KYBER768 @@ -25,6 +26,11 @@ build-scheme: nmake /f Makefile.Microsoft_nmake cd ..\..\..\test +clean-scheme: + cd $(SCHEME_DIR) + nmake /f Makefile.Microsoft_nmake clean + cd ..\..\..\test + $(DEST_DIR)\functest_$(SCHEME)_$(IMPLEMENTATION).EXE: build-scheme $(COMMON_OBJECTS) $(COMMON_DIR)\randombytes.obj -MKDIR $(DEST_DIR) $(CC) /c crypto_$(TYPE)\functest.c $(CFLAGS) /I $(SCHEME_DIR) /DPQCLEAN_NAMESPACE=PQCLEAN_$(SCHEME_UPPERCASE)_$(IMPLEMENTATION_UPPERCASE) diff --git a/test/helpers.py b/test/helpers.py index 6b558e7e..09a20985 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -1,11 +1,17 @@ +import os import subprocess -def run_subprocess(command, working_dir='.', expected_returncode=0): +def run_subprocess(command, working_dir='.', env=None, expected_returncode=0): """ Helper function to run a shell command and report success/failure depending on the exit status of the shell command. """ + if env is not None: + env_ = os.environ.copy() + env_.update(env) + env = env_ + # Note we need to capture stdout/stderr from the subprocess, # then print it, which nose/unittest will then capture and # buffer appropriately @@ -13,9 +19,29 @@ def run_subprocess(command, working_dir='.', expected_returncode=0): command, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, - cwd=working_dir + cwd=working_dir, + env=env, ) print(working_dir + " > " + " ".join(command)) print(result.stdout.decode('utf-8')) assert(result.returncode == expected_returncode) return result.stdout.decode('utf-8') + + +def make(*args, working_dir='.', env=None, **kwargs): + """ + Runs a make target in the specified working directory + + Usage: + make('clean', 'targetb', SCHEME='bla') + """ + make_command = 'make' + return run_subprocess( + [ + make_command, + *args, + *['{}={}'.format(k, v) for k, v in kwargs.items()], + ], + working_dir=working_dir, + env=env, + ) diff --git a/test/test_functest.py b/test/test_functest.py index c4d0fbbe..77846618 100644 --- a/test/test_functest.py +++ b/test/test_functest.py @@ -4,6 +4,9 @@ and executed for every scheme/implementation. """ import os +import platform +import unittest + import pqclean import helpers @@ -14,6 +17,12 @@ def test_functest(): yield check_functest, scheme.name, implementation.name +def test_functest_sanitizers(): + for scheme in pqclean.Scheme.all_schemes(): + for implementation in scheme.implementations: + yield check_functest_sanitizers, scheme.name, implementation.name + + def check_functest(scheme_name, implementation_name): implementation = pqclean.Implementation.by_name( scheme_name, implementation_name) @@ -30,6 +39,36 @@ def check_functest(scheme_name, implementation_name): ) +def check_functest_sanitizers(scheme_name, implementation_name): + env = None + if platform.machine() == 'ppc' and os.environ.get('CC', 'gcc') == 'clang': + raise unittest.SkipTest() + elif platform.machine() in ['armv7l', 'aarch64']: + env = {'ASAN_OPTIONS': 'detect_leaks=0'} + else: + print("Supported platform: {}".format(platform.machine())) + implementation = pqclean.Implementation.by_name( + scheme_name, implementation_name) + helpers.make('clean-scheme', 'functest', + TYPE=implementation.scheme.type, + SCHEME=scheme_name, + IMPLEMENTATION=implementation_name, + EXTRAFLAGS='-fsanitize=address,undefined', + working_dir=os.path.join('..', 'test'), + env=env) + helpers.run_subprocess( + ['./functest_{}_{}'.format(scheme_name, implementation_name)], + os.path.join('..', 'bin'), + env=env, + ) + # Remove files with ASAN library compiled in + helpers.make('clean-scheme', + TYPE=implementation.scheme.type, + SCHEME=scheme_name, + IMPLEMENTATION=implementation_name, + working_dir=os.path.join('..', 'test')) + + if __name__ == '__main__': try: import nose2 diff --git a/test/test_valgrind.py b/test/test_valgrind.py new file mode 100644 index 00000000..219791cd --- /dev/null +++ b/test/test_valgrind.py @@ -0,0 +1,40 @@ +""" +Runs the test files under valgrind to detect memory problems +""" + +import os +import platform +import unittest + +import pqclean +import helpers + + +def test_functest(): + for scheme in pqclean.Scheme.all_schemes(): + for implementation in scheme.implementations: + yield check_valgrind, implementation + + +def check_valgrind(implementation: pqclean.Implementation): + if (platform.machine() not in ('i386', 'x86_64') or + platform.system() != 'Linux'): + raise unittest.SkipTest() + + helpers.make(TYPE=implementation.scheme.type, + SCHEME=implementation.scheme.name, + IMPLEMENTATION=implementation.name, + working_dir=os.path.join('..', 'test')) + functest_name = './functest_{}_{}'.format(implementation.scheme.name, + implementation.name) + helpers.run_subprocess(['valgrind', functest_name], + os.path.join('..', 'bin')) + + +if __name__ == '__main__': + try: + import nose2 + nose2.main() + except ImportError: + import nose + nose.runmodule()