Merge pull request #64 from PQClean/valgrind_in_python

Valgrind in python

Makefile

@@ -1,9 +1,4 @@
-# This -Wall was supported by the European Commission through the ERC Starting Grant 805031 (EPOQUE)
-CFLAGS=-Wall -Wextra -Wpedantic -Werror -std=c99 -g $(EXTRAFLAGS)
-
 ALL_SCHEMES=$(filter-out crypto_%.c, $(wildcard crypto_*/*))
-COMMON_FILES = common/fips202.c common/sha2.c
-RANDOM_IMPL = common/randombytes.c
 
 default: help
 
@@ -14,60 +9,6 @@ ifndef SCHEME
 	$(error The SCHEME variable is not set. Example: SCHEME=crypto_kem/kyber768)
 endif
 
-bin/functest_$(subst /,_,$(SCHEME)): test/$(dir $(SCHEME))functest.c $(wildcard $(SCHEME)/clean/*.c) $(wildcard $(SCHEME)/clean/*.h) | require_scheme
-	mkdir -p bin
-	$(CC) $(CFLAGS) \
-		-DPQCLEAN_NAMESPACE=$(shell echo PQCLEAN_$(subst -,,$(notdir $(SCHEME)))_CLEAN | tr a-z A-Z) \
-		-iquote "./common/" \
-		-iquote "$(SCHEME)/clean/" \
-		-o bin/functest_$(subst /,_,$(SCHEME)) \
-		$(SCHEME)/clean/*.c \
-		$(COMMON_FILES) \
-		$(RANDOM_IMPL) \
-		$<
-
-.PHONY: functest
-functest: bin/functest_$(subst /,_,$(SCHEME))
-
-.PHONY: run-functest
-run-functest: bin/functest_$(subst /,_,$(SCHEME))
-	./$<
-
-.PHONY: run-valgrind
-run-valgrind: bin/functest_$(subst /,_,$(SCHEME))
-ifeq ($(shell uname -s),Linux)
-	valgrind --leak-check=full --error-exitcode=1 $<
-else
-	@echo "Valgrind not supported on this platform."
-endif
-
-bin/sanitizer_$(subst /,_,$(SCHEME)): test/$(dir $(SCHEME))functest.c $(wildcard $(SCHEME)/clean/*.c) $(wildcard $(SCHEME)/clean/*.h) | require_scheme
-	mkdir -p bin
-	$(CC) $(CFLAGS) -fsanitize=address,undefined \
-		-DPQCLEAN_NAMESPACE=$(shell echo PQCLEAN_$(subst -,,$(notdir $(SCHEME)))_CLEAN | tr a-z A-Z) \
-		-iquote "./common/" \
-		-iquote "$(SCHEME)/clean/" \
-		-o bin/sanitizer_$(subst /,_,$(SCHEME)) \
-		$(COMMON_FILES) \
-		$(RANDOM_IMPL) \
-		$(SCHEME)/clean/*.c \
-		$<
-
-.PHONY: sanitizer
-sanitizer: bin/sanitizer_$(subst /,_,$(SCHEME))
-
-bin/shared_$(subst /,_,$(SCHEME))_clean.so: $(wildcard $(SCHEME)/clean/*.c) | require_scheme
-	mkdir -p bin
-	$(CC) $(CFLAGS) \
-		-DPQCLEAN_NAMESPACE=$(shell echo PQCLEAN_$(subst -,,$(notdir $(SCHEME)))_CLEAN | tr a-z A-Z) \
-		-nostdlib \
-		-shared \
-		-fPIC \
-		-iquote "./common/" \
-		-iquote "$(SCHEME)/clean/" \
-		-o $@ \
-		$^
-
 .PHONY: clean
 clean:
 	rm -rf bin
@@ -98,14 +39,6 @@ apply-tidy:
 # The below should be outlined with ts=8
 .PHONY: help
 help:
-	@echo "make test-all				Run all tests"
-	@echo "make functest SCHEME=scheme		Build functional tests for SCHEME"
-	@echo "make functest-all			Build functional tests for all schemes"
-	@echo "make run-functest SCHEME=scheme		Run functional tests for SCHEME"
-	@echo "make run-functest-all			Run all functests"
-	@echo "make run-sanitizer-all			Run address sanitizer for all schemes"
-	@echo "make run-valgrind SCHEME=scheme		Run valgrind checks for SCHEME"
-	@echo "make run-valgrind-all			Run valgrind checks all schemes"
 	@echo "make clean				Clean up the bin/ folder"
 	@echo "make format				Automatically formats all the source code"
 	@echo "make tidy SCHEME=scheme  		Runs the clang-tidy linter against SCHEME"
@@ -114,42 +47,6 @@ help:
 	@echo "make apply-tidy-all			Tidy up all schemes"
 	@echo "make help				Displays this message"
 
-.PHONY: functest-all
-functest-all:
-	@for scheme in $(ALL_SCHEMES); do \
-	    $(MAKE) functest SCHEME=$$scheme || exit 1; \
-	done
-
-.PHONY: sanitizer-all
-sanitizer-all:
-	@for scheme in $(ALL_SCHEMES); do \
-	    $(MAKE) sanitizer SCHEME=$$scheme || exit 1; \
-	done
-
-.PHONY: run-valgrind-all
-run-valgrind-all:
-	@for scheme in $(ALL_SCHEMES); do \
-	    $(MAKE) run-valgrind SCHEME=$$scheme || exit 1; \
-	done
-
-.PHONY: run-functest-all
-run-functest-all: functest-all
-	@for functest in $$(find bin/ -maxdepth 1 -name 'functest_*' -not -type d) ; do \
-		echo ./$$functest ; \
-		./$$functest || exit 1 ;\
-	done
-	@echo Tests completed
-
-.PHONY: run-sanitizer-all
-run-sanitizer-all: sanitizer-all
-	@for sanitizer in $$(find bin/ -maxdepth 1 -name 'sanitizer_*' -not -type d) ; do \
-		echo ./$$sanitizer ; \
-		./$$sanitizer || exit 1 ;\
-	done
-	@echo Tests completed
-
-.PHONY: test-all
-test-all: run-functest-all run-valgrind-all run-sanitizer-all
 
 .PHONY: tidy-all
 tidy-all:

test/Makefile

@@ -14,6 +14,7 @@ COMMON_FILES=$(COMMON_DIR)/fips202.c $(COMMON_DIR)/sha2.c
 COMMON_HEADERS=$(COMMON_DIR)/fips202.h $(COMMON_DIR)/randombytes.h $(COMMON_DIR)/sha2.h
 DEST_DIR=../bin
 
+# This -Wall was supported by the European Commission through the ERC Starting Grant 805031 (EPOQUE)
 CFLAGS=-Wall -Wextra -Wpedantic -Werror -std=c99 -I$(COMMON_DIR) $(EXTRAFLAGS)
 
 all: $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION) $(DEST_DIR)/testvectors_$(SCHEME)_$(IMPLEMENTATION)
@@ -22,6 +23,16 @@ all: $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION) $(DEST_DIR)/testvectors_$(
 build-scheme:
 	cd $(SCHEME_DIR) && $(MAKE)
 
+.PHONY: clean-scheme
+clean-scheme:
+	cd $(SCHEME_DIR) && $(MAKE) clean
+
+.PHONY: functest
+functest: $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION)
+
+.PHONY: testvectors
+testvectors: $(DEST_DIR)/testvectors_$(SCHEME)_$(IMPLEMENTATION)
+
 $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION): build-scheme crypto_$(TYPE)/functest.c $(COMMON_FILES) $(COMMON_DIR)/randombytes.c $(COMMON_HEADERS)
 	mkdir -p $(DEST_DIR)
 	$(CC) $(CFLAGS) -DPQCLEAN_NAMESPACE=PQCLEAN_$(SCHEME_UPPERCASE)_$(IMPLEMENTATION_UPPERCASE) -I$(SCHEME_DIR) crypto_$(TYPE)/functest.c $(COMMON_FILES) $(COMMON_DIR)/notrandombytes.c -o $@ -L$(SCHEME_DIR) -l$(SCHEME)_$(IMPLEMENTATION)

test/Makefile.Microsoft_nmake

@@ -2,6 +2,7 @@
 #    nmake /f Makefile.Microsoft_nmake
 
 # override as desired
+# vim: set ts=4 sw=4 et:
 TYPE=kem
 SCHEME=kyber768
 SCHEME_UPPERCASE=KYBER768
@@ -25,6 +26,11 @@ build-scheme:
     nmake /f Makefile.Microsoft_nmake
     cd ..\..\..\test
 
+clean-scheme:
+    cd $(SCHEME_DIR)
+    nmake /f Makefile.Microsoft_nmake clean
+    cd ..\..\..\test
+
 $(DEST_DIR)\functest_$(SCHEME)_$(IMPLEMENTATION).EXE: build-scheme $(COMMON_OBJECTS) $(COMMON_DIR)\randombytes.obj
     -MKDIR $(DEST_DIR)
     $(CC) /c crypto_$(TYPE)\functest.c $(CFLAGS) /I $(SCHEME_DIR) /DPQCLEAN_NAMESPACE=PQCLEAN_$(SCHEME_UPPERCASE)_$(IMPLEMENTATION_UPPERCASE)

test/helpers.py

@@ -1,11 +1,17 @@
+import os
 import subprocess
 
 
-def run_subprocess(command, working_dir='.', expected_returncode=0):
+def run_subprocess(command, working_dir='.', env=None, expected_returncode=0):
     """
     Helper function to run a shell command and report success/failure
     depending on the exit status of the shell command.
     """
+    if env is not None:
+        env_ = os.environ.copy()
+        env_.update(env)
+        env = env_
+
     # Note we need to capture stdout/stderr from the subprocess,
     # then print it, which nose/unittest will then capture and
     # buffer appropriately
@@ -13,9 +19,29 @@ def run_subprocess(command, working_dir='.', expected_returncode=0):
         command,
         stdout=subprocess.PIPE,
         stderr=subprocess.STDOUT,
-        cwd=working_dir
+        cwd=working_dir,
+        env=env,
     )
     print(working_dir + " > " + " ".join(command))
     print(result.stdout.decode('utf-8'))
     assert(result.returncode == expected_returncode)
     return result.stdout.decode('utf-8')
+
+
+def make(*args, working_dir='.', env=None, **kwargs):
+    """
+    Runs a make target in the specified working directory
+
+    Usage:
+        make('clean', 'targetb', SCHEME='bla')
+    """
+    make_command = 'make'
+    return run_subprocess(
+        [
+            make_command,
+            *args,
+            *['{}={}'.format(k, v) for k, v in kwargs.items()],
+        ],
+        working_dir=working_dir,
+        env=env,
+    )

test/test_functest.py

@@ -4,6 +4,9 @@ and executed for every scheme/implementation.
 """
 
 import os
+import platform
+import unittest
+
 import pqclean
 import helpers
 
@@ -14,6 +17,12 @@ def test_functest():
             yield check_functest, scheme.name, implementation.name
 
 
+def test_functest_sanitizers():
+    for scheme in pqclean.Scheme.all_schemes():
+        for implementation in scheme.implementations:
+            yield check_functest_sanitizers, scheme.name, implementation.name
+
+
 def check_functest(scheme_name, implementation_name):
     implementation = pqclean.Implementation.by_name(
         scheme_name, implementation_name)
@@ -30,6 +39,36 @@ def check_functest(scheme_name, implementation_name):
     )
 
 
+def check_functest_sanitizers(scheme_name, implementation_name):
+    env = None
+    if platform.machine() == 'ppc' and os.environ.get('CC', 'gcc') == 'clang':
+        raise unittest.SkipTest()
+    elif platform.machine() in ['armv7l', 'aarch64']:
+        env = {'ASAN_OPTIONS': 'detect_leaks=0'}
+    else:
+        print("Supported platform: {}".format(platform.machine()))
+    implementation = pqclean.Implementation.by_name(
+        scheme_name, implementation_name)
+    helpers.make('clean-scheme', 'functest',
+                 TYPE=implementation.scheme.type,
+                 SCHEME=scheme_name,
+                 IMPLEMENTATION=implementation_name,
+                 EXTRAFLAGS='-fsanitize=address,undefined',
+                 working_dir=os.path.join('..', 'test'),
+                 env=env)
+    helpers.run_subprocess(
+        ['./functest_{}_{}'.format(scheme_name, implementation_name)],
+        os.path.join('..', 'bin'),
+        env=env,
+    )
+    # Remove files with ASAN library compiled in
+    helpers.make('clean-scheme',
+                 TYPE=implementation.scheme.type,
+                 SCHEME=scheme_name,
+                 IMPLEMENTATION=implementation_name,
+                 working_dir=os.path.join('..', 'test'))
+
+
 if __name__ == '__main__':
     try:
         import nose2

test/test_valgrind.py

@@ -0,0 +1,40 @@
+"""
+Runs the test files under valgrind to detect memory problems
+"""
+
+import os
+import platform
+import unittest
+
+import pqclean
+import helpers
+
+
+def test_functest():
+    for scheme in pqclean.Scheme.all_schemes():
+        for implementation in scheme.implementations:
+            yield check_valgrind, implementation
+
+
+def check_valgrind(implementation: pqclean.Implementation):
+    if (platform.machine() not in ('i386', 'x86_64') or
+            platform.system() != 'Linux'):
+        raise unittest.SkipTest()
+
+    helpers.make(TYPE=implementation.scheme.type,
+                 SCHEME=implementation.scheme.name,
+                 IMPLEMENTATION=implementation.name,
+                 working_dir=os.path.join('..', 'test'))
+    functest_name = './functest_{}_{}'.format(implementation.scheme.name,
+                                              implementation.name)
+    helpers.run_subprocess(['valgrind', functest_name],
+                           os.path.join('..', 'bin'))
+
+
+if __name__ == '__main__':
+    try:
+        import nose2
+        nose2.main()
+    except ImportError:
+        import nose
+        nose.runmodule()