From 17c9840b57ad2b2324cd05bf231fd3aa11bb4d29 Mon Sep 17 00:00:00 2001
From: Thom Wiggers <thom@thomwiggers.nl>
Date: Thu, 28 Feb 2019 14:05:55 +0100
Subject: [PATCH] Move valgrind tests to python-based testing framework

---
 Makefile              | 58 -------------------------------------------
 test/Makefile         |  1 +
 test/test_valgrind.py | 40 +++++++++++++++++++++++++++++
 3 files changed, 41 insertions(+), 58 deletions(-)
 create mode 100644 test/test_valgrind.py

diff --git a/Makefile b/Makefile
index f0b2eb5b..1abfdec7 100644
--- a/Makefile
+++ b/Makefile
@@ -1,9 +1,4 @@
-# This -Wall was supported by the European Commission through the ERC Starting Grant 805031 (EPOQUE)
-CFLAGS=-Wall -Wextra -Wpedantic -Werror -std=c99 -g $(EXTRAFLAGS)
-
 ALL_SCHEMES=$(filter-out crypto_%.c, $(wildcard crypto_*/*))
-COMMON_FILES = common/fips202.c common/sha2.c
-RANDOM_IMPL = common/randombytes.c
 
 default: help
 
@@ -14,45 +9,6 @@ ifndef SCHEME
 	$(error The SCHEME variable is not set. Example: SCHEME=crypto_kem/kyber768)
 endif
 
-bin/functest_$(subst /,_,$(SCHEME)): test/$(dir $(SCHEME))functest.c $(wildcard $(SCHEME)/clean/*.c) $(wildcard $(SCHEME)/clean/*.h) | require_scheme
-	mkdir -p bin
-	$(CC) $(CFLAGS) \
-		-DPQCLEAN_NAMESPACE=$(shell echo PQCLEAN_$(subst -,,$(notdir $(SCHEME)))_CLEAN | tr a-z A-Z) \
-		-iquote "./common/" \
-		-iquote "$(SCHEME)/clean/" \
-		-o bin/functest_$(subst /,_,$(SCHEME)) \
-		$(SCHEME)/clean/*.c \
-		$(COMMON_FILES) \
-		$(RANDOM_IMPL) \
-		$<
-
-.PHONY: functest
-functest: bin/functest_$(subst /,_,$(SCHEME))
-
-.PHONY: run-functest
-run-functest: bin/functest_$(subst /,_,$(SCHEME))
-	./$<
-
-.PHONY: run-valgrind
-run-valgrind: bin/functest_$(subst /,_,$(SCHEME))
-ifeq ($(shell uname -s),Linux)
-	valgrind --leak-check=full --error-exitcode=1 $<
-else
-	@echo "Valgrind not supported on this platform."
-endif
-
-bin/shared_$(subst /,_,$(SCHEME))_clean.so: $(wildcard $(SCHEME)/clean/*.c) | require_scheme
-	mkdir -p bin
-	$(CC) $(CFLAGS) \
-		-DPQCLEAN_NAMESPACE=$(shell echo PQCLEAN_$(subst -,,$(notdir $(SCHEME)))_CLEAN | tr a-z A-Z) \
-		-nostdlib \
-		-shared \
-		-fPIC \
-		-iquote "./common/" \
-		-iquote "$(SCHEME)/clean/" \
-		-o $@ \
-		$^
-
 .PHONY: clean
 clean:
 	rm -rf bin
@@ -83,11 +39,6 @@ apply-tidy:
 # The below should be outlined with ts=8
 .PHONY: help
 help:
-	@echo "make test-all				Run all tests"
-	@echo "make functest SCHEME=scheme		Build functional tests for SCHEME"
-	@echo "make run-functest SCHEME=scheme		Run functional tests for SCHEME"
-	@echo "make run-valgrind SCHEME=scheme		Run valgrind checks for SCHEME"
-	@echo "make run-valgrind-all			Run valgrind checks all schemes"
 	@echo "make clean				Clean up the bin/ folder"
 	@echo "make format				Automatically formats all the source code"
 	@echo "make tidy SCHEME=scheme  		Runs the clang-tidy linter against SCHEME"
@@ -97,15 +48,6 @@ help:
 	@echo "make help				Displays this message"
 
 
-.PHONY: run-valgrind-all
-run-valgrind-all:
-	@for scheme in $(ALL_SCHEMES); do \
-	    $(MAKE) run-valgrind SCHEME=$$scheme || exit 1; \
-	done
-
-.PHONY: test-all
-test-all: run-valgrind-all
-
 .PHONY: tidy-all
 tidy-all:
 	@for scheme in $(ALL_SCHEMES); do \
diff --git a/test/Makefile b/test/Makefile
index a048d7a6..5d184dff 100644
--- a/test/Makefile
+++ b/test/Makefile
@@ -14,6 +14,7 @@ COMMON_FILES=$(COMMON_DIR)/fips202.c $(COMMON_DIR)/sha2.c
 COMMON_HEADERS=$(COMMON_DIR)/fips202.h $(COMMON_DIR)/randombytes.h $(COMMON_DIR)/sha2.h
 DEST_DIR=../bin
 
+# This -Wall was supported by the European Commission through the ERC Starting Grant 805031 (EPOQUE)
 CFLAGS=-Wall -Wextra -Wpedantic -Werror -std=c99 -I$(COMMON_DIR) $(EXTRAFLAGS)
 
 all: $(DEST_DIR)/functest_$(SCHEME)_$(IMPLEMENTATION) $(DEST_DIR)/testvectors_$(SCHEME)_$(IMPLEMENTATION)
diff --git a/test/test_valgrind.py b/test/test_valgrind.py
new file mode 100644
index 00000000..219791cd
--- /dev/null
+++ b/test/test_valgrind.py
@@ -0,0 +1,40 @@
+"""
+Runs the test files under valgrind to detect memory problems
+"""
+
+import os
+import platform
+import unittest
+
+import pqclean
+import helpers
+
+
+def test_functest():
+    for scheme in pqclean.Scheme.all_schemes():
+        for implementation in scheme.implementations:
+            yield check_valgrind, implementation
+
+
+def check_valgrind(implementation: pqclean.Implementation):
+    if (platform.machine() not in ('i386', 'x86_64') or
+            platform.system() != 'Linux'):
+        raise unittest.SkipTest()
+
+    helpers.make(TYPE=implementation.scheme.type,
+                 SCHEME=implementation.scheme.name,
+                 IMPLEMENTATION=implementation.name,
+                 working_dir=os.path.join('..', 'test'))
+    functest_name = './functest_{}_{}'.format(implementation.scheme.name,
+                                              implementation.name)
+    helpers.run_subprocess(['valgrind', functest_name],
+                           os.path.join('..', 'bin'))
+
+
+if __name__ == '__main__':
+    try:
+        import nose2
+        nose2.main()
+    except ImportError:
+        import nose
+        nose.runmodule()