From 3e2c7b6b0555551a1c1023a65362118afc968a7e Mon Sep 17 00:00:00 2001 From: Thom Wiggers Date: Tue, 10 Nov 2020 17:51:33 +0100 Subject: [PATCH 1/4] Disable skipping slow tests --- test/helpers.py | 10 ---------- test/test_functest.py | 1 - test/test_valgrind.py | 1 - 3 files changed, 12 deletions(-) diff --git a/test/helpers.py b/test/helpers.py index 5eee1d29..db1c839d 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -156,16 +156,6 @@ def skip_windows(message="This test is not supported on Windows"): return wrapper -def slow_test(f): - @functools.wraps(f) - def wrapper(*args, **kwargs): - if ('CI' in os.environ and 'RUN_SLOW' not in os.environ and - os.environ.get('TRAVIS_EVENT_TYPE') != 'cron'): - raise unittest.SkipTest("Slow test skipped on CI run") - return f(*args, **kwargs) - return wrapper - - @lru_cache(maxsize=None) def ensure_available(executable): """ diff --git a/test/test_functest.py b/test/test_functest.py index 53171217..8249e6cd 100644 --- a/test/test_functest.py +++ b/test/test_functest.py @@ -50,7 +50,6 @@ def test_functest(implementation, impl_path, test_dir, ) @helpers.skip_windows() @helpers.filtered_test -@helpers.slow_test def test_functest_sanitizers(implementation, impl_path, test_dir, init, destr): dest_dir = os.path.join(test_dir, 'bin') diff --git a/test/test_valgrind.py b/test/test_valgrind.py index 32715b41..061758be 100644 --- a/test/test_valgrind.py +++ b/test/test_valgrind.py @@ -30,7 +30,6 @@ def valgrind_supports_exit_early(): for impl in pqclean.Scheme.all_supported_implementations()], ids=[str(impl) for impl in pqclean.Scheme.all_supported_implementations()], ) -@helpers.slow_test @helpers.filtered_test def test_valgrind(implementation: pqclean.Implementation, impl_path, test_dir, init, destr): From 7557be7471c6da75d9a77b00780ffb5bacf63e42 Mon Sep 17 00:00:00 2001 From: Thom Wiggers Date: Wed, 18 Nov 2020 09:27:46 +0100 Subject: [PATCH 2/4] Disable asan if clang is mentioned in CC (like in 'ccache clang') --- test/test_functest.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test_functest.py b/test/test_functest.py index 8249e6cd..c23f9b97 100644 --- a/test/test_functest.py +++ b/test/test_functest.py @@ -54,7 +54,7 @@ def test_functest_sanitizers(implementation, impl_path, test_dir, init, destr): dest_dir = os.path.join(test_dir, 'bin') env = None - if platform.machine() == 'ppc' and os.environ.get('CC', 'gcc') == 'clang': + if platform.machine() == 'ppc' and 'clang' in os.environ.get('CC', 'gcc'): raise unittest.SkipTest("Clang does not support ASAN on ppc") elif platform.machine() in ['armv7l', 'aarch64']: env = {'ASAN_OPTIONS': 'detect_leaks=0'} From fedc88be4b72dd8e657f220356a20e0650a77068 Mon Sep 17 00:00:00 2001 From: Thom Wiggers Date: Tue, 24 Nov 2020 11:20:24 +0100 Subject: [PATCH 3/4] Disable slowest tests on travis --- .travis.yml | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index f2a4fdd9..ad3750c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,7 +14,7 @@ matrix: - git checkout $TRAVIS_BRANCH - git reset --hard $COMMIT script: - - travis_wait 60 docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF --rm -v ~/.ccache:/ccache -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c " + - travis_wait 60 docker run -e CI=true -e PQCLEAN_SKIP_TESTS -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF --rm -v ~/.ccache:/ccache -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c " uname -a && export CCACHE_DIR=/ccache && export CCACHE_NOSTATS=1 && @@ -26,6 +26,7 @@ matrix: env: PQCLEAN_ONLY_TYPES: kem PQCLEAN_ONLY_DIFF: 1 + PQCLEAN_SKIP_TESTS: valgrind,sanitizer services: - docker - name: "KEMs: ARM64 + Clang" @@ -39,7 +40,7 @@ matrix: - git checkout $TRAVIS_BRANCH - git reset --hard $COMMIT script: - - travis_wait 60 docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF --rm -v ~/.ccache:/ccache -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c " + - travis_wait 60 docker run -e CI=true -e PQCLEAN_SKIP_TESTS -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF --rm -v ~/.ccache:/ccache -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c " uname -a && export CCACHE_NOSTATS=1 && export CCACHE_DIR=/ccache && @@ -51,6 +52,7 @@ matrix: env: PQCLEAN_ONLY_TYPES: kem PQCLEAN_ONLY_DIFF: 1 + PQCLEAN_SKIP_TESTS: valgrind,sanitizer services: - docker - name: "Signs: ARM64 + GCC" @@ -64,7 +66,7 @@ matrix: - git checkout $TRAVIS_BRANCH - git reset --hard $COMMIT script: - - travis_wait 60 docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF -e PQCLEAN_SKIP_SCHEMES --rm -v ~/.ccache:/ccache -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c " + - travis_wait 60 docker run -e CI=true -e PQCLEAN_SKIP_TESTS -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF -e PQCLEAN_SKIP_SCHEMES --rm -v ~/.ccache:/ccache -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c " uname -a && export CCACHE_NOSTATS=1 && export CCACHE_DIR=/ccache && @@ -77,6 +79,7 @@ matrix: PQCLEAN_ONLY_TYPES: sign PQCLEAN_ONLY_DIFF: 1 PQCLEAN_SKIP_SCHEMES: sphincs-haraka-128f-robust,sphincs-haraka-192s-robust,sphincs-sha256-128f-robust,sphincs-sha256-192s-robust,sphincs-shake256-128f-robust,sphincs-shake256-192s-robust,sphincs-haraka-128f-simple,sphincs-haraka-192s-simple,sphincs-sha256-128f-simple,sphincs-sha256-192s-simple,sphincs-shake256-128f-simple,sphincs-shake256-192s-simple,sphincs-haraka-128s-robust,sphincs-haraka-256f-robust,sphincs-sha256-128s-robust,sphincs-sha256-256f-robust,sphincs-shake256-128s-robust,sphincs-shake256-256f-robust,sphincs-haraka-128s-simple,sphincs-haraka-256f-simple,sphincs-sha256-128s-simple,sphincs-sha256-256f-simple,sphincs-shake256-128s-simple,sphincs-shake256-256f-simple,sphincs-haraka-192f-robust,sphincs-haraka-256s-robust,sphincs-sha256-192f-robust,sphincs-sha256-256s-robust,sphincs-shake256-192f-robust,sphincs-shake256-256s-robust,sphincs-haraka-192f-simple,sphincs-haraka-256s-simple,sphincs-sha256-192f-simple,sphincs-sha256-256s-simple,sphincs-shake256-192f-simple,sphincs-shake256-256s-simple + PQCLEAN_SKIP_TESTS: valgrind,sanitizer services: - docker - name: "signs: ARM64 + Clang" @@ -90,7 +93,7 @@ matrix: - git checkout $TRAVIS_BRANCH - git reset --hard $COMMIT script: - - travis_wait 60 docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF=1 -e PQCLEAN_SKIP_SCHEMES --rm -v ~/.ccache:/ccache -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c " + - travis_wait 60 docker run -e CI=true -e PQCLEAN_SKIP_TESTS -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF=1 -e PQCLEAN_SKIP_SCHEMES --rm -v ~/.ccache:/ccache -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c " uname -a && export CCACHE_NOSTATS=1 && export CCACHE_DIR=/ccache && @@ -103,6 +106,7 @@ matrix: PQCLEAN_ONLY_TYPES: sign PQCLEAN_ONLY_DIFF: 1 PQCLEAN_SKIP_SCHEMES: sphincs-haraka-128f-robust,sphincs-haraka-192s-robust,sphincs-sha256-128f-robust,sphincs-sha256-192s-robust,sphincs-shake256-128f-robust,sphincs-shake256-192s-robust,sphincs-haraka-128f-simple,sphincs-haraka-192s-simple,sphincs-sha256-128f-simple,sphincs-sha256-192s-simple,sphincs-shake256-128f-simple,sphincs-shake256-192s-simple,sphincs-haraka-128s-robust,sphincs-haraka-256f-robust,sphincs-sha256-128s-robust,sphincs-sha256-256f-robust,sphincs-shake256-128s-robust,sphincs-shake256-256f-robust,sphincs-haraka-128s-simple,sphincs-haraka-256f-simple,sphincs-sha256-128s-simple,sphincs-sha256-256f-simple,sphincs-shake256-128s-simple,sphincs-shake256-256f-simple,sphincs-haraka-192f-robust,sphincs-haraka-256s-robust,sphincs-sha256-192f-robust,sphincs-sha256-256s-robust,sphincs-shake256-192f-robust,sphincs-shake256-256s-robust,sphincs-haraka-192f-simple,sphincs-haraka-256s-simple,sphincs-sha256-192f-simple,sphincs-sha256-256s-simple,sphincs-shake256-192f-simple,sphincs-shake256-256s-simple + PQCLEAN_SKIP_TESTS: valgrind,sanitizer services: - docker From 0d88ed65d91a9064f6bfa047818bcddd6ca445ba Mon Sep 17 00:00:00 2001 From: Thom Wiggers Date: Tue, 1 Dec 2020 12:33:33 +0100 Subject: [PATCH 4/4] Disable SPHINCS+ test taking too much RAM in ASAN --- test/test_functest.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/test/test_functest.py b/test/test_functest.py index c23f9b97..c9a9db4b 100644 --- a/test/test_functest.py +++ b/test/test_functest.py @@ -54,6 +54,11 @@ def test_functest_sanitizers(implementation, impl_path, test_dir, init, destr): dest_dir = os.path.join(test_dir, 'bin') env = None + if (implementation.scheme.name == "sphincs-sha256-192s-robust" + and 'CI' in os.environ + and implementation.name == "clean" + and 'clang' in os.environ.get('CC', '')): + raise unittest.SkipTest("Clang makes this test use too much RAM") if platform.machine() == 'ppc' and 'clang' in os.environ.get('CC', 'gcc'): raise unittest.SkipTest("Clang does not support ASAN on ppc") elif platform.machine() in ['armv7l', 'aarch64']: