From 9e4d07dba133f0691e694ca16f657d0f97fa291e Mon Sep 17 00:00:00 2001 From: Thom Wiggers Date: Mon, 22 Jun 2020 04:10:07 +0200 Subject: [PATCH] Speed up test collection (#298) * don't do filesystem operations during collection * Greatly speed up test collection * fixup! Greatly speed up test collection * Silence junit warning * fixup! Greatly speed up test collection --- test/helpers.py | 45 +++++++++++++++++++++++++-------------------- test/pqclean.py | 15 +++++++++++++-- test/pytest.ini | 1 + 3 files changed, 39 insertions(+), 22 deletions(-) diff --git a/test/helpers.py b/test/helpers.py index a8b2a27b..5eee1d29 100644 --- a/test/helpers.py +++ b/test/helpers.py @@ -2,11 +2,13 @@ import atexit import functools import logging import os +import secrets import shutil +import string import subprocess import sys -import tempfile import unittest +from functools import lru_cache import pqclean @@ -22,6 +24,12 @@ def cleanup_testcases(): TEST_TEMPDIRS = [] +ALPHABET = string.ascii_letters + string.digits + '_' +def mktmpdir(parent, prefix): + """Returns a unique directory name""" + uniq = ''.join(secrets.choice(ALPHABET) for i in range(8)) + return os.path.join(parent, "{}_{}".format(prefix, uniq)) + def isolate_test_files(impl_path, test_prefix, dir=os.path.join('..', 'testcases')): @@ -34,24 +42,22 @@ def isolate_test_files(impl_path, test_prefix, os.mkdir(dir) except FileExistsError: pass - test_dir = tempfile.mkdtemp(prefix=test_prefix, dir=dir) + test_dir = mktmpdir(dir, test_prefix) test_dir = os.path.abspath(test_dir) TEST_TEMPDIRS.append(test_dir) - # Create layers in folder structure - nested_dir = os.path.join(test_dir, 'crypto_bla') - os.mkdir(nested_dir) - nested_dir = os.path.join(nested_dir, 'scheme') - os.mkdir(nested_dir) - - # Create test dependencies structure - os.mkdir(os.path.join(test_dir, 'test')) - # the implementation will go here. - new_impl_dir = os.path.abspath(os.path.join(nested_dir, 'impl')) + scheme_dir = os.path.join(test_dir, 'crypto_bla', 'scheme') + new_impl_dir = os.path.abspath(os.path.join(scheme_dir, 'impl')) def initializer(): """Isolate the files to be tested""" + # Create layers in folder structure + os.makedirs(scheme_dir) + + # Create test dependencies structure + os.mkdir(os.path.join(test_dir, 'test')) + # Copy common files (randombytes.c, aes.c, ...) shutil.copytree( os.path.join('..', 'common'), os.path.join(test_dir, 'common')) @@ -160,6 +166,7 @@ def slow_test(f): return wrapper +@lru_cache(maxsize=None) def ensure_available(executable): """ Checks if a command is available. @@ -278,18 +285,16 @@ def filtered_test(func): return wrapper -__CPUINFO = None - - +@lru_cache(maxsize=1) def get_cpu_info(): - global __CPUINFO - while __CPUINFO is None or 'flags' not in __CPUINFO: + the_info = None + while the_info is None or 'flags' not in the_info: import cpuinfo - __CPUINFO = cpuinfo.get_cpu_info() + the_info = cpuinfo.get_cpu_info() # CPUINFO is unreliable on Travis CI Macs if 'CI' in os.environ and sys.platform == 'darwin': - __CPUINFO['flags'] = [ + the_info['flags'] = [ 'aes', 'apic', 'avx1.0', 'clfsh', 'cmov', 'cx16', 'cx8', 'de', 'em64t', 'erms', 'f16c', 'fpu', 'fxsr', 'lahf', 'mca', 'mce', 'mmx', 'mon', 'msr', 'mtrr', 'osxsave', 'pae', 'pat', 'pcid', @@ -299,4 +304,4 @@ def get_cpu_info(): 'tsc_thread_offset', 'tsci', 'tsctmr', 'vme', 'vmm', 'x2apic', 'xd', 'xsave'] - return __CPUINFO + return the_info diff --git a/test/pqclean.py b/test/pqclean.py index 8713b5df..c1c27a89 100644 --- a/test/pqclean.py +++ b/test/pqclean.py @@ -1,6 +1,7 @@ import glob import os from typing import Optional +from functools import lru_cache import yaml import platform @@ -20,6 +21,7 @@ class Scheme: return 'PQCLEAN_{}_'.format(self.name.upper()).replace('-', '') @staticmethod + @lru_cache(maxsize=None) def by_name(scheme_name): for scheme in Scheme.all_schemes(): if scheme.name == scheme_name: @@ -27,6 +29,7 @@ class Scheme: raise KeyError() @staticmethod + @lru_cache(maxsize=1) def all_schemes(): schemes = [] schemes.extend(Scheme.all_schemes_of_type('kem')) @@ -34,6 +37,7 @@ class Scheme: return schemes @staticmethod + @lru_cache(maxsize=1) def all_implementations(): implementations = [] for scheme in Scheme.all_schemes(): @@ -41,11 +45,13 @@ class Scheme: return implementations @staticmethod + @lru_cache(maxsize=1) def all_supported_implementations(): return [impl for impl in Scheme.all_implementations() if impl.supported_on_current_platform()] @staticmethod + @lru_cache(maxsize=32) def all_schemes_of_type(type: str) -> list: schemes = [] p = os.path.join('..', 'crypto_' + type) @@ -60,12 +66,13 @@ class Scheme: assert('Unknown type') return schemes + @lru_cache(maxsize=None) def metadata(self): metafile = os.path.join(self.path(), 'META.yml') try: with open(metafile, encoding='utf-8') as f: - metadata = yaml.safe_load(f.read()) - return metadata + metadata = yaml.safe_load(f) + return metadata except Exception as e: print("Can't open {}: {}".format(metafile, e)) return None @@ -80,6 +87,7 @@ class Implementation: self.scheme = scheme self.name = name + @lru_cache(maxsize=None) def metadata(self): for i in self.scheme.metadata()['implementations']: if i['name'] == self.name: @@ -104,6 +112,7 @@ class Implementation: '*.o' if os.name != 'nt' else '*.obj')) @staticmethod + @lru_cache(maxsize=None) def by_name(scheme_name, implementation_name): scheme = Scheme.by_name(scheme_name) for implementation in scheme.implementations: @@ -112,6 +121,7 @@ class Implementation: raise KeyError() @staticmethod + @lru_cache(maxsize=None) def all_implementations(scheme: Scheme) -> list: implementations = [] for d in os.listdir(scheme.path()): @@ -143,6 +153,7 @@ class Implementation: return True + @lru_cache(maxsize=10000) def supported_on_current_platform(self) -> bool: if 'supported_platforms' not in self.metadata(): return True diff --git a/test/pytest.ini b/test/pytest.ini index 204dfc00..c61c29d1 100644 --- a/test/pytest.ini +++ b/test/pytest.ini @@ -2,3 +2,4 @@ norecursedirs = .git * empty_parameter_set_mark = fail_at_collect junit_log_passing_tests = False +junit_family=xunit2