From 5f0ddcad78c6b2dd4b3ef7e4cde63ddc353097a7 Mon Sep 17 00:00:00 2001 From: Thom Wiggers Date: Thu, 12 Dec 2019 11:35:19 +0100 Subject: [PATCH] Add ccache to speed up builds (#257) * Add ccache to speed up builds * Fix circleci config * fixup! Fix circleci config * Try again with environment variables * Set up ccache on MacOS --- .circleci/config.yml | 26 ++++++++++++++++++++++--- .travis.yml | 45 ++++++++++++++++++++++++++++++++++++-------- 2 files changed, 60 insertions(+), 11 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index a248d631..8b693585 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -14,16 +14,26 @@ version: 2.1 - run: name: Install the emulation handlers command: docker run --rm --privileged multiarch/qemu-user-static:register --reset + - restore_cache: + keys: + - v1-ccache-{{ .Environment.CIRCLE_JOB }} - run: name: Run the tests in a container command: | - docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF=1 -e PQCLEAN_SKIP_SCHEMES=sphincs-haraka-128f-robust,sphincs-haraka-192s-robust,sphincs-sha256-128f-robust,sphincs-sha256-192s-robust,sphincs-shake256-128f-robust,sphincs-shake256-192s-robust,sphincs-haraka-128f-simple,sphincs-haraka-192s-simple,sphincs-sha256-128f-simple,sphincs-sha256-192s-simple,sphincs-shake256-128f-simple,sphincs-shake256-192s-simple,sphincs-haraka-128s-robust,sphincs-haraka-256f-robust,sphincs-sha256-128s-robust,sphincs-sha256-256f-robust,sphincs-shake256-128s-robust,sphincs-shake256-256f-robust,sphincs-haraka-128s-simple,sphincs-haraka-256f-simple,sphincs-sha256-128s-simple,sphincs-sha256-256f-simple,sphincs-shake256-128s-simple,sphincs-shake256-256f-simple,sphincs-haraka-192f-robust,sphincs-haraka-256s-robust,sphincs-sha256-192f-robust,sphincs-sha256-256s-robust,sphincs-shake256-192f-robust,sphincs-shake256-256s-robust,sphincs-haraka-192f-simple,sphincs-haraka-256s-simple,sphincs-sha256-192f-simple,sphincs-sha256-256s-simple,sphincs-shake256-192f-simple,sphincs-shake256-256s-simple --rm -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:$ARCH" /bin/bash -c " + docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF=1 -e PQCLEAN_SKIP_SCHEMES=sphincs-haraka-128f-robust,sphincs-haraka-192s-robust,sphincs-sha256-128f-robust,sphincs-sha256-192s-robust,sphincs-shake256-128f-robust,sphincs-shake256-192s-robust,sphincs-haraka-128f-simple,sphincs-haraka-192s-simple,sphincs-sha256-128f-simple,sphincs-sha256-192s-simple,sphincs-shake256-128f-simple,sphincs-shake256-192s-simple,sphincs-haraka-128s-robust,sphincs-haraka-256f-robust,sphincs-sha256-128s-robust,sphincs-sha256-256f-robust,sphincs-shake256-128s-robust,sphincs-shake256-256f-robust,sphincs-haraka-128s-simple,sphincs-haraka-256f-simple,sphincs-sha256-128s-simple,sphincs-sha256-256f-simple,sphincs-shake256-128s-simple,sphincs-shake256-256f-simple,sphincs-haraka-192f-robust,sphincs-haraka-256s-robust,sphincs-sha256-192f-robust,sphincs-sha256-256s-robust,sphincs-shake256-192f-robust,sphincs-shake256-256s-robust,sphincs-haraka-192f-simple,sphincs-haraka-256s-simple,sphincs-sha256-192f-simple,sphincs-sha256-256s-simple,sphincs-shake256-192f-simple,sphincs-shake256-256s-simple --rm -v `pwd`:`pwd` -w `pwd` -v ~/ccache:/ccache "pqclean/ci-container:$ARCH" /bin/bash -c " uname -a && - export CC=${CC} && + export CCACHE_NOSTATS=1 && + export CCACHE_DIR=/ccache && + export CCACHE_SLOPPINESS=include_file_mtime && + export CC=\"ccache ${CC}\" && pip3 install -r requirements.txt && mkdir test-results && cd test && python3 -m pytest --verbose --junitxml=test-results/pytest/results.xml --numprocesses=auto" no_output_timeout: 2h + - save_cache: + key: v1-ccache-{{ .Environment.CIRCLE_JOB }} + paths: + - ~/ccache - store_test_results: path: test/test-results @@ -34,10 +44,16 @@ version: 2.1 - checkout - run: <<: *checkouttask + - restore_cache: + keys: + - v1-ccache-{{ .Environment.CIRCLE_JOB }} - run: name: Run tests command: | - export CC=${CC} + export CCACHE_NOSTATS=1 + export CCACHE_DIR=/ccache + export CCACHE_SLOPPINESS=include_file_mtime + export CC="ccache ${CC}" export PQCLEAN_ONLY_DIFF=1 export PQCLEAN_SKIP_SCHEMES=sphincs-haraka-128f-robust,sphincs-haraka-192s-robust,sphincs-sha256-128f-robust,sphincs-sha256-192s-robust,sphincs-shake256-128f-robust,sphincs-shake256-192s-robust,sphincs-haraka-128f-simple,sphincs-haraka-192s-simple,sphincs-sha256-128f-simple,sphincs-sha256-192s-simple,sphincs-shake256-128f-simple,sphincs-shake256-192s-simple,sphincs-haraka-128s-robust,sphincs-haraka-256f-robust,sphincs-sha256-128s-robust,sphincs-sha256-256f-robust,sphincs-shake256-128s-robust,sphincs-shake256-256f-robust,sphincs-haraka-128s-simple,sphincs-haraka-256f-simple,sphincs-sha256-128s-simple,sphincs-sha256-256f-simple,sphincs-shake256-128s-simple,sphincs-shake256-256f-simple,sphincs-haraka-192f-robust,sphincs-haraka-256s-robust,sphincs-sha256-192f-robust,sphincs-sha256-256s-robust,sphincs-shake256-192f-robust,sphincs-shake256-256s-robust,sphincs-haraka-192f-simple,sphincs-haraka-256s-simple,sphincs-sha256-192f-simple,sphincs-sha256-256s-simple,sphincs-shake256-192f-simple,sphincs-shake256-256s-simple pip3 install -r requirements.txt @@ -49,6 +65,10 @@ version: 2.1 path: test/test-results - store_artifacts: path: test/test-results + - save_cache: + key: v1-ccache-{{ .Environment.CIRCLE_JOB }} + paths: + - /ccache jobs: # First the KEM jobs diff --git a/.travis.yml b/.travis.yml index e061e843..1aaccabb 100644 --- a/.travis.yml +++ b/.travis.yml @@ -14,9 +14,12 @@ matrix: - git checkout $TRAVIS_BRANCH - git reset --hard $COMMIT script: - - travis_wait 60 docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF --rm -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c " + - travis_wait 60 docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF --rm -v ~/.ccache:/ccache -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c " uname -a && - export CC=${CC} && + export CCACHE_DIR=/ccache && + export CCACHE_NOSTATS=1 && + export CCACHE_SLOPPINESS=include_file_mtime && + export CC=\"ccache ${CC}\" && pip3 install -r requirements.txt && mkdir test-results && cd test && python3 -m pytest --verbose --junitxml=test-results/pytest/results.xml --numprocesses=auto" @@ -36,9 +39,12 @@ matrix: - git checkout $TRAVIS_BRANCH - git reset --hard $COMMIT script: - - travis_wait 60 docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF --rm -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c " + - travis_wait 60 docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF --rm -v ~/.ccache:/ccache -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c " uname -a && - export CC=${CC} && + export CCACHE_NOSTATS=1 && + export CCACHE_DIR=/ccache && + export CCACHE_SLOPPINESS=include_file_mtime && + export CC=\"ccache ${CC}\" && pip3 install -r requirements.txt && mkdir test-results && cd test && python3 -m pytest --verbose --junitxml=test-results/pytest/results.xml --numprocesses=auto" @@ -58,9 +64,12 @@ matrix: - git checkout $TRAVIS_BRANCH - git reset --hard $COMMIT script: - - travis_wait 60 docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF -e PQCLEAN_SKIP_SCHEMES --rm -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c " + - travis_wait 60 docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF -e PQCLEAN_SKIP_SCHEMES --rm -v ~/.ccache:/ccache -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c " uname -a && - export CC=${CC} && + export CCACHE_NOSTATS=1 && + export CCACHE_DIR=/ccache && + export CCACHE_SLOPPINESS=include_file_mtime && + export CC=\"ccache ${CC}\" && pip3 install -r requirements.txt && mkdir test-results && cd test && python3 -m pytest --verbose --junitxml=test-results/pytest/results.xml --numprocesses=auto" @@ -81,9 +90,12 @@ matrix: - git checkout $TRAVIS_BRANCH - git reset --hard $COMMIT script: - - travis_wait 60 docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF=1 -e PQCLEAN_SKIP_SCHEMES --rm -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c " + - travis_wait 60 docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF=1 -e PQCLEAN_SKIP_SCHEMES --rm -v ~/.ccache:/ccache -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c " uname -a && - export CC=${CC} && + export CCACHE_NOSTATS=1 && + export CCACHE_DIR=/ccache && + export CCACHE_SLOPPINESS=include_file_mtime && + export CC=\"ccache ${CC}\" && pip3 install -r requirements.txt && mkdir test-results && cd test && python3 -m pytest --verbose --junitxml=test-results/pytest/results.xml --numprocesses=auto" @@ -102,6 +114,9 @@ matrix: before_install: - pip3 install -r requirements.txt before_script: + - export CCACHE_NOSTATS=1 + - export CCACHE_SLOPPINESS=include_file_mtime + - export PATH="/usr/local/opt/ccache/libexec:$PATH" - export COMMIT=$(git rev-parse HEAD) - git config --replace-all remote.origin.fetch +refs/heads/*:refs/remotes/origin/* - git fetch --unshallow @@ -118,6 +133,7 @@ matrix: homebrew: packages: - astyle + - ccache - name: "KEMs: MacOS + GCC9" os: osx osx_image: xcode11.2 @@ -126,12 +142,15 @@ matrix: homebrew: packages: - astyle + - ccache - gcc@9 env: PQCLEAN_ONLY_TYPES: kem PQCLEAN_ONLY_DIFF: 1 PQCLEAN_SKIP_SCHEMES: sphincs-haraka-128f-robust,sphincs-haraka-192s-robust,sphincs-sha256-128f-robust,sphincs-sha256-192s-robust,sphincs-shake256-128f-robust,sphincs-shake256-192s-robust,sphincs-haraka-128f-simple,sphincs-haraka-192s-simple,sphincs-sha256-128f-simple,sphincs-sha256-192s-simple,sphincs-shake256-128f-simple,sphincs-shake256-192s-simple,sphincs-haraka-128s-robust,sphincs-haraka-256f-robust,sphincs-sha256-128s-robust,sphincs-sha256-256f-robust,sphincs-shake256-128s-robust,sphincs-shake256-256f-robust,sphincs-haraka-128s-simple,sphincs-haraka-256f-simple,sphincs-sha256-128s-simple,sphincs-sha256-256f-simple,sphincs-shake256-128s-simple,sphincs-shake256-256f-simple,sphincs-haraka-192f-robust,sphincs-haraka-256s-robust,sphincs-sha256-192f-robust,sphincs-sha256-256s-robust,sphincs-shake256-192f-robust,sphincs-shake256-256s-robust,sphincs-haraka-192f-simple,sphincs-haraka-256s-simple,sphincs-sha256-192f-simple,sphincs-sha256-256s-simple,sphincs-shake256-192f-simple,sphincs-shake256-256s-simple before_install: + - export CCACHE_NOSTATS=1 + - export CCACHE_SLOPPINESS=include_file_mtime - export COMMIT=$(git rev-parse HEAD) - git config --replace-all remote.origin.fetch +refs/heads/*:refs/remotes/origin/* - git fetch --unshallow @@ -140,6 +159,7 @@ matrix: - pip3 install -r requirements.txt - brew link gcc - export PATH="/usr/local/bin:$PATH" + - export CC="ccache gcc" - ln -s /usr/local/bin/gcc-9 /usr/local/bin/gcc - gcc --version script: @@ -152,11 +172,14 @@ matrix: before_install: - pip3 install -r requirements.txt before_script: + - export CCACHE_NOSTATS=1 + - export CCACHE_SLOPPINESS=include_file_mtime - export COMMIT=$(git rev-parse HEAD) - git config --replace-all remote.origin.fetch +refs/heads/*:refs/remotes/origin/* - git fetch --unshallow - git checkout $TRAVIS_BRANCH - git reset --hard $COMMIT + - export PATH="/usr/local/opt/ccache/libexec:$PATH" script: # Use travis-wait to allow slower tests to run - "cd test && travis_wait 60 python3 -m pytest --numprocesses=auto" @@ -168,6 +191,7 @@ matrix: homebrew: packages: - astyle + - ccache - name: "SIGs on MacOS + GCC9" os: osx osx_image: xcode11.2 @@ -176,12 +200,15 @@ matrix: homebrew: packages: - astyle + - ccache - gcc@9 env: PQCLEAN_ONLY_TYPES: sign PQCLEAN_ONLY_DIFF: 1 PQCLEAN_SKIP_SCHEMES: sphincs-haraka-128f-robust,sphincs-haraka-192s-robust,sphincs-sha256-128f-robust,sphincs-sha256-192s-robust,sphincs-shake256-128f-robust,sphincs-shake256-192s-robust,sphincs-haraka-128f-simple,sphincs-haraka-192s-simple,sphincs-sha256-128f-simple,sphincs-sha256-192s-simple,sphincs-shake256-128f-simple,sphincs-shake256-192s-simple,sphincs-haraka-128s-robust,sphincs-haraka-256f-robust,sphincs-sha256-128s-robust,sphincs-sha256-256f-robust,sphincs-shake256-128s-robust,sphincs-shake256-256f-robust,sphincs-haraka-128s-simple,sphincs-haraka-256f-simple,sphincs-sha256-128s-simple,sphincs-sha256-256f-simple,sphincs-shake256-128s-simple,sphincs-shake256-256f-simple,sphincs-haraka-192f-robust,sphincs-haraka-256s-robust,sphincs-sha256-192f-robust,sphincs-sha256-256s-robust,sphincs-shake256-192f-robust,sphincs-shake256-256s-robust,sphincs-haraka-192f-simple,sphincs-haraka-256s-simple,sphincs-sha256-192f-simple,sphincs-sha256-256s-simple,sphincs-shake256-192f-simple,sphincs-shake256-256s-simple before_install: + - export CCACHE_NOSTATS=1 + - export CCACHE_SLOPPINESS=include_file_mtime - export COMMIT=$(git rev-parse HEAD) - git config --replace-all remote.origin.fetch +refs/heads/*:refs/remotes/origin/* - git fetch --unshallow @@ -190,6 +217,7 @@ matrix: - pip3 install -r requirements.txt - brew link gcc - export PATH="/usr/local/bin:$PATH" + - export CC="ccache gcc" - ln -s /usr/local/bin/gcc-9 /usr/local/bin/gcc - gcc --version script: @@ -201,6 +229,7 @@ cache: directories: - $HOME/Library/Caches/pip - $HOME/Library/Caches/Homebrew + - $HOME/.ccache before_cache: - brew cleanup