Add ccache to speed up builds (#257)

* Add ccache to speed up builds

* Fix circleci config

* fixup! Fix circleci config

* Try again with environment variables

* Set up ccache on MacOS

.circleci/config.yml

@@ -14,16 +14,26 @@ version: 2.1
     - run:
         name: Install the emulation handlers
         command: docker run --rm --privileged multiarch/qemu-user-static:register --reset
+    - restore_cache:
+        keys:
+          - v1-ccache-{{ .Environment.CIRCLE_JOB }}
     - run:
         name: Run the tests in a container
         command: |
-          docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF=1 -e PQCLEAN_SKIP_SCHEMES=sphincs-haraka-128f-robust,sphincs-haraka-192s-robust,sphincs-sha256-128f-robust,sphincs-sha256-192s-robust,sphincs-shake256-128f-robust,sphincs-shake256-192s-robust,sphincs-haraka-128f-simple,sphincs-haraka-192s-simple,sphincs-sha256-128f-simple,sphincs-sha256-192s-simple,sphincs-shake256-128f-simple,sphincs-shake256-192s-simple,sphincs-haraka-128s-robust,sphincs-haraka-256f-robust,sphincs-sha256-128s-robust,sphincs-sha256-256f-robust,sphincs-shake256-128s-robust,sphincs-shake256-256f-robust,sphincs-haraka-128s-simple,sphincs-haraka-256f-simple,sphincs-sha256-128s-simple,sphincs-sha256-256f-simple,sphincs-shake256-128s-simple,sphincs-shake256-256f-simple,sphincs-haraka-192f-robust,sphincs-haraka-256s-robust,sphincs-sha256-192f-robust,sphincs-sha256-256s-robust,sphincs-shake256-192f-robust,sphincs-shake256-256s-robust,sphincs-haraka-192f-simple,sphincs-haraka-256s-simple,sphincs-sha256-192f-simple,sphincs-sha256-256s-simple,sphincs-shake256-192f-simple,sphincs-shake256-256s-simple --rm -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:$ARCH" /bin/bash -c "
+          docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF=1 -e PQCLEAN_SKIP_SCHEMES=sphincs-haraka-128f-robust,sphincs-haraka-192s-robust,sphincs-sha256-128f-robust,sphincs-sha256-192s-robust,sphincs-shake256-128f-robust,sphincs-shake256-192s-robust,sphincs-haraka-128f-simple,sphincs-haraka-192s-simple,sphincs-sha256-128f-simple,sphincs-sha256-192s-simple,sphincs-shake256-128f-simple,sphincs-shake256-192s-simple,sphincs-haraka-128s-robust,sphincs-haraka-256f-robust,sphincs-sha256-128s-robust,sphincs-sha256-256f-robust,sphincs-shake256-128s-robust,sphincs-shake256-256f-robust,sphincs-haraka-128s-simple,sphincs-haraka-256f-simple,sphincs-sha256-128s-simple,sphincs-sha256-256f-simple,sphincs-shake256-128s-simple,sphincs-shake256-256f-simple,sphincs-haraka-192f-robust,sphincs-haraka-256s-robust,sphincs-sha256-192f-robust,sphincs-sha256-256s-robust,sphincs-shake256-192f-robust,sphincs-shake256-256s-robust,sphincs-haraka-192f-simple,sphincs-haraka-256s-simple,sphincs-sha256-192f-simple,sphincs-sha256-256s-simple,sphincs-shake256-192f-simple,sphincs-shake256-256s-simple --rm -v `pwd`:`pwd` -w `pwd` -v ~/ccache:/ccache "pqclean/ci-container:$ARCH" /bin/bash -c "
           uname -a &&
-          export CC=${CC} &&
+          export CCACHE_NOSTATS=1 &&
+          export CCACHE_DIR=/ccache &&
+          export CCACHE_SLOPPINESS=include_file_mtime &&
+          export CC=\"ccache ${CC}\" &&
           pip3 install -r requirements.txt &&
           mkdir test-results &&
           cd test && python3 -m pytest --verbose --junitxml=test-results/pytest/results.xml --numprocesses=auto"
         no_output_timeout: 2h
+    - save_cache:
+        key: v1-ccache-{{ .Environment.CIRCLE_JOB }}
+        paths:
+          - ~/ccache
     - store_test_results:
         path: test/test-results
 
@@ -34,10 +44,16 @@ version: 2.1
     - checkout
     - run:
         <<: *checkouttask
+    - restore_cache:
+        keys:
+          - v1-ccache-{{ .Environment.CIRCLE_JOB }}
     - run:
         name: Run tests
         command: |
-          export CC=${CC}
+          export CCACHE_NOSTATS=1
+          export CCACHE_DIR=/ccache
+          export CCACHE_SLOPPINESS=include_file_mtime
+          export CC="ccache ${CC}"
           export PQCLEAN_ONLY_DIFF=1
           export PQCLEAN_SKIP_SCHEMES=sphincs-haraka-128f-robust,sphincs-haraka-192s-robust,sphincs-sha256-128f-robust,sphincs-sha256-192s-robust,sphincs-shake256-128f-robust,sphincs-shake256-192s-robust,sphincs-haraka-128f-simple,sphincs-haraka-192s-simple,sphincs-sha256-128f-simple,sphincs-sha256-192s-simple,sphincs-shake256-128f-simple,sphincs-shake256-192s-simple,sphincs-haraka-128s-robust,sphincs-haraka-256f-robust,sphincs-sha256-128s-robust,sphincs-sha256-256f-robust,sphincs-shake256-128s-robust,sphincs-shake256-256f-robust,sphincs-haraka-128s-simple,sphincs-haraka-256f-simple,sphincs-sha256-128s-simple,sphincs-sha256-256f-simple,sphincs-shake256-128s-simple,sphincs-shake256-256f-simple,sphincs-haraka-192f-robust,sphincs-haraka-256s-robust,sphincs-sha256-192f-robust,sphincs-sha256-256s-robust,sphincs-shake256-192f-robust,sphincs-shake256-256s-robust,sphincs-haraka-192f-simple,sphincs-haraka-256s-simple,sphincs-sha256-192f-simple,sphincs-sha256-256s-simple,sphincs-shake256-192f-simple,sphincs-shake256-256s-simple
           pip3 install -r requirements.txt
@@ -49,6 +65,10 @@ version: 2.1
         path: test/test-results
     - store_artifacts:
         path: test/test-results
+    - save_cache:
+        key: v1-ccache-{{ .Environment.CIRCLE_JOB }}
+        paths:
+          - /ccache
 
 jobs:
   # First the KEM jobs

.travis.yml

@@ -14,9 +14,12 @@ matrix:
         - git checkout $TRAVIS_BRANCH
         - git reset --hard $COMMIT
       script:
-        - travis_wait 60 docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF --rm -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c "
+        - travis_wait 60 docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF --rm -v ~/.ccache:/ccache -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c "
           uname -a &&
-          export CC=${CC} &&
+          export CCACHE_DIR=/ccache &&
+          export CCACHE_NOSTATS=1 &&
+          export CCACHE_SLOPPINESS=include_file_mtime &&
+          export CC=\"ccache ${CC}\" &&
           pip3 install -r requirements.txt &&
           mkdir test-results &&
           cd test && python3 -m pytest --verbose --junitxml=test-results/pytest/results.xml --numprocesses=auto"
@@ -36,9 +39,12 @@ matrix:
         - git checkout $TRAVIS_BRANCH
         - git reset --hard $COMMIT
       script:
-        - travis_wait 60 docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF --rm -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c "
+        - travis_wait 60 docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF --rm -v ~/.ccache:/ccache -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c "
           uname -a &&
-          export CC=${CC} &&
+          export CCACHE_NOSTATS=1 &&
+          export CCACHE_DIR=/ccache &&
+          export CCACHE_SLOPPINESS=include_file_mtime &&
+          export CC=\"ccache ${CC}\" &&
           pip3 install -r requirements.txt &&
           mkdir test-results &&
           cd test && python3 -m pytest --verbose --junitxml=test-results/pytest/results.xml --numprocesses=auto"
@@ -58,9 +64,12 @@ matrix:
         - git checkout $TRAVIS_BRANCH
         - git reset --hard $COMMIT
       script:
-        - travis_wait 60 docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF -e PQCLEAN_SKIP_SCHEMES --rm -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c "
+        - travis_wait 60 docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF -e PQCLEAN_SKIP_SCHEMES --rm -v ~/.ccache:/ccache -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c "
           uname -a &&
-          export CC=${CC} &&
+          export CCACHE_NOSTATS=1 &&
+          export CCACHE_DIR=/ccache &&
+          export CCACHE_SLOPPINESS=include_file_mtime &&
+          export CC=\"ccache ${CC}\" &&
           pip3 install -r requirements.txt &&
           mkdir test-results &&
           cd test && python3 -m pytest --verbose --junitxml=test-results/pytest/results.xml --numprocesses=auto"
@@ -81,9 +90,12 @@ matrix:
         - git checkout $TRAVIS_BRANCH
         - git reset --hard $COMMIT
       script:
-        - travis_wait 60 docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF=1 -e PQCLEAN_SKIP_SCHEMES --rm -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c "
+        - travis_wait 60 docker run -e CI=true -e PQCLEAN_ONLY_TYPES -e PQCLEAN_ONLY_DIFF=1 -e PQCLEAN_SKIP_SCHEMES --rm -v ~/.ccache:/ccache -v `pwd`:`pwd` -w `pwd` "pqclean/ci-container:arm64" /bin/bash -c "
           uname -a &&
-          export CC=${CC} &&
+          export CCACHE_NOSTATS=1 &&
+          export CCACHE_DIR=/ccache &&
+          export CCACHE_SLOPPINESS=include_file_mtime &&
+          export CC=\"ccache ${CC}\" &&
           pip3 install -r requirements.txt &&
           mkdir test-results &&
           cd test && python3 -m pytest --verbose --junitxml=test-results/pytest/results.xml --numprocesses=auto"
@@ -102,6 +114,9 @@ matrix:
       before_install:
         - pip3 install -r requirements.txt
       before_script:
+        - export CCACHE_NOSTATS=1
+        - export CCACHE_SLOPPINESS=include_file_mtime
+        - export PATH="/usr/local/opt/ccache/libexec:$PATH"
         - export COMMIT=$(git rev-parse HEAD)
         - git config --replace-all remote.origin.fetch +refs/heads/*:refs/remotes/origin/*
         - git fetch --unshallow
@@ -118,6 +133,7 @@ matrix:
         homebrew:
           packages:
             - astyle
+            - ccache
     - name: "KEMs: MacOS + GCC9"
       os: osx
       osx_image: xcode11.2
@@ -126,12 +142,15 @@ matrix:
         homebrew:
           packages:
             - astyle
+            - ccache
             - gcc@9
       env:
         PQCLEAN_ONLY_TYPES: kem
         PQCLEAN_ONLY_DIFF: 1
         PQCLEAN_SKIP_SCHEMES: sphincs-haraka-128f-robust,sphincs-haraka-192s-robust,sphincs-sha256-128f-robust,sphincs-sha256-192s-robust,sphincs-shake256-128f-robust,sphincs-shake256-192s-robust,sphincs-haraka-128f-simple,sphincs-haraka-192s-simple,sphincs-sha256-128f-simple,sphincs-sha256-192s-simple,sphincs-shake256-128f-simple,sphincs-shake256-192s-simple,sphincs-haraka-128s-robust,sphincs-haraka-256f-robust,sphincs-sha256-128s-robust,sphincs-sha256-256f-robust,sphincs-shake256-128s-robust,sphincs-shake256-256f-robust,sphincs-haraka-128s-simple,sphincs-haraka-256f-simple,sphincs-sha256-128s-simple,sphincs-sha256-256f-simple,sphincs-shake256-128s-simple,sphincs-shake256-256f-simple,sphincs-haraka-192f-robust,sphincs-haraka-256s-robust,sphincs-sha256-192f-robust,sphincs-sha256-256s-robust,sphincs-shake256-192f-robust,sphincs-shake256-256s-robust,sphincs-haraka-192f-simple,sphincs-haraka-256s-simple,sphincs-sha256-192f-simple,sphincs-sha256-256s-simple,sphincs-shake256-192f-simple,sphincs-shake256-256s-simple
       before_install:
+        - export CCACHE_NOSTATS=1
+        - export CCACHE_SLOPPINESS=include_file_mtime
         - export COMMIT=$(git rev-parse HEAD)
         - git config --replace-all remote.origin.fetch +refs/heads/*:refs/remotes/origin/*
         - git fetch --unshallow
@@ -140,6 +159,7 @@ matrix:
         - pip3 install -r requirements.txt
         - brew link gcc
         - export PATH="/usr/local/bin:$PATH"
+        - export CC="ccache gcc"
         - ln -s /usr/local/bin/gcc-9 /usr/local/bin/gcc
         - gcc --version
       script:
@@ -152,11 +172,14 @@ matrix:
       before_install:
         - pip3 install -r requirements.txt
       before_script:
+        - export CCACHE_NOSTATS=1
+        - export CCACHE_SLOPPINESS=include_file_mtime
         - export COMMIT=$(git rev-parse HEAD)
         - git config --replace-all remote.origin.fetch +refs/heads/*:refs/remotes/origin/*
         - git fetch --unshallow
         - git checkout $TRAVIS_BRANCH
         - git reset --hard $COMMIT
+        - export PATH="/usr/local/opt/ccache/libexec:$PATH"
       script:
         # Use travis-wait to allow slower tests to run
         - "cd test && travis_wait 60 python3 -m pytest --numprocesses=auto"
@@ -168,6 +191,7 @@ matrix:
         homebrew:
           packages:
             - astyle
+            - ccache
     - name: "SIGs on MacOS + GCC9"
       os: osx
       osx_image: xcode11.2
@@ -176,12 +200,15 @@ matrix:
         homebrew:
           packages:
             - astyle
+            - ccache
             - gcc@9
       env:
         PQCLEAN_ONLY_TYPES: sign
         PQCLEAN_ONLY_DIFF: 1
         PQCLEAN_SKIP_SCHEMES: sphincs-haraka-128f-robust,sphincs-haraka-192s-robust,sphincs-sha256-128f-robust,sphincs-sha256-192s-robust,sphincs-shake256-128f-robust,sphincs-shake256-192s-robust,sphincs-haraka-128f-simple,sphincs-haraka-192s-simple,sphincs-sha256-128f-simple,sphincs-sha256-192s-simple,sphincs-shake256-128f-simple,sphincs-shake256-192s-simple,sphincs-haraka-128s-robust,sphincs-haraka-256f-robust,sphincs-sha256-128s-robust,sphincs-sha256-256f-robust,sphincs-shake256-128s-robust,sphincs-shake256-256f-robust,sphincs-haraka-128s-simple,sphincs-haraka-256f-simple,sphincs-sha256-128s-simple,sphincs-sha256-256f-simple,sphincs-shake256-128s-simple,sphincs-shake256-256f-simple,sphincs-haraka-192f-robust,sphincs-haraka-256s-robust,sphincs-sha256-192f-robust,sphincs-sha256-256s-robust,sphincs-shake256-192f-robust,sphincs-shake256-256s-robust,sphincs-haraka-192f-simple,sphincs-haraka-256s-simple,sphincs-sha256-192f-simple,sphincs-sha256-256s-simple,sphincs-shake256-192f-simple,sphincs-shake256-256s-simple
       before_install:
+        - export CCACHE_NOSTATS=1
+        - export CCACHE_SLOPPINESS=include_file_mtime
         - export COMMIT=$(git rev-parse HEAD)
         - git config --replace-all remote.origin.fetch +refs/heads/*:refs/remotes/origin/*
         - git fetch --unshallow
@@ -190,6 +217,7 @@ matrix:
         - pip3 install -r requirements.txt
         - brew link gcc
         - export PATH="/usr/local/bin:$PATH"
+        - export CC="ccache gcc"
         - ln -s /usr/local/bin/gcc-9 /usr/local/bin/gcc
         - gcc --version
       script:
@@ -201,6 +229,7 @@ cache:
   directories:
     - $HOME/Library/Caches/pip
     - $HOME/Library/Caches/Homebrew
+    - $HOME/.ccache
 
 before_cache:
   - brew cleanup