diff options
author | Wladimir J. van der Laan <laanwj@protonmail.com> | 2019-08-29 12:52:19 +0200 |
---|---|---|
committer | Wladimir J. van der Laan <laanwj@protonmail.com> | 2019-08-29 12:52:41 +0200 |
commit | 1f77c5c3e7c02db27d1046d02e4a6129d236b8a8 (patch) | |
tree | f075b395bbbdd51cd405074ee7464a40d4fa7d65 | |
parent | fc5b756bae26db5e88bd6ed15e2eb45b05e76bdd (diff) | |
parent | fa27372e65359f12bea9ac0680d6bcb96680dda8 (diff) | |
download | bitcoin-1f77c5c3e7c02db27d1046d02e4a6129d236b8a8.tar.xz |
Merge #16739: ci: Pass down $MAKEJOBS to test_runner.py, other improvements
fa27372e65359f12bea9ac0680d6bcb96680dda8 ci: Move CCACHE_DIR and test_runner tmp dir into ./ci/scratch/ (MarcoFalke)
fa60583d23653ca02c8071934a35964bd31fcc2b ci: Pass down $MAKEJOBS to test_runner.py (MarcoFalke)
Pull request description:
Some changes to the ci system:
* Pass down MAKEJOBS to the test_runner, instead of falling back to the default of 4. Passing it down avoids OOM on weak machines and allows better use of resources on beefy machines.
* Move CCACHE_DIR to ./ci/scratch/ subfolder: `ccache` is executed with root permissions inside the docker, so the cache files are created with root as owner. So it might be wise to not put them in the $HOME of the host
* Use the scratch dir as prefix for the test runner, as opposed to `/tmp/`, which is often a ramdisk and thus leads to OOM on the host when either a lot of tests are run in parallel or when a lot of tests fail and the datadirs are not cleaned.
ACKs for top commit:
fanquake:
ACK fa27372e65359f12bea9ac0680d6bcb96680dda8
laanwj:
ACK fa27372e65359f12bea9ac0680d6bcb96680dda8
Tree-SHA512: 67834fbab282051ec81c319d460528b32870507e53df2b8a1ce9a1f3f6a685aaf8eb8ba03f5406918ca4a33adf736e6a4adad7134c54cf3a9e47a26c64a13442
-rw-r--r-- | .travis.yml | 2 | ||||
-rwxr-xr-x | ci/test/00_setup_env.sh | 7 | ||||
-rwxr-xr-x | ci/test/04_install.sh | 1 | ||||
-rwxr-xr-x | ci/test/06_script_b.sh | 2 |
4 files changed, 8 insertions, 4 deletions
diff --git a/.travis.yml b/.travis.yml index f515ab2b87..04308a5fa6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -33,7 +33,7 @@ cache: directories: - $TRAVIS_BUILD_DIR/depends/built - $TRAVIS_BUILD_DIR/depends/sdk-sources - - $HOME/.ccache + - $TRAVIS_BUILD_DIR/ci/scratch/.ccache stages: - lint - test diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 09b37f8240..51b5cfdd3f 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -11,7 +11,10 @@ echo "Setting default values in env" BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) export BASE_ROOT_DIR -export MAKEJOBS=${MAKEJOBS:--j3} +# The number of parallel jobs to pass down to make and test_runner.py +export MAKEJOBS=${MAKEJOBS:--j4} +# A folder for the ci system to put temporary files (ccache, datadirs for tests, ...) +export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch/} export HOST=${HOST:-x86_64-unknown-linux-gnu} export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true} @@ -21,7 +24,7 @@ export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1$TRAVIS_BUILD_ID} export CCACHE_SIZE=${CCACHE_SIZE:-100M} export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp} export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} -export CCACHE_DIR=${CCACHE_DIR:-$HOME/.ccache} +export CCACHE_DIR=${CCACHE_DIR:-$BASE_SCRATCH_DIR/.ccache} export BASE_BUILD_DIR=${BASE_BUILD_DIR:-${TRAVIS_BUILD_DIR:-$BASE_ROOT_DIR}} export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST} export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 3535746e83..54d7a9b814 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -6,6 +6,7 @@ export LC_ALL=C.UTF-8 +mkdir -p "${BASE_SCRATCH_DIR}" ccache echo "Creating ccache dir if it didn't already exist" if [ ! -d ${DIR_QA_ASSETS} ]; then diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index d179ce81c0..ea7beae85f 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -16,7 +16,7 @@ fi if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then BEGIN_FOLD functional-tests - DOCKER_EXEC test/functional/test_runner.py --ci --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --quiet --failfast + DOCKER_EXEC test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 ${TEST_RUNNER_EXTRA} --quiet --failfast END_FOLD fi |