diff options
-rw-r--r-- | .cirrus.yml | 24 | ||||
-rw-r--r-- | .travis.yml | 39 | ||||
-rw-r--r-- | .travis/README.md | 8 | ||||
-rwxr-xr-x | .travis/test_04_install.sh | 37 | ||||
-rw-r--r-- | ci/README.md | 25 | ||||
-rwxr-xr-x | ci/extended_lint/04_install.sh (renamed from .travis/extended_lint_04_install.sh) | 0 | ||||
-rwxr-xr-x | ci/extended_lint/06_script.sh (renamed from .travis/extended_lint_06_script.sh) | 0 | ||||
-rwxr-xr-x | ci/lint/04_install.sh (renamed from .travis/lint_04_install.sh) | 0 | ||||
-rwxr-xr-x | ci/lint/05_before_script.sh (renamed from .travis/lint_05_before_script.sh) | 0 | ||||
-rwxr-xr-x | ci/lint/06_script.sh (renamed from .travis/lint_06_script.sh) | 0 | ||||
-rw-r--r-- | ci/retry/README.md | 123 | ||||
-rwxr-xr-x | ci/retry/retry | 163 | ||||
-rwxr-xr-x | ci/test/00_setup_env.sh | 33 | ||||
-rwxr-xr-x | ci/test/03_before_install.sh (renamed from .travis/test_03_before_install.sh) | 1 | ||||
-rwxr-xr-x | ci/test/04_install.sh | 49 | ||||
-rwxr-xr-x | ci/test/05_before_script.sh (renamed from .travis/test_05_before_script.sh) | 0 | ||||
-rwxr-xr-x | ci/test/06_script_a.sh (renamed from .travis/test_06_script_a.sh) | 12 | ||||
-rwxr-xr-x | ci/test/06_script_b.sh (renamed from .travis/test_06_script_b.sh) | 4 | ||||
-rwxr-xr-x | ci/test_run_all.sh | 16 |
19 files changed, 452 insertions, 82 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index ce17a223b1..1e6e6937da 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -27,3 +27,27 @@ task: - gmake check ${MAKEJOBS} VERBOSE=1 functional_test_script: - ./test/functional/test_runner.py --jobs 9 --ci --extended --exclude feature_dbcrash --combinedlogslen=1000 --quiet --failfast +task: + name: "x86_64 Linux [GOAL: install] [bionic] [Using ./ci/ system]" + container: + image: ubuntu:18.04 + cpu: 8 + memory: 8G + timeout_in: 60m + env: + MAKEJOBS: "-j9" + RUN_CI_ON_HOST: "1" + CCACHE_SIZE: "200M" + CCACHE_DIR: "/tmp/ccache_dir" + ccache_cache: + folder: "/tmp/ccache_dir" + depends_built_cache: + folder: "/tmp/cirrus-ci-build/depends/built" + depends_sdk_cache: + folder: "/tmp/cirrus-ci-build/depends/sdk-sources" + install_script: + - apt-get update + - apt-get -y install git bash ccache + - ccache --max-size=${CCACHE_SIZE} + ci_script: + - ./ci/test_run_all.sh diff --git a/.travis.yml b/.travis.yml index d49c116a5e..7753dcc51f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -40,38 +40,25 @@ stages: - extended-lint env: global: - - MAKEJOBS=-j3 - - RUN_UNIT_TESTS=true - - RUN_FUNCTIONAL_TESTS=true - - RUN_FUZZ_TESTS=false - - DOCKER_NAME_TAG=ubuntu:18.04 - - BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID - - CCACHE_SIZE=100M - - CCACHE_TEMPDIR=/tmp/.ccache-temp - - CCACHE_COMPRESS=1 - - CCACHE_DIR=$HOME/.ccache - - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out - - SDK_URL=https://bitcoincore.org/depends-sources/sdks - - WINEDEBUG=fixme-all - - DOCKER_PACKAGES="build-essential libtool autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache" + - CI_RETRY_EXE="travis_retry" - CACHE_ERR_MSG="Error! Initial build successful, but not enough time remains to run later build stages and tests. Please manually re-run this job by using the travis restart button or asking a bitcoin maintainer to restart. The next run should not time out because the build cache has been saved." before_install: - - set -o errexit; source .travis/test_03_before_install.sh + - set -o errexit; source ./ci/test/00_setup_env.sh + - set -o errexit; source ./ci/test/03_before_install.sh install: - - set -o errexit; source .travis/test_04_install.sh + - set -o errexit; source ./ci/test/04_install.sh before_script: - - set -o errexit; source .travis/test_05_before_script.sh + - set -o errexit; source ./ci/test/05_before_script.sh script: - export CONTINUE=1 - if [ $SECONDS -gt 1200 ]; then export CONTINUE=0; fi # Likely the depends build took very long - if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi # Whitelisted repo (90 minutes build time) - - if [ $CONTINUE = "1" ]; then set -o errexit; source .travis/test_06_script_a.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi + - if [ $CONTINUE = "1" ]; then set -o errexit; source ./ci/test/06_script_a.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi - if [ $SECONDS -gt 2000 ]; then export CONTINUE=0; fi # Likely the build took very long; The tests take about 1000s, so we should abort if we have less than 50*60-1000=2000s left - if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi # Whitelisted repo (90 minutes build time) - - if [ $CONTINUE = "1" ]; then set -o errexit; source .travis/test_06_script_b.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi + - if [ $CONTINUE = "1" ]; then set -o errexit; source ./ci/test/06_script_b.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi after_script: - echo $TRAVIS_COMMIT_RANGE - - echo $TRAVIS_COMMIT_LOG jobs: include: @@ -82,11 +69,11 @@ jobs: language: python python: '3.5' # Oldest supported version according to doc/dependencies.md install: - - set -o errexit; source .travis/lint_04_install.sh + - set -o errexit; source ./ci/lint/04_install.sh before_script: - - set -o errexit; source .travis/lint_05_before_script.sh + - set -o errexit; source ./ci/lint/05_before_script.sh script: - - set -o errexit; source .travis/lint_06_script.sh + - set -o errexit; source ./ci/lint/06_script.sh - stage: extended-lint name: 'extended lint [runtime >= 60 seconds]' @@ -95,11 +82,11 @@ jobs: language: python python: '3.5' install: - - set -o errexit; source .travis/extended_lint_04_install.sh + - set -o errexit; source ./ci/extended_lint/04_install.sh before_script: - - set -o errexit; source .travis/lint_05_before_script.sh + - set -o errexit; source ./ci/lint/05_before_script.sh script: - - set -o errexit; source .travis/extended_lint_06_script.sh + - set -o errexit; source ./ci/extended_lint/06_script.sh - stage: test name: 'ARM [GOAL: install] [no unit or functional tests]' diff --git a/.travis/README.md b/.travis/README.md deleted file mode 100644 index 21d1b9cc03..0000000000 --- a/.travis/README.md +++ /dev/null @@ -1,8 +0,0 @@ -## travis build scripts - -The `.travis` directory contains scripts for each build step in each build stage. -Currently the travis build defines two stages `lint` and `test`. Each stage has -it's own [lifecycle](https://docs.travis-ci.com/user/customizing-the-build/#the-build-lifecycle). -Every script in here is named and numbered according to which stage and lifecycle -step it belongs to. - diff --git a/.travis/test_04_install.sh b/.travis/test_04_install.sh deleted file mode 100755 index 319f2c5b21..0000000000 --- a/.travis/test_04_install.sh +++ /dev/null @@ -1,37 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -free -m -h -echo "Number of CPUs (nproc): $(nproc)" - -travis_retry docker pull "$DOCKER_NAME_TAG" - -export DIR_FUZZ_IN=${TRAVIS_BUILD_DIR}/qa-assets -git clone https://github.com/bitcoin-core/qa-assets ${DIR_FUZZ_IN} -export DIR_FUZZ_IN=${DIR_FUZZ_IN}/fuzz_seed_corpus/ - -mkdir -p "${TRAVIS_BUILD_DIR}/sanitizer-output/" -export ASAN_OPTIONS="" -export LSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/lsan" -export TSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${TRAVIS_BUILD_DIR}/sanitizer-output/tsan" -export UBSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1" -env | grep -E '^(BITCOIN_CONFIG|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env -if [[ $HOST = *-mingw32 ]]; then - DOCKER_ADMIN="--cap-add SYS_ADMIN" -elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) - DOCKER_ADMIN="--cap-add SYS_PTRACE" -fi -DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$TRAVIS_BUILD_DIR,dst=$TRAVIS_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $TRAVIS_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG) - -DOCKER_EXEC () { - docker exec $DOCKER_ID bash -c "cd $PWD && $*" -} - -travis_retry DOCKER_EXEC apt-get update -travis_retry DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES - diff --git a/ci/README.md b/ci/README.md new file mode 100644 index 0000000000..754cbc7c95 --- /dev/null +++ b/ci/README.md @@ -0,0 +1,25 @@ +## ci scripts + +This directory contains scripts for each build step in each build stage. + +Currently three stages `lint`, `extended_lint` and `test` are defined. Each stage has its own lifecycle, similar to the +[Travis CI lifecycle](https://docs.travis-ci.com/user/job-lifecycle#the-job-lifecycle). Every script in here is named +and numbered according to which stage and lifecycle step it belongs to. + +### Running a stage locally + +To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage +requires `docker` to be installed. To install all requirements on Ubuntu, run + +``` +sudo apt install docker.io ccache bash git +``` + +To run the test stage, + +``` +./ci/test_run_all.sh +``` + +Be aware that the tests will be build and run in-place, so please run at your own risk. +If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first. diff --git a/.travis/extended_lint_04_install.sh b/ci/extended_lint/04_install.sh index 123d874a84..123d874a84 100755 --- a/.travis/extended_lint_04_install.sh +++ b/ci/extended_lint/04_install.sh diff --git a/.travis/extended_lint_06_script.sh b/ci/extended_lint/06_script.sh index e8228c9c4d..e8228c9c4d 100755 --- a/.travis/extended_lint_06_script.sh +++ b/ci/extended_lint/06_script.sh diff --git a/.travis/lint_04_install.sh b/ci/lint/04_install.sh index 20bff368a5..20bff368a5 100755 --- a/.travis/lint_04_install.sh +++ b/ci/lint/04_install.sh diff --git a/.travis/lint_05_before_script.sh b/ci/lint/05_before_script.sh index 28bcbb47f7..28bcbb47f7 100755 --- a/.travis/lint_05_before_script.sh +++ b/ci/lint/05_before_script.sh diff --git a/.travis/lint_06_script.sh b/ci/lint/06_script.sh index c7dea599dc..c7dea599dc 100755 --- a/.travis/lint_06_script.sh +++ b/ci/lint/06_script.sh diff --git a/ci/retry/README.md b/ci/retry/README.md new file mode 100644 index 0000000000..983a498070 --- /dev/null +++ b/ci/retry/README.md @@ -0,0 +1,123 @@ +retry - The command line retry tool +------------------------------------------ + +Retry any shell command with exponential backoff or constant delay. + +### Instructions + +Install: + +retry is a shell script, so drop it somewhere and make sure it's added to your $PATH. Or you can use the following one-liner: + +```sh +sudo sh -c "curl https://raw.githubusercontent.com/kadwanev/retry/master/retry -o /usr/local/bin/retry && chmod +x /usr/local/bin/retry" +``` + +If you're on OS X, retry is also on Homebrew: + +``` +brew pull 27283 +brew install retry +``` +Not popular enough for homebrew-core. Please star this project to help. + +### Usage + +Help: + +`retry -?` + + Usage: retry [options] -- execute command + -h, -?, --help + -v, --verbose Verbose output + -t, --tries=# Set max retries: Default 10 + -s, --sleep=secs Constant sleep amount (seconds) + -m, --min=secs Exponenetial Backoff: minimum sleep amount (seconds): Default 0.3 + -x, --max=secs Exponenetial Backoff: maximum sleep amount (seconds): Default 60 + -f, --fail="script +cmds" Fail Script: run in case of final failure + +### Examples + +No problem: + +`retry echo u work good` + + u work good + +Test functionality: + +`retry 'echo "y u no work"; false'` + + y u no work + Before retry #1: sleeping 0.3 seconds + y u no work + Before retry #2: sleeping 0.6 seconds + y u no work + Before retry #3: sleeping 1.2 seconds + y u no work + Before retry #4: sleeping 2.4 seconds + y u no work + Before retry #5: sleeping 4.8 seconds + y u no work + Before retry #6: sleeping 9.6 seconds + y u no work + Before retry #7: sleeping 19.2 seconds + y u no work + Before retry #8: sleeping 38.4 seconds + y u no work + Before retry #9: sleeping 60.0 seconds + y u no work + Before retry #10: sleeping 60.0 seconds + y u no work + etc.. + +Limit retries: + +`retry -t 4 'echo "y u no work"; false'` + + y u no work + Before retry #1: sleeping 0.3 seconds + y u no work + Before retry #2: sleeping 0.6 seconds + y u no work + Before retry #3: sleeping 1.2 seconds + y u no work + Before retry #4: sleeping 2.4 seconds + y u no work + Retries exhausted + +Bad command: + +`retry poop` + + bash: poop: command not found + +Fail command: + +`retry -t 3 -f 'echo "oh poopsickles"' 'echo "y u no work"; false'` + + y u no work + Before retry #1: sleeping 0.3 seconds + y u no work + Before retry #2: sleeping 0.6 seconds + y u no work + Before retry #3: sleeping 1.2 seconds + y u no work + Retries exhausted, running fail script + oh poopsickles + +Last attempt passed: + +`retry -t 3 -- 'if [ $RETRY_ATTEMPT -eq 3 ]; then echo Passed at attempt $RETRY_ATTEMPT; true; else echo Failed at attempt $RETRY_ATTEMPT; false; fi;'` + + Failed at attempt 0 + Before retry #1: sleeping 0.3 seconds + Failed at attempt 1 + Before retry #2: sleeping 0.6 seconds + Failed at attempt 2 + Before retry #3: sleeping 1.2 seconds + Passed at attempt 3 + +### License + +Apache 2.0 - go nuts diff --git a/ci/retry/retry b/ci/retry/retry new file mode 100755 index 0000000000..0e5f6e9701 --- /dev/null +++ b/ci/retry/retry @@ -0,0 +1,163 @@ +#!/usr/bin/env bash + +GETOPT_BIN=$IN_GETOPT_BIN +GETOPT_BIN=${GETOPT_BIN:-getopt} + +__sleep_amount() { + if [ -n "$constant_sleep" ]; then + sleep_time=$constant_sleep + else + #TODO: check for awk + #TODO: check if user would rather use one of the other possible dependencies: python, ruby, bc, dc + sleep_time=`awk "BEGIN {t = $min_sleep * $(( (1<<($attempts -1)) )); print (t > $max_sleep ? $max_sleep : t)}"` + fi +} + +__log_out() { + echo "$1" 1>&2 +} + +# Paramters: max_tries min_sleep max_sleep constant_sleep fail_script EXECUTION_COMMAND +retry() +{ + local max_tries="$1"; shift + local min_sleep="$1"; shift + local max_sleep="$1"; shift + local constant_sleep="$1"; shift + local fail_script="$1"; shift + if [ -n "$VERBOSE" ]; then + __log_out "Retry Parameters: max_tries=$max_tries min_sleep=$min_sleep max_sleep=$max_sleep constant_sleep=$constant_sleep" + if [ -n "$fail_script" ]; then __log_out "Fail script: $fail_script"; fi + __log_out "" + __log_out "Execution Command: $*" + __log_out "" + fi + + local attempts=0 + local return_code=1 + + + while [[ $return_code -ne 0 && $attempts -le $max_tries ]]; do + if [ $attempts -gt 0 ]; then + __sleep_amount + __log_out "Before retry #$attempts: sleeping $sleep_time seconds" + sleep $sleep_time + fi + + P="$1" + for param in "${@:2}"; do P="$P '$param'"; done + #TODO: replace single quotes in each arg with '"'"' ? + export RETRY_ATTEMPT=$attempts + bash -c "$P" + return_code=$? + #__log_out "Process returned $return_code on attempt $attempts" + if [ $return_code -eq 127 ]; then + # command not found + exit $return_code + elif [ $return_code -ne 0 ]; then + attempts=$[$attempts +1] + fi + done + + if [ $attempts -gt $max_tries ]; then + if [ -n "$fail_script" ]; then + __log_out "Retries exhausted, running fail script" + eval $fail_script + else + __log_out "Retries exhausted" + fi + fi + + exit $return_code +} + +# If we're being sourced, don't worry about such things +if [ "$BASH_SOURCE" == "$0" ]; then + # Prints the help text + help() + { + local retry=$(basename $0) + cat <<EOF +Usage: $retry [options] -- execute command + -h, -?, --help + -v, --verbose Verbose output + -t, --tries=# Set max retries: Default 10 + -s, --sleep=secs Constant sleep amount (seconds) + -m, --min=secs Exponenetial Backoff: minimum sleep amount (seconds): Default 0.3 + -x, --max=secs Exponenetial Backoff: maximum sleep amount (seconds): Default 60 + -f, --fail="script +cmds" Fail Script: run in case of final failure +EOF + } + + # show help for no arguments if stdin is a terminal + if { [ -z "$1" ] && [ -t 0 ] ; } || [ "$1" == '-h' ] || [ "$1" == '-?' ] || [ "$1" == '--help' ] + then + help + exit 0 + fi + + $GETOPT_BIN --test > /dev/null + if [[ $? -ne 4 ]]; then + echo "I’m sorry, 'getopt --test' failed in this environment. Please load GNU getopt." + exit 1 + fi + + OPTIONS=vt:s:m:x:f: + LONGOPTIONS=verbose,tries:,sleep:,min:,max:,fail: + + PARSED=$($GETOPT_BIN --options="$OPTIONS" --longoptions="$LONGOPTIONS" --name "$0" -- "$@") + if [[ $? -ne 0 ]]; then + # e.g. $? == 1 + # then getopt has complained about wrong arguments to stdout + exit 2 + fi + # read getopt’s output this way to handle the quoting right: + eval set -- "$PARSED" + + max_tries=10 + min_sleep=0.3 + max_sleep=60.0 + constant_sleep= + fail_script= + + # now enjoy the options in order and nicely split until we see -- + while true; do + case "$1" in + -v|--verbose) + VERBOSE=true + shift + ;; + -t|--tries) + max_tries="$2" + shift 2 + ;; + -s|--sleep) + constant_sleep="$2" + shift 2 + ;; + -m|--min) + min_sleep="$2" + shift 2 + ;; + -x|--max) + max_sleep="$2" + shift 2 + ;; + -f|--fail) + fail_script="$2" + shift 2 + ;; + --) + shift + break + ;; + *) + echo "Programming error" + exit 3 + ;; + esac + done + + retry "$max_tries" "$min_sleep" "$max_sleep" "$constant_sleep" "$fail_script" "$@" + +fi diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh new file mode 100755 index 0000000000..fae48ba176 --- /dev/null +++ b/ci/test/00_setup_env.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +echo "Setting default values in env" + +BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd ) +export BASE_ROOT_DIR + +export MAKEJOBS=${MAKEJOBS:--j3} +export HOST=${HOST:-x86_64-unknown-linux-gnu} +export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true} +export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true} +export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} +export DOCKER_NAME_TAG=${DOCKER_NAME_TAG:-ubuntu:18.04} +export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1$TRAVIS_BUILD_ID} +export CCACHE_SIZE=${CCACHE_SIZE:-100M} +export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp} +export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1} +export CCACHE_DIR=${CCACHE_DIR:-$HOME/.ccache} +export BASE_BUILD_DIR=${BASE_BUILD_DIR:-${TRAVIS_BUILD_DIR:-$BASE_ROOT_DIR}} +export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST} +export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} +export WINEDEBUG=${WINEDEBUG:-fixme-all} +export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3} +export GOAL=${GOAL:-install} +export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets} +export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH +export CI_RETRY_EXE=${CI_RETRY_EXE:retry} diff --git a/.travis/test_03_before_install.sh b/ci/test/03_before_install.sh index 3c9fcf3f98..5086114ba1 100755 --- a/.travis/test_03_before_install.sh +++ b/ci/test/03_before_install.sh @@ -6,7 +6,6 @@ export LC_ALL=C.UTF-8 -PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g") # Add llvm-symbolizer directory to PATH. Needed to get symbolized stack traces from the sanitizers. PATH=$PATH:/usr/lib/llvm-6.0/bin/ export PATH diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh new file mode 100755 index 0000000000..3535746e83 --- /dev/null +++ b/ci/test/04_install.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2018 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +ccache echo "Creating ccache dir if it didn't already exist" + +if [ ! -d ${DIR_QA_ASSETS} ]; then + git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} +fi +export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ + +mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/" +export ASAN_OPTIONS="" +export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan" +export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_BUILD_DIR}/sanitizer-output/tsan" +export UBSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1" +env | grep -E '^(BITCOIN_CONFIG|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env +if [[ $HOST = *-mingw32 ]]; then + DOCKER_ADMIN="--cap-add SYS_ADMIN" +elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) + DOCKER_ADMIN="--cap-add SYS_PTRACE" +fi + +if [ -z "$RUN_CI_ON_HOST" ]; then + echo "Creating $DOCKER_NAME_TAG container to run in" + ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG" + + DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$BASE_BUILD_DIR,dst=$BASE_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $BASE_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG) + + DOCKER_EXEC () { + docker exec $DOCKER_ID bash -c "cd $PWD && $*" + } +else + echo "Running on host system without docker wrapper" + DOCKER_EXEC () { + bash -c "cd $PWD && $*" + } +fi + +DOCKER_EXEC free -m -h +DOCKER_EXEC echo "Number of CPUs \(nproc\): $(nproc)" + +${CI_RETRY_EXE} DOCKER_EXEC apt-get update +${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES + diff --git a/.travis/test_05_before_script.sh b/ci/test/05_before_script.sh index 516d3fc042..516d3fc042 100755 --- a/.travis/test_05_before_script.sh +++ b/ci/test/05_before_script.sh diff --git a/.travis/test_06_script_a.sh b/ci/test/06_script_a.sh index 8cc593f936..eb6ade7919 100755 --- a/.travis/test_06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -6,11 +6,7 @@ export LC_ALL=C.UTF-8 -TRAVIS_COMMIT_LOG=$(git log --format=fuller -1) -export TRAVIS_COMMIT_LOG - -OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST -BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib" +BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$BASE_BUILD_DIR/depends/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" if [ -z "$NO_DEPENDS" ]; then DOCKER_EXEC ccache --max-size=$CCACHE_SIZE fi @@ -23,7 +19,7 @@ else fi END_FOLD -mkdir build +mkdir -p build cd build || (echo "could not enter build directory"; exit 1) BEGIN_FOLD configure @@ -41,10 +37,10 @@ DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOI END_FOLD set -o errtrace -trap 'DOCKER_EXEC "cat ${TRAVIS_BUILD_DIR}/sanitizer-output/* 2> /dev/null"' ERR +trap 'DOCKER_EXEC "cat ${BASE_BUILD_DIR}/sanitizer-output/* 2> /dev/null"' ERR BEGIN_FOLD build DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false ) END_FOLD -cd ${TRAVIS_BUILD_DIR} || (echo "could not enter travis build dir $TRAVIS_BUILD_DIR"; exit 1) +cd ${BASE_BUILD_DIR} || (echo "could not enter travis build dir $BASE_BUILD_DIR"; exit 1) diff --git a/.travis/test_06_script_b.sh b/ci/test/06_script_b.sh index e40055a6ee..fafb6e0493 100755 --- a/.travis/test_06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -10,7 +10,7 @@ cd "build/bitcoin-$HOST" || (echo "could not enter distdir build/bitcoin-$HOST"; if [ "$RUN_UNIT_TESTS" = "true" ]; then BEGIN_FOLD unit-tests - DOCKER_EXEC LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1 + DOCKER_EXEC LD_LIBRARY_PATH=$BASE_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1 END_FOLD fi @@ -26,4 +26,4 @@ if [ "$RUN_FUZZ_TESTS" = "true" ]; then END_FOLD fi -cd ${TRAVIS_BUILD_DIR} || (echo "could not enter travis build dir $TRAVIS_BUILD_DIR"; exit 1) +cd ${BASE_BUILD_DIR} || (echo "could not enter travis build dir $BASE_BUILD_DIR"; exit 1) diff --git a/ci/test_run_all.sh b/ci/test_run_all.sh new file mode 100755 index 0000000000..a39f1f9f09 --- /dev/null +++ b/ci/test_run_all.sh @@ -0,0 +1,16 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +echo "Setting default values in env" + +set -o errexit; source ./ci/test/00_setup_env.sh +set -o errexit; source ./ci/test/03_before_install.sh +set -o errexit; source ./ci/test/04_install.sh +set -o errexit; source ./ci/test/05_before_script.sh +set -o errexit; source ./ci/test/06_script_a.sh +set -o errexit; source ./ci/test/06_script_b.sh |