aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
-rw-r--r--.cirrus.yml24
-rw-r--r--.travis.yml39
-rw-r--r--.travis/README.md8
-rwxr-xr-x.travis/test_04_install.sh37
-rw-r--r--ci/README.md25
-rwxr-xr-xci/extended_lint/04_install.sh (renamed from .travis/extended_lint_04_install.sh)0
-rwxr-xr-xci/extended_lint/06_script.sh (renamed from .travis/extended_lint_06_script.sh)0
-rwxr-xr-xci/lint/04_install.sh (renamed from .travis/lint_04_install.sh)0
-rwxr-xr-xci/lint/05_before_script.sh (renamed from .travis/lint_05_before_script.sh)0
-rwxr-xr-xci/lint/06_script.sh (renamed from .travis/lint_06_script.sh)0
-rw-r--r--ci/retry/README.md123
-rwxr-xr-xci/retry/retry163
-rwxr-xr-xci/test/00_setup_env.sh33
-rwxr-xr-xci/test/03_before_install.sh (renamed from .travis/test_03_before_install.sh)1
-rwxr-xr-xci/test/04_install.sh49
-rwxr-xr-xci/test/05_before_script.sh (renamed from .travis/test_05_before_script.sh)0
-rwxr-xr-xci/test/06_script_a.sh (renamed from .travis/test_06_script_a.sh)12
-rwxr-xr-xci/test/06_script_b.sh (renamed from .travis/test_06_script_b.sh)4
-rwxr-xr-xci/test_run_all.sh16
19 files changed, 452 insertions, 82 deletions
diff --git a/.cirrus.yml b/.cirrus.yml
index ce17a223b1..1e6e6937da 100644
--- a/.cirrus.yml
+++ b/.cirrus.yml
@@ -27,3 +27,27 @@ task:
- gmake check ${MAKEJOBS} VERBOSE=1
functional_test_script:
- ./test/functional/test_runner.py --jobs 9 --ci --extended --exclude feature_dbcrash --combinedlogslen=1000 --quiet --failfast
+task:
+ name: "x86_64 Linux [GOAL: install] [bionic] [Using ./ci/ system]"
+ container:
+ image: ubuntu:18.04
+ cpu: 8
+ memory: 8G
+ timeout_in: 60m
+ env:
+ MAKEJOBS: "-j9"
+ RUN_CI_ON_HOST: "1"
+ CCACHE_SIZE: "200M"
+ CCACHE_DIR: "/tmp/ccache_dir"
+ ccache_cache:
+ folder: "/tmp/ccache_dir"
+ depends_built_cache:
+ folder: "/tmp/cirrus-ci-build/depends/built"
+ depends_sdk_cache:
+ folder: "/tmp/cirrus-ci-build/depends/sdk-sources"
+ install_script:
+ - apt-get update
+ - apt-get -y install git bash ccache
+ - ccache --max-size=${CCACHE_SIZE}
+ ci_script:
+ - ./ci/test_run_all.sh
diff --git a/.travis.yml b/.travis.yml
index d49c116a5e..7753dcc51f 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -40,38 +40,25 @@ stages:
- extended-lint
env:
global:
- - MAKEJOBS=-j3
- - RUN_UNIT_TESTS=true
- - RUN_FUNCTIONAL_TESTS=true
- - RUN_FUZZ_TESTS=false
- - DOCKER_NAME_TAG=ubuntu:18.04
- - BOOST_TEST_RANDOM=1$TRAVIS_BUILD_ID
- - CCACHE_SIZE=100M
- - CCACHE_TEMPDIR=/tmp/.ccache-temp
- - CCACHE_COMPRESS=1
- - CCACHE_DIR=$HOME/.ccache
- - BASE_OUTDIR=$TRAVIS_BUILD_DIR/out
- - SDK_URL=https://bitcoincore.org/depends-sources/sdks
- - WINEDEBUG=fixme-all
- - DOCKER_PACKAGES="build-essential libtool autotools-dev automake pkg-config bsdmainutils curl git ca-certificates ccache"
+ - CI_RETRY_EXE="travis_retry"
- CACHE_ERR_MSG="Error! Initial build successful, but not enough time remains to run later build stages and tests. Please manually re-run this job by using the travis restart button or asking a bitcoin maintainer to restart. The next run should not time out because the build cache has been saved."
before_install:
- - set -o errexit; source .travis/test_03_before_install.sh
+ - set -o errexit; source ./ci/test/00_setup_env.sh
+ - set -o errexit; source ./ci/test/03_before_install.sh
install:
- - set -o errexit; source .travis/test_04_install.sh
+ - set -o errexit; source ./ci/test/04_install.sh
before_script:
- - set -o errexit; source .travis/test_05_before_script.sh
+ - set -o errexit; source ./ci/test/05_before_script.sh
script:
- export CONTINUE=1
- if [ $SECONDS -gt 1200 ]; then export CONTINUE=0; fi # Likely the depends build took very long
- if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi # Whitelisted repo (90 minutes build time)
- - if [ $CONTINUE = "1" ]; then set -o errexit; source .travis/test_06_script_a.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi
+ - if [ $CONTINUE = "1" ]; then set -o errexit; source ./ci/test/06_script_a.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi
- if [ $SECONDS -gt 2000 ]; then export CONTINUE=0; fi # Likely the build took very long; The tests take about 1000s, so we should abort if we have less than 50*60-1000=2000s left
- if [ $TRAVIS_REPO_SLUG = "bitcoin/bitcoin" ]; then export CONTINUE=1; fi # Whitelisted repo (90 minutes build time)
- - if [ $CONTINUE = "1" ]; then set -o errexit; source .travis/test_06_script_b.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi
+ - if [ $CONTINUE = "1" ]; then set -o errexit; source ./ci/test/06_script_b.sh; else set +o errexit; echo "$CACHE_ERR_MSG"; false; fi
after_script:
- echo $TRAVIS_COMMIT_RANGE
- - echo $TRAVIS_COMMIT_LOG
jobs:
include:
@@ -82,11 +69,11 @@ jobs:
language: python
python: '3.5' # Oldest supported version according to doc/dependencies.md
install:
- - set -o errexit; source .travis/lint_04_install.sh
+ - set -o errexit; source ./ci/lint/04_install.sh
before_script:
- - set -o errexit; source .travis/lint_05_before_script.sh
+ - set -o errexit; source ./ci/lint/05_before_script.sh
script:
- - set -o errexit; source .travis/lint_06_script.sh
+ - set -o errexit; source ./ci/lint/06_script.sh
- stage: extended-lint
name: 'extended lint [runtime >= 60 seconds]'
@@ -95,11 +82,11 @@ jobs:
language: python
python: '3.5'
install:
- - set -o errexit; source .travis/extended_lint_04_install.sh
+ - set -o errexit; source ./ci/extended_lint/04_install.sh
before_script:
- - set -o errexit; source .travis/lint_05_before_script.sh
+ - set -o errexit; source ./ci/lint/05_before_script.sh
script:
- - set -o errexit; source .travis/extended_lint_06_script.sh
+ - set -o errexit; source ./ci/extended_lint/06_script.sh
- stage: test
name: 'ARM [GOAL: install] [no unit or functional tests]'
diff --git a/.travis/README.md b/.travis/README.md
deleted file mode 100644
index 21d1b9cc03..0000000000
--- a/.travis/README.md
+++ /dev/null
@@ -1,8 +0,0 @@
-## travis build scripts
-
-The `.travis` directory contains scripts for each build step in each build stage.
-Currently the travis build defines two stages `lint` and `test`. Each stage has
-it's own [lifecycle](https://docs.travis-ci.com/user/customizing-the-build/#the-build-lifecycle).
-Every script in here is named and numbered according to which stage and lifecycle
-step it belongs to.
-
diff --git a/.travis/test_04_install.sh b/.travis/test_04_install.sh
deleted file mode 100755
index 319f2c5b21..0000000000
--- a/.travis/test_04_install.sh
+++ /dev/null
@@ -1,37 +0,0 @@
-#!/usr/bin/env bash
-#
-# Copyright (c) 2018 The Bitcoin Core developers
-# Distributed under the MIT software license, see the accompanying
-# file COPYING or http://www.opensource.org/licenses/mit-license.php.
-
-export LC_ALL=C.UTF-8
-
-free -m -h
-echo "Number of CPUs (nproc): $(nproc)"
-
-travis_retry docker pull "$DOCKER_NAME_TAG"
-
-export DIR_FUZZ_IN=${TRAVIS_BUILD_DIR}/qa-assets
-git clone https://github.com/bitcoin-core/qa-assets ${DIR_FUZZ_IN}
-export DIR_FUZZ_IN=${DIR_FUZZ_IN}/fuzz_seed_corpus/
-
-mkdir -p "${TRAVIS_BUILD_DIR}/sanitizer-output/"
-export ASAN_OPTIONS=""
-export LSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/lsan"
-export TSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${TRAVIS_BUILD_DIR}/sanitizer-output/tsan"
-export UBSAN_OPTIONS="suppressions=${TRAVIS_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1"
-env | grep -E '^(BITCOIN_CONFIG|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env
-if [[ $HOST = *-mingw32 ]]; then
- DOCKER_ADMIN="--cap-add SYS_ADMIN"
-elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764)
- DOCKER_ADMIN="--cap-add SYS_PTRACE"
-fi
-DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$TRAVIS_BUILD_DIR,dst=$TRAVIS_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $TRAVIS_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG)
-
-DOCKER_EXEC () {
- docker exec $DOCKER_ID bash -c "cd $PWD && $*"
-}
-
-travis_retry DOCKER_EXEC apt-get update
-travis_retry DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES
-
diff --git a/ci/README.md b/ci/README.md
new file mode 100644
index 0000000000..754cbc7c95
--- /dev/null
+++ b/ci/README.md
@@ -0,0 +1,25 @@
+## ci scripts
+
+This directory contains scripts for each build step in each build stage.
+
+Currently three stages `lint`, `extended_lint` and `test` are defined. Each stage has its own lifecycle, similar to the
+[Travis CI lifecycle](https://docs.travis-ci.com/user/job-lifecycle#the-job-lifecycle). Every script in here is named
+and numbered according to which stage and lifecycle step it belongs to.
+
+### Running a stage locally
+
+To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage
+requires `docker` to be installed. To install all requirements on Ubuntu, run
+
+```
+sudo apt install docker.io ccache bash git
+```
+
+To run the test stage,
+
+```
+./ci/test_run_all.sh
+```
+
+Be aware that the tests will be build and run in-place, so please run at your own risk.
+If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first.
diff --git a/.travis/extended_lint_04_install.sh b/ci/extended_lint/04_install.sh
index 123d874a84..123d874a84 100755
--- a/.travis/extended_lint_04_install.sh
+++ b/ci/extended_lint/04_install.sh
diff --git a/.travis/extended_lint_06_script.sh b/ci/extended_lint/06_script.sh
index e8228c9c4d..e8228c9c4d 100755
--- a/.travis/extended_lint_06_script.sh
+++ b/ci/extended_lint/06_script.sh
diff --git a/.travis/lint_04_install.sh b/ci/lint/04_install.sh
index 20bff368a5..20bff368a5 100755
--- a/.travis/lint_04_install.sh
+++ b/ci/lint/04_install.sh
diff --git a/.travis/lint_05_before_script.sh b/ci/lint/05_before_script.sh
index 28bcbb47f7..28bcbb47f7 100755
--- a/.travis/lint_05_before_script.sh
+++ b/ci/lint/05_before_script.sh
diff --git a/.travis/lint_06_script.sh b/ci/lint/06_script.sh
index c7dea599dc..c7dea599dc 100755
--- a/.travis/lint_06_script.sh
+++ b/ci/lint/06_script.sh
diff --git a/ci/retry/README.md b/ci/retry/README.md
new file mode 100644
index 0000000000..983a498070
--- /dev/null
+++ b/ci/retry/README.md
@@ -0,0 +1,123 @@
+retry - The command line retry tool
+------------------------------------------
+
+Retry any shell command with exponential backoff or constant delay.
+
+### Instructions
+
+Install:
+
+retry is a shell script, so drop it somewhere and make sure it's added to your $PATH. Or you can use the following one-liner:
+
+```sh
+sudo sh -c "curl https://raw.githubusercontent.com/kadwanev/retry/master/retry -o /usr/local/bin/retry && chmod +x /usr/local/bin/retry"
+```
+
+If you're on OS X, retry is also on Homebrew:
+
+```
+brew pull 27283
+brew install retry
+```
+Not popular enough for homebrew-core. Please star this project to help.
+
+### Usage
+
+Help:
+
+`retry -?`
+
+ Usage: retry [options] -- execute command
+ -h, -?, --help
+ -v, --verbose Verbose output
+ -t, --tries=# Set max retries: Default 10
+ -s, --sleep=secs Constant sleep amount (seconds)
+ -m, --min=secs Exponenetial Backoff: minimum sleep amount (seconds): Default 0.3
+ -x, --max=secs Exponenetial Backoff: maximum sleep amount (seconds): Default 60
+ -f, --fail="script +cmds" Fail Script: run in case of final failure
+
+### Examples
+
+No problem:
+
+`retry echo u work good`
+
+ u work good
+
+Test functionality:
+
+`retry 'echo "y u no work"; false'`
+
+ y u no work
+ Before retry #1: sleeping 0.3 seconds
+ y u no work
+ Before retry #2: sleeping 0.6 seconds
+ y u no work
+ Before retry #3: sleeping 1.2 seconds
+ y u no work
+ Before retry #4: sleeping 2.4 seconds
+ y u no work
+ Before retry #5: sleeping 4.8 seconds
+ y u no work
+ Before retry #6: sleeping 9.6 seconds
+ y u no work
+ Before retry #7: sleeping 19.2 seconds
+ y u no work
+ Before retry #8: sleeping 38.4 seconds
+ y u no work
+ Before retry #9: sleeping 60.0 seconds
+ y u no work
+ Before retry #10: sleeping 60.0 seconds
+ y u no work
+ etc..
+
+Limit retries:
+
+`retry -t 4 'echo "y u no work"; false'`
+
+ y u no work
+ Before retry #1: sleeping 0.3 seconds
+ y u no work
+ Before retry #2: sleeping 0.6 seconds
+ y u no work
+ Before retry #3: sleeping 1.2 seconds
+ y u no work
+ Before retry #4: sleeping 2.4 seconds
+ y u no work
+ Retries exhausted
+
+Bad command:
+
+`retry poop`
+
+ bash: poop: command not found
+
+Fail command:
+
+`retry -t 3 -f 'echo "oh poopsickles"' 'echo "y u no work"; false'`
+
+ y u no work
+ Before retry #1: sleeping 0.3 seconds
+ y u no work
+ Before retry #2: sleeping 0.6 seconds
+ y u no work
+ Before retry #3: sleeping 1.2 seconds
+ y u no work
+ Retries exhausted, running fail script
+ oh poopsickles
+
+Last attempt passed:
+
+`retry -t 3 -- 'if [ $RETRY_ATTEMPT -eq 3 ]; then echo Passed at attempt $RETRY_ATTEMPT; true; else echo Failed at attempt $RETRY_ATTEMPT; false; fi;'`
+
+ Failed at attempt 0
+ Before retry #1: sleeping 0.3 seconds
+ Failed at attempt 1
+ Before retry #2: sleeping 0.6 seconds
+ Failed at attempt 2
+ Before retry #3: sleeping 1.2 seconds
+ Passed at attempt 3
+
+### License
+
+Apache 2.0 - go nuts
diff --git a/ci/retry/retry b/ci/retry/retry
new file mode 100755
index 0000000000..0e5f6e9701
--- /dev/null
+++ b/ci/retry/retry
@@ -0,0 +1,163 @@
+#!/usr/bin/env bash
+
+GETOPT_BIN=$IN_GETOPT_BIN
+GETOPT_BIN=${GETOPT_BIN:-getopt}
+
+__sleep_amount() {
+ if [ -n "$constant_sleep" ]; then
+ sleep_time=$constant_sleep
+ else
+ #TODO: check for awk
+ #TODO: check if user would rather use one of the other possible dependencies: python, ruby, bc, dc
+ sleep_time=`awk "BEGIN {t = $min_sleep * $(( (1<<($attempts -1)) )); print (t > $max_sleep ? $max_sleep : t)}"`
+ fi
+}
+
+__log_out() {
+ echo "$1" 1>&2
+}
+
+# Paramters: max_tries min_sleep max_sleep constant_sleep fail_script EXECUTION_COMMAND
+retry()
+{
+ local max_tries="$1"; shift
+ local min_sleep="$1"; shift
+ local max_sleep="$1"; shift
+ local constant_sleep="$1"; shift
+ local fail_script="$1"; shift
+ if [ -n "$VERBOSE" ]; then
+ __log_out "Retry Parameters: max_tries=$max_tries min_sleep=$min_sleep max_sleep=$max_sleep constant_sleep=$constant_sleep"
+ if [ -n "$fail_script" ]; then __log_out "Fail script: $fail_script"; fi
+ __log_out ""
+ __log_out "Execution Command: $*"
+ __log_out ""
+ fi
+
+ local attempts=0
+ local return_code=1
+
+
+ while [[ $return_code -ne 0 && $attempts -le $max_tries ]]; do
+ if [ $attempts -gt 0 ]; then
+ __sleep_amount
+ __log_out "Before retry #$attempts: sleeping $sleep_time seconds"
+ sleep $sleep_time
+ fi
+
+ P="$1"
+ for param in "${@:2}"; do P="$P '$param'"; done
+ #TODO: replace single quotes in each arg with '"'"' ?
+ export RETRY_ATTEMPT=$attempts
+ bash -c "$P"
+ return_code=$?
+ #__log_out "Process returned $return_code on attempt $attempts"
+ if [ $return_code -eq 127 ]; then
+ # command not found
+ exit $return_code
+ elif [ $return_code -ne 0 ]; then
+ attempts=$[$attempts +1]
+ fi
+ done
+
+ if [ $attempts -gt $max_tries ]; then
+ if [ -n "$fail_script" ]; then
+ __log_out "Retries exhausted, running fail script"
+ eval $fail_script
+ else
+ __log_out "Retries exhausted"
+ fi
+ fi
+
+ exit $return_code
+}
+
+# If we're being sourced, don't worry about such things
+if [ "$BASH_SOURCE" == "$0" ]; then
+ # Prints the help text
+ help()
+ {
+ local retry=$(basename $0)
+ cat <<EOF
+Usage: $retry [options] -- execute command
+ -h, -?, --help
+ -v, --verbose Verbose output
+ -t, --tries=# Set max retries: Default 10
+ -s, --sleep=secs Constant sleep amount (seconds)
+ -m, --min=secs Exponenetial Backoff: minimum sleep amount (seconds): Default 0.3
+ -x, --max=secs Exponenetial Backoff: maximum sleep amount (seconds): Default 60
+ -f, --fail="script +cmds" Fail Script: run in case of final failure
+EOF
+ }
+
+ # show help for no arguments if stdin is a terminal
+ if { [ -z "$1" ] && [ -t 0 ] ; } || [ "$1" == '-h' ] || [ "$1" == '-?' ] || [ "$1" == '--help' ]
+ then
+ help
+ exit 0
+ fi
+
+ $GETOPT_BIN --test > /dev/null
+ if [[ $? -ne 4 ]]; then
+ echo "I’m sorry, 'getopt --test' failed in this environment. Please load GNU getopt."
+ exit 1
+ fi
+
+ OPTIONS=vt:s:m:x:f:
+ LONGOPTIONS=verbose,tries:,sleep:,min:,max:,fail:
+
+ PARSED=$($GETOPT_BIN --options="$OPTIONS" --longoptions="$LONGOPTIONS" --name "$0" -- "$@")
+ if [[ $? -ne 0 ]]; then
+ # e.g. $? == 1
+ # then getopt has complained about wrong arguments to stdout
+ exit 2
+ fi
+ # read getopt’s output this way to handle the quoting right:
+ eval set -- "$PARSED"
+
+ max_tries=10
+ min_sleep=0.3
+ max_sleep=60.0
+ constant_sleep=
+ fail_script=
+
+ # now enjoy the options in order and nicely split until we see --
+ while true; do
+ case "$1" in
+ -v|--verbose)
+ VERBOSE=true
+ shift
+ ;;
+ -t|--tries)
+ max_tries="$2"
+ shift 2
+ ;;
+ -s|--sleep)
+ constant_sleep="$2"
+ shift 2
+ ;;
+ -m|--min)
+ min_sleep="$2"
+ shift 2
+ ;;
+ -x|--max)
+ max_sleep="$2"
+ shift 2
+ ;;
+ -f|--fail)
+ fail_script="$2"
+ shift 2
+ ;;
+ --)
+ shift
+ break
+ ;;
+ *)
+ echo "Programming error"
+ exit 3
+ ;;
+ esac
+ done
+
+ retry "$max_tries" "$min_sleep" "$max_sleep" "$constant_sleep" "$fail_script" "$@"
+
+fi
diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh
new file mode 100755
index 0000000000..fae48ba176
--- /dev/null
+++ b/ci/test/00_setup_env.sh
@@ -0,0 +1,33 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+echo "Setting default values in env"
+
+BASE_ROOT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )"/../../ >/dev/null 2>&1 && pwd )
+export BASE_ROOT_DIR
+
+export MAKEJOBS=${MAKEJOBS:--j3}
+export HOST=${HOST:-x86_64-unknown-linux-gnu}
+export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true}
+export RUN_FUNCTIONAL_TESTS=${RUN_FUNCTIONAL_TESTS:-true}
+export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false}
+export DOCKER_NAME_TAG=${DOCKER_NAME_TAG:-ubuntu:18.04}
+export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1$TRAVIS_BUILD_ID}
+export CCACHE_SIZE=${CCACHE_SIZE:-100M}
+export CCACHE_TEMPDIR=${CCACHE_TEMPDIR:-/tmp/.ccache-temp}
+export CCACHE_COMPRESS=${CCACHE_COMPRESS:-1}
+export CCACHE_DIR=${CCACHE_DIR:-$HOME/.ccache}
+export BASE_BUILD_DIR=${BASE_BUILD_DIR:-${TRAVIS_BUILD_DIR:-$BASE_ROOT_DIR}}
+export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST}
+export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
+export WINEDEBUG=${WINEDEBUG:-fixme-all}
+export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3}
+export GOAL=${GOAL:-install}
+export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets}
+export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH
+export CI_RETRY_EXE=${CI_RETRY_EXE:retry}
diff --git a/.travis/test_03_before_install.sh b/ci/test/03_before_install.sh
index 3c9fcf3f98..5086114ba1 100755
--- a/.travis/test_03_before_install.sh
+++ b/ci/test/03_before_install.sh
@@ -6,7 +6,6 @@
export LC_ALL=C.UTF-8
-PATH=$(echo $PATH | tr ':' "\n" | sed '/\/opt\/python/d' | tr "\n" ":" | sed "s|::|:|g")
# Add llvm-symbolizer directory to PATH. Needed to get symbolized stack traces from the sanitizers.
PATH=$PATH:/usr/lib/llvm-6.0/bin/
export PATH
diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh
new file mode 100755
index 0000000000..3535746e83
--- /dev/null
+++ b/ci/test/04_install.sh
@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2018 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+ccache echo "Creating ccache dir if it didn't already exist"
+
+if [ ! -d ${DIR_QA_ASSETS} ]; then
+ git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS}
+fi
+export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/
+
+mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/"
+export ASAN_OPTIONS=""
+export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan"
+export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_BUILD_DIR}/sanitizer-output/tsan"
+export UBSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1"
+env | grep -E '^(BITCOIN_CONFIG|CCACHE_|WINEDEBUG|LC_ALL|BOOST_TEST_RANDOM|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS)' | tee /tmp/env
+if [[ $HOST = *-mingw32 ]]; then
+ DOCKER_ADMIN="--cap-add SYS_ADMIN"
+elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764)
+ DOCKER_ADMIN="--cap-add SYS_PTRACE"
+fi
+
+if [ -z "$RUN_CI_ON_HOST" ]; then
+ echo "Creating $DOCKER_NAME_TAG container to run in"
+ ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG"
+
+ DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$BASE_BUILD_DIR,dst=$BASE_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $BASE_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG)
+
+ DOCKER_EXEC () {
+ docker exec $DOCKER_ID bash -c "cd $PWD && $*"
+ }
+else
+ echo "Running on host system without docker wrapper"
+ DOCKER_EXEC () {
+ bash -c "cd $PWD && $*"
+ }
+fi
+
+DOCKER_EXEC free -m -h
+DOCKER_EXEC echo "Number of CPUs \(nproc\): $(nproc)"
+
+${CI_RETRY_EXE} DOCKER_EXEC apt-get update
+${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -qq $PACKAGES $DOCKER_PACKAGES
+
diff --git a/.travis/test_05_before_script.sh b/ci/test/05_before_script.sh
index 516d3fc042..516d3fc042 100755
--- a/.travis/test_05_before_script.sh
+++ b/ci/test/05_before_script.sh
diff --git a/.travis/test_06_script_a.sh b/ci/test/06_script_a.sh
index 8cc593f936..eb6ade7919 100755
--- a/.travis/test_06_script_a.sh
+++ b/ci/test/06_script_a.sh
@@ -6,11 +6,7 @@
export LC_ALL=C.UTF-8
-TRAVIS_COMMIT_LOG=$(git log --format=fuller -1)
-export TRAVIS_COMMIT_LOG
-
-OUTDIR=$BASE_OUTDIR/$TRAVIS_PULL_REQUEST/$TRAVIS_JOB_NUMBER-$HOST
-BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$TRAVIS_BUILD_DIR/depends/$HOST --bindir=$OUTDIR/bin --libdir=$OUTDIR/lib"
+BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$BASE_BUILD_DIR/depends/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib"
if [ -z "$NO_DEPENDS" ]; then
DOCKER_EXEC ccache --max-size=$CCACHE_SIZE
fi
@@ -23,7 +19,7 @@ else
fi
END_FOLD
-mkdir build
+mkdir -p build
cd build || (echo "could not enter build directory"; exit 1)
BEGIN_FOLD configure
@@ -41,10 +37,10 @@ DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOI
END_FOLD
set -o errtrace
-trap 'DOCKER_EXEC "cat ${TRAVIS_BUILD_DIR}/sanitizer-output/* 2> /dev/null"' ERR
+trap 'DOCKER_EXEC "cat ${BASE_BUILD_DIR}/sanitizer-output/* 2> /dev/null"' ERR
BEGIN_FOLD build
DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false )
END_FOLD
-cd ${TRAVIS_BUILD_DIR} || (echo "could not enter travis build dir $TRAVIS_BUILD_DIR"; exit 1)
+cd ${BASE_BUILD_DIR} || (echo "could not enter travis build dir $BASE_BUILD_DIR"; exit 1)
diff --git a/.travis/test_06_script_b.sh b/ci/test/06_script_b.sh
index e40055a6ee..fafb6e0493 100755
--- a/.travis/test_06_script_b.sh
+++ b/ci/test/06_script_b.sh
@@ -10,7 +10,7 @@ cd "build/bitcoin-$HOST" || (echo "could not enter distdir build/bitcoin-$HOST";
if [ "$RUN_UNIT_TESTS" = "true" ]; then
BEGIN_FOLD unit-tests
- DOCKER_EXEC LD_LIBRARY_PATH=$TRAVIS_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1
+ DOCKER_EXEC LD_LIBRARY_PATH=$BASE_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1
END_FOLD
fi
@@ -26,4 +26,4 @@ if [ "$RUN_FUZZ_TESTS" = "true" ]; then
END_FOLD
fi
-cd ${TRAVIS_BUILD_DIR} || (echo "could not enter travis build dir $TRAVIS_BUILD_DIR"; exit 1)
+cd ${BASE_BUILD_DIR} || (echo "could not enter travis build dir $BASE_BUILD_DIR"; exit 1)
diff --git a/ci/test_run_all.sh b/ci/test_run_all.sh
new file mode 100755
index 0000000000..a39f1f9f09
--- /dev/null
+++ b/ci/test_run_all.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+echo "Setting default values in env"
+
+set -o errexit; source ./ci/test/00_setup_env.sh
+set -o errexit; source ./ci/test/03_before_install.sh
+set -o errexit; source ./ci/test/04_install.sh
+set -o errexit; source ./ci/test/05_before_script.sh
+set -o errexit; source ./ci/test/06_script_a.sh
+set -o errexit; source ./ci/test/06_script_b.sh