15 files changed, 71 insertions, 38 deletions
diff --git a/ci/README.md b/ci/README.md
index fb1cd7460b..880e49b459 100644
--- a/ci/README.md
+++ b/ci/README.md
@@ -8,11 +8,21 @@ and numbered according to which stage and lifecycle step it belongs to.
 
 ### Running a stage locally
 
+Be aware that the tests will be built and run in-place, so please run at your own risk.
+If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first.
+
+The ci needs to perform various sysadmin tasks such as installing packages or writing to the user's home directory.
+While most of the actions are done inside a docker container, this is not possible for all. Thus, cache directories,
+such as the depends cache or ccache, are mounted as read-write into the docker container. While it should be fine to run
+the ci system locally on you development box, the ci scripts can generally be assumed to have received less review and
+testing compared to other parts of the codebase. If you want to keep the work tree clean, you might want to run the ci
+system in a virtual machine with a Linux operating system of your choice.
+
 To allow for a wide range of tested environments, but also ensure reproducibility to some extent, the test stage
 requires `docker` to be installed. To install all requirements on Ubuntu, run
 
 ```
-sudo apt install docker.io bash git
+sudo apt install docker.io bash
 ```
 
 To run the default test stage,
@@ -26,6 +36,3 @@ To run the test stage with a specific configuration,
 ```
 FILE_ENV="./ci/test/00_setup_env_arm.sh" ./ci/test_run_all.sh
 ```
-
-Be aware that the tests will be build and run in-place, so please run at your own risk.
-If the repository is not a fresh git clone, you might have to clean files from previous builds or test runs first.
diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh
index 8bc50da2c3..2919072fe4 100755
--- a/ci/test/00_setup_env.sh
+++ b/ci/test/00_setup_env.sh
@@ -21,7 +21,10 @@ echo "Fallback to default values in env (if not yet set)"
 export MAKEJOBS=${MAKEJOBS:--j4}
 # A folder for the ci system to put temporary files (ccache, datadirs for tests, ...)
 export BASE_SCRATCH_DIR=${BASE_SCRATCH_DIR:-$BASE_ROOT_DIR/ci/scratch/}
-export HOST=${HOST:-x86_64-unknown-linux-gnu}
+# What host to compile for. See also ./depends/README.md
+# Tests that need cross-compilation export the appropriate HOST.
+# Tests that run natively guess the host
+export HOST=${HOST:-$("$BASE_ROOT_DIR/depends/config.guess")}
 # Whether to prefer BusyBox over GNU utilities
 export USE_BUSY_BOX=${USE_BUSY_BOX:-false}
 export RUN_UNIT_TESTS=${RUN_UNIT_TESTS:-true}
@@ -41,7 +44,7 @@ export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_ROOT_DIR}
 export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_BUILD_DIR/out/$HOST}
 export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks}
 export WINEDEBUG=${WINEDEBUG:-fixme-all}
-export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3}
+export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git}
 export GOAL=${GOAL:-install}
 export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_BUILD_DIR}/qa-assets}
 export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH
diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh
index f384ba9263..af166b6ca7 100644
--- a/ci/test/00_setup_env_mac.sh
+++ b/ci/test/00_setup_env_mac.sh
@@ -6,7 +6,7 @@
 
 export LC_ALL=C.UTF-8
 
-export HOST=x86_64-apple-darwin14
+export HOST=x86_64-apple-darwin16
 export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools"
 export OSX_SDK=10.11
 export RUN_UNIT_TESTS=false
diff --git a/ci/test/00_setup_env_mac_host.sh b/ci/test/00_setup_env_mac_host.sh
index aa68a5700e..0b437a723f 100644
--- a/ci/test/00_setup_env_mac_host.sh
+++ b/ci/test/00_setup_env_mac_host.sh
@@ -6,7 +6,7 @@
 
 export LC_ALL=C.UTF-8
 
-export HOST=x86_64-apple-darwin14
+export HOST=x86_64-apple-darwin16
 export BREW_PACKAGES="automake berkeley-db4 libtool boost miniupnpc pkg-config qt qrencode python3 ccache zeromq"
 export PIP_PACKAGES="zmq"
 export RUN_CI_ON_HOST=true
diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh
new file mode 100644
index 0000000000..b354940d35
--- /dev/null
+++ b/ci/test/00_setup_env_native_asan.sh
@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+#
+# Copyright (c) 2019 The Bitcoin Core developers
+# Distributed under the MIT software license, see the accompanying
+# file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+export LC_ALL=C.UTF-8
+
+export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev"
+export NO_DEPENDS=1
+export GOAL="install"
+export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++"
diff --git a/ci/test/00_setup_env_amd64_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh
index 7bdfc2c320..b0405bb762 100644
--- a/ci/test/00_setup_env_amd64_fuzz.sh
+++ b/ci/test/00_setup_env_native_fuzz.sh
@@ -6,11 +6,11 @@
 
 export LC_ALL=C.UTF-8
 
-export HOST=x86_64-unknown-linux-gnu
-export PACKAGES="clang llvm python3 libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev"
+export PACKAGES="clang-8 llvm-8 python3 libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev"
 export NO_DEPENDS=1
 export RUN_UNIT_TESTS=false
 export RUN_FUNCTIONAL_TESTS=false
 export RUN_FUZZ_TESTS=true
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang CXX=clang++"
+export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang-8 CXX=clang++-8"
+# Use clang-8, instead of default clang on bionic, which is clang-6 and does not come with libfuzzer on aarch64
diff --git a/ci/test/00_setup_env_amd64_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh
index d5a2ba3111..53348559be 100644
--- a/ci/test/00_setup_env_amd64_nowallet.sh
+++ b/ci/test/00_setup_env_native_nowallet.sh
@@ -6,7 +6,6 @@
 
 export LC_ALL=C.UTF-8
 
-export HOST=x86_64-unknown-linux-gnu
 export PACKAGES="python3-zmq"
 export DEP_OPTS="NO_WALLET=1"
 export GOAL="install"
diff --git a/ci/test/00_setup_env_amd64_qt5.sh b/ci/test/00_setup_env_native_qt5.sh
index 55820ea835..622ec3cfe1 100644
--- a/ci/test/00_setup_env_amd64_qt5.sh
+++ b/ci/test/00_setup_env_native_qt5.sh
@@ -6,7 +6,6 @@
 
 export LC_ALL=C.UTF-8
 
-export HOST=x86_64-unknown-linux-gnu
 export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev"
 export DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1"
 export TEST_RUNNER_EXTRA="--coverage --extended --exclude feature_dbcrash"  # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash
diff --git a/ci/test/00_setup_env_amd64_trusty.sh b/ci/test/00_setup_env_native_trusty.sh
index 51e98788c7..893c1f6b53 100644
--- a/ci/test/00_setup_env_amd64_trusty.sh
+++ b/ci/test/00_setup_env_native_trusty.sh
@@ -6,9 +6,8 @@
 
 export LC_ALL=C.UTF-8
 
-export HOST=x86_64-unknown-linux-gnu
 export DOCKER_NAME_TAG=ubuntu:14.04
-export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libicu-dev libpng-dev libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.1++-dev libzmq3-dev libqrencode-dev"
+export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libicu-dev libpng-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.1++-dev libzmq3-dev libqrencode-dev"
 export NO_DEPENDS=1
 export RUN_FUNCTIONAL_TESTS=false
 export GOAL="install"
diff --git a/ci/test/00_setup_env_amd64_tsan.sh b/ci/test/00_setup_env_native_tsan.sh
index 82ac988c41..f7a16206ab 100644
--- a/ci/test/00_setup_env_amd64_tsan.sh
+++ b/ci/test/00_setup_env_native_tsan.sh
@@ -6,9 +6,8 @@
 
 export LC_ALL=C.UTF-8
 
-export HOST=x86_64-unknown-linux-gnu
 export DOCKER_NAME_TAG=ubuntu:16.04
-export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libssl-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev"
+export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev"
 export NO_DEPENDS=1
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=thread --disable-hardening --disable-asm CC=clang CXX=clang++"
+export BITCOIN_CONFIG="--enable-zmq --disable-wallet --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=thread --disable-hardening --disable-asm CC=clang CXX=clang++"
diff --git a/ci/test/00_setup_env_amd64_asan.sh b/ci/test/00_setup_env_s390x.sh
index 46b870e145..b41d44c61a 100644
--- a/ci/test/00_setup_env_amd64_asan.sh
+++ b/ci/test/00_setup_env_s390x.sh
@@ -6,8 +6,13 @@
 
 export LC_ALL=C.UTF-8
 
-export HOST=x86_64-unknown-linux-gnu
+export HOST=s390x-unknown-linux-gnu
+export DOCKER_NAME_TAG=s390x/ubuntu:18.04
 export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libssl1.0-dev libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-chrono-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev"
 export NO_DEPENDS=1
+export RUN_UNIT_TESTS=true
+export RUN_FUNCTIONAL_TESTS=false
 export GOAL="install"
-export BITCOIN_CONFIG="--enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS=-DDEBUG_LOCKORDER --with-sanitizers=address,integer,undefined CC=clang CXX=clang++"
+export BITCOIN_CONFIG="--enable-reduce-exports --with-incompatible-bdb"
+
+lscpu
diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh
index 271ae82e5c..ce41c37b6e 100755
--- a/ci/test/04_install.sh
+++ b/ci/test/04_install.sh
@@ -25,8 +25,6 @@ if [ "$TRAVIS_OS_NAME" == "osx" ]; then
   done
 
   export PATH="/usr/local/opt/ccache/libexec:$PATH"
-  OPENSSL_PKG_CONFIG_PATH="/usr/local/opt/openssl@1.1/lib/pkgconfig"
-  export PKG_CONFIG_PATH=$OPENSSL_PKG_CONFIG_PATH:$PKG_CONFIG_PATH
 
   ${CI_RETRY_EXE} pip3 install $PIP_PACKAGES
 
@@ -35,12 +33,6 @@ fi
 mkdir -p "${BASE_SCRATCH_DIR}"
 mkdir -p "${CCACHE_DIR}"
 
-if [ ! -d ${DIR_QA_ASSETS} ]; then
-  git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS}
-fi
-export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/
-
-mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/"
 export ASAN_OPTIONS="detect_stack_use_after_return=1"
 export LSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/lsan"
 export TSAN_OPTIONS="suppressions=${BASE_BUILD_DIR}/test/sanitizer_suppressions/tsan:log_path=${BASE_BUILD_DIR}/sanitizer-output/tsan"
@@ -56,7 +48,13 @@ if [ -z "$RUN_CI_ON_HOST" ]; then
   echo "Creating $DOCKER_NAME_TAG container to run in"
   ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG"
 
-  DOCKER_ID=$(docker run $DOCKER_ADMIN -idt --mount type=bind,src=$BASE_BUILD_DIR,dst=$BASE_BUILD_DIR --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR -w $BASE_BUILD_DIR --env-file /tmp/env $DOCKER_NAME_TAG)
+  DOCKER_ID=$(docker run $DOCKER_ADMIN -idt \
+                  --mount type=bind,src=$BASE_BUILD_DIR,dst=/ro_base,readonly \
+                  --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \
+                  --mount type=bind,src=$BASE_BUILD_DIR/depends,dst=$BASE_BUILD_DIR/depends \
+                  -w $BASE_BUILD_DIR \
+                  --env-file /tmp/env \
+                  $DOCKER_NAME_TAG)
 
   DOCKER_EXEC () {
     docker exec $DOCKER_ID bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd $PWD && $*"
@@ -85,6 +83,18 @@ if [ "$TRAVIS_OS_NAME" != "osx" ]; then
   ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES
 fi
 
+if [ ! -d ${DIR_QA_ASSETS} ]; then
+  DOCKER_EXEC git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS}
+fi
+export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/
+
+DOCKER_EXEC mkdir -p "${BASE_BUILD_DIR}/sanitizer-output/"
+
+if [ -z "$RUN_CI_ON_HOST" ]; then
+  echo "Create $BASE_BUILD_DIR"
+  DOCKER_EXEC rsync -a /ro_base/ $BASE_BUILD_DIR
+fi
+
 if [ "$USE_BUSY_BOX" = "true" ]; then
   echo "Setup to use BusyBox utils"
   DOCKER_EXEC mkdir -p $BASE_SCRATCH_DIR/bins/
diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh
index a0488f0807..3f0bba19ac 100755
--- a/ci/test/05_before_script.sh
+++ b/ci/test/05_before_script.sh
@@ -13,13 +13,13 @@ else
   DOCKER_EXEC echo \> \$HOME/.bitcoin
 fi
 
-mkdir -p depends/SDKs depends/sdk-sources
+DOCKER_EXEC mkdir -p depends/SDKs depends/sdk-sources
 
 if [ -n "$OSX_SDK" ] && [ ! -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
   curl --location --fail $SDK_URL/MacOSX${OSX_SDK}.sdk.tar.gz -o depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
 fi
 if [ -n "$OSX_SDK" ] && [ -f depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz ]; then
-  tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
+  DOCKER_EXEC tar -C depends/SDKs -xf depends/sdk-sources/MacOSX${OSX_SDK}.sdk.tar.gz
 fi
 if [[ $HOST = *-mingw32 ]]; then
   DOCKER_EXEC update-alternatives --set $HOST-g++ \$\(which $HOST-g++-posix\)
diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh
index c4dc22bdd8..34b8477197 100755
--- a/ci/test/06_script_a.sh
+++ b/ci/test/06_script_a.sh
@@ -19,7 +19,9 @@ else
 fi
 END_FOLD
 
+# Create folder on host and docker, so that `cd` works
 mkdir -p build
+DOCKER_EXEC mkdir -p build
 
 # Temporarily disable errexit, because Travis macOS fails without error message
 set +o errexit
@@ -27,10 +29,12 @@ cd build || (echo "could not enter build directory"; exit 1)
 set -o errexit
 
 BEGIN_FOLD configure
-DOCKER_EXEC ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false)
+DOCKER_EXEC ../configure --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false)
 END_FOLD
 
 BEGIN_FOLD distdir
+# Create folder on host and docker, so that `cd` works
+mkdir -p "bitcoin-$HOST"
 DOCKER_EXEC make distdir VERSION=$HOST
 END_FOLD
 
@@ -39,7 +43,7 @@ cd "bitcoin-$HOST" || (echo "could not enter distdir bitcoin-$HOST"; exit 1)
 set -o errexit
 
 BEGIN_FOLD configure
-DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( cat config.log && false)
+DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false)
 END_FOLD
 
 set -o errtrace
diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh
index 048fe06c84..49962bb49b 100755
--- a/ci/test/06_script_b.sh
+++ b/ci/test/06_script_b.sh
@@ -32,7 +32,7 @@ fi
 
 if [ "$RUN_UNIT_TESTS" = "true" ]; then
   BEGIN_FOLD unit-tests
-  bash -c "while sleep 500; do echo .; done" &  # Print dots in case the unit tests take a long time to run
+  bash -c "${CI_WAIT}" &  # Print dots in case the unit tests take a long time to run
   DOCKER_EXEC LD_LIBRARY_PATH=$BASE_BUILD_DIR/depends/$HOST/lib make $MAKEJOBS check VERBOSE=1
   END_FOLD
 fi
@@ -48,7 +48,3 @@ if [ "$RUN_FUZZ_TESTS" = "true" ]; then
   DOCKER_EXEC test/fuzz/test_runner.py -l DEBUG ${DIR_FUZZ_IN}
   END_FOLD
 fi
-
-set +o errexit
-cd ${BASE_BUILD_DIR} || (echo "could not enter travis build dir $BASE_BUILD_DIR"; exit 1)
-set -o errexit