diff options
Diffstat (limited to 'ci/test')
25 files changed, 122 insertions, 79 deletions
diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 5a150d5f80..07c20f632d 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -48,7 +48,7 @@ export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} export EXPECTED_TESTS_DURATION_IN_SECONDS=${EXPECTED_TESTS_DURATION_IN_SECONDS:-1000} export CONTAINER_NAME=${CONTAINER_NAME:-ci_unnamed} -export DOCKER_NAME_TAG=${DOCKER_NAME_TAG:-ubuntu:20.04} +export CI_IMAGE_NAME_TAG=${CI_IMAGE_NAME_TAG:-ubuntu:20.04} # Randomize test order. # See https://www.boost.org/doc/libs/1_71_0/libs/test/doc/html/boost_test/utf_reference/rt_param_reference/random.html export BOOST_TEST_RANDOM=${BOOST_TEST_RANDOM:-1} @@ -66,7 +66,7 @@ export BASE_OUTDIR=${BASE_OUTDIR:-$BASE_SCRATCH_DIR/out/$HOST} export BASE_BUILD_DIR=${BASE_BUILD_DIR:-$BASE_SCRATCH_DIR/build} export PREVIOUS_RELEASES_DIR=${PREVIOUS_RELEASES_DIR:-$BASE_ROOT_DIR/releases/$HOST} export SDK_URL=${SDK_URL:-https://bitcoincore.org/depends-sources/sdks} -export DOCKER_PACKAGES=${DOCKER_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison} +export CI_BASE_PACKAGES=${CI_BASE_PACKAGES:-build-essential libtool autotools-dev automake pkg-config bsdmainutils curl ca-certificates ccache python3 rsync git procps bison} export GOAL=${GOAL:-install} export DIR_QA_ASSETS=${DIR_QA_ASSETS:-${BASE_SCRATCH_DIR}/qa-assets} export PATH=${BASE_ROOT_DIR}/ci/retry:$PATH diff --git a/ci/test/00_setup_env_android.sh b/ci/test/00_setup_env_android.sh index 6732db36ad..e1830b4f49 100755 --- a/ci/test/00_setup_env_android.sh +++ b/ci/test/00_setup_env_android.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8 export HOST=aarch64-linux-android export PACKAGES="unzip openjdk-8-jdk gradle" export CONTAINER_NAME=ci_android -export DOCKER_NAME_TAG="ubuntu:focal" +export CI_IMAGE_NAME_TAG="ubuntu:focal" export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index 932be4b43d..ac0c0be96a 100755 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -18,7 +18,7 @@ if [ -n "$QEMU_USER_CMD" ]; then fi export CONTAINER_NAME=ci_arm_linux # Use debian to avoid 404 apt errors when cross compiling -export DOCKER_NAME_TAG="debian:bullseye" +export CI_IMAGE_NAME_TAG="debian:bullseye" export USE_BUSY_BOX=true export RUN_UNIT_TESTS=true export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh index 1ce3261f44..e8f963c8a9 100755 --- a/ci/test/00_setup_env_i686_centos.sh +++ b/ci/test/00_setup_env_i686_centos.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2021 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,8 +8,8 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_centos -export DOCKER_NAME_TAG=quay.io/centos/centos:stream8 -export DOCKER_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison" +export CI_IMAGE_NAME_TAG=quay.io/centos/centos:stream8 +export CI_BASE_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python38 python38-pip which patch lbzip2 xz procps-ng dash rsync coreutils bison" export PIP_PACKAGES="pyzmq" export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-reduce-exports" diff --git a/ci/test/00_setup_env_i686_multiprocess.sh b/ci/test/00_setup_env_i686_multiprocess.sh index 76de87d955..9e3ea0d383 100755 --- a/ci/test/00_setup_env_i686_multiprocess.sh +++ b/ci/test/00_setup_env_i686_multiprocess.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2021 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu export CONTAINER_NAME=ci_i686_multiprocess -export DOCKER_NAME_TAG=ubuntu:20.04 +export CI_IMAGE_NAME_TAG=ubuntu:20.04 export PACKAGES="cmake python3 llvm clang g++-multilib" export DEP_OPTS="DEBUG=1 MULTIPROCESS=1" export GOAL="install" diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index c4f22c8f9e..fe42871c31 100755 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross -export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos +export CI_IMAGE_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos export HOST=x86_64-apple-darwin export PACKAGES="cmake libz-dev libtinfo5 python3-setuptools xorriso" export XCODE_VERSION=12.2 diff --git a/ci/test/00_setup_env_mac_native_arm64.sh b/ci/test/00_setup_env_mac_native_arm64.sh index cb0e13e77c..a6799d7b88 100755 --- a/ci/test/00_setup_env_mac_native_arm64.sh +++ b/ci/test/00_setup_env_mac_native_arm64.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 4f1792a9f0..bb3f6997f3 100755 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -1,19 +1,26 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -# We install an up-to-date 'bpfcc-tools' package from an untrusted PPA. -# This can be dropped with the next Ubuntu or Debian release that includes up-to-date packages. -# See the if-then in ci/test/04_install.sh too. -export ADD_UNTRUSTED_BPFCC_PPA=true +# Only install BCC tracing packages in Cirrus CI. +if [[ "${CIRRUS_CI}" == "true" ]]; then + # We install an up-to-date 'bpfcc-tools' package from an untrusted PPA. + # This can be dropped with the next Ubuntu or Debian release that includes up-to-date packages. + # See the if-then in ci/test/04_install.sh too. + export ADD_UNTRUSTED_BPFCC_PPA=true + export BPFCC_PACKAGE="bpfcc-tools" +else + export ADD_UNTRUSTED_BPFCC_PPA=false + export BPFCC_PACKAGE="" +fi export CONTAINER_NAME=ci_native_asan -export PACKAGES="systemtap-sdt-dev bpfcc-tools clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev" -export DOCKER_NAME_TAG=ubuntu:22.04 # May not run in docker unless --enable-usdt is dropped +export PACKAGES="systemtap-sdt-dev clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev ${BPFCC_PACKAGE}" +export CI_IMAGE_NAME_TAG=ubuntu:22.04 export NO_DEPENDS=1 export GOAL="install" -export BITCOIN_CONFIG="--enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" +export BITCOIN_CONFIG="--enable-c++20 --enable-usdt --enable-zmq --with-incompatible-bdb --with-gui=qt5 CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' --with-sanitizers=address,integer,undefined CC=clang CXX=clang++" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index d7caec8359..a2d393bb5a 100755 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:22.04" +export CI_IMAGE_NAME_TAG="ubuntu:22.04" export CONTAINER_NAME=ci_native_fuzz export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev" export NO_DEPENDS=1 diff --git a/ci/test/00_setup_env_native_fuzz_with_msan.sh b/ci/test/00_setup_env_native_fuzz_with_msan.sh index 071bac8fb3..d35701160a 100755 --- a/ci/test/00_setup_env_native_fuzz_with_msan.sh +++ b/ci/test/00_setup_env_native_fuzz_with_msan.sh @@ -1,23 +1,23 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2021 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:20.04" +export CI_IMAGE_NAME_TAG="ubuntu:20.04" LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" export MSAN_AND_LIBCXX_FLAGS="${MSAN_FLAGS} ${LIBCXX_FLAGS}" -export CONTAINER_NAME="ci_native_msan" +export CONTAINER_NAME="ci_native_fuzz_msan" export PACKAGES="clang-12 llvm-12 cmake" # BDB generates false-positives and will be removed in future export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" export RUN_UNIT_TESTS="false" export RUN_FUNCTIONAL_TESTS="false" diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index 97c530e19e..75c10046b7 100755 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:22.04" +export CI_IMAGE_NAME_TAG="ubuntu:22.04" export CONTAINER_NAME=ci_native_fuzz_valgrind export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libsqlite3-dev valgrind" export NO_DEPENDS=1 diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index 34a792ec8f..48049a0a3c 100755 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2021 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:20.04" +export CI_IMAGE_NAME_TAG="ubuntu:20.04" LIBCXX_DIR="${BASE_SCRATCH_DIR}/msan/build/" export MSAN_FLAGS="-fsanitize=memory -fsanitize-memory-track-origins=2 -fno-omit-frame-pointer -g -O1 -fno-optimize-sibling-calls" LIBCXX_FLAGS="-nostdinc++ -stdlib=libc++ -L${LIBCXX_DIR}lib -lc++abi -I${LIBCXX_DIR}include -I${LIBCXX_DIR}include/c++/v1 -lpthread -Wl,-rpath,${LIBCXX_DIR}lib -Wno-unused-command-line-argument" @@ -17,7 +17,7 @@ export PACKAGES="clang-12 llvm-12 cmake" # BDB generates false-positives and will be removed in future export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' libevent_cflags='${MSAN_FLAGS}' sqlite_cflags='${MSAN_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" -export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no --prefix=${DEPENDS_DIR}/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" +export BITCOIN_CONFIG="--with-sanitizers=memory --disable-hardening --with-asm=no CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}'" export USE_MEMORY_SANITIZER="true" export RUN_FUNCTIONAL_TESTS="false" export CCACHE_SIZE=250M diff --git a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh index 63560a5f5c..08bb5d1eab 100755 --- a/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh +++ b/ci/test/00_setup_env_native_nowallet_libbitcoinkernel.sh @@ -1,14 +1,16 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet_libbitcoinkernel -export DOCKER_NAME_TAG=ubuntu:18.04 # Use bionic to have one config run the tests in python3.6, see doc/dependencies.md -export PACKAGES="python3-zmq clang-8 llvm-8 libc++abi-8-dev libc++-8-dev" # Use clang-8 to test C++17 compatibility, see doc/dependencies.md +export CI_IMAGE_NAME_TAG=debian:buster +# Use minimum supported python3.7 and clang-8, see doc/dependencies.md +export PACKAGES="-t buster-backports python3-zmq clang-8 llvm-8 libc++abi-8-dev libc++-8-dev" +export APPEND_APT_SOURCES_LIST="deb http://deb.debian.org/debian buster-backports main" export DEP_OPTS="NO_WALLET=1 CC=clang-8 CXX='clang++-8 -stdlib=libc++'" export GOAL="install" export BITCOIN_CONFIG="--enable-reduce-exports CC=clang-8 CXX='clang++-8 -stdlib=libc++' --enable-experimental-util-chainstate --with-experimental-kernel-lib --enable-shared" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index 8a6ea62d5c..3f39185ae8 100755 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -1,13 +1,14 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_qt5 -export DOCKER_NAME_TAG=debian:buster # Check that buster gcc-8 can compile our C++17 and run our functional tests in python3, see doc/dependencies.md +export CI_IMAGE_NAME_TAG=debian:buster +# Use minimum supported python3.7 and gcc-8, see doc/dependencies.md export PACKAGES="gcc-8 g++-8 python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" export DEP_OPTS="NO_QT=1 NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1 CC=gcc-8 CXX=g++-8" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash diff --git a/ci/test/00_setup_env_native_tidy.sh b/ci/test/00_setup_env_native_tidy.sh index 11a12e336a..06ffe14636 100755 --- a/ci/test/00_setup_env_native_tidy.sh +++ b/ci/test/00_setup_env_native_tidy.sh @@ -6,7 +6,7 @@ export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:22.04" +export CI_IMAGE_NAME_TAG="ubuntu:22.04" export CONTAINER_NAME=ci_native_tidy export PACKAGES="clang libclang-dev llvm-dev clang-tidy bear cmake libevent-dev libboost-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev systemtap-sdt-dev libqt5gui5 libqt5core5a libqt5dbus5 qttools5-dev qttools5-dev-tools libqrencode-dev libsqlite3-dev libdb++-dev" export NO_DEPENDS=1 @@ -15,5 +15,5 @@ export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=false export RUN_TIDY=true export GOAL="install" -export BITCOIN_CONFIG="CC=clang CXX=clang++ --enable-c++20 --with-incompatible-bdb --disable-hardening CFLAGS='-O0 -g0' CXXFLAGS='-O0 -g0'" +export BITCOIN_CONFIG="CC=clang CXX=clang++ --with-incompatible-bdb --disable-hardening CFLAGS='-O0 -g0' CXXFLAGS='-O0 -g0'" export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index 6bf8391209..61bcd98f0a 100755 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -1,14 +1,14 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_tsan -export DOCKER_NAME_TAG=ubuntu:22.04 +export CI_IMAGE_NAME_TAG=ubuntu:22.04 export PACKAGES="clang-13 llvm-13 libc++abi-13-dev libc++-13-dev python3-zmq" export DEP_OPTS="CC=clang-13 CXX='clang++-13 -stdlib=libc++'" export GOAL="install" -export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION' CXXFLAGS='-g' --with-sanitizers=thread CC=clang-13 CXX='clang++-13 -stdlib=libc++'" +export BITCOIN_CONFIG="--enable-zmq CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER -DDEBUG_LOCKCONTENTION' CXXFLAGS='-g' --with-sanitizers=thread" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index d8c08fca39..039a22b02e 100755 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -export DOCKER_NAME_TAG="ubuntu:22.04" +export CI_IMAGE_NAME_TAG="ubuntu:22.04" export CONTAINER_NAME=ci_native_valgrind export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" export USE_VALGRIND=1 diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index 136edb6662..af18703ce1 100755 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,7 +18,7 @@ if [ -n "$QEMU_USER_CMD" ]; then fi # Use debian to avoid 404 apt errors export CONTAINER_NAME=ci_s390x -export DOCKER_NAME_TAG="debian:bookworm" +export CI_IMAGE_NAME_TAG="debian:bookworm" export TEST_RUNNER_ENV="LC_ALL=C" export TEST_RUNNER_EXTRA="--exclude feature_init,rpc_bind,feature_bind_extra" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 export RUN_FUNCTIONAL_TESTS=true diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index 3600113551..817ee724c2 100755 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -1,13 +1,13 @@ #!/usr/bin/env bash # -# Copyright (c) 2019-2021 The Bitcoin Core developers +# Copyright (c) 2019-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win64 -export DOCKER_NAME_TAG=ubuntu:22.04 # Check that Jammy can cross-compile to win64 +export CI_IMAGE_NAME_TAG=ubuntu:22.04 # Check that Jammy can cross-compile to win64 export HOST=x86_64-w64-mingw32 export DPKG_ADD_ARCH="i386" export PACKAGES="python3 nsis g++-mingw-w64-x86-64-posix wine-binfmt wine64 wine32 file" diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index a4f1a8a7ff..9bfe555243 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2021 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -20,14 +20,20 @@ export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/t export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|DEBIAN_FRONTEND|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|PREVIOUS_RELEASES_DIR)' | tee /tmp/env if [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) - DOCKER_ADMIN="--cap-add SYS_PTRACE" + CI_CONTAINER_CAP="--cap-add SYS_PTRACE" fi export P_CI_DIR="$PWD" +export BINS_SCRATCH_DIR="${BASE_SCRATCH_DIR}/bins/" if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then - echo "Creating $DOCKER_NAME_TAG container to run in" - ${CI_RETRY_EXE} docker pull "$DOCKER_NAME_TAG" + echo "Creating $CI_IMAGE_NAME_TAG container to run in" + LOCAL_UID=$(id -u) + LOCAL_GID=$(id -g) + + # the name isn't important, so long as we use the same UID + LOCAL_USER=nonroot + ${CI_RETRY_EXE} docker pull "$CI_IMAGE_NAME_TAG" if [ -n "${RESTART_CI_DOCKER_BEFORE_RUN}" ] ; then echo "Restart docker before run to stop and clear all containers started with --rm" @@ -35,7 +41,7 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then fi # shellcheck disable=SC2086 - DOCKER_ID=$(docker run $DOCKER_ADMIN --rm --interactive --detach --tty \ + CI_CONTAINER_ID=$(docker run $CI_CONTAINER_CAP --rm --interactive --detach --tty \ --mount type=bind,src=$BASE_ROOT_DIR,dst=/ro_base,readonly \ --mount type=bind,src=$CCACHE_DIR,dst=$CCACHE_DIR \ --mount type=bind,src=$DEPENDS_DIR,dst=$DEPENDS_DIR \ @@ -43,24 +49,40 @@ if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then -w $BASE_ROOT_DIR \ --env-file /tmp/env \ --name $CONTAINER_NAME \ - $DOCKER_NAME_TAG) - export DOCKER_CI_CMD_PREFIX="docker exec $DOCKER_ID" + $CI_IMAGE_NAME_TAG) + export CI_CONTAINER_ID + + # Create a non-root user inside the container which matches the local user. + # + # This prevents the root user in the container modifying the local file system permissions + # on the mounted directories + docker exec "$CI_CONTAINER_ID" useradd -u "$LOCAL_UID" -o -m "$LOCAL_USER" + docker exec "$CI_CONTAINER_ID" groupmod -o -g "$LOCAL_GID" "$LOCAL_USER" + docker exec "$CI_CONTAINER_ID" chown -R "$LOCAL_USER":"$LOCAL_USER" "${BASE_ROOT_DIR}" + export CI_EXEC_CMD_PREFIX_ROOT="docker exec -u 0 $CI_CONTAINER_ID" + export CI_EXEC_CMD_PREFIX="docker exec -u $LOCAL_UID $CI_CONTAINER_ID" else echo "Running on host system without docker wrapper" fi CI_EXEC () { - $DOCKER_CI_CMD_PREFIX bash -c "export PATH=$BASE_SCRATCH_DIR/bins/:\$PATH && cd \"$P_CI_DIR\" && $*" + $CI_EXEC_CMD_PREFIX bash -c "export PATH=${BINS_SCRATCH_DIR}:\$PATH && cd \"$P_CI_DIR\" && $*" +} +CI_EXEC_ROOT () { + $CI_EXEC_CMD_PREFIX_ROOT bash -c "export PATH=${BINS_SCRATCH_DIR}:\$PATH && cd \"$P_CI_DIR\" && $*" } export -f CI_EXEC +export -f CI_EXEC_ROOT + +CI_EXEC mkdir -p "${BINS_SCRATCH_DIR}" if [ -n "$DPKG_ADD_ARCH" ]; then - CI_EXEC dpkg --add-architecture "$DPKG_ADD_ARCH" + CI_EXEC_ROOT dpkg --add-architecture "$DPKG_ADD_ARCH" fi -if [[ $DOCKER_NAME_TAG == *centos* ]]; then - ${CI_RETRY_EXE} CI_EXEC dnf -y install epel-release - ${CI_RETRY_EXE} CI_EXEC dnf -y --allowerasing install "$DOCKER_PACKAGES" "$PACKAGES" +if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then + ${CI_RETRY_EXE} CI_EXEC_ROOT dnf -y install epel-release + ${CI_RETRY_EXE} CI_EXEC_ROOT dnf -y --allowerasing install "$CI_BASE_PACKAGES" "$PACKAGES" elif [ "$CI_USE_APT_INSTALL" != "no" ]; then if [[ "${ADD_UNTRUSTED_BPFCC_PPA}" == "true" ]]; then # Ubuntu 22.04 LTS and Debian 11 both have an outdated bpfcc-tools packages. @@ -68,10 +90,13 @@ elif [ "$CI_USE_APT_INSTALL" != "no" ]; then # packages. Meanwhile, use an untrusted PPA to install an up-to-date version of the bpfcc-tools # package. # TODO: drop this once we can use newer images in GCE - CI_EXEC add-apt-repository ppa:hadret/bpfcc + CI_EXEC_ROOT add-apt-repository ppa:hadret/bpfcc + fi + if [[ -n "${APPEND_APT_SOURCES_LIST}" ]]; then + CI_EXEC_ROOT echo "${APPEND_APT_SOURCES_LIST}" >> /etc/apt/sources.list fi - ${CI_RETRY_EXE} CI_EXEC apt-get update - ${CI_RETRY_EXE} CI_EXEC apt-get install --no-install-recommends --no-upgrade -y "$PACKAGES" "$DOCKER_PACKAGES" + ${CI_RETRY_EXE} CI_EXEC_ROOT apt-get update + ${CI_RETRY_EXE} CI_EXEC_ROOT apt-get install --no-install-recommends --no-upgrade -y "$PACKAGES" "$CI_BASE_PACKAGES" fi if [ -n "$PIP_PACKAGES" ]; then @@ -112,8 +137,8 @@ fi CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then - CI_EXEC "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-12) 100" - CI_EXEC "update-alternatives --install /usr/bin/clang clang \$(which clang-12) 100" + CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang++ clang++ \$(which clang++-12) 100" + CI_EXEC_ROOT "update-alternatives --install /usr/bin/clang clang \$(which clang-12) 100" CI_EXEC "mkdir -p ${BASE_SCRATCH_DIR}/msan/build/" CI_EXEC "git clone --depth=1 https://github.com/llvm/llvm-project -b llvmorg-12.0.0 ${BASE_SCRATCH_DIR}/msan/llvm-project" CI_EXEC "cd ${BASE_SCRATCH_DIR}/msan/build/ && cmake -DLLVM_ENABLE_PROJECTS='libcxx;libcxxabi' -DCMAKE_BUILD_TYPE=Release -DLLVM_USE_SANITIZER=MemoryWithOrigins -DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++ -DLLVM_TARGETS_TO_BUILD=X86 ../llvm-project/llvm/" @@ -126,7 +151,7 @@ if [[ "${RUN_TIDY}" == "true" ]]; then CI_EXEC "mkdir -p ${DIR_IWYU}/build/" CI_EXEC "git clone --depth=1 https://github.com/include-what-you-use/include-what-you-use -b clang_14 ${DIR_IWYU}/include-what-you-use" CI_EXEC "cd ${DIR_IWYU}/build && cmake -G 'Unix Makefiles' -DCMAKE_PREFIX_PATH=/usr/lib/llvm-14 ../include-what-you-use" - CI_EXEC "cd ${DIR_IWYU}/build && make install $MAKEJOBS" + CI_EXEC_ROOT "cd ${DIR_IWYU}/build && make install $MAKEJOBS" fi fi @@ -137,12 +162,10 @@ fi if [ "$USE_BUSY_BOX" = "true" ]; then echo "Setup to use BusyBox utils" - CI_EXEC mkdir -p "${BASE_SCRATCH_DIR}/bins/" # tar excluded for now because it requires passing in the exact archive type in ./depends (fixed in later BusyBox version) - # find excluded for now because it does not recognize the -delete option in ./depends (fixed in later BusyBox version) # ar excluded for now because it does not recognize the -q option in ./depends (unknown if fixed) # shellcheck disable=SC1010 - CI_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \| grep -v "^find$"\)\; do ln -s \$\(command -v busybox\) "${BASE_SCRATCH_DIR}/bins/\$util"\; done + CI_EXEC for util in \$\(busybox --list \| grep -v "^ar$" \| grep -v "^tar$" \)\; do ln -s \$\(command -v busybox\) "${BINS_SCRATCH_DIR}/\$util"\; done # Print BusyBox version CI_EXEC patch --help fi diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index ef3dff86ca..f49305597d 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2021 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -11,6 +11,7 @@ if [ "$CI_OS_NAME" == "macos" ]; then echo > "${HOME}/Library/Application Support/Bitcoin" else CI_EXEC echo \> \$HOME/.bitcoin + CI_EXEC_ROOT echo \> \$HOME/.bitcoin fi CI_EXEC mkdir -p "${DEPENDS_DIR}/SDKs" "${DEPENDS_DIR}/sdk-sources" @@ -37,7 +38,7 @@ if [ -n "$ANDROID_HOME" ] && [ ! -d "$ANDROID_HOME" ]; then fi if [ -z "$NO_DEPENDS" ]; then - if [[ $DOCKER_NAME_TAG == *centos* ]]; then + if [[ $CI_IMAGE_NAME_TAG == *centos* ]]; then # CentOS has problems building the depends if the config shell is not explicitly set # (i.e. for libevent a Makefile with an empty SHELL variable is generated, leading to # an error as the first command is executed) diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index 13693a2ecf..5856d33d2d 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -1,12 +1,15 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2021 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 -BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST" +BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking" +if [ -z "$NO_DEPENDS" ]; then + BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} CONFIG_SITE=$DEPENDS_DIR/$HOST/share/config.site" +fi if [ -z "$NO_WERROR" ]; then BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror" fi @@ -23,7 +26,7 @@ if [ -n "$ANDROID_TOOLS_URL" ]; then exit 0 fi -BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" +BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-external-signer --prefix=$BASE_OUTDIR" if [ -n "$CONFIG_SHELL" ]; then CI_EXEC "$CONFIG_SHELL" -c "./autogen.sh" diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 6177736722..f792a9f192 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2021 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -31,7 +31,7 @@ if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then fi if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then - CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${TEST_RUNNER_ENV}" test/functional/test_runner.py --ci "$MAKEJOBS" --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" "${TEST_RUNNER_EXTRA}" --quiet --failfast + CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" "${TEST_RUNNER_ENV}" test/functional/test_runner.py --ci "$MAKEJOBS" --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=99999999 --timeout-factor="${TEST_RUNNER_TIMEOUT_FACTOR}" "${TEST_RUNNER_EXTRA}" --quiet --failfast fi if [ "${RUN_TIDY}" = "true" ]; then @@ -56,7 +56,7 @@ if [ "${RUN_TIDY}" = "true" ]; then " src/rpc/fees.cpp"\ " src/rpc/signmessage.cpp"\ " src/test/fuzz/txorphan.cpp"\ - " src/threadinterrupt.cpp"\ + " src/test/fuzz/util/"\ " src/util/bip32.cpp"\ " src/util/bytevectorhash.cpp"\ " src/util/check.cpp"\ @@ -70,6 +70,7 @@ if [ "${RUN_TIDY}" = "true" ]; then " src/util/strencodings.cpp"\ " src/util/string.cpp"\ " src/util/syserror.cpp"\ + " src/util/threadinterrupt.cpp"\ " src/zmq"\ " -p . ${MAKEJOBS} -- -Xiwyu --cxx17ns -Xiwyu --mapping_file=${BASE_BUILD_DIR}/bitcoin-$HOST/contrib/devtools/iwyu/bitcoin.core.imp" fi @@ -81,3 +82,8 @@ fi if [ "$RUN_FUZZ_TESTS" = "true" ]; then CI_EXEC LD_LIBRARY_PATH="${DEPENDS_DIR}/${HOST}/lib" test/fuzz/test_runner.py "${FUZZ_TESTS_CONFIG}" "$MAKEJOBS" -l DEBUG "${DIR_FUZZ_IN}" fi + +if [ -z "$DANGER_RUN_CI_ON_HOST" ]; then + echo "Stop and remove CI container by ID" + docker container kill "${CI_CONTAINER_ID}" +fi diff --git a/ci/test/wrap-qemu.sh b/ci/test/wrap-qemu.sh index eb31edbce8..e028ede378 100755 --- a/ci/test/wrap-qemu.sh +++ b/ci/test/wrap-qemu.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2021 The Bitcoin Core developers +# Copyright (c) 2018-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/wrap-wine.sh b/ci/test/wrap-wine.sh index 1662f8f6a3..90e53887bc 100755 --- a/ci/test/wrap-wine.sh +++ b/ci/test/wrap-wine.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2020-2021 The Bitcoin Core developers +# Copyright (c) 2020-2022 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. |