diff options
Diffstat (limited to 'ci')
28 files changed, 120 insertions, 133 deletions
diff --git a/ci/lint/04_install.sh b/ci/lint/04_install.sh index fae424051d..343b82a1ad 100755 --- a/ci/lint/04_install.sh +++ b/ci/lint/04_install.sh @@ -1,19 +1,21 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C -travis_retry sudo apt update && sudo apt install -y clang-format-9 -sudo update-alternatives --install /usr/bin/clang-format clang-format $(which clang-format-9 ) 100 -sudo update-alternatives --install /usr/bin/clang-format-diff clang-format-diff $(which clang-format-diff-9) 100 +${CI_RETRY_EXE} apt-get update +${CI_RETRY_EXE} apt-get install -y clang-format-9 python3-pip curl git gawk jq +update-alternatives --install /usr/bin/clang-format clang-format $(which clang-format-9 ) 100 +update-alternatives --install /usr/bin/clang-format-diff clang-format-diff $(which clang-format-diff-9) 100 -travis_retry pip3 install codespell==1.17.1 -travis_retry pip3 install flake8==3.8.3 -travis_retry pip3 install yq -travis_retry pip3 install mypy==0.781 +${CI_RETRY_EXE} pip3 install codespell==2.0.0 +${CI_RETRY_EXE} pip3 install flake8==3.8.3 +${CI_RETRY_EXE} pip3 install yq +${CI_RETRY_EXE} pip3 install mypy==0.781 +${CI_RETRY_EXE} pip3 install vulture==2.3 SHELLCHECK_VERSION=v0.7.1 curl -sL "https://github.com/koalaman/shellcheck/releases/download/${SHELLCHECK_VERSION}/shellcheck-${SHELLCHECK_VERSION}.linux.x86_64.tar.xz" | tar --xz -xf - --directory /tmp/ diff --git a/ci/lint/05_before_script.sh b/ci/lint/05_before_script.sh deleted file mode 100755 index 2987812c8e..0000000000 --- a/ci/lint/05_before_script.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C - -git fetch --unshallow diff --git a/ci/lint/06_script.sh b/ci/lint/06_script.sh index 003bdf3c29..e38cfe8eef 100755 --- a/ci/lint/06_script.sh +++ b/ci/lint/06_script.sh @@ -1,26 +1,33 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C -if [ "$TRAVIS_EVENT_TYPE" = "pull_request" ]; then - test/lint/commit-script-check.sh $TRAVIS_COMMIT_RANGE +GIT_HEAD=$(git rev-parse HEAD) +if [ -n "$CIRRUS_PR" ]; then + COMMIT_RANGE="$CIRRUS_BASE_SHA..$GIT_HEAD" + test/lint/commit-script-check.sh $COMMIT_RANGE fi +export COMMIT_RANGE +# This only checks that the trees are pure subtrees, it is not doing a full +# check with -r to not have to fetch all the remotes. test/lint/git-subtree-check.sh src/crypto/ctaes test/lint/git-subtree-check.sh src/secp256k1 test/lint/git-subtree-check.sh src/univalue test/lint/git-subtree-check.sh src/leveldb test/lint/git-subtree-check.sh src/crc32c test/lint/check-doc.py -test/lint/check-rpc-mappings.py . test/lint/lint-all.sh -if [ "$TRAVIS_REPO_SLUG" = "bitcoin/bitcoin" ] && [ "$TRAVIS_EVENT_TYPE" = "cron" ]; then +if [ "$CIRRUS_REPO_FULL_NAME" = "bitcoin/bitcoin" ] && [ -n "$CIRRUS_CRON" ]; then git log --merges --before="2 days ago" -1 --format='%H' > ./contrib/verify-commits/trusted-sha512-root-commit - travis_retry gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys $(<contrib/verify-commits/trusted-keys) && + ${CI_RETRY_EXE} gpg --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys $(<contrib/verify-commits/trusted-keys) && ./contrib/verify-commits/verify-commits.py --clean-merge=2; fi + +echo +git log --no-merges --oneline $COMMIT_RANGE diff --git a/ci/lint_run_all.sh b/ci/lint_run_all.sh new file mode 100755 index 0000000000..7adfe71674 --- /dev/null +++ b/ci/lint_run_all.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2019-2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +set -o errexit; source ./ci/test/00_setup_env.sh +set -o errexit; source ./ci/lint/04_install.sh +set -o errexit; source ./ci/lint/06_script.sh diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 702e881862..72e29141a6 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -38,7 +38,7 @@ export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false} # By how much to scale the test_runner timeouts (option --timeout-factor). # This is needed because some ci machines have slow CPU or disk, so sanitizers # might be slow or a reindex might be waiting on disk IO. -export TEST_RUNNER_TIMEOUT_FACTOR=${TEST_RUNNER_TIMEOUT_FACTOR:-4} +export TEST_RUNNER_TIMEOUT_FACTOR=${TEST_RUNNER_TIMEOUT_FACTOR:-40} export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-} export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} export EXPECTED_TESTS_DURATION_IN_SECONDS=${EXPECTED_TESTS_DURATION_IN_SECONDS:-1000} diff --git a/ci/test/00_setup_env_arm.sh b/ci/test/00_setup_env_arm.sh index 2e445c126d..42783197a9 100644 --- a/ci/test/00_setup_env_arm.sh +++ b/ci/test/00_setup_env_arm.sh @@ -21,8 +21,8 @@ export CONTAINER_NAME=ci_arm_linux export DOCKER_NAME_TAG="debian:buster" export USE_BUSY_BOX=true export RUN_UNIT_TESTS=true -export RUN_FUNCTIONAL_TESTS=true +export RUN_FUNCTIONAL_TESTS=false export GOAL="install" # -Wno-psabi is to disable ABI warnings: "note: parameter passing for argument of type ... changed in GCC 7.1" # This could be removed once the ABI change warning does not show up by default -export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CXXFLAGS=-Wno-psabi --enable-werror --with-boost-process" +export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CXXFLAGS=-Wno-psabi --with-boost-process" diff --git a/ci/test/00_setup_env_i686_centos.sh b/ci/test/00_setup_env_i686_centos.sh index e58003ab19..1ec44a7a1a 100644 --- a/ci/test/00_setup_env_i686_centos.sh +++ b/ci/test/00_setup_env_i686_centos.sh @@ -7,9 +7,10 @@ export LC_ALL=C.UTF-8 export HOST=i686-pc-linux-gnu -export CONTAINER_NAME=ci_i686_centos_7 -export DOCKER_NAME_TAG=centos:7 -export DOCKER_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python36-zmq which patch lbzip2 dash" +export CONTAINER_NAME=ci_i686_centos_8 +export DOCKER_NAME_TAG=centos:8 +export DOCKER_PACKAGES="gcc-c++ glibc-devel.x86_64 libstdc++-devel.x86_64 glibc-devel.i686 libstdc++-devel.i686 ccache libtool make git python3 python3-zmq which patch lbzip2 dash rsync coreutils" export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-reduce-exports --with-boost-process" export CONFIG_SHELL="/bin/dash" +export TEST_RUNNER_ENV="LC_ALL=en_US.UTF-8" diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index b62f1603f4..6da011d19b 100644 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -7,11 +7,12 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross -export HOST=x86_64-apple-darwin16 -export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools" +export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to macos (Focal is used in the gitian build as well) +export HOST=x86_64-apple-darwin18 +export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools libtinfo5 python3-dev python3-setuptools xorriso" export XCODE_VERSION=11.3.1 export XCODE_BUILD_ID=11C505 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export GOAL="deploy" -export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --enable-werror --with-boost-process" +export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --with-boost-process" diff --git a/ci/test/00_setup_env_mac_host.sh b/ci/test/00_setup_env_mac_host.sh index 5fb127b762..274a0d1b7c 100644 --- a/ci/test/00_setup_env_mac_host.sh +++ b/ci/test/00_setup_env_mac_host.sh @@ -6,17 +6,13 @@ export LC_ALL=C.UTF-8 -export HOST=x86_64-apple-darwin16 -export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to macos (bionic is used in the gitian build as well) +export HOST=x86_64-apple-darwin18 export PIP_PACKAGES="zmq" export GOAL="install" -export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --enable-werror --with-boost-process" +export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --with-boost-process" +export CI_OS_NAME="macos" export NO_DEPENDS=1 export OSX_SDK="" export CCACHE_SIZE=300M export RUN_SECURITY_TESTS="true" -if [ "$TRAVIS_REPO_SLUG" != "bitcoin/bitcoin" ]; then - export RUN_FUNCTIONAL_TESTS="false" - export EXPECTED_TESTS_DURATION_IN_SECONDS=200 -fi diff --git a/ci/test/00_setup_env_native_asan.sh b/ci/test/00_setup_env_native_asan.sh index 251ece7984..e47119e6fa 100644 --- a/ci/test/00_setup_env_native_asan.sh +++ b/ci/test/00_setup_env_native_asan.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_asan -export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev libqrencode-dev" +export PACKAGES="clang llvm python3-zmq qtbase5-dev qttools5-dev-tools libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libqrencode-dev libsqlite3-dev" export DOCKER_NAME_TAG=ubuntu:20.04 export NO_DEPENDS=1 export GOAL="install" diff --git a/ci/test/00_setup_env_native_fuzz.sh b/ci/test/00_setup_env_native_fuzz.sh index a32de4a6b5..ebb5a1cabe 100644 --- a/ci/test/00_setup_env_native_fuzz.sh +++ b/ci/test/00_setup_env_native_fuzz.sh @@ -8,11 +8,11 @@ export LC_ALL=C.UTF-8 export DOCKER_NAME_TAG="ubuntu:20.04" export CONTAINER_NAME=ci_native_fuzz -export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev" +export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false export RUN_FUZZ_TESTS=true export GOAL="install" -export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined CC=clang CXX=clang++ --with-boost-process" +export BITCOIN_CONFIG="--enable-fuzz --with-sanitizers=fuzzer,address,undefined,integer CC=clang CXX=clang++ --with-boost-process" export CCACHE_SIZE=200M diff --git a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh index e06a40eb23..2cf672b91e 100644 --- a/ci/test/00_setup_env_native_fuzz_with_valgrind.sh +++ b/ci/test/00_setup_env_native_fuzz_with_valgrind.sh @@ -8,7 +8,7 @@ export LC_ALL=C.UTF-8 export DOCKER_NAME_TAG="ubuntu:20.04" export CONTAINER_NAME=ci_native_fuzz_valgrind -export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev valgrind" +export PACKAGES="clang llvm python3 libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev valgrind" export NO_DEPENDS=1 export RUN_UNIT_TESTS=false export RUN_FUNCTIONAL_TESTS=false diff --git a/ci/test/00_setup_env_native_msan.sh b/ci/test/00_setup_env_native_msan.sh index 6a4979990b..3ce50f816f 100644 --- a/ci/test/00_setup_env_native_msan.sh +++ b/ci/test/00_setup_env_native_msan.sh @@ -15,7 +15,7 @@ export BDB_PREFIX="${BASE_ROOT_DIR}/db4" export CONTAINER_NAME="ci_native_msan" export PACKAGES="clang-9 llvm-9 cmake" -export DEP_OPTS="NO_WALLET=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++11 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' zeromq_cxxflags='-std=c++11 ${MSAN_AND_LIBCXX_FLAGS}'" +export DEP_OPTS="NO_BDB=1 NO_QT=1 CC='clang' CXX='clang++' CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' boost_cxxflags='-std=c++17 -fvisibility=hidden -fPIC ${MSAN_AND_LIBCXX_FLAGS}' zeromq_cxxflags='-std=c++17 ${MSAN_AND_LIBCXX_FLAGS}'" export GOAL="install" export BITCOIN_CONFIG="--enable-wallet --with-sanitizers=memory --with-asm=no --prefix=${BASE_ROOT_DIR}/depends/x86_64-pc-linux-gnu/ CC=clang CXX=clang++ CFLAGS='${MSAN_FLAGS}' CXXFLAGS='${MSAN_AND_LIBCXX_FLAGS}' BDB_LIBS='-L${BDB_PREFIX}/lib -ldb_cxx-4.8' BDB_CFLAGS='-I${BDB_PREFIX}/include'" export USE_MEMORY_SANITIZER="true" diff --git a/ci/test/00_setup_env_native_multiprocess.sh b/ci/test/00_setup_env_native_multiprocess.sh index 522a5d9fc2..c5692d786a 100644 --- a/ci/test/00_setup_env_native_multiprocess.sh +++ b/ci/test/00_setup_env_native_multiprocess.sh @@ -13,3 +13,4 @@ export DEP_OPTS="MULTIPROCESS=1" export GOAL="install" export BITCOIN_CONFIG="--with-boost-process" export TEST_RUNNER_ENV="BITCOIND=bitcoin-node" +export RUN_SECURITY_TESTS="true" diff --git a/ci/test/00_setup_env_native_nowallet.sh b/ci/test/00_setup_env_native_nowallet.sh index 0a09bfe230..7ff044c020 100644 --- a/ci/test/00_setup_env_native_nowallet.sh +++ b/ci/test/00_setup_env_native_nowallet.sh @@ -7,8 +7,8 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_nowallet -export DOCKER_NAME_TAG=ubuntu:16.04 # Use xenial to have one config run the tests in python3.5, see doc/dependencies.md -export PACKAGES="python3-zmq clang-3.8 llvm-3.8" # Use clang-3.8 to test C++11 compatibility, see doc/dependencies.md +export DOCKER_NAME_TAG=ubuntu:18.04 # Use bionic to have one config run the tests in python3.6, see doc/dependencies.md +export PACKAGES="python3-zmq clang-5.0 llvm-5.0" # Use clang-5 to test C++17 compatibility, see doc/dependencies.md export DEP_OPTS="NO_WALLET=1" export GOAL="install" -export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CC=clang-3.8 CXX=clang++-3.8 --with-boost-process" +export BITCOIN_CONFIG="--enable-glibc-back-compat --enable-reduce-exports CC=clang-5.0 CXX=clang++-5.0 --with-boost-process" diff --git a/ci/test/00_setup_env_native_qt5.sh b/ci/test/00_setup_env_native_qt5.sh index f9d869b4fd..4c42605e9a 100644 --- a/ci/test/00_setup_env_native_qt5.sh +++ b/ci/test/00_setup_env_native_qt5.sh @@ -7,13 +7,13 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_qt5 -export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can compile our c++17 and run our functional tests in python3 +export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic gcc-7 can compile our c++17 and run our functional tests in python3, see doc/dependencies.md export PACKAGES="python3-zmq qtbase5-dev qttools5-dev-tools libdbus-1-dev libharfbuzz-dev" -export DEP_OPTS="NO_QT=1 NO_UPNP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" +export DEP_OPTS="NO_QT=1 NO_UPNP=1 NO_NATPMP=1 DEBUG=1 ALLOW_HOST_PACKAGES=1" export TEST_RUNNER_EXTRA="--previous-releases --coverage --extended --exclude feature_dbcrash" # Run extended tests so that coverage does not fail, but exclude the very slow dbcrash -export RUN_SECURITY_TESTS="true" export RUN_UNIT_TESTS_SEQUENTIAL="true" export RUN_UNIT_TESTS="false" export GOAL="install" -export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.1 v0.18.1 v0.19.1" -export BITCOIN_CONFIG="--enable-zmq --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports --enable-c++17 --enable-debug CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" --with-boost-process" +export PREVIOUS_RELEASES_TO_DOWNLOAD="v0.15.2 v0.16.3 v0.17.2 v0.18.1 v0.19.1" +export BITCOIN_CONFIG="--enable-zmq --with-libs=no --with-gui=qt5 --enable-glibc-back-compat --enable-reduce-exports +--enable-debug --disable-fuzz-binary CFLAGS=\"-g0 -O2 -funsigned-char\" CXXFLAGS=\"-g0 -O2 -funsigned-char\" --with-boost-process" diff --git a/ci/test/00_setup_env_native_tsan.sh b/ci/test/00_setup_env_native_tsan.sh index b14a46562c..182e42ee7d 100644 --- a/ci/test/00_setup_env_native_tsan.sh +++ b/ci/test/00_setup_env_native_tsan.sh @@ -10,6 +10,5 @@ export CONTAINER_NAME=ci_native_tsan export DOCKER_NAME_TAG=ubuntu:20.04 export PACKAGES="clang llvm libc++abi-dev libc++-dev python3-zmq" export DEP_OPTS="CC=clang CXX='clang++ -stdlib=libc++'" -export TEST_RUNNER_EXTRA="--exclude feature_block" # Low memory on Travis machines, exclude feature_block. export GOAL="install" export BITCOIN_CONFIG="--enable-zmq --with-gui=no CPPFLAGS='-DARENA_DEBUG -DDEBUG_LOCKORDER' CXXFLAGS='-g' --with-sanitizers=thread CC=clang CXX='clang++ -stdlib=libc++' --with-boost-process" diff --git a/ci/test/00_setup_env_native_valgrind.sh b/ci/test/00_setup_env_native_valgrind.sh index 710d9e1011..e079a7057c 100644 --- a/ci/test/00_setup_env_native_valgrind.sh +++ b/ci/test/00_setup_env_native_valgrind.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_native_valgrind -export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-system-dev libboost-filesystem-dev libboost-test-dev libboost-thread-dev libdb5.3++-dev libminiupnpc-dev libzmq3-dev" +export PACKAGES="valgrind clang llvm python3-zmq libevent-dev bsdmainutils libboost-dev libboost-system-dev libboost-filesystem-dev libboost-test-dev libdb5.3++-dev libminiupnpc-dev libnatpmp-dev libzmq3-dev libsqlite3-dev" export USE_VALGRIND=1 export NO_DEPENDS=1 export TEST_RUNNER_EXTRA="--exclude rpc_bind" # Excluded for now, see https://github.com/bitcoin/bitcoin/issues/17765#issuecomment-602068547 diff --git a/ci/test/00_setup_env_s390x.sh b/ci/test/00_setup_env_s390x.sh index fe330920d0..accbd07e22 100644 --- a/ci/test/00_setup_env_s390x.sh +++ b/ci/test/00_setup_env_s390x.sh @@ -20,6 +20,7 @@ fi export CONTAINER_NAME=ci_s390x export DOCKER_NAME_TAG="debian:buster" export RUN_UNIT_TESTS=true +export TEST_RUNNER_ENV="LC_ALL=C" export RUN_FUNCTIONAL_TESTS=true export GOAL="install" export BITCOIN_CONFIG="--enable-reduce-exports --with-incompatible-bdb --with-boost-process" diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index 2b351dff6d..1e68d2a61a 100644 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -7,10 +7,14 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win64 -export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to win64 (bionic is used in the gitian build as well) +export DOCKER_NAME_TAG=ubuntu:20.04 # Check that Focal can cross-compile to win64 (Focal is used in the gitian build as well) export HOST=x86_64-w64-mingw32 -export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64" +export DPKG_ADD_ARCH="i386" +export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64 wine32 file" export RUN_FUNCTIONAL_TESTS=false -export RUN_SECURITY_TESTS="true" export GOAL="deploy" export BITCOIN_CONFIG="--enable-reduce-exports --disable-gui-tests --without-boost-process" + +# Compiler for MinGW-w64 causes false -Wreturn-type warning. +# See https://sourceforge.net/p/mingw-w64/bugs/306/ +export NO_WERROR=1 diff --git a/ci/test/03_before_install.sh b/ci/test/03_before_install.sh deleted file mode 100755 index 80806aab75..0000000000 --- a/ci/test/03_before_install.sh +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash -# -# Copyright (c) 2018-2019 The Bitcoin Core developers -# Distributed under the MIT software license, see the accompanying -# file COPYING or http://www.opensource.org/licenses/mit-license.php. - -export LC_ALL=C.UTF-8 - -BEGIN_FOLD () { - echo "" - CURRENT_FOLD_NAME=$1 - echo "travis_fold:start:${CURRENT_FOLD_NAME}" -} - -END_FOLD () { - RET=$? - echo "travis_fold:end:${CURRENT_FOLD_NAME}" - if [ $RET != 0 ]; then - echo "${CURRENT_FOLD_NAME} failed with status code ${RET}" - fi -} - diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index d3566914ac..f0ed314d19 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -6,15 +6,12 @@ export LC_ALL=C.UTF-8 -if [[ $DOCKER_NAME_TAG == centos* ]]; then - export LC_ALL=en_US.utf8 -fi if [[ $QEMU_USER_CMD == qemu-s390* ]]; then export LC_ALL=C fi -if [ "$TRAVIS_OS_NAME" == "osx" ]; then - ${CI_RETRY_EXE} pip3 install $PIP_PACKAGES +if [ "$CI_OS_NAME" == "macos" ]; then + IN_GETOPT_BIN="/usr/local/opt/gnu-getopt/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES fi # Create folders that are mounted into the docker @@ -26,9 +23,7 @@ export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/l export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|DEBIAN_FRONTEND|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|PREVIOUS_RELEASES_DIR)' | tee /tmp/env -if [[ $HOST = *-mingw32 ]]; then - DOCKER_ADMIN="--cap-add SYS_ADMIN" -elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) +if [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) DOCKER_ADMIN="--cap-add SYS_PTRACE" fi @@ -62,30 +57,29 @@ if [ -n "$DPKG_ADD_ARCH" ]; then fi if [[ $DOCKER_NAME_TAG == centos* ]]; then - ${CI_RETRY_EXE} DOCKER_EXEC yum -y install epel-release - ${CI_RETRY_EXE} DOCKER_EXEC yum -y install $DOCKER_PACKAGES $PACKAGES + ${CI_RETRY_EXE} DOCKER_EXEC dnf -y install epel-release + ${CI_RETRY_EXE} DOCKER_EXEC dnf -y --allowerasing install $DOCKER_PACKAGES $PACKAGES elif [ "$CI_USE_APT_INSTALL" != "no" ]; then ${CI_RETRY_EXE} DOCKER_EXEC apt-get update ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES fi -if [ "$TRAVIS_OS_NAME" == "osx" ]; then +if [ "$CI_OS_NAME" == "macos" ]; then top -l 1 -s 0 | awk ' /PhysMem/ {print}' echo "Number of CPUs: $(sysctl -n hw.logicalcpu)" else DOCKER_EXEC free -m -h DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) DOCKER_EXEC echo $(lscpu | grep Endian) - DOCKER_EXEC echo "Free disk space:" - DOCKER_EXEC df -h fi +DOCKER_EXEC echo "Free disk space:" +DOCKER_EXEC df -h if [ ! -d ${DIR_QA_ASSETS} ]; then - if [ "$RUN_FUZZ_TESTS" = "true" ]; then - DOCKER_EXEC git clone https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} - fi + DOCKER_EXEC git clone --depth=1 https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} fi export DIR_FUZZ_IN=${DIR_QA_ASSETS}/fuzz_seed_corpus/ +export DIR_UNIT_TEST_DATA=${DIR_QA_ASSETS}/unit_test_data/ DOCKER_EXEC mkdir -p "${BASE_SCRATCH_DIR}/sanitizer-output/" diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index 131ea21677..f69afd8a26 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 # Make sure default datadir does not exist and is never read by creating a dummy file -if [ "$TRAVIS_OS_NAME" == "osx" ]; then +if [ "$CI_OS_NAME" == "macos" ]; then echo > $HOME/Library/Application\ Support/Bitcoin else DOCKER_EXEC echo \> \$HOME/.bitcoin @@ -19,7 +19,7 @@ OSX_SDK_BASENAME="Xcode-${XCODE_VERSION}-${XCODE_BUILD_ID}-extracted-SDK-with-li OSX_SDK_PATH="${DEPENDS_DIR}/sdk-sources/${OSX_SDK_BASENAME}" if [ -n "$XCODE_VERSION" ] && [ ! -f "$OSX_SDK_PATH" ]; then - curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH" + DOCKER_EXEC curl --location --fail "${SDK_URL}/${OSX_SDK_BASENAME}" -o "$OSX_SDK_PATH" fi if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then @@ -40,14 +40,12 @@ if [ -z "$NO_DEPENDS" ]; then # CentOS has problems building the depends if the config shell is not explicitly set # (i.e. for libevent a Makefile with an empty SHELL variable is generated, leading to # an error as the first command is executed) - SHELL_OPTS="CONFIG_SHELL=/bin/bash" + SHELL_OPTS="LC_ALL=en_US.UTF-8 CONFIG_SHELL=/bin/bash" else SHELL_OPTS="CONFIG_SHELL=" fi DOCKER_EXEC $SHELL_OPTS make $MAKEJOBS -C depends HOST=$HOST $DEP_OPTS fi if [ -n "$PREVIOUS_RELEASES_TO_DOWNLOAD" ]; then - BEGIN_FOLD previous-versions - DOCKER_EXEC contrib/devtools/previous_release.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}" - END_FOLD + DOCKER_EXEC test/get_previous_releases.py -b -t "$PREVIOUS_RELEASES_DIR" "${PREVIOUS_RELEASES_TO_DOWNLOAD}" fi diff --git a/ci/test/06_script_a.sh b/ci/test/06_script_a.sh index 17d765b862..7986f7665a 100755 --- a/ci/test/06_script_a.sh +++ b/ci/test/06_script_a.sh @@ -6,33 +6,28 @@ export LC_ALL=C.UTF-8 -BITCOIN_CONFIG_ALL="--disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" +BITCOIN_CONFIG_ALL="--enable-suppress-external-warnings --disable-dependency-tracking --prefix=$DEPENDS_DIR/$HOST --bindir=$BASE_OUTDIR/bin --libdir=$BASE_OUTDIR/lib" +if [ -z "$NO_WERROR" ]; then + BITCOIN_CONFIG_ALL="${BITCOIN_CONFIG_ALL} --enable-werror" +fi DOCKER_EXEC "ccache --zero-stats --max-size=$CCACHE_SIZE" -BEGIN_FOLD autogen if [ -n "$CONFIG_SHELL" ]; then DOCKER_EXEC "$CONFIG_SHELL" -c "./autogen.sh" else DOCKER_EXEC ./autogen.sh fi -END_FOLD DOCKER_EXEC mkdir -p "${BASE_BUILD_DIR}" export P_CI_DIR="${BASE_BUILD_DIR}" -BEGIN_FOLD configure DOCKER_EXEC "${BASE_ROOT_DIR}/configure" --cache-file=config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) -END_FOLD -BEGIN_FOLD distdir DOCKER_EXEC make distdir VERSION=$HOST -END_FOLD export P_CI_DIR="${BASE_BUILD_DIR}/bitcoin-$HOST" -BEGIN_FOLD configure DOCKER_EXEC ./configure --cache-file=../config.cache $BITCOIN_CONFIG_ALL $BITCOIN_CONFIG || ( (DOCKER_EXEC cat config.log) && false) -END_FOLD set -o errtrace trap 'DOCKER_EXEC "cat ${BASE_SCRATCH_DIR}/sanitizer-output/* 2> /dev/null"' ERR @@ -45,12 +40,8 @@ if [[ ${USE_MEMORY_SANITIZER} == "true" ]]; then DOCKER_EXEC 'grep -v HAVE_SYS_GETRANDOM src/config/bitcoin-config.h > src/config/bitcoin-config.h.tmp && mv src/config/bitcoin-config.h.tmp src/config/bitcoin-config.h' fi -BEGIN_FOLD build DOCKER_EXEC make $MAKEJOBS $GOAL || ( echo "Build failure. Verbose build follows." && DOCKER_EXEC make $GOAL V=1 ; false ) -END_FOLD -BEGIN_FOLD cache_stats DOCKER_EXEC "ccache --version | head -n 1 && ccache --show-stats" DOCKER_EXEC du -sh "${DEPENDS_DIR}"/*/ DOCKER_EXEC du -sh "${PREVIOUS_RELEASES_DIR}" -END_FOLD diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 6c14a3dfbe..194b14beab 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -6,47 +6,40 @@ export LC_ALL=C.UTF-8 +if [[ $HOST = *-mingw32 ]]; then + # Generate all binaries, so that they can be wrapped + DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1 + DOCKER_EXEC make $MAKEJOBS -C src/univalue VERBOSE=1 + DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" +fi + if [ -n "$QEMU_USER_CMD" ]; then - BEGIN_FOLD wrap-qemu # Generate all binaries, so that they can be wrapped DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1 DOCKER_EXEC make $MAKEJOBS -C src/univalue VERBOSE=1 DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-qemu.sh" - END_FOLD fi if [ -n "$USE_VALGRIND" ]; then - BEGIN_FOLD wrap-valgrind DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-valgrind.sh" - END_FOLD fi if [ "$RUN_UNIT_TESTS" = "true" ]; then - BEGIN_FOLD unit-tests - DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib make $MAKEJOBS check VERBOSE=1 - END_FOLD + DOCKER_EXEC ${TEST_RUNNER_ENV} DIR_UNIT_TEST_DATA=${DIR_UNIT_TEST_DATA} LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib make $MAKEJOBS check VERBOSE=1 fi if [ "$RUN_UNIT_TESTS_SEQUENTIAL" = "true" ]; then - BEGIN_FOLD unit-tests-seq - DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib "${BASE_BUILD_DIR}/bitcoin-*/src/test/test_bitcoin*" --catch_system_errors=no -l test_suite - END_FOLD + DOCKER_EXEC ${TEST_RUNNER_ENV} DIR_UNIT_TEST_DATA=${DIR_UNIT_TEST_DATA} LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib "${BASE_BUILD_DIR}/bitcoin-*/src/test/test_bitcoin*" --catch_system_errors=no -l test_suite fi if [ "$RUN_FUNCTIONAL_TESTS" = "true" ]; then - BEGIN_FOLD functional-tests DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib ${TEST_RUNNER_ENV} test/functional/test_runner.py --ci $MAKEJOBS --tmpdirprefix "${BASE_SCRATCH_DIR}/test_runner/" --ansi --combinedlogslen=4000 --timeout-factor=${TEST_RUNNER_TIMEOUT_FACTOR} ${TEST_RUNNER_EXTRA} --quiet --failfast - END_FOLD fi if [ "$RUN_SECURITY_TESTS" = "true" ]; then - BEGIN_FOLD security-tests DOCKER_EXEC make test-security-check - END_FOLD fi if [ "$RUN_FUZZ_TESTS" = "true" ]; then - BEGIN_FOLD fuzz-tests DOCKER_EXEC LD_LIBRARY_PATH=$DEPENDS_DIR/$HOST/lib test/fuzz/test_runner.py ${FUZZ_TESTS_CONFIG} $MAKEJOBS -l DEBUG ${DIR_FUZZ_IN} - END_FOLD fi diff --git a/ci/test/wrap-qemu.sh b/ci/test/wrap-qemu.sh index be7d7fcc1f..2cd7c8cec2 100755 --- a/ci/test/wrap-qemu.sh +++ b/ci/test/wrap-qemu.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash # -# Copyright (c) 2018-2019 The Bitcoin Core developers +# Copyright (c) 2018-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. diff --git a/ci/test/wrap-wine.sh b/ci/test/wrap-wine.sh new file mode 100755 index 0000000000..82964897e1 --- /dev/null +++ b/ci/test/wrap-wine.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}.exe; do + # shellcheck disable=SC2044 + for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename $b_name)"); do + if (file "$b" | grep "Windows"); then + echo "Wrap $b ..." + mv "$b" "${b}_orig" + echo '#!/usr/bin/env bash' > "$b" + echo "( wine \"${b}_orig\" \"\$@\" ) || ( sleep 1 && wine \"${b}_orig\" \"\$@\" )" >> "$b" + chmod +x "$b" + fi + done +done diff --git a/ci/test_run_all.sh b/ci/test_run_all.sh index a1d4bd1952..93b07aab1e 100755 --- a/ci/test_run_all.sh +++ b/ci/test_run_all.sh @@ -1,13 +1,12 @@ #!/usr/bin/env bash # -# Copyright (c) 2019 The Bitcoin Core developers +# Copyright (c) 2019-2020 The Bitcoin Core developers # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. export LC_ALL=C.UTF-8 set -o errexit; source ./ci/test/00_setup_env.sh -set -o errexit; source ./ci/test/03_before_install.sh set -o errexit; source ./ci/test/04_install.sh set -o errexit; source ./ci/test/05_before_script.sh set -o errexit; source ./ci/test/06_script_a.sh |