diff options
33 files changed, 253 insertions, 121 deletions
diff --git a/.cirrus.yml b/.cirrus.yml index 916f172a6a..415aa549f1 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -8,7 +8,7 @@ container: memory: 8G # Set to 8GB to avoid OOM. https://cirrus-ci.org/guide/linux/#linux-containers kvm: true # Use kvm to avoid spurious CI failures in the default virtualization cluster, see https://github.com/bitcoin/bitcoin/issues/20093 env: - PACKAGE_MANAGER_INSTALL : "apt-get update && apt-get install -y" + PACKAGE_MANAGER_INSTALL: "apt-get update && apt-get install -y" MAKEJOBS: "-j4" DANGER_RUN_CI_ON_HOST: "1" # Containers will be discarded after the run, so there is no risk that the ci scripts modify the system TEST_RUNNER_PORT_MIN: "14000" # Must be larger than 12321, which is used for the http cache. See https://cirrus-ci.org/guide/writing-tasks/#http-cache @@ -54,6 +54,22 @@ global_task_template: &GLOBAL_TASK_TEMPLATE # - choco install python --version=3.7.7 -y task: + name: 'ARM [GOAL: install] [buster] [unit tests, no functional tests]' + << : *GLOBAL_TASK_TEMPLATE + container: + image: debian:buster + env: + FILE_ENV: "./ci/test/00_setup_env_arm.sh" + +task: + name: 'Win64 [GOAL: deploy] [unit tests, no gui, no boost::process, no functional tests]' + << : *GLOBAL_TASK_TEMPLATE + container: + image: ubuntu:bionic + env: + FILE_ENV: "./ci/test/00_setup_env_win64.sh" + +task: name: 'x86_64 Linux [GOAL: install] [bionic] [C++17, previous releases, uses qt5 dev package and some depends packages] [unsigned char]' << : *GLOBAL_TASK_TEMPLATE container: @@ -97,14 +113,6 @@ task: FILE_ENV: "./ci/test/00_setup_env_native_fuzz.sh" task: - name: 'x86_64 Linux [GOAL: install] [focal] [no depends, only system libs, fuzzers under valgrind]' - << : *GLOBAL_TASK_TEMPLATE - container: - image: ubuntu:focal - env: - FILE_ENV: "./ci/test/00_setup_env_native_fuzz_with_valgrind.sh" - -task: name: 'x86_64 Linux [GOAL: install] [focal] [multiprocess]' << : *GLOBAL_TASK_TEMPLATE container: @@ -119,3 +127,17 @@ task: image: ubuntu:bionic env: FILE_ENV: "./ci/test/00_setup_env_mac.sh" + +task: + name: 'macOS 10.14 native [GOAL: install] [GUI] [no depends]' + macos_brew_addon_script: + - brew install boost libevent berkeley-db4 qt miniupnpc ccache zeromq qrencode sqlite libtool automake pkg-config gnu-getopt + << : *GLOBAL_TASK_TEMPLATE + osx_instance: + # Use latest image, but hardcode version to avoid silent upgrades (and breaks) + image: catalina-xcode-12.1 # https://cirrus-ci.org/guide/macOS + env: + DANGER_RUN_CI_ON_HOST: "true" + CI_USE_APT_INSTALL: "no" + PACKAGE_MANAGER_INSTALL: "echo" # Nothing to do + FILE_ENV: "./ci/test/00_setup_env_mac_host.sh" diff --git a/.travis.yml b/.travis.yml index 55b34ae422..656eed9871 100644 --- a/.travis.yml +++ b/.travis.yml @@ -66,18 +66,6 @@ jobs: - set -o errexit; source ./ci/lint/06_script.sh - stage: test - name: 'ARM [GOAL: install] [buster] [unit tests, no functional tests]' - arch: arm64 # Can disable QEMU_USER_CMD and run the tests natively without qemu - env: >- - FILE_ENV="./ci/test/00_setup_env_arm.sh" - QEMU_USER_CMD="" - - - stage: test - name: 'Win64 [GOAL: deploy] [unit tests, no gui, no boost::process, no functional tests]' - env: >- - FILE_ENV="./ci/test/00_setup_env_win64.sh" - - - stage: test name: '32-bit + dash [GOAL: install] [CentOS 7] [gui]' env: >- FILE_ENV="./ci/test/00_setup_env_i686_centos.sh" @@ -86,24 +74,3 @@ jobs: name: 'x86_64 Linux [GOAL: install] [xenial] [no wallet]' env: >- FILE_ENV="./ci/test/00_setup_env_native_nowallet.sh" - - - stage: test - name: 'macOS 10.14 native [GOAL: install] [GUI] [no depends]' - os: osx - # Use the most recent version: - # Xcode 11.3.1, macOS 10.14, SDK 10.15 - # https://docs.travis-ci.com/user/reference/osx/#macos-version - osx_image: xcode11.3 - addons: - homebrew: - packages: - - berkeley-db4 - - miniupnpc - - qrencode - - sqlite - - ccache - - zeromq - env: >- - DANGER_RUN_CI_ON_HOST=true - CI_USE_APT_INSTALL=no - FILE_ENV="./ci/test/00_setup_env_mac_host.sh" diff --git a/ci/test/00_setup_env.sh b/ci/test/00_setup_env.sh index 702e881862..72e29141a6 100755 --- a/ci/test/00_setup_env.sh +++ b/ci/test/00_setup_env.sh @@ -38,7 +38,7 @@ export RUN_SECURITY_TESTS=${RUN_SECURITY_TESTS:-false} # By how much to scale the test_runner timeouts (option --timeout-factor). # This is needed because some ci machines have slow CPU or disk, so sanitizers # might be slow or a reindex might be waiting on disk IO. -export TEST_RUNNER_TIMEOUT_FACTOR=${TEST_RUNNER_TIMEOUT_FACTOR:-4} +export TEST_RUNNER_TIMEOUT_FACTOR=${TEST_RUNNER_TIMEOUT_FACTOR:-40} export TEST_RUNNER_ENV=${TEST_RUNNER_ENV:-} export RUN_FUZZ_TESTS=${RUN_FUZZ_TESTS:-false} export EXPECTED_TESTS_DURATION_IN_SECONDS=${EXPECTED_TESTS_DURATION_IN_SECONDS:-1000} diff --git a/ci/test/00_setup_env_mac.sh b/ci/test/00_setup_env_mac.sh index b62f1603f4..e4450a65ce 100644 --- a/ci/test/00_setup_env_mac.sh +++ b/ci/test/00_setup_env_mac.sh @@ -7,6 +7,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_macos_cross +export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to macos (bionic is used in the gitian build as well) export HOST=x86_64-apple-darwin16 export PACKAGES="cmake imagemagick libcap-dev librsvg2-bin libz-dev libbz2-dev libtiff-tools python3-dev python3-setuptools" export XCODE_VERSION=11.3.1 diff --git a/ci/test/00_setup_env_mac_host.sh b/ci/test/00_setup_env_mac_host.sh index 5fb127b762..7c25a34cfe 100644 --- a/ci/test/00_setup_env_mac_host.sh +++ b/ci/test/00_setup_env_mac_host.sh @@ -7,16 +7,12 @@ export LC_ALL=C.UTF-8 export HOST=x86_64-apple-darwin16 -export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to macos (bionic is used in the gitian build as well) export PIP_PACKAGES="zmq" export GOAL="install" export BITCOIN_CONFIG="--with-gui --enable-reduce-exports --enable-werror --with-boost-process" +export CI_OS_NAME="macos" export NO_DEPENDS=1 export OSX_SDK="" export CCACHE_SIZE=300M export RUN_SECURITY_TESTS="true" -if [ "$TRAVIS_REPO_SLUG" != "bitcoin/bitcoin" ]; then - export RUN_FUNCTIONAL_TESTS="false" - export EXPECTED_TESTS_DURATION_IN_SECONDS=200 -fi diff --git a/ci/test/00_setup_env_win64.sh b/ci/test/00_setup_env_win64.sh index 2b351dff6d..72cc3f63c4 100644 --- a/ci/test/00_setup_env_win64.sh +++ b/ci/test/00_setup_env_win64.sh @@ -9,7 +9,7 @@ export LC_ALL=C.UTF-8 export CONTAINER_NAME=ci_win64 export DOCKER_NAME_TAG=ubuntu:18.04 # Check that bionic can cross-compile to win64 (bionic is used in the gitian build as well) export HOST=x86_64-w64-mingw32 -export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64" +export PACKAGES="python3 nsis g++-mingw-w64-x86-64 wine-binfmt wine64 file" export RUN_FUNCTIONAL_TESTS=false export RUN_SECURITY_TESTS="true" export GOAL="deploy" diff --git a/ci/test/04_install.sh b/ci/test/04_install.sh index 632bccf574..db74fe6569 100755 --- a/ci/test/04_install.sh +++ b/ci/test/04_install.sh @@ -13,8 +13,8 @@ if [[ $QEMU_USER_CMD == qemu-s390* ]]; then export LC_ALL=C fi -if [ "$TRAVIS_OS_NAME" == "osx" ]; then - ${CI_RETRY_EXE} pip3 install $PIP_PACKAGES +if [ "$CI_OS_NAME" == "macos" ]; then + IN_GETOPT_BIN="/usr/local/opt/gnu-getopt/bin/getopt" ${CI_RETRY_EXE} pip3 install --user $PIP_PACKAGES fi # Create folders that are mounted into the docker @@ -26,9 +26,7 @@ export LSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/l export TSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/tsan:halt_on_error=1:log_path=${BASE_SCRATCH_DIR}/sanitizer-output/tsan" export UBSAN_OPTIONS="suppressions=${BASE_ROOT_DIR}/test/sanitizer_suppressions/ubsan:print_stacktrace=1:halt_on_error=1:report_error_type=1" env | grep -E '^(BITCOIN_CONFIG|BASE_|QEMU_|CCACHE_|LC_ALL|BOOST_TEST_RANDOM|DEBIAN_FRONTEND|CONFIG_SHELL|(ASAN|LSAN|TSAN|UBSAN)_OPTIONS|PREVIOUS_RELEASES_DIR)' | tee /tmp/env -if [[ $HOST = *-mingw32 ]]; then - DOCKER_ADMIN="--cap-add SYS_ADMIN" -elif [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) +if [[ $BITCOIN_CONFIG = *--with-sanitizers=*address* ]]; then # If ran with (ASan + LSan), Docker needs access to ptrace (https://github.com/google/sanitizers/issues/764) DOCKER_ADMIN="--cap-add SYS_PTRACE" fi @@ -69,16 +67,16 @@ elif [ "$CI_USE_APT_INSTALL" != "no" ]; then ${CI_RETRY_EXE} DOCKER_EXEC apt-get install --no-install-recommends --no-upgrade -y $PACKAGES $DOCKER_PACKAGES fi -if [ "$TRAVIS_OS_NAME" == "osx" ]; then +if [ "$CI_OS_NAME" == "macos" ]; then top -l 1 -s 0 | awk ' /PhysMem/ {print}' echo "Number of CPUs: $(sysctl -n hw.logicalcpu)" else DOCKER_EXEC free -m -h DOCKER_EXEC echo "Number of CPUs \(nproc\):" \$\(nproc\) DOCKER_EXEC echo $(lscpu | grep Endian) - DOCKER_EXEC echo "Free disk space:" - DOCKER_EXEC df -h fi +DOCKER_EXEC echo "Free disk space:" +DOCKER_EXEC df -h if [ ! -d ${DIR_QA_ASSETS} ]; then DOCKER_EXEC git clone --depth=1 https://github.com/bitcoin-core/qa-assets ${DIR_QA_ASSETS} diff --git a/ci/test/05_before_script.sh b/ci/test/05_before_script.sh index 8ce839fc04..42c244c2f5 100755 --- a/ci/test/05_before_script.sh +++ b/ci/test/05_before_script.sh @@ -7,7 +7,7 @@ export LC_ALL=C.UTF-8 # Make sure default datadir does not exist and is never read by creating a dummy file -if [ "$TRAVIS_OS_NAME" == "osx" ]; then +if [ "$CI_OS_NAME" == "macos" ]; then echo > $HOME/Library/Application\ Support/Bitcoin else DOCKER_EXEC echo \> \$HOME/.bitcoin diff --git a/ci/test/06_script_b.sh b/ci/test/06_script_b.sh index 87e9f31d0f..7aea21f257 100755 --- a/ci/test/06_script_b.sh +++ b/ci/test/06_script_b.sh @@ -6,6 +6,15 @@ export LC_ALL=C.UTF-8 +if [[ $HOST = *-mingw32 ]]; then + BEGIN_FOLD wrap-wine + # Generate all binaries, so that they can be wrapped + DOCKER_EXEC make $MAKEJOBS -C src/secp256k1 VERBOSE=1 + DOCKER_EXEC make $MAKEJOBS -C src/univalue VERBOSE=1 + DOCKER_EXEC "${BASE_ROOT_DIR}/ci/test/wrap-wine.sh" + END_FOLD +fi + if [ -n "$QEMU_USER_CMD" ]; then BEGIN_FOLD wrap-qemu # Generate all binaries, so that they can be wrapped diff --git a/ci/test/wrap-wine.sh b/ci/test/wrap-wine.sh new file mode 100755 index 0000000000..58a8983e6e --- /dev/null +++ b/ci/test/wrap-wine.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# +# Copyright (c) 2020 The Bitcoin Core developers +# Distributed under the MIT software license, see the accompanying +# file COPYING or http://www.opensource.org/licenses/mit-license.php. + +export LC_ALL=C.UTF-8 + +for b_name in {"${BASE_OUTDIR}/bin"/*,src/secp256k1/*tests,src/univalue/{no_nul,test_json,unitester,object}}.exe; do + # shellcheck disable=SC2044 + for b in $(find "${BASE_ROOT_DIR}" -executable -type f -name "$(basename $b_name)"); do + if (file "$b" | grep "Windows"); then + echo "Wrap $b ..." + mv "$b" "${b}_orig" + echo '#!/usr/bin/env bash' > "$b" + echo "wine64 \"${b}_orig\" \"\$@\"" >> "$b" + chmod +x "$b" + fi + done +done diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml index 65f9a2e5c9..ec716cf2f3 100644 --- a/contrib/gitian-descriptors/gitian-linux.yml +++ b/contrib/gitian-descriptors/gitian-linux.yml @@ -112,7 +112,7 @@ script: | # Create the source tarball mkdir -p "$(dirname "$GIT_ARCHIVE")" - git archive --output="$GIT_ARCHIVE" HEAD + git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD ORIGPATH="$PATH" # Extract the git archive into a dir for each host and build @@ -129,7 +129,7 @@ script: | cd distsrc-${i} INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} - tar -xf $GIT_ARCHIVE + tar --strip-components=1 -xf "${GIT_ARCHIVE}" ./autogen.sh CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" LDFLAGS="${HOST_LDFLAGS}" diff --git a/contrib/gitian-descriptors/gitian-osx.yml b/contrib/gitian-descriptors/gitian-osx.yml index e0aaafc15a..df50f45188 100644 --- a/contrib/gitian-descriptors/gitian-osx.yml +++ b/contrib/gitian-descriptors/gitian-osx.yml @@ -111,7 +111,7 @@ script: | # Create the source tarball mkdir -p "$(dirname "$GIT_ARCHIVE")" - git archive --output="$GIT_ARCHIVE" HEAD + git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD ORIGPATH="$PATH" # Extract the git archive into a dir for each host and build @@ -121,7 +121,7 @@ script: | cd distsrc-${i} INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} - tar -xf $GIT_ARCHIVE + tar --strip-components=1 -xf "${GIT_ARCHIVE}" ./autogen.sh CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} diff --git a/contrib/gitian-descriptors/gitian-win.yml b/contrib/gitian-descriptors/gitian-win.yml index 5f671b95ce..4d61cded74 100644 --- a/contrib/gitian-descriptors/gitian-win.yml +++ b/contrib/gitian-descriptors/gitian-win.yml @@ -116,7 +116,7 @@ script: | # Create the source tarball mkdir -p "$(dirname "$GIT_ARCHIVE")" - git archive --output="$GIT_ARCHIVE" HEAD + git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD ORIGPATH="$PATH" # Extract the git archive into a dir for each host and build @@ -126,7 +126,7 @@ script: | cd distsrc-${i} INSTALLPATH="${PWD}/installed/${DISTNAME}" mkdir -p ${INSTALLPATH} - tar -xf $GIT_ARCHIVE + tar --strip-components=1 -xf "${GIT_ARCHIVE}" ./autogen.sh CONFIG_SITE=${BASEPREFIX}/${i}/share/config.site ./configure --prefix=/ --disable-ccache --disable-maintainer-mode --disable-dependency-tracking ${CONFIGFLAGS} CFLAGS="${HOST_CFLAGS}" CXXFLAGS="${HOST_CXXFLAGS}" diff --git a/contrib/guix/libexec/build.sh b/contrib/guix/libexec/build.sh index 5be3baeefa..d658c4f6a6 100644 --- a/contrib/guix/libexec/build.sh +++ b/contrib/guix/libexec/build.sh @@ -158,7 +158,7 @@ GIT_ARCHIVE="${OUTDIR}/src/${DISTNAME}.tar.gz" # Create the source tarball if not already there if [ ! -e "$GIT_ARCHIVE" ]; then mkdir -p "$(dirname "$GIT_ARCHIVE")" - git archive --output="$GIT_ARCHIVE" HEAD + git archive --prefix="${DISTNAME}/" --output="$GIT_ARCHIVE" HEAD fi ########################### @@ -193,7 +193,7 @@ export PATH="${BASEPREFIX}/${HOST}/native/bin:${PATH}" cd "$DISTSRC" # Extract the source tarball - tar -xf "${GIT_ARCHIVE}" + tar --strip-components=1 -xf "${GIT_ARCHIVE}" ./autogen.sh diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index d7bc73defb..0621da8ddf 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -15,6 +15,7 @@ TEST_UTIL_H = \ test/util/setup_common.h \ test/util/str.h \ test/util/transaction_utils.h \ + test/util/validation.h \ test/util/wallet.h libtest_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) @@ -27,6 +28,7 @@ libtest_util_a_SOURCES = \ test/util/setup_common.cpp \ test/util/str.cpp \ test/util/transaction_utils.cpp \ + test/util/validation.cpp \ test/util/wallet.cpp \ $(TEST_UTIL_H) diff --git a/src/netbase.cpp b/src/netbase.cpp index 0273839017..264029d8a2 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -52,14 +52,20 @@ enum Network ParseNetwork(const std::string& net_in) { return NET_UNROUTABLE; } -std::string GetNetworkName(enum Network net) { - switch(net) - { +std::string GetNetworkName(enum Network net) +{ + switch (net) { + case NET_UNROUTABLE: return "unroutable"; case NET_IPV4: return "ipv4"; case NET_IPV6: return "ipv6"; case NET_ONION: return "onion"; - default: return ""; - } + case NET_I2P: return "i2p"; + case NET_CJDNS: return "cjdns"; + case NET_INTERNAL: return "internal"; + case NET_MAX: assert(false); + } // no default case, so the compiler can warn about missing cases + + assert(false); } bool static LookupIntern(const std::string& name, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup) diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index b81e6414a5..f98ea63782 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -497,11 +497,9 @@ static RPCHelpMan getnettotals() static UniValue GetNetworksInfo() { UniValue networks(UniValue::VARR); - for(int n=0; n<NET_MAX; ++n) - { + for (int n = 0; n < NET_MAX; ++n) { enum Network network = static_cast<enum Network>(n); - if(network == NET_UNROUTABLE || network == NET_INTERNAL) - continue; + if (network == NET_UNROUTABLE || network == NET_I2P || network == NET_CJDNS || network == NET_INTERNAL) continue; proxyType proxy; UniValue obj(UniValue::VOBJ); GetProxy(network, proxy); diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index 9803fdc882..8ca5366c8a 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -15,6 +15,7 @@ #include <net.h> #include <netbase.h> #include <node/utxo_snapshot.h> +#include <optional.h> #include <primitives/block.h> #include <protocol.h> #include <psbt.h> @@ -61,15 +62,19 @@ T Deserialize(CDataStream ds) } template <typename T> -void DeserializeFromFuzzingInput(const std::vector<uint8_t>& buffer, T& obj) +void DeserializeFromFuzzingInput(const std::vector<uint8_t>& buffer, T& obj, const Optional<int> protocol_version = nullopt) { CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); - try { - int version; - ds >> version; - ds.SetVersion(version); - } catch (const std::ios_base::failure&) { - throw invalid_fuzzing_input_exception(); + if (protocol_version) { + ds.SetVersion(*protocol_version); + } else { + try { + int version; + ds >> version; + ds.SetVersion(version); + } catch (const std::ios_base::failure&) { + throw invalid_fuzzing_input_exception(); + } } try { ds >> obj; @@ -125,9 +130,15 @@ void test_one_input(const std::vector<uint8_t>& buffer) CScript script; DeserializeFromFuzzingInput(buffer, script); #elif SUB_NET_DESERIALIZE - CSubNet sub_net; - DeserializeFromFuzzingInput(buffer, sub_net); - AssertEqualAfterSerializeDeserialize(sub_net); + CSubNet sub_net_1; + DeserializeFromFuzzingInput(buffer, sub_net_1, INIT_PROTO_VERSION); + AssertEqualAfterSerializeDeserialize(sub_net_1, INIT_PROTO_VERSION); + CSubNet sub_net_2; + DeserializeFromFuzzingInput(buffer, sub_net_2, INIT_PROTO_VERSION | ADDRV2_FORMAT); + AssertEqualAfterSerializeDeserialize(sub_net_2, INIT_PROTO_VERSION | ADDRV2_FORMAT); + CSubNet sub_net_3; + DeserializeFromFuzzingInput(buffer, sub_net_3); + AssertEqualAfterSerializeDeserialize(sub_net_3, INIT_PROTO_VERSION | ADDRV2_FORMAT); #elif TX_IN_DESERIALIZE CTxIn tx_in; DeserializeFromFuzzingInput(buffer, tx_in); @@ -195,6 +206,13 @@ void test_one_input(const std::vector<uint8_t>& buffer) AssertEqualAfterSerializeDeserialize(s); } AssertEqualAfterSerializeDeserialize(s, INIT_PROTO_VERSION | ADDRV2_FORMAT); + CService s1; + DeserializeFromFuzzingInput(buffer, s1, INIT_PROTO_VERSION); + AssertEqualAfterSerializeDeserialize(s1, INIT_PROTO_VERSION); + assert(s1.IsAddrV1Compatible()); + CService s2; + DeserializeFromFuzzingInput(buffer, s2, INIT_PROTO_VERSION | ADDRV2_FORMAT); + AssertEqualAfterSerializeDeserialize(s2, INIT_PROTO_VERSION | ADDRV2_FORMAT); #elif MESSAGEHEADER_DESERIALIZE CMessageHeader mh; DeserializeFromFuzzingInput(buffer, mh); diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 3ef03137ec..9390399878 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -16,6 +16,7 @@ #include <test/util/mining.h> #include <test/util/net.h> #include <test/util/setup_common.h> +#include <test/util/validation.h> #include <util/memory.h> #include <validationinterface.h> #include <version.h> @@ -63,10 +64,14 @@ void test_one_input(const std::vector<uint8_t>& buffer) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); ConnmanTestMsg& connman = *(ConnmanTestMsg*)g_setup->m_node.connman.get(); + TestChainState& chainstate = *(TestChainState*)&g_setup->m_node.chainman->ActiveChainstate(); + chainstate.ResetIbd(); const std::string random_message_type{fuzzed_data_provider.ConsumeBytesAsString(CMessageHeader::COMMAND_SIZE).c_str()}; if (!LIMIT_TO_MESSAGE_TYPE.empty() && random_message_type != LIMIT_TO_MESSAGE_TYPE) { return; } + const bool jump_out_of_ibd{fuzzed_data_provider.ConsumeBool()}; + if (jump_out_of_ibd) chainstate.JumpOutOfIbd(); CDataStream random_bytes_data_stream{fuzzed_data_provider.ConsumeRemainingBytes<unsigned char>(), SER_NETWORK, PROTOCOL_VERSION}; CNode& p2p_node = *MakeUnique<CNode>(0, ServiceFlags(NODE_NETWORK | NODE_WITNESS | NODE_BLOOM), 0, INVALID_SOCKET, CAddress{CService{in_addr{0x0100007f}, 7777}, NODE_NETWORK}, 0, 0, CAddress{}, std::string{}, ConnectionType::OUTBOUND_FULL_RELAY).release(); p2p_node.fSuccessfullyConnected = true; @@ -76,7 +81,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) g_setup->m_node.peerman->InitializeNode(&p2p_node); try { g_setup->m_node.peerman->ProcessMessage(p2p_node, random_message_type, random_bytes_data_stream, - GetTime<std::chrono::microseconds>(), std::atomic<bool>{false}); + GetTime<std::chrono::microseconds>(), std::atomic<bool>{false}); } catch (const std::ios_base::failure&) { } SyncWithValidationInterfaceQueue(); diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index f722eeac3a..19ea92b750 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -12,6 +12,7 @@ #include <test/util/mining.h> #include <test/util/net.h> #include <test/util/setup_common.h> +#include <test/util/validation.h> #include <util/memory.h> #include <validation.h> #include <validationinterface.h> @@ -39,7 +40,10 @@ void test_one_input(const std::vector<uint8_t>& buffer) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); ConnmanTestMsg& connman = *(ConnmanTestMsg*)g_setup->m_node.connman.get(); + TestChainState& chainstate = *(TestChainState*)&g_setup->m_node.chainman->ActiveChainstate(); + chainstate.ResetIbd(); std::vector<CNode*> peers; + bool jump_out_of_ibd{false}; const auto num_peers_to_add = fuzzed_data_provider.ConsumeIntegralInRange(1, 3); for (int i = 0; i < num_peers_to_add; ++i) { @@ -58,6 +62,8 @@ void test_one_input(const std::vector<uint8_t>& buffer) } while (fuzzed_data_provider.ConsumeBool()) { + if (!jump_out_of_ibd) jump_out_of_ibd = fuzzed_data_provider.ConsumeBool(); + if (jump_out_of_ibd && chainstate.IsInitialBlockDownload()) chainstate.JumpOutOfIbd(); const std::string random_message_type{fuzzed_data_provider.ConsumeBytesAsString(CMessageHeader::COMMAND_SIZE).c_str()}; CSerializedNetMsg net_msg; diff --git a/src/test/util/validation.cpp b/src/test/util/validation.cpp new file mode 100644 index 0000000000..1aed492c3c --- /dev/null +++ b/src/test/util/validation.cpp @@ -0,0 +1,22 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/validation.h> + +#include <util/check.h> +#include <util/time.h> +#include <validation.h> + +void TestChainState::ResetIbd() +{ + m_cached_finished_ibd = false; + assert(IsInitialBlockDownload()); +} + +void TestChainState::JumpOutOfIbd() +{ + Assert(IsInitialBlockDownload()); + m_cached_finished_ibd = true; + Assert(!IsInitialBlockDownload()); +} diff --git a/src/test/util/validation.h b/src/test/util/validation.h new file mode 100644 index 0000000000..b13aa0be60 --- /dev/null +++ b/src/test/util/validation.h @@ -0,0 +1,17 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_VALIDATION_H +#define BITCOIN_TEST_UTIL_VALIDATION_H + +#include <validation.h> + +struct TestChainState : public CChainState { + /** Reset the ibd cache to its initial state */ + void ResetIbd(); + /** Toggle IsInitialBlockDownload from true to false */ + void JumpOutOfIbd(); +}; + +#endif // BITCOIN_TEST_UTIL_VALIDATION_H diff --git a/src/validation.h b/src/validation.h index 3d9fa92c15..ffb038ad75 100644 --- a/src/validation.h +++ b/src/validation.h @@ -503,9 +503,9 @@ enum class CoinsCacheSizeState * whereas block information and metadata independent of the current tip is * kept in `BlockMetadataManager`. */ -class CChainState { -private: - +class CChainState +{ +protected: /** * Every received block is assigned a unique and increasing identifier, so we * know which one to give priority in case of a fork. diff --git a/src/wallet/rpcdump.cpp b/src/wallet/rpcdump.cpp index 884ab58497..6b46868d10 100644 --- a/src/wallet/rpcdump.cpp +++ b/src/wallet/rpcdump.cpp @@ -1523,7 +1523,9 @@ static UniValue ProcessDescriptorImport(CWallet * const pwallet, const UniValue& // Need to ExpandPrivate to check if private keys are available for all pubkeys FlatSigningProvider expand_keys; std::vector<CScript> scripts; - parsed_desc->Expand(0, keys, scripts, expand_keys); + if (!parsed_desc->Expand(0, keys, scripts, expand_keys)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Cannot expand descriptor. Probably because of hardened derivations without private keys provided"); + } parsed_desc->ExpandPrivate(0, keys, expand_keys); // Check if all private keys are provided @@ -1559,7 +1561,7 @@ static UniValue ProcessDescriptorImport(CWallet * const pwallet, const UniValue& } // Add descriptor to the wallet - auto spk_manager = pwallet->AddWalletDescriptor(w_desc, keys, label); + auto spk_manager = pwallet->AddWalletDescriptor(w_desc, keys, label, internal); if (spk_manager == nullptr) { throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Could not add descriptor '%s'", descriptor)); } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index eb92142aa9..c92ff73e35 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -4515,7 +4515,7 @@ DescriptorScriptPubKeyMan* CWallet::GetDescriptorScriptPubKeyMan(const WalletDes return nullptr; } -ScriptPubKeyMan* CWallet::AddWalletDescriptor(WalletDescriptor& desc, const FlatSigningProvider& signing_provider, const std::string& label) +ScriptPubKeyMan* CWallet::AddWalletDescriptor(WalletDescriptor& desc, const FlatSigningProvider& signing_provider, const std::string& label, bool internal) { if (!IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { WalletLogPrintf("Cannot add WalletDescriptor to a non-descriptor wallet\n"); @@ -4560,7 +4560,10 @@ ScriptPubKeyMan* CWallet::AddWalletDescriptor(WalletDescriptor& desc, const Flat } // Top up key pool, the manager will generate new scriptPubKeys internally - new_spk_man->TopUp(); + if (!new_spk_man->TopUp()) { + WalletLogPrintf("Could not top up scriptPubKeys\n"); + return nullptr; + } // Apply the label if necessary // Note: we disable labels for ranged descriptors @@ -4572,7 +4575,7 @@ ScriptPubKeyMan* CWallet::AddWalletDescriptor(WalletDescriptor& desc, const Flat } CTxDestination dest; - if (ExtractDestination(script_pub_keys.at(0), dest)) { + if (!internal && ExtractDestination(script_pub_keys.at(0), dest)) { SetAddressBook(dest, label, "receive"); } } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 74de55dcb5..00e0e3c84d 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -1280,7 +1280,7 @@ public: DescriptorScriptPubKeyMan* GetDescriptorScriptPubKeyMan(const WalletDescriptor& desc) const; //! Add a descriptor to the wallet, return a ScriptPubKeyMan & associated output type - ScriptPubKeyMan* AddWalletDescriptor(WalletDescriptor& desc, const FlatSigningProvider& signing_provider, const std::string& label); + ScriptPubKeyMan* AddWalletDescriptor(WalletDescriptor& desc, const FlatSigningProvider& signing_provider, const std::string& label, bool internal); }; /** diff --git a/test/functional/feature_proxy.py b/test/functional/feature_proxy.py index dfae58e860..05b658ed87 100755 --- a/test/functional/feature_proxy.py +++ b/test/functional/feature_proxy.py @@ -26,6 +26,8 @@ addnode connect to IPv4 addnode connect to IPv6 addnode connect to onion addnode connect to generic DNS name + +- Test getnetworkinfo for each node """ import socket @@ -41,12 +43,16 @@ from test_framework.util import ( from test_framework.netutil import test_ipv6_local RANGE_BEGIN = PORT_MIN + 2 * PORT_RANGE # Start after p2p and rpc ports -# From GetNetworkName() in netbase.cpp: -NET_UNROUTABLE = "" + +# Networks returned by RPC getpeerinfo, defined in src/netbase.cpp::GetNetworkName() +NET_UNROUTABLE = "unroutable" NET_IPV4 = "ipv4" NET_IPV6 = "ipv6" NET_ONION = "onion" +# Networks returned by RPC getnetworkinfo, defined in src/rpc/net.cpp::GetNetworksInfo() +NETWORKS = frozenset({NET_IPV4, NET_IPV6, NET_ONION}) + class ProxyTest(BitcoinTestFramework): def set_test_params(self): @@ -84,14 +90,14 @@ class ProxyTest(BitcoinTestFramework): self.serv3 = Socks5Server(self.conf3) self.serv3.start() - # Note: proxies are not used to connect to local nodes - # this is because the proxy to use is based on CService.GetNetwork(), which return NET_UNROUTABLE for localhost + # Note: proxies are not used to connect to local nodes. This is because the proxy to + # use is based on CService.GetNetwork(), which returns NET_UNROUTABLE for localhost. args = [ ['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-proxyrandomize=1'], ['-listen', '-proxy=%s:%i' % (self.conf1.addr),'-onion=%s:%i' % (self.conf2.addr),'-proxyrandomize=0'], ['-listen', '-proxy=%s:%i' % (self.conf2.addr),'-proxyrandomize=1'], [] - ] + ] if self.have_ipv6: args[3] = ['-listen', '-proxy=[%s]:%i' % (self.conf3.addr),'-proxyrandomize=0', '-noonion'] self.add_nodes(self.num_nodes, extra_args=args) @@ -189,15 +195,17 @@ class ProxyTest(BitcoinTestFramework): r[x['name']] = x return r - # test RPC getnetworkinfo + self.log.info("Test RPC getnetworkinfo") n0 = networks_dict(self.nodes[0].getnetworkinfo()) - for net in ['ipv4','ipv6','onion']: + assert_equal(NETWORKS, n0.keys()) + for net in NETWORKS: assert_equal(n0[net]['proxy'], '%s:%i' % (self.conf1.addr)) assert_equal(n0[net]['proxy_randomize_credentials'], True) assert_equal(n0['onion']['reachable'], True) n1 = networks_dict(self.nodes[1].getnetworkinfo()) - for net in ['ipv4','ipv6']: + assert_equal(NETWORKS, n1.keys()) + for net in ['ipv4', 'ipv6']: assert_equal(n1[net]['proxy'], '%s:%i' % (self.conf1.addr)) assert_equal(n1[net]['proxy_randomize_credentials'], False) assert_equal(n1['onion']['proxy'], '%s:%i' % (self.conf2.addr)) @@ -205,14 +213,16 @@ class ProxyTest(BitcoinTestFramework): assert_equal(n1['onion']['reachable'], True) n2 = networks_dict(self.nodes[2].getnetworkinfo()) - for net in ['ipv4','ipv6','onion']: + assert_equal(NETWORKS, n2.keys()) + for net in NETWORKS: assert_equal(n2[net]['proxy'], '%s:%i' % (self.conf2.addr)) assert_equal(n2[net]['proxy_randomize_credentials'], True) assert_equal(n2['onion']['reachable'], True) if self.have_ipv6: n3 = networks_dict(self.nodes[3].getnetworkinfo()) - for net in ['ipv4','ipv6']: + assert_equal(NETWORKS, n3.keys()) + for net in NETWORKS: assert_equal(n3[net]['proxy'], '[%s]:%i' % (self.conf3.addr)) assert_equal(n3[net]['proxy_randomize_credentials'], False) assert_equal(n3['onion']['reachable'], False) diff --git a/test/functional/feature_taproot.py b/test/functional/feature_taproot.py index 3e47e24a3b..6e28cfb265 100755 --- a/test/functional/feature_taproot.py +++ b/test/functional/feature_taproot.py @@ -9,6 +9,7 @@ from test_framework.blocktools import ( create_block, add_witness_commitment, MAX_BLOCK_SIGOPS_WEIGHT, + NORMAL_GBT_REQUEST_PARAMS, WITNESS_SCALE_FACTOR, ) from test_framework.messages import ( @@ -1199,7 +1200,7 @@ class TaprootTest(BitcoinTestFramework): self.num_nodes = 2 self.setup_clean_chain = True # Node 0 has Taproot inactive, Node 1 active. - self.extra_args = [["-whitelist=127.0.0.1", "-par=1", "-vbparams=taproot:1:1"], ["-whitelist=127.0.0.1", "-par=1"]] + self.extra_args = [["-par=1", "-vbparams=taproot:1:1"], ["-par=1"]] def block_submit(self, node, txs, msg, err_msg, cb_pubkey=None, fees=0, sigops_weight=0, witness=False, accept=False): @@ -1218,7 +1219,7 @@ class TaprootTest(BitcoinTestFramework): witness and add_witness_commitment(block) block.rehash() block.solve() - block_response = node.submitblock(block.serialize(True).hex()) + block_response = node.submitblock(block.serialize().hex()) if err_msg is not None: assert block_response is not None and err_msg in block_response, "Missing error message '%s' from block response '%s': %s" % (err_msg, "(None)" if block_response is None else block_response, msg) if (accept): @@ -1436,17 +1437,27 @@ class TaprootTest(BitcoinTestFramework): self.log.info(" - Done") def run_test(self): - self.connect_nodes(0, 1) - # Post-taproot activation tests go first (pre-taproot tests' blocks are invalid post-taproot). self.log.info("Post-activation tests...") self.nodes[1].generate(101) self.test_spenders(self.nodes[1], spenders_taproot_active(), input_counts=[1, 2, 2, 2, 2, 3]) - # Transfer % of funds to pre-taproot node. + # Transfer funds to pre-taproot node. addr = self.nodes[0].getnewaddress() - self.nodes[1].sendtoaddress(address=addr, amount=int(self.nodes[1].getbalance() * 70000000) / 100000000) - self.nodes[1].generate(1) + rawtx = self.nodes[1].createrawtransaction( + inputs=[{ + 'txid': i['txid'], + 'vout': i['vout'] + } for i in self.nodes[1].listunspent()], + outputs={addr: self.nodes[1].getbalance()}, + ) + rawtx = self.nodes[1].signrawtransactionwithwallet(rawtx)['hex'] + # Transaction is too large to fit into the mempool, so put it into a block + block = create_block(tmpl=self.nodes[1].getblocktemplate(NORMAL_GBT_REQUEST_PARAMS), txlist=[rawtx]) + add_witness_commitment(block) + block.rehash() + block.solve() + assert_equal(None, self.nodes[1].submitblock(block.serialize().hex())) self.sync_blocks() # Pre-taproot activation tests. diff --git a/test/functional/test_framework/blocktools.py b/test/functional/test_framework/blocktools.py index 64bc7e0485..6b7214f03a 100644 --- a/test/functional/test_framework/blocktools.py +++ b/test/functional/test_framework/blocktools.py @@ -5,7 +5,6 @@ """Utilities for manipulating blocks and transactions.""" from binascii import a2b_hex -import io import struct import time import unittest @@ -45,7 +44,6 @@ from .script import ( hash160, ) from .util import assert_equal -from io import BytesIO WITNESS_SCALE_FACTOR = 4 MAX_BLOCK_SIGOPS = 20000 @@ -78,9 +76,7 @@ def create_block(hashprev=None, coinbase=None, ntime=None, *, version=None, tmpl if txlist: for tx in txlist: if not hasattr(tx, 'calc_sha256'): - txo = CTransaction() - txo.deserialize(io.BytesIO(tx)) - tx = txo + tx = FromHex(CTransaction(), tx) block.vtx.append(tx) block.hashMerkleRoot = block.calc_merkle_root() block.calc_sha256() @@ -166,8 +162,7 @@ def create_transaction(node, txid, to_address, *, amount): sign for the output that is being spent. """ raw_tx = create_raw_transaction(node, txid, to_address, amount=amount) - tx = CTransaction() - tx.deserialize(BytesIO(hex_str_to_bytes(raw_tx))) + tx = FromHex(CTransaction(), raw_tx) return tx def create_raw_transaction(node, txid, to_address, *, amount): diff --git a/test/functional/test_framework/key.py b/test/functional/test_framework/key.py index abf2507154..f3d13c049b 100644 --- a/test/functional/test_framework/key.py +++ b/test/functional/test_framework/key.py @@ -10,7 +10,6 @@ import csv import hashlib import os import random -import sys import unittest from .util import modinv @@ -22,6 +21,7 @@ def TaggedHash(tag, data): return hashlib.sha256(ss).digest() def xor_bytes(b0, b1): + assert len(b0) == len(b1) return bytes(x ^ y for (x, y) in zip(b0, b1)) def jacobi_symbol(n, k): @@ -523,7 +523,8 @@ class TestFrameworkKey(unittest.TestCase): def test_schnorr_testvectors(self): """Implement the BIP340 test vectors (read from bip340_test_vectors.csv).""" num_tests = 0 - with open(os.path.join(sys.path[0], 'test_framework', 'bip340_test_vectors.csv'), newline='', encoding='utf8') as csvfile: + vectors_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), 'bip340_test_vectors.csv') + with open(vectors_file, newline='', encoding='utf8') as csvfile: reader = csv.reader(csvfile) next(reader) for row in reader: diff --git a/test/functional/wallet_importdescriptors.py b/test/functional/wallet_importdescriptors.py index 949adeb703..2903a84998 100755 --- a/test/functional/wallet_importdescriptors.py +++ b/test/functional/wallet_importdescriptors.py @@ -15,6 +15,7 @@ variants. - `test_address()` is called to call getaddressinfo for an address on node1 and test the values returned.""" +from test_framework.address import key_to_p2pkh from test_framework.test_framework import BitcoinTestFramework from test_framework.descriptors import descsum_create from test_framework.util import ( @@ -107,6 +108,17 @@ class ImportDescriptorsTest(BitcoinTestFramework): error_code=-8, error_message="Internal addresses should not have a label") + self.log.info("Internal addresses should be detected as such") + key = get_generate_key() + addr = key_to_p2pkh(key.pubkey) + self.test_importdesc({"desc": descsum_create("pkh(" + key.pubkey + ")"), + "timestamp": "now", + "internal": True}, + success=True) + info = w1.getaddressinfo(addr) + assert_equal(info["ismine"], True) + assert_equal(info["ischange"], True) + # # Test importing of a P2SH-P2WPKH descriptor key = get_generate_key() self.log.info("Should not import a p2sh-p2wpkh descriptor without checksum") @@ -209,6 +221,15 @@ class ImportDescriptorsTest(BitcoinTestFramework): success=False, error_code=-4, error_message='Cannot import private keys to a wallet with private keys disabled') + + self.log.info("Should not import a descriptor with hardened derivations when private keys are disabled") + self.test_importdesc({"desc": descsum_create("wpkh(" + xpub + "/1h/*)"), + "timestamp": "now", + "range": 1}, + success=False, + error_code=-4, + error_message='Cannot expand descriptor. Probably because of hardened derivations without private keys provided') + for address in addresses: test_address(w1, address, diff --git a/test/functional/wallet_listsinceblock.py b/test/functional/wallet_listsinceblock.py index 07c14db6b1..6a1b9097c5 100755 --- a/test/functional/wallet_listsinceblock.py +++ b/test/functional/wallet_listsinceblock.py @@ -191,6 +191,7 @@ class ListSinceBlockTest(BitcoinTestFramework): address = key_to_p2wpkh(eckey.get_pubkey().get_bytes()) self.nodes[2].sendtoaddress(address, 10) self.nodes[2].generate(6) + self.sync_all() self.nodes[2].importprivkey(privkey) utxos = self.nodes[2].listunspent() utxo = [u for u in utxos if u["address"] == address][0] diff --git a/test/sanitizer_suppressions/tsan b/test/sanitizer_suppressions/tsan index 625085c55b..c07f405944 100644 --- a/test/sanitizer_suppressions/tsan +++ b/test/sanitizer_suppressions/tsan @@ -47,3 +47,4 @@ deadlock:src/qt/test/* # External libraries deadlock:libdb race:libzmq +race:epoll_ctl # https://github.com/bitcoin/bitcoin/pull/20218 |