diff options
27 files changed, 440 insertions, 190 deletions
diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index f78bebf114..68ebb5def1 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -14,6 +14,10 @@ When complete, it will have produced `Bitcoin-Qt.dmg`. ## SDK Extraction +Our current macOS SDK (`macOSX10.14.sdk`) can be extracted from +[Xcode_10.2.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_10.2.1/Xcode_10.2.1.xip). +An Apple ID is needed to download this. + `Xcode.app` is packaged in a `.xip` archive. This makes the SDK less-trivial to extract on non-macOS machines. One approach (tested on Debian Buster) is outlined below: @@ -38,14 +42,14 @@ xar -xf Xcode_10.2.1.xip -C . ./pbzx/pbzx -n Content | cpio -i -find Xcode.app -type d -name MacOSX.sdk -execdir sh -c 'tar -c MacOSX.sdk/ | gzip -9n > /MacOSX10.14.sdk.tar.gz' \; +find Xcode.app -type d -name MacOSX.sdk -exec sh -c 'tar --transform="s/MacOSX.sdk/MacOSX10.14.sdk/" -c -C$(dirname {}) MacOSX.sdk/ | gzip -9n > MacOSX10.14.sdk.tar.gz' \; ``` on macOS the process is more straightforward: ```bash xip -x Xcode_10.2.1.xip -tar -C Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.14.sdk.tar.gz MacOSX.sdk +tar -s "/MacOSX.sdk/MacOSX10.14.sdk/" -C Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.14.sdk.tar.gz MacOSX.sdk ``` Our previously used macOS SDK (`MacOSX10.11.sdk`) can be extracted from diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 9642337821..419b1db44e 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -133,3 +133,25 @@ $ afl/afl-fuzz -i inputs/ -o outputs/ -- src/test/fuzz/bech32 ``` Read the [`afl-fuzz` documentation](https://github.com/google/afl) for more information. + +# Fuzzing Bitcoin Core using Honggfuzz + +## Quickstart guide + +To quickly get started fuzzing Bitcoin Core using [Honggfuzz](https://github.com/google/honggfuzz): + +```sh +$ git clone https://github.com/bitcoin/bitcoin +$ cd bitcoin/ +$ ./autogen.sh +$ git clone https://github.com/google/honggfuzz +$ cd honggfuzz/ +$ make +$ cd .. +$ CC=$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang CXX=$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang++ ./configure --enable-fuzz --with-sanitizers=address,undefined +$ make +$ mkdir -p inputs/ +$ honggfuzz/honggfuzz -i inputs/ -- src/test/fuzz/process_message +``` + +Read the [Honggfuzz documentation](https://github.com/google/honggfuzz/blob/master/docs/USAGE.md) for more information. diff --git a/doc/init.md b/doc/init.md index 0b327aab58..99aa0a0def 100644 --- a/doc/init.md +++ b/doc/init.md @@ -20,9 +20,9 @@ The macOS configuration assumes bitcoind will be set up for the current user. Configuration --------------------------------- -At a bare minimum, bitcoind requires that the rpcpassword setting be set -when running as a daemon. If the configuration file does not exist or this -setting is not set, bitcoind will shut down promptly after startup. +Running bitcoind as a daemon does not require any manual configuration. You may +set the `rpcauth` setting in the `bitcoin.conf` configuration file to override +the default behaviour of using a special cookie for authentication. This password does not have to be remembered or typed as it is mostly used as a fixed token that bitcoind and client programs read from the configuration diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index eae8b1fcd1..766c0fca54 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -12,6 +12,7 @@ GENERATED_BENCH_FILES = $(RAW_BENCH_FILES:.raw=.raw.h) bench_bench_bitcoin_SOURCES = \ $(RAW_BENCH_FILES) \ + bench/addrman.cpp \ bench/bench_bitcoin.cpp \ bench/bench.cpp \ bench/bench.h \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 059ffd6964..06dc59cf5c 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -69,6 +69,7 @@ FUZZ_TARGETS = \ test/fuzz/partially_signed_transaction_deserialize \ test/fuzz/pow \ test/fuzz/prefilled_transaction_deserialize \ + test/fuzz/primitives_transaction \ test/fuzz/process_messages \ test/fuzz/process_message \ test/fuzz/process_message_addr \ @@ -686,6 +687,12 @@ test_fuzz_prefilled_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_prefilled_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_prefilled_transaction_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_primitives_transaction_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_primitives_transaction_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_primitives_transaction_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_primitives_transaction_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_primitives_transaction_SOURCES = test/fuzz/primitives_transaction.cpp + test_fuzz_process_messages_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_process_messages_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_messages_LDADD = $(FUZZ_SUITE_LD_COMMON) diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp new file mode 100644 index 0000000000..cc260df2b8 --- /dev/null +++ b/src/bench/addrman.cpp @@ -0,0 +1,140 @@ +// Copyright (c) 2020-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <addrman.h> +#include <bench/bench.h> +#include <random.h> +#include <util/time.h> + +#include <vector> + +/* A "source" is a source address from which we have received a bunch of other addresses. */ + +static constexpr size_t NUM_SOURCES = 64; +static constexpr size_t NUM_ADDRESSES_PER_SOURCE = 256; + +static std::vector<CAddress> g_sources; +static std::vector<std::vector<CAddress>> g_addresses; + +static void CreateAddresses() +{ + if (g_sources.size() > 0) { // already created + return; + } + + FastRandomContext rng(uint256(std::vector<unsigned char>(32, 123))); + + auto randAddr = [&rng]() { + in6_addr addr; + memcpy(&addr, rng.randbytes(sizeof(addr)).data(), sizeof(addr)); + + uint16_t port; + memcpy(&port, rng.randbytes(sizeof(port)).data(), sizeof(port)); + if (port == 0) { + port = 1; + } + + CAddress ret(CService(addr, port), NODE_NETWORK); + + ret.nTime = GetAdjustedTime(); + + return ret; + }; + + for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { + g_sources.emplace_back(randAddr()); + g_addresses.emplace_back(); + for (size_t addr_i = 0; addr_i < NUM_ADDRESSES_PER_SOURCE; ++addr_i) { + g_addresses[source_i].emplace_back(randAddr()); + } + } +} + +static void AddAddressesToAddrMan(CAddrMan& addrman) +{ + for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { + addrman.Add(g_addresses[source_i], g_sources[source_i]); + } +} + +static void FillAddrMan(CAddrMan& addrman) +{ + CreateAddresses(); + + AddAddressesToAddrMan(addrman); +} + +/* Benchmarks */ + +static void AddrManAdd(benchmark::State& state) +{ + CreateAddresses(); + + CAddrMan addrman; + + while (state.KeepRunning()) { + AddAddressesToAddrMan(addrman); + addrman.Clear(); + } +} + +static void AddrManSelect(benchmark::State& state) +{ + CAddrMan addrman; + + FillAddrMan(addrman); + + while (state.KeepRunning()) { + const auto& address = addrman.Select(); + assert(address.GetPort() > 0); + } +} + +static void AddrManGetAddr(benchmark::State& state) +{ + CAddrMan addrman; + + FillAddrMan(addrman); + + while (state.KeepRunning()) { + const auto& addresses = addrman.GetAddr(); + assert(addresses.size() > 0); + } +} + +static void AddrManGood(benchmark::State& state) +{ + /* Create many CAddrMan objects - one to be modified at each loop iteration. + * This is necessary because the CAddrMan::Good() method modifies the + * object, affecting the timing of subsequent calls to the same method and + * we want to do the same amount of work in every loop iteration. */ + + const uint64_t numLoops = state.m_num_iters * state.m_num_evals; + + std::vector<CAddrMan> addrmans(numLoops); + for (auto& addrman : addrmans) { + FillAddrMan(addrman); + } + + auto markSomeAsGood = [](CAddrMan& addrman) { + for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { + for (size_t addr_i = 0; addr_i < NUM_ADDRESSES_PER_SOURCE; ++addr_i) { + if (addr_i % 32 == 0) { + addrman.Good(g_addresses[source_i][addr_i]); + } + } + } + }; + + uint64_t i = 0; + while (state.KeepRunning()) { + markSomeAsGood(addrmans.at(i)); + ++i; + } +} + +BENCHMARK(AddrManAdd, 5); +BENCHMARK(AddrManSelect, 1000000); +BENCHMARK(AddrManGetAddr, 500); +BENCHMARK(AddrManGood, 2); @@ -45,16 +45,10 @@ static const bool DEFAULT_WHITELISTRELAY = true; /** Default for -whitelistforcerelay. */ static const bool DEFAULT_WHITELISTFORCERELAY = false; -/** Time between pings automatically sent out for latency probing and keepalive (in seconds). */ -static const int PING_INTERVAL = 2 * 60; /** Time after which to disconnect, after waiting for a ping response (or inactivity). */ static const int TIMEOUT_INTERVAL = 20 * 60; /** Run the feeler connection loop once every 2 minutes or 120 seconds. **/ static const int FEELER_INTERVAL = 120; -/** The maximum number of entries in an 'inv' protocol message */ -static const unsigned int MAX_INV_SZ = 50000; -/** The maximum number of entries in a locator */ -static const unsigned int MAX_LOCATOR_SZ = 101; /** The maximum number of new addresses to accumulate before announcing. */ static const unsigned int MAX_ADDR_TO_SEND = 1000; /** Maximum length of incoming protocol messages (no message over 4 MB is currently acceptable). */ diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 96e9178b6e..26327ac6eb 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -64,6 +64,12 @@ static constexpr int STALE_RELAY_AGE_LIMIT = 30 * 24 * 60 * 60; /// Age after which a block is considered historical for purposes of rate /// limiting block relay. Set to one week, denominated in seconds. static constexpr int HISTORICAL_BLOCK_AGE = 7 * 24 * 60 * 60; +/** Time between pings automatically sent out for latency probing and keepalive (in seconds). */ +static const int PING_INTERVAL = 2 * 60; +/** The maximum number of entries in a locator */ +static const unsigned int MAX_LOCATOR_SZ = 101; +/** The maximum number of entries in an 'inv' protocol message */ +static const unsigned int MAX_INV_SZ = 50000; /** Maximum number of in-flight transactions from a peer */ static constexpr int32_t MAX_PEER_TX_IN_FLIGHT = 100; /** Maximum number of announced transactions from a peer */ @@ -80,7 +86,47 @@ static_assert(INBOUND_PEER_TX_DELAY >= MAX_GETDATA_RANDOM_DELAY, "To preserve security, MAX_GETDATA_RANDOM_DELAY should not exceed INBOUND_PEER_DELAY"); /** Limit to avoid sending big packets. Not used in processing incoming GETDATA for compatibility */ static const unsigned int MAX_GETDATA_SZ = 1000; - +/** Number of blocks that can be requested at any given time from a single peer. */ +static const int MAX_BLOCKS_IN_TRANSIT_PER_PEER = 16; +/** Timeout in seconds during which a peer must stall block download progress before being disconnected. */ +static const unsigned int BLOCK_STALLING_TIMEOUT = 2; +/** Number of headers sent in one getheaders result. We rely on the assumption that if a peer sends + * less than this number, we reached its tip. Changing this value is a protocol upgrade. */ +static const unsigned int MAX_HEADERS_RESULTS = 2000; +/** Maximum depth of blocks we're willing to serve as compact blocks to peers + * when requested. For older blocks, a regular BLOCK response will be sent. */ +static const int MAX_CMPCTBLOCK_DEPTH = 5; +/** Maximum depth of blocks we're willing to respond to GETBLOCKTXN requests for. */ +static const int MAX_BLOCKTXN_DEPTH = 10; +/** Size of the "block download window": how far ahead of our current height do we fetch? + * Larger windows tolerate larger download speed differences between peer, but increase the potential + * degree of disordering of blocks on disk (which make reindexing and pruning harder). We'll probably + * want to make this a per-peer adaptive value at some point. */ +static const unsigned int BLOCK_DOWNLOAD_WINDOW = 1024; +/** Block download timeout base, expressed in millionths of the block interval (i.e. 10 min) */ +static const int64_t BLOCK_DOWNLOAD_TIMEOUT_BASE = 1000000; +/** Additional block download timeout per parallel downloading peer (i.e. 5 min) */ +static const int64_t BLOCK_DOWNLOAD_TIMEOUT_PER_PEER = 500000; +/** Maximum number of headers to announce when relaying blocks with headers message.*/ +static const unsigned int MAX_BLOCKS_TO_ANNOUNCE = 8; +/** Maximum number of unconnecting headers announcements before DoS score */ +static const int MAX_UNCONNECTING_HEADERS = 10; +/** Minimum blocks required to signal NODE_NETWORK_LIMITED */ +static const unsigned int NODE_NETWORK_LIMITED_MIN_BLOCKS = 288; +/** Average delay between local address broadcasts */ +static constexpr std::chrono::hours AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL{24}; +/** Average delay between peer address broadcasts */ +static constexpr std::chrono::seconds AVG_ADDRESS_BROADCAST_INTERVAL{30}; +/** Average delay between trickled inventory transmissions in seconds. + * Blocks and whitelisted receivers bypass this, outbound peers get half this delay. */ +static const unsigned int INVENTORY_BROADCAST_INTERVAL = 5; +/** Maximum number of inventory items to send per transmission. + * Limits the impact of low-fee transaction floods. */ +static constexpr unsigned int INVENTORY_BROADCAST_MAX = 7 * INVENTORY_BROADCAST_INTERVAL; +/** Average delay between feefilter broadcasts in seconds. */ +static constexpr unsigned int AVG_FEEFILTER_BROADCAST_INTERVAL = 10 * 60; +/** Maximum feefilter broadcast delay after significant change. */ +static constexpr unsigned int MAX_FEEFILTER_CHANGE_DELAY = 5 * 60; struct COrphanTx { // When modifying, adapt the copy of this definition in tests/DoS_tests. @@ -97,21 +143,6 @@ void EraseOrphansFor(NodeId peer); /** Increase a node's misbehavior score. */ void Misbehaving(NodeId nodeid, int howmuch, const std::string& message="") EXCLUSIVE_LOCKS_REQUIRED(cs_main); -/** Average delay between local address broadcasts */ -static constexpr std::chrono::hours AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL{24}; -/** Average delay between peer address broadcasts */ -static constexpr std::chrono::seconds AVG_ADDRESS_BROADCAST_INTERVAL{30}; -/** Average delay between trickled inventory transmissions in seconds. - * Blocks and whitelisted receivers bypass this, outbound peers get half this delay. */ -static const unsigned int INVENTORY_BROADCAST_INTERVAL = 5; -/** Maximum number of inventory items to send per transmission. - * Limits the impact of low-fee transaction floods. */ -static constexpr unsigned int INVENTORY_BROADCAST_MAX = 7 * INVENTORY_BROADCAST_INTERVAL; -/** Average delay between feefilter broadcasts in seconds. */ -static constexpr unsigned int AVG_FEEFILTER_BROADCAST_INTERVAL = 10 * 60; -/** Maximum feefilter broadcast delay after significant change. */ -static constexpr unsigned int MAX_FEEFILTER_CHANGE_DELAY = 5 * 60; - // Internal stuff namespace { /** Number of nodes with fSyncStarted. */ diff --git a/src/test/fuzz/block.cpp b/src/test/fuzz/block.cpp index d1e92ce607..f30fa03e0b 100644 --- a/src/test/fuzz/block.cpp +++ b/src/test/fuzz/block.cpp @@ -62,4 +62,8 @@ void test_one_input(const std::vector<uint8_t>& buffer) const size_t raw_memory_size = RecursiveDynamicUsage(block); const size_t raw_memory_size_as_shared_ptr = RecursiveDynamicUsage(std::make_shared<CBlock>(block)); assert(raw_memory_size_as_shared_ptr > raw_memory_size); + CBlock block_copy = block; + block_copy.SetNull(); + const bool is_null = block_copy.IsNull(); + assert(is_null); } diff --git a/src/test/fuzz/primitives_transaction.cpp b/src/test/fuzz/primitives_transaction.cpp new file mode 100644 index 0000000000..2e5ba6bdb0 --- /dev/null +++ b/src/test/fuzz/primitives_transaction.cpp @@ -0,0 +1,34 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <optional.h> +#include <primitives/transaction.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const CScript script = ConsumeScript(fuzzed_data_provider); + const Optional<COutPoint> out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); + if (out_point) { + const CTxIn tx_in{*out_point, script, fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; + (void)tx_in; + } + const CTxOut tx_out_1{ConsumeMoney(fuzzed_data_provider), script}; + const CTxOut tx_out_2{ConsumeMoney(fuzzed_data_provider), ConsumeScript(fuzzed_data_provider)}; + assert((tx_out_1 == tx_out_2) != (tx_out_1 != tx_out_2)); + const Optional<CMutableTransaction> mutable_tx_1 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const Optional<CMutableTransaction> mutable_tx_2 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (mutable_tx_1 && mutable_tx_2) { + const CTransaction tx_1{*mutable_tx_1}; + const CTransaction tx_2{*mutable_tx_2}; + assert((tx_1 == tx_2) != (tx_1 != tx_2)); + } +} diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 0ed3721636..c03365199a 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -19,16 +19,13 @@ #include <validationinterface.h> #include <version.h> -#include <algorithm> #include <atomic> #include <cassert> #include <chrono> #include <cstdint> #include <iosfwd> #include <iostream> -#include <map> #include <memory> -#include <set> #include <string> #include <vector> @@ -44,19 +41,6 @@ const std::string LIMIT_TO_MESSAGE_TYPE{TO_STRING(MESSAGE_TYPE)}; const std::string LIMIT_TO_MESSAGE_TYPE; #endif -const std::map<std::string, std::set<std::string>> EXPECTED_DESERIALIZATION_EXCEPTIONS = { - {"CDataStream::read(): end of data: iostream error", {"addr", "block", "blocktxn", "cmpctblock", "feefilter", "filteradd", "filterload", "getblocks", "getblocktxn", "getdata", "getheaders", "headers", "inv", "notfound", "ping", "sendcmpct", "tx"}}, - {"CompactSize exceeds limit of type: iostream error", {"cmpctblock"}}, - {"differential value overflow: iostream error", {"getblocktxn"}}, - {"index overflowed 16 bits: iostream error", {"getblocktxn"}}, - {"index overflowed 16-bits: iostream error", {"cmpctblock"}}, - {"indexes overflowed 16 bits: iostream error", {"getblocktxn"}}, - {"non-canonical ReadCompactSize(): iostream error", {"addr", "block", "blocktxn", "cmpctblock", "filteradd", "filterload", "getblocks", "getblocktxn", "getdata", "getheaders", "headers", "inv", "notfound", "tx"}}, - {"ReadCompactSize(): size too large: iostream error", {"addr", "block", "blocktxn", "cmpctblock", "filteradd", "filterload", "getblocks", "getblocktxn", "getdata", "getheaders", "headers", "inv", "notfound", "tx"}}, - {"Superfluous witness record: iostream error", {"block", "blocktxn", "cmpctblock", "tx"}}, - {"Unknown transaction optional data: iostream error", {"block", "blocktxn", "cmpctblock", "tx"}}, -}; - const TestingSetup* g_setup; } // namespace @@ -91,13 +75,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) g_setup->m_node.peer_logic->InitializeNode(&p2p_node); try { (void)ProcessMessage(&p2p_node, random_message_type, random_bytes_data_stream, GetTimeMillis(), Params(), *g_setup->m_node.mempool, g_setup->m_node.connman.get(), g_setup->m_node.banman.get(), std::atomic<bool>{false}); - } catch (const std::ios_base::failure& e) { - const std::string exception_message{e.what()}; - const auto p = EXPECTED_DESERIALIZATION_EXCEPTIONS.find(exception_message); - if (p == EXPECTED_DESERIALIZATION_EXCEPTIONS.cend() || p->second.count(random_message_type) == 0) { - std::cout << "Unexpected exception when processing message type \"" << random_message_type << "\": " << exception_message << std::endl; - assert(false); - } + } catch (const std::ios_base::failure&) { } SyncWithValidationInterfaceQueue(); } diff --git a/src/validation.cpp b/src/validation.cpp index fb635b4202..25975e3e31 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -59,6 +59,25 @@ #define MICRO 0.000001 #define MILLI 0.001 +/** + * An extra transaction can be added to a package, as long as it only has one + * ancestor and is no larger than this. Not really any reason to make this + * configurable as it doesn't materially change DoS parameters. + */ +static const unsigned int EXTRA_DESCENDANT_TX_SIZE_LIMIT = 10000; +/** Maximum kilobytes for transactions to store for processing during reorg */ +static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; +/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ +static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB +/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ +static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB +/** Time to wait (in seconds) between writing blocks/block index to disk. */ +static const unsigned int DATABASE_WRITE_INTERVAL = 60 * 60; +/** Time to wait (in seconds) between flushing chainstate to disk. */ +static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; +/** Maximum age of our tip in seconds for us to be considered current for fee estimation */ +static const int64_t MAX_FEE_ESTIMATION_TIP_AGE = 3 * 60 * 60; + bool CBlockIndexWorkComparator::operator()(const CBlockIndex *pa, const CBlockIndex *pb) const { // First sort by most total work, ... if (pa->nChainWork > pb->nChainWork) return false; diff --git a/src/validation.h b/src/validation.h index dbf7aa28db..91b1ba6497 100644 --- a/src/validation.h +++ b/src/validation.h @@ -60,57 +60,15 @@ static const unsigned int DEFAULT_ANCESTOR_SIZE_LIMIT = 101; static const unsigned int DEFAULT_DESCENDANT_LIMIT = 25; /** Default for -limitdescendantsize, maximum kilobytes of in-mempool descendants */ static const unsigned int DEFAULT_DESCENDANT_SIZE_LIMIT = 101; -/** - * An extra transaction can be added to a package, as long as it only has one - * ancestor and is no larger than this. Not really any reason to make this - * configurable as it doesn't materially change DoS parameters. - */ -static const unsigned int EXTRA_DESCENDANT_TX_SIZE_LIMIT = 10000; /** Default for -mempoolexpiry, expiration time for mempool transactions in hours */ static const unsigned int DEFAULT_MEMPOOL_EXPIRY = 336; -/** Maximum kilobytes for transactions to store for processing during reorg */ -static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; /** The maximum size of a blk?????.dat file (since 0.8) */ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB -/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ -static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB -/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ -static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB - /** Maximum number of dedicated script-checking threads allowed */ static const int MAX_SCRIPTCHECK_THREADS = 15; /** -par default (number of script-checking threads, 0 = auto) */ static const int DEFAULT_SCRIPTCHECK_THREADS = 0; -/** Number of blocks that can be requested at any given time from a single peer. */ -static const int MAX_BLOCKS_IN_TRANSIT_PER_PEER = 16; -/** Timeout in seconds during which a peer must stall block download progress before being disconnected. */ -static const unsigned int BLOCK_STALLING_TIMEOUT = 2; -/** Number of headers sent in one getheaders result. We rely on the assumption that if a peer sends - * less than this number, we reached its tip. Changing this value is a protocol upgrade. */ -static const unsigned int MAX_HEADERS_RESULTS = 2000; -/** Maximum depth of blocks we're willing to serve as compact blocks to peers - * when requested. For older blocks, a regular BLOCK response will be sent. */ -static const int MAX_CMPCTBLOCK_DEPTH = 5; -/** Maximum depth of blocks we're willing to respond to GETBLOCKTXN requests for. */ -static const int MAX_BLOCKTXN_DEPTH = 10; -/** Size of the "block download window": how far ahead of our current height do we fetch? - * Larger windows tolerate larger download speed differences between peer, but increase the potential - * degree of disordering of blocks on disk (which make reindexing and pruning harder). We'll probably - * want to make this a per-peer adaptive value at some point. */ -static const unsigned int BLOCK_DOWNLOAD_WINDOW = 1024; -/** Time to wait (in seconds) between writing blocks/block index to disk. */ -static const unsigned int DATABASE_WRITE_INTERVAL = 60 * 60; -/** Time to wait (in seconds) between flushing chainstate to disk. */ -static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; -/** Block download timeout base, expressed in millionths of the block interval (i.e. 10 min) */ -static const int64_t BLOCK_DOWNLOAD_TIMEOUT_BASE = 1000000; -/** Additional block download timeout per parallel downloading peer (i.e. 5 min) */ -static const int64_t BLOCK_DOWNLOAD_TIMEOUT_PER_PEER = 500000; - static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60; -/** Maximum age of our tip in seconds for us to be considered current for fee estimation */ -static const int64_t MAX_FEE_ESTIMATION_TIP_AGE = 3 * 60 * 60; - static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; @@ -119,15 +77,21 @@ static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; static const bool DEFAULT_PERSIST_MEMPOOL = true; /** Default for using fee filter */ static const bool DEFAULT_FEEFILTER = true; - -/** Maximum number of headers to announce when relaying blocks with headers message.*/ -static const unsigned int MAX_BLOCKS_TO_ANNOUNCE = 8; - -/** Maximum number of unconnecting headers announcements before DoS score */ -static const int MAX_UNCONNECTING_HEADERS = 10; - /** Default for -stopatheight */ static const int DEFAULT_STOPATHEIGHT = 0; +/** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ::ChainActive().Tip() will not be pruned. */ +static const unsigned int MIN_BLOCKS_TO_KEEP = 288; +static const signed int DEFAULT_CHECKBLOCKS = 6; +static const unsigned int DEFAULT_CHECKLEVEL = 3; +// Require that user allocate at least 550 MiB for block & undo files (blk???.dat and rev???.dat) +// At 1MB per block, 288 blocks = 288MB. +// Add 15% for Undo data = 331MB +// Add 20% for Orphan block rate = 397MB +// We want the low water mark after pruning to be at least 397 MB and since we prune in +// full block file chunks, we need the high water mark which triggers the prune to be +// one 128MB block file + added 15% undo data = 147MB greater for a total of 545MB +// Setting the target to >= 550 MiB will make it likely we can respect the target. +static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024; struct BlockHasher { @@ -175,23 +139,6 @@ extern bool fHavePruned; extern bool fPruneMode; /** Number of MiB of block files that we're trying to stay below. */ extern uint64_t nPruneTarget; -/** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ::ChainActive().Tip() will not be pruned. */ -static const unsigned int MIN_BLOCKS_TO_KEEP = 288; -/** Minimum blocks required to signal NODE_NETWORK_LIMITED */ -static const unsigned int NODE_NETWORK_LIMITED_MIN_BLOCKS = 288; - -static const signed int DEFAULT_CHECKBLOCKS = 6; -static const unsigned int DEFAULT_CHECKLEVEL = 3; - -// Require that user allocate at least 550 MiB for block & undo files (blk???.dat and rev???.dat) -// At 1MB per block, 288 blocks = 288MB. -// Add 15% for Undo data = 331MB -// Add 20% for Orphan block rate = 397MB -// We want the low water mark after pruning to be at least 397 MB and since we prune in -// full block file chunks, we need the high water mark which triggers the prune to be -// one 128MB block file + added 15% undo data = 147MB greater for a total of 545MB -// Setting the target to >= 550 MiB will make it likely we can respect the target. -static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024; /** * Process an incoming block. This only returns after the best known valid diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index fe45ed34b5..549e8b2029 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -324,7 +324,7 @@ class BIP68Test(BitcoinTestFramework): block.solve() tip = block.sha256 height += 1 - self.nodes[0].submitblock(ToHex(block)) + assert_equal(None if i == 1 else 'inconclusive', self.nodes[0].submitblock(ToHex(block))) cur_time += 1 mempool = self.nodes[0].getrawmempool() @@ -381,7 +381,7 @@ class BIP68Test(BitcoinTestFramework): add_witness_commitment(block) block.solve() - self.nodes[0].submitblock(block.serialize().hex()) + assert_equal(None, self.nodes[0].submitblock(block.serialize().hex())) assert_equal(self.nodes[0].getbestblockhash(), block.hash) def activateCSV(self): diff --git a/test/functional/feature_nulldummy.py b/test/functional/feature_nulldummy.py index 12471c5088..ff55cb76d9 100755 --- a/test/functional/feature_nulldummy.py +++ b/test/functional/feature_nulldummy.py @@ -111,7 +111,7 @@ class NULLDUMMYTest(BitcoinTestFramework): witness and add_witness_commitment(block) block.rehash() block.solve() - node.submitblock(block.serialize().hex()) + assert_equal(None if accept else 'block-validation-failed', node.submitblock(block.serialize().hex())) if (accept): assert_equal(node.getbestblockhash(), block.hash) self.tip = block.sha256 diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 2e80d7a248..1c94305220 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -3,6 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoin-cli""" +from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_process_error, get_auth_cookie @@ -51,7 +52,7 @@ class TestBitcoinCli(BitcoinTestFramework): self.log.info("Test -getinfo returns expected network and blockchain info") if self.is_wallet_compiled(): self.nodes[0].encryptwallet(password) - cli_get_info = self.nodes[0].cli().send_cli('-getinfo') + cli_get_info = self.nodes[0].cli('-getinfo').send_cli() network_info = self.nodes[0].getnetworkinfo() blockchain_info = self.nodes[0].getblockchaininfo() assert_equal(cli_get_info['version'], network_info['version']) @@ -72,12 +73,52 @@ class TestBitcoinCli(BitcoinTestFramework): assert_equal(cli_get_info['paytxfee'], wallet_info['paytxfee']) assert_equal(cli_get_info['relayfee'], network_info['relayfee']) assert_equal(self.nodes[0].cli.getwalletinfo(), wallet_info) + + # Setup to test -getinfo and -rpcwallet= with multiple wallets. + wallets = ['', 'Encrypted', 'secret'] + amounts = [Decimal('59.999928'), Decimal(9), Decimal(31)] + self.nodes[0].createwallet(wallet_name=wallets[1]) + self.nodes[0].createwallet(wallet_name=wallets[2]) + w1 = self.nodes[0].get_wallet_rpc(wallets[0]) + w2 = self.nodes[0].get_wallet_rpc(wallets[1]) + w3 = self.nodes[0].get_wallet_rpc(wallets[2]) + w1.walletpassphrase(password, self.rpc_timeout) + w1.sendtoaddress(w2.getnewaddress(), amounts[1]) + w1.sendtoaddress(w3.getnewaddress(), amounts[2]) + + # Mine a block to confirm; adds a block reward (50 BTC) to the default wallet. + self.nodes[0].generate(1) + + self.log.info("Test -getinfo with multiple wallets loaded returns no balance") + assert_equal(set(self.nodes[0].listwallets()), set(wallets)) + assert 'balance' not in self.nodes[0].cli('-getinfo').send_cli().keys() + + self.log.info("Test -getinfo with multiple wallets and -rpcwallet returns specified wallet balance") + for i in range(len(wallets)): + cli_get_info = self.nodes[0].cli('-getinfo').send_cli('-rpcwallet={}'.format(wallets[i])) + assert_equal(cli_get_info['balance'], amounts[i]) + + self.log.info("Test -getinfo with multiple wallets and -rpcwallet=non-existing-wallet returns no balance") + assert 'balance' not in self.nodes[0].cli('-getinfo').send_cli('-rpcwallet=does-not-exist').keys() + + self.log.info("Test -getinfo after unloading all wallets except a non-default one returns its balance") + self.nodes[0].unloadwallet(wallets[0]) + self.nodes[0].unloadwallet(wallets[2]) + assert_equal(self.nodes[0].listwallets(), [wallets[1]]) + assert_equal(self.nodes[0].cli('-getinfo').send_cli()['balance'], amounts[1]) + + self.log.info("Test -getinfo -rpcwallet=remaining-non-default-wallet returns its balance") + assert_equal(self.nodes[0].cli('-getinfo').send_cli('-rpcwallet={}'.format(wallets[1]))['balance'], amounts[1]) + + self.log.info("Test -getinfo with -rpcwallet=unloaded wallet returns no balance") + assert 'balance' not in self.nodes[0].cli('-getinfo').send_cli('-rpcwallet={}'.format(wallets[2])).keys() else: self.log.info("*** Wallet not compiled; cli getwalletinfo and -getinfo wallet tests skipped") + self.nodes[0].generate(1) # maintain block parity with the wallet_compiled conditional branch self.log.info("Test -version with node stopped") self.stop_node(0) - cli_response = self.nodes[0].cli().send_cli('-version') + cli_response = self.nodes[0].cli('-version').send_cli() assert "{} RPC client version".format(self.config['environment']['PACKAGE_NAME']) in cli_response self.log.info("Test -rpcwait option successfully waits for RPC connection") @@ -85,7 +126,7 @@ class TestBitcoinCli(BitcoinTestFramework): self.nodes[0].wait_for_cookie_credentials() # ensure cookie file is available to avoid race condition blocks = self.nodes[0].cli('-rpcwait').send_cli('getblockcount') self.nodes[0].wait_for_rpc_connection() - assert_equal(blocks, BLOCKS) + assert_equal(blocks, BLOCKS + 1) if __name__ == '__main__': diff --git a/test/functional/mempool_reorg.py b/test/functional/mempool_reorg.py index 2d23343fd5..8e1f87e42c 100755 --- a/test/functional/mempool_reorg.py +++ b/test/functional/mempool_reorg.py @@ -16,6 +16,12 @@ from test_framework.util import assert_equal, assert_raises_rpc_error class MempoolCoinbaseTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + self.extra_args = [ + [ + '-whitelist=noban@127.0.0.1', # immediate tx relay + ], + [] + ] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -38,17 +44,21 @@ class MempoolCoinbaseTest(BitcoinTestFramework): # 3. Indirect (coinbase and child both in chain) : spend_103 and spend_103_1 # Use invalidatblock to make all of the above coinbase spends invalid (immature coinbase), # and make sure the mempool code behaves correctly. - b = [ self.nodes[0].getblockhash(n) for n in range(101, 105) ] - coinbase_txids = [ self.nodes[0].getblock(h)['tx'][0] for h in b ] + b = [self.nodes[0].getblockhash(n) for n in range(101, 105)] + coinbase_txids = [self.nodes[0].getblock(h)['tx'][0] for h in b] spend_101_raw = create_raw_transaction(self.nodes[0], coinbase_txids[1], node1_address, amount=49.99) spend_102_raw = create_raw_transaction(self.nodes[0], coinbase_txids[2], node0_address, amount=49.99) spend_103_raw = create_raw_transaction(self.nodes[0], coinbase_txids[3], node0_address, amount=49.99) # Create a transaction which is time-locked to two blocks in the future - timelock_tx = self.nodes[0].createrawtransaction([{"txid": coinbase_txids[0], "vout": 0}], {node0_address: 49.99}) - # Set the time lock - timelock_tx = timelock_tx.replace("ffffffff", "11111191", 1) - timelock_tx = timelock_tx[:-8] + hex(self.nodes[0].getblockcount() + 2)[2:] + "000000" + timelock_tx = self.nodes[0].createrawtransaction( + inputs=[{ + "txid": coinbase_txids[0], + "vout": 0, + }], + outputs={node0_address: 49.99}, + locktime=self.nodes[0].getblockcount() + 2, + ) timelock_tx = self.nodes[0].signrawtransactionwithwallet(timelock_tx)["hex"] # This will raise an exception because the timelock transaction is too immature to spend assert_raises_rpc_error(-26, "non-final", self.nodes[0].sendrawtransaction, timelock_tx) @@ -67,6 +77,10 @@ class MempoolCoinbaseTest(BitcoinTestFramework): # Broadcast and mine 103_1: spend_103_1_id = self.nodes[0].sendrawtransaction(spend_103_1_raw) last_block = self.nodes[0].generate(1) + # Sync blocks, so that peer 1 gets the block before timelock_tx + # Otherwise, peer 1 would put the timelock_tx in recentRejects + self.sync_all() + # Time-locked transaction can now be spent timelock_tx_id = self.nodes[0].sendrawtransaction(timelock_tx) diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index 7905cf5018..66e6f8c424 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -378,8 +378,6 @@ class CompactBlocksTest(BitcoinTestFramework): # request for announce in ["inv", "header"]: block = self.build_block_on_tip(node, segwit=segwit) - with mininode_lock: - test_node.last_message.pop("getdata", None) if announce == "inv": test_node.send_message(msg_inv([CInv(2, block.sha256)])) @@ -387,10 +385,8 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_header_for_blocks([block]) else: test_node.send_header_for_blocks([block]) - wait_until(lambda: "getdata" in test_node.last_message, timeout=30, lock=mininode_lock) - assert_equal(len(test_node.last_message["getdata"].inv), 1) + test_node.wait_for_getdata([block.sha256], timeout=30) assert_equal(test_node.last_message["getdata"].inv[0].type, 4) - assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) # Send back a compactblock message that omits the coinbase comp_block = HeaderAndShortIDs() @@ -567,10 +563,8 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) # We should receive a getdata request - wait_until(lambda: "getdata" in test_node.last_message, timeout=10, lock=mininode_lock) - assert_equal(len(test_node.last_message["getdata"].inv), 1) + test_node.wait_for_getdata([block.sha256], timeout=10) assert test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2 | MSG_WITNESS_FLAG - assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) # Deliver the block if version == 2: diff --git a/test/functional/p2p_fingerprint.py b/test/functional/p2p_fingerprint.py index fab0887197..c9fbb830c8 100755 --- a/test/functional/p2p_fingerprint.py +++ b/test/functional/p2p_fingerprint.py @@ -90,7 +90,7 @@ class P2PFingerprintTest(BitcoinTestFramework): # Force reorg to a longer chain node0.send_message(msg_headers(new_blocks)) - node0.wait_for_getdata() + node0.wait_for_getdata([x.sha256 for x in new_blocks]) for block in new_blocks: node0.send_and_ping(msg_block(block)) diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index 09089371d3..dbdce6552a 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -159,7 +159,7 @@ class TestP2PConn(P2PInterface): self.last_message.pop("getdata", None) self.send_message(msg_inv(inv=[CInv(1, tx.sha256)])) if success: - self.wait_for_getdata(timeout) + self.wait_for_getdata([tx.sha256], timeout) else: time.sleep(timeout) assert not self.last_message.get("getdata") @@ -176,7 +176,7 @@ class TestP2PConn(P2PInterface): self.send_message(msg_inv(inv=[CInv(2, block.sha256)])) self.wait_for_getheaders() self.send_message(msg) - self.wait_for_getdata() + self.wait_for_getdata([block.sha256]) def request_block(self, blockhash, inv_type, timeout=60): with mininode_lock: @@ -862,13 +862,13 @@ class SegWitTest(BitcoinTestFramework): # We can't send over the p2p network, because this is too big to relay # TODO: repeat this test with a block that can be relayed - self.nodes[0].submitblock(block.serialize().hex()) + assert_equal('bad-witness-nonce-size', self.nodes[0].submitblock(block.serialize().hex())) assert self.nodes[0].getbestblockhash() != block.hash block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.pop() assert get_virtual_size(block) < MAX_BLOCK_BASE_SIZE - self.nodes[0].submitblock(block.serialize().hex()) + assert_equal(None, self.nodes[0].submitblock(block.serialize().hex())) assert self.nodes[0].getbestblockhash() == block.hash @@ -975,14 +975,14 @@ class SegWitTest(BitcoinTestFramework): add_witness_commitment(block, nonce=1) block.vtx[0].wit = CTxWitness() # drop the nonce block.solve() - self.nodes[0].submitblock(block.serialize().hex()) + assert_equal('bad-witness-merkle-match', self.nodes[0].submitblock(block.serialize().hex())) assert self.nodes[0].getbestblockhash() != block.hash # Now redo commitment with the standard nonce, but let bitcoind fill it in. add_witness_commitment(block, nonce=0) block.vtx[0].wit = CTxWitness() block.solve() - self.nodes[0].submitblock(block.serialize().hex()) + assert_equal(None, self.nodes[0].submitblock(block.serialize().hex())) assert_equal(self.nodes[0].getbestblockhash(), block.hash) # This time, add a tx with non-empty witness, but don't supply @@ -997,7 +997,7 @@ class SegWitTest(BitcoinTestFramework): block_2.vtx[0].vout.pop() block_2.vtx[0].wit = CTxWitness() - self.nodes[0].submitblock(block_2.serialize().hex()) + assert_equal('bad-txnmrklroot', self.nodes[0].submitblock(block_2.serialize().hex())) # Tip should not advance! assert self.nodes[0].getbestblockhash() != block_2.hash diff --git a/test/functional/p2p_sendheaders.py b/test/functional/p2p_sendheaders.py index 74d5536f5f..a8fba306a7 100755 --- a/test/functional/p2p_sendheaders.py +++ b/test/functional/p2p_sendheaders.py @@ -144,13 +144,6 @@ class BaseNode(P2PInterface): getblocks_message.locator.vHave = locator self.send_message(getblocks_message) - def wait_for_getdata(self, hash_list, timeout=60): - if hash_list == []: - return - - test_function = lambda: "getdata" in self.last_message and [x.hash for x in self.last_message["getdata"].inv] == hash_list - wait_until(test_function, timeout=timeout, lock=mininode_lock) - def wait_for_block_announcement(self, block_hash, timeout=60): test_function = lambda: self.last_blockhash_announced == block_hash wait_until(test_function, timeout=timeout, lock=mininode_lock) diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 33fba1c69a..4855f62a8f 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -603,16 +603,16 @@ class CBlock(CBlockHeader): __slots__ = ("vtx",) def __init__(self, header=None): - super(CBlock, self).__init__(header) + super().__init__(header) self.vtx = [] def deserialize(self, f): - super(CBlock, self).deserialize(f) + super().deserialize(f) self.vtx = deser_vector(f, CTransaction) def serialize(self, with_witness=True): r = b"" - r += super(CBlock, self).serialize() + r += super().serialize() if with_witness: r += ser_vector(self.vtx, "serialize_with_witness") else: @@ -752,7 +752,7 @@ class P2PHeaderAndShortIDs: class P2PHeaderAndShortWitnessIDs(P2PHeaderAndShortIDs): __slots__ = () def serialize(self): - return super(P2PHeaderAndShortWitnessIDs, self).serialize(with_witness=True) + return super().serialize(with_witness=True) # Calculate the BIP 152-compact blocks shortid for a given transaction hash def calculate_shortid(k0, k1, tx_hash): diff --git a/test/functional/test_framework/mininode.py b/test/functional/test_framework/mininode.py index ea078fd81c..6aa73623e6 100755 --- a/test/functional/test_framework/mininode.py +++ b/test/functional/test_framework/mininode.py @@ -406,17 +406,17 @@ class P2PInterface(P2PConnection): wait_until(test_function, timeout=timeout, lock=mininode_lock) - def wait_for_getdata(self, timeout=60): + def wait_for_getdata(self, hash_list, timeout=60): """Waits for a getdata message. - Receiving any getdata message will satisfy the predicate. the last_message["getdata"] - value must be explicitly cleared before calling this method, or this will return - immediately with success. TODO: change this method to take a hash value and only - return true if the correct block/tx has been requested.""" + The object hashes in the inventory vector must match the provided hash_list.""" def test_function(): assert self.is_connected - return self.last_message.get("getdata") + last_data = self.last_message.get("getdata") + if not last_data: + return False + return [x.hash for x in last_data.inv] == hash_list wait_until(test_function, timeout=timeout, lock=mininode_lock) diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 016a2b4f0f..e475ed8596 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -97,7 +97,7 @@ class CScriptOp(int): return _opcode_instances[n] except IndexError: assert len(_opcode_instances) == n - _opcode_instances.append(super(CScriptOp, cls).__new__(cls, n)) + _opcode_instances.append(super().__new__(cls, n)) return _opcode_instances[n] # Populate opcode instance table @@ -372,7 +372,7 @@ class CScriptTruncatedPushDataError(CScriptInvalidError): """Invalid pushdata due to truncation""" def __init__(self, msg, data): self.data = data - super(CScriptTruncatedPushDataError, self).__init__(msg) + super().__init__(msg) # This is used, eg, for blockchain heights in coinbase scripts (bip34) @@ -458,14 +458,14 @@ class CScript(bytes): def __new__(cls, value=b''): if isinstance(value, bytes) or isinstance(value, bytearray): - return super(CScript, cls).__new__(cls, value) + return super().__new__(cls, value) else: def coerce_iterable(iterable): for instance in iterable: yield cls.__coerce_instance(instance) # Annoyingly on both python2 and python3 bytes.join() always # returns a bytes instance even when subclassed. - return super(CScript, cls).__new__(cls, b''.join(coerce_iterable(value))) + return super().__new__(cls, b''.join(coerce_iterable(value))) def raw_iter(self): """Raw iteration diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index 64f39b8cfe..507a0cff60 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -580,7 +580,7 @@ class TestNodeCLI(): if command is not None: p_args += [command] p_args += pos_args + named_args - self.log.debug("Running bitcoin-cli command: %s" % command) + self.log.debug("Running bitcoin-cli {}".format(p_args[2:])) process = subprocess.Popen(p_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) cli_stdout, cli_stderr = process.communicate(input=self.input) returncode = process.poll() diff --git a/test/functional/wallet_txn_clone.py b/test/functional/wallet_txn_clone.py index 99559090ee..ad23206c90 100755 --- a/test/functional/wallet_txn_clone.py +++ b/test/functional/wallet_txn_clone.py @@ -29,7 +29,7 @@ class TxnMallTest(BitcoinTestFramework): def setup_network(self): # Start with split network: - super(TxnMallTest, self).setup_network() + super().setup_network() disconnect_nodes(self.nodes[1], 2) disconnect_nodes(self.nodes[2], 1) diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index 2455d3a3c3..e2454c4237 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -5,12 +5,13 @@ """Run fuzz test targets. """ +from concurrent.futures import ThreadPoolExecutor, as_completed import argparse import configparser +import logging import os -import sys import subprocess -import logging +import sys def main(): @@ -36,6 +37,12 @@ def main(): help="A comma-separated list of targets to exclude", ) parser.add_argument( + '--par', + type=int, + default=4, + help='How many targets to merge or execute in parallel.', + ) + parser.add_argument( 'seed_dir', help='The seed corpus to run on (must contain subfolders for each fuzz target).', ) @@ -124,25 +131,29 @@ def main(): logging.error("subprocess timed out: Currently only libFuzzer is supported") sys.exit(1) - if args.m_dir: - merge_inputs( + with ThreadPoolExecutor(max_workers=args.par) as fuzz_pool: + if args.m_dir: + merge_inputs( + fuzz_pool=fuzz_pool, + corpus=args.seed_dir, + test_list=test_list_selection, + build_dir=config["environment"]["BUILDDIR"], + merge_dir=args.m_dir, + ) + return + + run_once( + fuzz_pool=fuzz_pool, corpus=args.seed_dir, test_list=test_list_selection, build_dir=config["environment"]["BUILDDIR"], - merge_dir=args.m_dir, + use_valgrind=args.valgrind, ) - return - - run_once( - corpus=args.seed_dir, - test_list=test_list_selection, - build_dir=config["environment"]["BUILDDIR"], - use_valgrind=args.valgrind, - ) -def merge_inputs(*, corpus, test_list, build_dir, merge_dir): +def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir): logging.info("Merge the inputs in the passed dir into the seed_dir. Passed dir {}".format(merge_dir)) + jobs = [] for t in test_list: args = [ os.path.join(build_dir, 'src', 'test', 'fuzz', t), @@ -153,12 +164,20 @@ def merge_inputs(*, corpus, test_list, build_dir, merge_dir): ] os.makedirs(os.path.join(corpus, t), exist_ok=True) os.makedirs(os.path.join(merge_dir, t), exist_ok=True) - logging.debug('Run {} with args {}'.format(t, args)) - output = subprocess.run(args, check=True, stderr=subprocess.PIPE, universal_newlines=True).stderr - logging.debug('Output: {}'.format(output)) + def job(t, args): + output = 'Run {} with args {}\n'.format(t, " ".join(args)) + output += subprocess.run(args, check=True, stderr=subprocess.PIPE, universal_newlines=True).stderr + logging.debug(output) + + jobs.append(fuzz_pool.submit(job, t, args)) + + for future in as_completed(jobs): + future.result() -def run_once(*, corpus, test_list, build_dir, use_valgrind): + +def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind): + jobs = [] for t in test_list: corpus_path = os.path.join(corpus, t) os.makedirs(corpus_path, exist_ok=True) @@ -169,10 +188,18 @@ def run_once(*, corpus, test_list, build_dir, use_valgrind): ] if use_valgrind: args = ['valgrind', '--quiet', '--error-exitcode=1'] + args - logging.debug('Run {} with args {}'.format(t, args)) - result = subprocess.run(args, stderr=subprocess.PIPE, universal_newlines=True) - output = result.stderr - logging.debug('Output: {}'.format(output)) + + def job(t, args): + output = 'Run {} with args {}'.format(t, args) + result = subprocess.run(args, stderr=subprocess.PIPE, universal_newlines=True) + output += result.stderr + return output, result + + jobs.append(fuzz_pool.submit(job, t, args)) + + for future in as_completed(jobs): + output, result = future.result() + logging.debug(output) try: result.check_returncode() except subprocess.CalledProcessError as e: @@ -180,7 +207,7 @@ def run_once(*, corpus, test_list, build_dir, use_valgrind): logging.info(e.stdout) if e.stderr: logging.info(e.stderr) - logging.info("Target \"{}\" failed with exit code {}: {}".format(t, e.returncode, " ".join(args))) + logging.info("Target \"{}\" failed with exit code {}".format(" ".join(result.args), e.returncode)) sys.exit(1) |