diff options
29 files changed, 456 insertions, 206 deletions
diff --git a/contrib/macdeploy/README.md b/contrib/macdeploy/README.md index f78bebf114..68ebb5def1 100644 --- a/contrib/macdeploy/README.md +++ b/contrib/macdeploy/README.md @@ -14,6 +14,10 @@ When complete, it will have produced `Bitcoin-Qt.dmg`. ## SDK Extraction +Our current macOS SDK (`macOSX10.14.sdk`) can be extracted from +[Xcode_10.2.1.xip](https://download.developer.apple.com/Developer_Tools/Xcode_10.2.1/Xcode_10.2.1.xip). +An Apple ID is needed to download this. + `Xcode.app` is packaged in a `.xip` archive. This makes the SDK less-trivial to extract on non-macOS machines. One approach (tested on Debian Buster) is outlined below: @@ -38,14 +42,14 @@ xar -xf Xcode_10.2.1.xip -C . ./pbzx/pbzx -n Content | cpio -i -find Xcode.app -type d -name MacOSX.sdk -execdir sh -c 'tar -c MacOSX.sdk/ | gzip -9n > /MacOSX10.14.sdk.tar.gz' \; +find Xcode.app -type d -name MacOSX.sdk -exec sh -c 'tar --transform="s/MacOSX.sdk/MacOSX10.14.sdk/" -c -C$(dirname {}) MacOSX.sdk/ | gzip -9n > MacOSX10.14.sdk.tar.gz' \; ``` on macOS the process is more straightforward: ```bash xip -x Xcode_10.2.1.xip -tar -C Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.14.sdk.tar.gz MacOSX.sdk +tar -s "/MacOSX.sdk/MacOSX10.14.sdk/" -C Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Developer/SDKs/ -czf MacOSX10.14.sdk.tar.gz MacOSX.sdk ``` Our previously used macOS SDK (`MacOSX10.11.sdk`) can be extracted from diff --git a/doc/fuzzing.md b/doc/fuzzing.md index 9642337821..419b1db44e 100644 --- a/doc/fuzzing.md +++ b/doc/fuzzing.md @@ -133,3 +133,25 @@ $ afl/afl-fuzz -i inputs/ -o outputs/ -- src/test/fuzz/bech32 ``` Read the [`afl-fuzz` documentation](https://github.com/google/afl) for more information. + +# Fuzzing Bitcoin Core using Honggfuzz + +## Quickstart guide + +To quickly get started fuzzing Bitcoin Core using [Honggfuzz](https://github.com/google/honggfuzz): + +```sh +$ git clone https://github.com/bitcoin/bitcoin +$ cd bitcoin/ +$ ./autogen.sh +$ git clone https://github.com/google/honggfuzz +$ cd honggfuzz/ +$ make +$ cd .. +$ CC=$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang CXX=$(pwd)/honggfuzz/hfuzz_cc/hfuzz-clang++ ./configure --enable-fuzz --with-sanitizers=address,undefined +$ make +$ mkdir -p inputs/ +$ honggfuzz/honggfuzz -i inputs/ -- src/test/fuzz/process_message +``` + +Read the [Honggfuzz documentation](https://github.com/google/honggfuzz/blob/master/docs/USAGE.md) for more information. diff --git a/doc/init.md b/doc/init.md index 0b327aab58..99aa0a0def 100644 --- a/doc/init.md +++ b/doc/init.md @@ -20,9 +20,9 @@ The macOS configuration assumes bitcoind will be set up for the current user. Configuration --------------------------------- -At a bare minimum, bitcoind requires that the rpcpassword setting be set -when running as a daemon. If the configuration file does not exist or this -setting is not set, bitcoind will shut down promptly after startup. +Running bitcoind as a daemon does not require any manual configuration. You may +set the `rpcauth` setting in the `bitcoin.conf` configuration file to override +the default behaviour of using a special cookie for authentication. This password does not have to be remembered or typed as it is mostly used as a fixed token that bitcoind and client programs read from the configuration diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index eae8b1fcd1..766c0fca54 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -12,6 +12,7 @@ GENERATED_BENCH_FILES = $(RAW_BENCH_FILES:.raw=.raw.h) bench_bench_bitcoin_SOURCES = \ $(RAW_BENCH_FILES) \ + bench/addrman.cpp \ bench/bench_bitcoin.cpp \ bench/bench.cpp \ bench/bench.h \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 059ffd6964..06dc59cf5c 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -69,6 +69,7 @@ FUZZ_TARGETS = \ test/fuzz/partially_signed_transaction_deserialize \ test/fuzz/pow \ test/fuzz/prefilled_transaction_deserialize \ + test/fuzz/primitives_transaction \ test/fuzz/process_messages \ test/fuzz/process_message \ test/fuzz/process_message_addr \ @@ -686,6 +687,12 @@ test_fuzz_prefilled_transaction_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_prefilled_transaction_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_prefilled_transaction_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_primitives_transaction_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_primitives_transaction_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_primitives_transaction_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_primitives_transaction_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_primitives_transaction_SOURCES = test/fuzz/primitives_transaction.cpp + test_fuzz_process_messages_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_process_messages_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_process_messages_LDADD = $(FUZZ_SUITE_LD_COMMON) diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp new file mode 100644 index 0000000000..cc260df2b8 --- /dev/null +++ b/src/bench/addrman.cpp @@ -0,0 +1,140 @@ +// Copyright (c) 2020-2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <addrman.h> +#include <bench/bench.h> +#include <random.h> +#include <util/time.h> + +#include <vector> + +/* A "source" is a source address from which we have received a bunch of other addresses. */ + +static constexpr size_t NUM_SOURCES = 64; +static constexpr size_t NUM_ADDRESSES_PER_SOURCE = 256; + +static std::vector<CAddress> g_sources; +static std::vector<std::vector<CAddress>> g_addresses; + +static void CreateAddresses() +{ + if (g_sources.size() > 0) { // already created + return; + } + + FastRandomContext rng(uint256(std::vector<unsigned char>(32, 123))); + + auto randAddr = [&rng]() { + in6_addr addr; + memcpy(&addr, rng.randbytes(sizeof(addr)).data(), sizeof(addr)); + + uint16_t port; + memcpy(&port, rng.randbytes(sizeof(port)).data(), sizeof(port)); + if (port == 0) { + port = 1; + } + + CAddress ret(CService(addr, port), NODE_NETWORK); + + ret.nTime = GetAdjustedTime(); + + return ret; + }; + + for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { + g_sources.emplace_back(randAddr()); + g_addresses.emplace_back(); + for (size_t addr_i = 0; addr_i < NUM_ADDRESSES_PER_SOURCE; ++addr_i) { + g_addresses[source_i].emplace_back(randAddr()); + } + } +} + +static void AddAddressesToAddrMan(CAddrMan& addrman) +{ + for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { + addrman.Add(g_addresses[source_i], g_sources[source_i]); + } +} + +static void FillAddrMan(CAddrMan& addrman) +{ + CreateAddresses(); + + AddAddressesToAddrMan(addrman); +} + +/* Benchmarks */ + +static void AddrManAdd(benchmark::State& state) +{ + CreateAddresses(); + + CAddrMan addrman; + + while (state.KeepRunning()) { + AddAddressesToAddrMan(addrman); + addrman.Clear(); + } +} + +static void AddrManSelect(benchmark::State& state) +{ + CAddrMan addrman; + + FillAddrMan(addrman); + + while (state.KeepRunning()) { + const auto& address = addrman.Select(); + assert(address.GetPort() > 0); + } +} + +static void AddrManGetAddr(benchmark::State& state) +{ + CAddrMan addrman; + + FillAddrMan(addrman); + + while (state.KeepRunning()) { + const auto& addresses = addrman.GetAddr(); + assert(addresses.size() > 0); + } +} + +static void AddrManGood(benchmark::State& state) +{ + /* Create many CAddrMan objects - one to be modified at each loop iteration. + * This is necessary because the CAddrMan::Good() method modifies the + * object, affecting the timing of subsequent calls to the same method and + * we want to do the same amount of work in every loop iteration. */ + + const uint64_t numLoops = state.m_num_iters * state.m_num_evals; + + std::vector<CAddrMan> addrmans(numLoops); + for (auto& addrman : addrmans) { + FillAddrMan(addrman); + } + + auto markSomeAsGood = [](CAddrMan& addrman) { + for (size_t source_i = 0; source_i < NUM_SOURCES; ++source_i) { + for (size_t addr_i = 0; addr_i < NUM_ADDRESSES_PER_SOURCE; ++addr_i) { + if (addr_i % 32 == 0) { + addrman.Good(g_addresses[source_i][addr_i]); + } + } + } + }; + + uint64_t i = 0; + while (state.KeepRunning()) { + markSomeAsGood(addrmans.at(i)); + ++i; + } +} + +BENCHMARK(AddrManAdd, 5); +BENCHMARK(AddrManSelect, 1000000); +BENCHMARK(AddrManGetAddr, 500); +BENCHMARK(AddrManGood, 2); diff --git a/src/init.cpp b/src/init.cpp index 83b3f2f47f..a213dacbe0 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -520,7 +520,8 @@ void SetupServerArgs(NodeContext& node) gArgs.AddArg("-limitdescendantsize=<n>", strprintf("Do not accept transactions if any ancestor would have more than <n> kilobytes of in-mempool descendants (default: %u).", DEFAULT_DESCENDANT_SIZE_LIMIT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-addrmantest", "Allows to test address relay on localhost", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-debug=<category>", "Output debugging information (default: -nodebug, supplying <category> is optional). " - "If <category> is not supplied or if <category> = 1, output all debugging information. <category> can be: " + ListLogCategories() + ".", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); + "If <category> is not supplied or if <category> = 1, output all debugging information. <category> can be: " + LogInstance().LogCategoriesString() + ".", + ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-debugexclude=<category>", strprintf("Exclude debugging information for a category. Can be used in conjunction with -debug=1 to output debug logs for all categories except one or more specified categories."), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-logips", strprintf("Include IP addresses in debug output (default: %u)", DEFAULT_LOGIPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); gArgs.AddArg("-logtimestamps", strprintf("Prepend debug output with timestamp (default: %u)", DEFAULT_LOGTIMESTAMPS), ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); diff --git a/src/logging.cpp b/src/logging.cpp index 6fd916b603..eb9da06d9b 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -182,30 +182,15 @@ bool GetLogCategory(BCLog::LogFlags& flag, const std::string& str) return false; } -std::string ListLogCategories() +std::vector<LogCategory> BCLog::Logger::LogCategoriesList() { - std::string ret; - int outcount = 0; + std::vector<LogCategory> ret; for (const CLogCategoryDesc& category_desc : LogCategories) { // Omit the special cases. if (category_desc.flag != BCLog::NONE && category_desc.flag != BCLog::ALL) { - if (outcount != 0) ret += ", "; - ret += category_desc.category; - outcount++; - } - } - return ret; -} - -std::vector<CLogCategoryActive> ListActiveLogCategories() -{ - std::vector<CLogCategoryActive> ret; - for (const CLogCategoryDesc& category_desc : LogCategories) { - // Omit the special cases. - if (category_desc.flag != BCLog::NONE && category_desc.flag != BCLog::ALL) { - CLogCategoryActive catActive; + LogCategory catActive; catActive.category = category_desc.category; - catActive.active = LogAcceptCategory(category_desc.flag); + catActive.active = WillLogCategory(category_desc.flag); ret.push_back(catActive); } } diff --git a/src/logging.h b/src/logging.h index b2fde1b9ea..ab07010316 100644 --- a/src/logging.h +++ b/src/logging.h @@ -8,6 +8,7 @@ #include <fs.h> #include <tinyformat.h> +#include <util/string.h> #include <atomic> #include <cstdint> @@ -24,8 +25,7 @@ extern const char * const DEFAULT_DEBUGLOGFILE; extern bool fLogIPs; -struct CLogCategoryActive -{ +struct LogCategory { std::string category; bool active; }; @@ -132,6 +132,13 @@ namespace BCLog { bool DisableCategory(const std::string& str); bool WillLogCategory(LogFlags category) const; + /** Returns a vector of the log categories */ + std::vector<LogCategory> LogCategoriesList(); + /** Returns a string with the log categories */ + std::string LogCategoriesString() + { + return Join(LogCategoriesList(), ", ", [&](const LogCategory& i) { return i.category; }); + }; bool DefaultShrinkDebugFile() const; }; @@ -146,12 +153,6 @@ static inline bool LogAcceptCategory(BCLog::LogFlags category) return LogInstance().WillLogCategory(category); } -/** Returns a string with the log categories. */ -std::string ListLogCategories(); - -/** Returns a vector of the active log categories. */ -std::vector<CLogCategoryActive> ListActiveLogCategories(); - /** Return true if str parses as a log category and set the flag */ bool GetLogCategory(BCLog::LogFlags& flag, const std::string& str); @@ -45,16 +45,10 @@ static const bool DEFAULT_WHITELISTRELAY = true; /** Default for -whitelistforcerelay. */ static const bool DEFAULT_WHITELISTFORCERELAY = false; -/** Time between pings automatically sent out for latency probing and keepalive (in seconds). */ -static const int PING_INTERVAL = 2 * 60; /** Time after which to disconnect, after waiting for a ping response (or inactivity). */ static const int TIMEOUT_INTERVAL = 20 * 60; /** Run the feeler connection loop once every 2 minutes or 120 seconds. **/ static const int FEELER_INTERVAL = 120; -/** The maximum number of entries in an 'inv' protocol message */ -static const unsigned int MAX_INV_SZ = 50000; -/** The maximum number of entries in a locator */ -static const unsigned int MAX_LOCATOR_SZ = 101; /** The maximum number of new addresses to accumulate before announcing. */ static const unsigned int MAX_ADDR_TO_SEND = 1000; /** Maximum length of incoming protocol messages (no message over 4 MB is currently acceptable). */ diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 96e9178b6e..26327ac6eb 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -64,6 +64,12 @@ static constexpr int STALE_RELAY_AGE_LIMIT = 30 * 24 * 60 * 60; /// Age after which a block is considered historical for purposes of rate /// limiting block relay. Set to one week, denominated in seconds. static constexpr int HISTORICAL_BLOCK_AGE = 7 * 24 * 60 * 60; +/** Time between pings automatically sent out for latency probing and keepalive (in seconds). */ +static const int PING_INTERVAL = 2 * 60; +/** The maximum number of entries in a locator */ +static const unsigned int MAX_LOCATOR_SZ = 101; +/** The maximum number of entries in an 'inv' protocol message */ +static const unsigned int MAX_INV_SZ = 50000; /** Maximum number of in-flight transactions from a peer */ static constexpr int32_t MAX_PEER_TX_IN_FLIGHT = 100; /** Maximum number of announced transactions from a peer */ @@ -80,7 +86,47 @@ static_assert(INBOUND_PEER_TX_DELAY >= MAX_GETDATA_RANDOM_DELAY, "To preserve security, MAX_GETDATA_RANDOM_DELAY should not exceed INBOUND_PEER_DELAY"); /** Limit to avoid sending big packets. Not used in processing incoming GETDATA for compatibility */ static const unsigned int MAX_GETDATA_SZ = 1000; - +/** Number of blocks that can be requested at any given time from a single peer. */ +static const int MAX_BLOCKS_IN_TRANSIT_PER_PEER = 16; +/** Timeout in seconds during which a peer must stall block download progress before being disconnected. */ +static const unsigned int BLOCK_STALLING_TIMEOUT = 2; +/** Number of headers sent in one getheaders result. We rely on the assumption that if a peer sends + * less than this number, we reached its tip. Changing this value is a protocol upgrade. */ +static const unsigned int MAX_HEADERS_RESULTS = 2000; +/** Maximum depth of blocks we're willing to serve as compact blocks to peers + * when requested. For older blocks, a regular BLOCK response will be sent. */ +static const int MAX_CMPCTBLOCK_DEPTH = 5; +/** Maximum depth of blocks we're willing to respond to GETBLOCKTXN requests for. */ +static const int MAX_BLOCKTXN_DEPTH = 10; +/** Size of the "block download window": how far ahead of our current height do we fetch? + * Larger windows tolerate larger download speed differences between peer, but increase the potential + * degree of disordering of blocks on disk (which make reindexing and pruning harder). We'll probably + * want to make this a per-peer adaptive value at some point. */ +static const unsigned int BLOCK_DOWNLOAD_WINDOW = 1024; +/** Block download timeout base, expressed in millionths of the block interval (i.e. 10 min) */ +static const int64_t BLOCK_DOWNLOAD_TIMEOUT_BASE = 1000000; +/** Additional block download timeout per parallel downloading peer (i.e. 5 min) */ +static const int64_t BLOCK_DOWNLOAD_TIMEOUT_PER_PEER = 500000; +/** Maximum number of headers to announce when relaying blocks with headers message.*/ +static const unsigned int MAX_BLOCKS_TO_ANNOUNCE = 8; +/** Maximum number of unconnecting headers announcements before DoS score */ +static const int MAX_UNCONNECTING_HEADERS = 10; +/** Minimum blocks required to signal NODE_NETWORK_LIMITED */ +static const unsigned int NODE_NETWORK_LIMITED_MIN_BLOCKS = 288; +/** Average delay between local address broadcasts */ +static constexpr std::chrono::hours AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL{24}; +/** Average delay between peer address broadcasts */ +static constexpr std::chrono::seconds AVG_ADDRESS_BROADCAST_INTERVAL{30}; +/** Average delay between trickled inventory transmissions in seconds. + * Blocks and whitelisted receivers bypass this, outbound peers get half this delay. */ +static const unsigned int INVENTORY_BROADCAST_INTERVAL = 5; +/** Maximum number of inventory items to send per transmission. + * Limits the impact of low-fee transaction floods. */ +static constexpr unsigned int INVENTORY_BROADCAST_MAX = 7 * INVENTORY_BROADCAST_INTERVAL; +/** Average delay between feefilter broadcasts in seconds. */ +static constexpr unsigned int AVG_FEEFILTER_BROADCAST_INTERVAL = 10 * 60; +/** Maximum feefilter broadcast delay after significant change. */ +static constexpr unsigned int MAX_FEEFILTER_CHANGE_DELAY = 5 * 60; struct COrphanTx { // When modifying, adapt the copy of this definition in tests/DoS_tests. @@ -97,21 +143,6 @@ void EraseOrphansFor(NodeId peer); /** Increase a node's misbehavior score. */ void Misbehaving(NodeId nodeid, int howmuch, const std::string& message="") EXCLUSIVE_LOCKS_REQUIRED(cs_main); -/** Average delay between local address broadcasts */ -static constexpr std::chrono::hours AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL{24}; -/** Average delay between peer address broadcasts */ -static constexpr std::chrono::seconds AVG_ADDRESS_BROADCAST_INTERVAL{30}; -/** Average delay between trickled inventory transmissions in seconds. - * Blocks and whitelisted receivers bypass this, outbound peers get half this delay. */ -static const unsigned int INVENTORY_BROADCAST_INTERVAL = 5; -/** Maximum number of inventory items to send per transmission. - * Limits the impact of low-fee transaction floods. */ -static constexpr unsigned int INVENTORY_BROADCAST_MAX = 7 * INVENTORY_BROADCAST_INTERVAL; -/** Average delay between feefilter broadcasts in seconds. */ -static constexpr unsigned int AVG_FEEFILTER_BROADCAST_INTERVAL = 10 * 60; -/** Maximum feefilter broadcast delay after significant change. */ -static constexpr unsigned int MAX_FEEFILTER_CHANGE_DELAY = 5 * 60; - // Internal stuff namespace { /** Number of nodes with fSyncStarted. */ diff --git a/src/rpc/misc.cpp b/src/rpc/misc.cpp index 0d51c6ae1e..f3c5fed858 100644 --- a/src/rpc/misc.cpp +++ b/src/rpc/misc.cpp @@ -516,7 +516,7 @@ UniValue logging(const JSONRPCRequest& request) "When called with arguments, adds or removes categories from debug logging and return the lists above.\n" "The arguments are evaluated in order \"include\", \"exclude\".\n" "If an item is both included and excluded, it will thus end up being excluded.\n" - "The valid logging categories are: " + ListLogCategories() + "\n" + "The valid logging categories are: " + LogInstance().LogCategoriesString() + "\n" "In addition, the following are available as category names with special meanings:\n" " - \"all\", \"1\" : represent all logging categories.\n" " - \"none\", \"0\" : even if other logging categories are specified, ignore all of them.\n" @@ -568,8 +568,7 @@ UniValue logging(const JSONRPCRequest& request) } UniValue result(UniValue::VOBJ); - std::vector<CLogCategoryActive> vLogCatActive = ListActiveLogCategories(); - for (const auto& logCatActive : vLogCatActive) { + for (const auto& logCatActive : LogInstance().LogCategoriesList()) { result.pushKV(logCatActive.category, logCatActive.active); } diff --git a/src/test/fuzz/block.cpp b/src/test/fuzz/block.cpp index d1e92ce607..f30fa03e0b 100644 --- a/src/test/fuzz/block.cpp +++ b/src/test/fuzz/block.cpp @@ -62,4 +62,8 @@ void test_one_input(const std::vector<uint8_t>& buffer) const size_t raw_memory_size = RecursiveDynamicUsage(block); const size_t raw_memory_size_as_shared_ptr = RecursiveDynamicUsage(std::make_shared<CBlock>(block)); assert(raw_memory_size_as_shared_ptr > raw_memory_size); + CBlock block_copy = block; + block_copy.SetNull(); + const bool is_null = block_copy.IsNull(); + assert(is_null); } diff --git a/src/test/fuzz/primitives_transaction.cpp b/src/test/fuzz/primitives_transaction.cpp new file mode 100644 index 0000000000..2e5ba6bdb0 --- /dev/null +++ b/src/test/fuzz/primitives_transaction.cpp @@ -0,0 +1,34 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <optional.h> +#include <primitives/transaction.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> + +#include <cstdint> +#include <string> +#include <vector> + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const CScript script = ConsumeScript(fuzzed_data_provider); + const Optional<COutPoint> out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); + if (out_point) { + const CTxIn tx_in{*out_point, script, fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; + (void)tx_in; + } + const CTxOut tx_out_1{ConsumeMoney(fuzzed_data_provider), script}; + const CTxOut tx_out_2{ConsumeMoney(fuzzed_data_provider), ConsumeScript(fuzzed_data_provider)}; + assert((tx_out_1 == tx_out_2) != (tx_out_1 != tx_out_2)); + const Optional<CMutableTransaction> mutable_tx_1 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const Optional<CMutableTransaction> mutable_tx_2 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + if (mutable_tx_1 && mutable_tx_2) { + const CTransaction tx_1{*mutable_tx_1}; + const CTransaction tx_2{*mutable_tx_2}; + assert((tx_1 == tx_2) != (tx_1 != tx_2)); + } +} diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 0ed3721636..c03365199a 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -19,16 +19,13 @@ #include <validationinterface.h> #include <version.h> -#include <algorithm> #include <atomic> #include <cassert> #include <chrono> #include <cstdint> #include <iosfwd> #include <iostream> -#include <map> #include <memory> -#include <set> #include <string> #include <vector> @@ -44,19 +41,6 @@ const std::string LIMIT_TO_MESSAGE_TYPE{TO_STRING(MESSAGE_TYPE)}; const std::string LIMIT_TO_MESSAGE_TYPE; #endif -const std::map<std::string, std::set<std::string>> EXPECTED_DESERIALIZATION_EXCEPTIONS = { - {"CDataStream::read(): end of data: iostream error", {"addr", "block", "blocktxn", "cmpctblock", "feefilter", "filteradd", "filterload", "getblocks", "getblocktxn", "getdata", "getheaders", "headers", "inv", "notfound", "ping", "sendcmpct", "tx"}}, - {"CompactSize exceeds limit of type: iostream error", {"cmpctblock"}}, - {"differential value overflow: iostream error", {"getblocktxn"}}, - {"index overflowed 16 bits: iostream error", {"getblocktxn"}}, - {"index overflowed 16-bits: iostream error", {"cmpctblock"}}, - {"indexes overflowed 16 bits: iostream error", {"getblocktxn"}}, - {"non-canonical ReadCompactSize(): iostream error", {"addr", "block", "blocktxn", "cmpctblock", "filteradd", "filterload", "getblocks", "getblocktxn", "getdata", "getheaders", "headers", "inv", "notfound", "tx"}}, - {"ReadCompactSize(): size too large: iostream error", {"addr", "block", "blocktxn", "cmpctblock", "filteradd", "filterload", "getblocks", "getblocktxn", "getdata", "getheaders", "headers", "inv", "notfound", "tx"}}, - {"Superfluous witness record: iostream error", {"block", "blocktxn", "cmpctblock", "tx"}}, - {"Unknown transaction optional data: iostream error", {"block", "blocktxn", "cmpctblock", "tx"}}, -}; - const TestingSetup* g_setup; } // namespace @@ -91,13 +75,7 @@ void test_one_input(const std::vector<uint8_t>& buffer) g_setup->m_node.peer_logic->InitializeNode(&p2p_node); try { (void)ProcessMessage(&p2p_node, random_message_type, random_bytes_data_stream, GetTimeMillis(), Params(), *g_setup->m_node.mempool, g_setup->m_node.connman.get(), g_setup->m_node.banman.get(), std::atomic<bool>{false}); - } catch (const std::ios_base::failure& e) { - const std::string exception_message{e.what()}; - const auto p = EXPECTED_DESERIALIZATION_EXCEPTIONS.find(exception_message); - if (p == EXPECTED_DESERIALIZATION_EXCEPTIONS.cend() || p->second.count(random_message_type) == 0) { - std::cout << "Unexpected exception when processing message type \"" << random_message_type << "\": " << exception_message << std::endl; - assert(false); - } + } catch (const std::ios_base::failure&) { } SyncWithValidationInterfaceQueue(); } diff --git a/src/validation.cpp b/src/validation.cpp index fb635b4202..25975e3e31 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -59,6 +59,25 @@ #define MICRO 0.000001 #define MILLI 0.001 +/** + * An extra transaction can be added to a package, as long as it only has one + * ancestor and is no larger than this. Not really any reason to make this + * configurable as it doesn't materially change DoS parameters. + */ +static const unsigned int EXTRA_DESCENDANT_TX_SIZE_LIMIT = 10000; +/** Maximum kilobytes for transactions to store for processing during reorg */ +static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; +/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ +static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB +/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ +static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB +/** Time to wait (in seconds) between writing blocks/block index to disk. */ +static const unsigned int DATABASE_WRITE_INTERVAL = 60 * 60; +/** Time to wait (in seconds) between flushing chainstate to disk. */ +static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; +/** Maximum age of our tip in seconds for us to be considered current for fee estimation */ +static const int64_t MAX_FEE_ESTIMATION_TIP_AGE = 3 * 60 * 60; + bool CBlockIndexWorkComparator::operator()(const CBlockIndex *pa, const CBlockIndex *pb) const { // First sort by most total work, ... if (pa->nChainWork > pb->nChainWork) return false; diff --git a/src/validation.h b/src/validation.h index dbf7aa28db..91b1ba6497 100644 --- a/src/validation.h +++ b/src/validation.h @@ -60,57 +60,15 @@ static const unsigned int DEFAULT_ANCESTOR_SIZE_LIMIT = 101; static const unsigned int DEFAULT_DESCENDANT_LIMIT = 25; /** Default for -limitdescendantsize, maximum kilobytes of in-mempool descendants */ static const unsigned int DEFAULT_DESCENDANT_SIZE_LIMIT = 101; -/** - * An extra transaction can be added to a package, as long as it only has one - * ancestor and is no larger than this. Not really any reason to make this - * configurable as it doesn't materially change DoS parameters. - */ -static const unsigned int EXTRA_DESCENDANT_TX_SIZE_LIMIT = 10000; /** Default for -mempoolexpiry, expiration time for mempool transactions in hours */ static const unsigned int DEFAULT_MEMPOOL_EXPIRY = 336; -/** Maximum kilobytes for transactions to store for processing during reorg */ -static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; /** The maximum size of a blk?????.dat file (since 0.8) */ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB -/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ -static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB -/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ -static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB - /** Maximum number of dedicated script-checking threads allowed */ static const int MAX_SCRIPTCHECK_THREADS = 15; /** -par default (number of script-checking threads, 0 = auto) */ static const int DEFAULT_SCRIPTCHECK_THREADS = 0; -/** Number of blocks that can be requested at any given time from a single peer. */ -static const int MAX_BLOCKS_IN_TRANSIT_PER_PEER = 16; -/** Timeout in seconds during which a peer must stall block download progress before being disconnected. */ -static const unsigned int BLOCK_STALLING_TIMEOUT = 2; -/** Number of headers sent in one getheaders result. We rely on the assumption that if a peer sends - * less than this number, we reached its tip. Changing this value is a protocol upgrade. */ -static const unsigned int MAX_HEADERS_RESULTS = 2000; -/** Maximum depth of blocks we're willing to serve as compact blocks to peers - * when requested. For older blocks, a regular BLOCK response will be sent. */ -static const int MAX_CMPCTBLOCK_DEPTH = 5; -/** Maximum depth of blocks we're willing to respond to GETBLOCKTXN requests for. */ -static const int MAX_BLOCKTXN_DEPTH = 10; -/** Size of the "block download window": how far ahead of our current height do we fetch? - * Larger windows tolerate larger download speed differences between peer, but increase the potential - * degree of disordering of blocks on disk (which make reindexing and pruning harder). We'll probably - * want to make this a per-peer adaptive value at some point. */ -static const unsigned int BLOCK_DOWNLOAD_WINDOW = 1024; -/** Time to wait (in seconds) between writing blocks/block index to disk. */ -static const unsigned int DATABASE_WRITE_INTERVAL = 60 * 60; -/** Time to wait (in seconds) between flushing chainstate to disk. */ -static const unsigned int DATABASE_FLUSH_INTERVAL = 24 * 60 * 60; -/** Block download timeout base, expressed in millionths of the block interval (i.e. 10 min) */ -static const int64_t BLOCK_DOWNLOAD_TIMEOUT_BASE = 1000000; -/** Additional block download timeout per parallel downloading peer (i.e. 5 min) */ -static const int64_t BLOCK_DOWNLOAD_TIMEOUT_PER_PEER = 500000; - static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60; -/** Maximum age of our tip in seconds for us to be considered current for fee estimation */ -static const int64_t MAX_FEE_ESTIMATION_TIP_AGE = 3 * 60 * 60; - static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; @@ -119,15 +77,21 @@ static const unsigned int DEFAULT_BANSCORE_THRESHOLD = 100; static const bool DEFAULT_PERSIST_MEMPOOL = true; /** Default for using fee filter */ static const bool DEFAULT_FEEFILTER = true; - -/** Maximum number of headers to announce when relaying blocks with headers message.*/ -static const unsigned int MAX_BLOCKS_TO_ANNOUNCE = 8; - -/** Maximum number of unconnecting headers announcements before DoS score */ -static const int MAX_UNCONNECTING_HEADERS = 10; - /** Default for -stopatheight */ static const int DEFAULT_STOPATHEIGHT = 0; +/** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ::ChainActive().Tip() will not be pruned. */ +static const unsigned int MIN_BLOCKS_TO_KEEP = 288; +static const signed int DEFAULT_CHECKBLOCKS = 6; +static const unsigned int DEFAULT_CHECKLEVEL = 3; +// Require that user allocate at least 550 MiB for block & undo files (blk???.dat and rev???.dat) +// At 1MB per block, 288 blocks = 288MB. +// Add 15% for Undo data = 331MB +// Add 20% for Orphan block rate = 397MB +// We want the low water mark after pruning to be at least 397 MB and since we prune in +// full block file chunks, we need the high water mark which triggers the prune to be +// one 128MB block file + added 15% undo data = 147MB greater for a total of 545MB +// Setting the target to >= 550 MiB will make it likely we can respect the target. +static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024; struct BlockHasher { @@ -175,23 +139,6 @@ extern bool fHavePruned; extern bool fPruneMode; /** Number of MiB of block files that we're trying to stay below. */ extern uint64_t nPruneTarget; -/** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ::ChainActive().Tip() will not be pruned. */ -static const unsigned int MIN_BLOCKS_TO_KEEP = 288; -/** Minimum blocks required to signal NODE_NETWORK_LIMITED */ -static const unsigned int NODE_NETWORK_LIMITED_MIN_BLOCKS = 288; - -static const signed int DEFAULT_CHECKBLOCKS = 6; -static const unsigned int DEFAULT_CHECKLEVEL = 3; - -// Require that user allocate at least 550 MiB for block & undo files (blk???.dat and rev???.dat) -// At 1MB per block, 288 blocks = 288MB. -// Add 15% for Undo data = 331MB -// Add 20% for Orphan block rate = 397MB -// We want the low water mark after pruning to be at least 397 MB and since we prune in -// full block file chunks, we need the high water mark which triggers the prune to be -// one 128MB block file + added 15% undo data = 147MB greater for a total of 545MB -// Setting the target to >= 550 MiB will make it likely we can respect the target. -static const uint64_t MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024; /** * Process an incoming block. This only returns after the best known valid diff --git a/test/functional/feature_bip68_sequence.py b/test/functional/feature_bip68_sequence.py index fe45ed34b5..549e8b2029 100755 --- a/test/functional/feature_bip68_sequence.py +++ b/test/functional/feature_bip68_sequence.py @@ -324,7 +324,7 @@ class BIP68Test(BitcoinTestFramework): block.solve() tip = block.sha256 height += 1 - self.nodes[0].submitblock(ToHex(block)) + assert_equal(None if i == 1 else 'inconclusive', self.nodes[0].submitblock(ToHex(block))) cur_time += 1 mempool = self.nodes[0].getrawmempool() @@ -381,7 +381,7 @@ class BIP68Test(BitcoinTestFramework): add_witness_commitment(block) block.solve() - self.nodes[0].submitblock(block.serialize().hex()) + assert_equal(None, self.nodes[0].submitblock(block.serialize().hex())) assert_equal(self.nodes[0].getbestblockhash(), block.hash) def activateCSV(self): diff --git a/test/functional/feature_nulldummy.py b/test/functional/feature_nulldummy.py index 12471c5088..ff55cb76d9 100755 --- a/test/functional/feature_nulldummy.py +++ b/test/functional/feature_nulldummy.py @@ -111,7 +111,7 @@ class NULLDUMMYTest(BitcoinTestFramework): witness and add_witness_commitment(block) block.rehash() block.solve() - node.submitblock(block.serialize().hex()) + assert_equal(None if accept else 'block-validation-failed', node.submitblock(block.serialize().hex())) if (accept): assert_equal(node.getbestblockhash(), block.hash) self.tip = block.sha256 diff --git a/test/functional/interface_bitcoin_cli.py b/test/functional/interface_bitcoin_cli.py index 2e80d7a248..1c94305220 100755 --- a/test/functional/interface_bitcoin_cli.py +++ b/test/functional/interface_bitcoin_cli.py @@ -3,6 +3,7 @@ # Distributed under the MIT software license, see the accompanying # file COPYING or http://www.opensource.org/licenses/mit-license.php. """Test bitcoin-cli""" +from decimal import Decimal from test_framework.test_framework import BitcoinTestFramework from test_framework.util import assert_equal, assert_raises_process_error, get_auth_cookie @@ -51,7 +52,7 @@ class TestBitcoinCli(BitcoinTestFramework): self.log.info("Test -getinfo returns expected network and blockchain info") if self.is_wallet_compiled(): self.nodes[0].encryptwallet(password) - cli_get_info = self.nodes[0].cli().send_cli('-getinfo') + cli_get_info = self.nodes[0].cli('-getinfo').send_cli() network_info = self.nodes[0].getnetworkinfo() blockchain_info = self.nodes[0].getblockchaininfo() assert_equal(cli_get_info['version'], network_info['version']) @@ -72,12 +73,52 @@ class TestBitcoinCli(BitcoinTestFramework): assert_equal(cli_get_info['paytxfee'], wallet_info['paytxfee']) assert_equal(cli_get_info['relayfee'], network_info['relayfee']) assert_equal(self.nodes[0].cli.getwalletinfo(), wallet_info) + + # Setup to test -getinfo and -rpcwallet= with multiple wallets. + wallets = ['', 'Encrypted', 'secret'] + amounts = [Decimal('59.999928'), Decimal(9), Decimal(31)] + self.nodes[0].createwallet(wallet_name=wallets[1]) + self.nodes[0].createwallet(wallet_name=wallets[2]) + w1 = self.nodes[0].get_wallet_rpc(wallets[0]) + w2 = self.nodes[0].get_wallet_rpc(wallets[1]) + w3 = self.nodes[0].get_wallet_rpc(wallets[2]) + w1.walletpassphrase(password, self.rpc_timeout) + w1.sendtoaddress(w2.getnewaddress(), amounts[1]) + w1.sendtoaddress(w3.getnewaddress(), amounts[2]) + + # Mine a block to confirm; adds a block reward (50 BTC) to the default wallet. + self.nodes[0].generate(1) + + self.log.info("Test -getinfo with multiple wallets loaded returns no balance") + assert_equal(set(self.nodes[0].listwallets()), set(wallets)) + assert 'balance' not in self.nodes[0].cli('-getinfo').send_cli().keys() + + self.log.info("Test -getinfo with multiple wallets and -rpcwallet returns specified wallet balance") + for i in range(len(wallets)): + cli_get_info = self.nodes[0].cli('-getinfo').send_cli('-rpcwallet={}'.format(wallets[i])) + assert_equal(cli_get_info['balance'], amounts[i]) + + self.log.info("Test -getinfo with multiple wallets and -rpcwallet=non-existing-wallet returns no balance") + assert 'balance' not in self.nodes[0].cli('-getinfo').send_cli('-rpcwallet=does-not-exist').keys() + + self.log.info("Test -getinfo after unloading all wallets except a non-default one returns its balance") + self.nodes[0].unloadwallet(wallets[0]) + self.nodes[0].unloadwallet(wallets[2]) + assert_equal(self.nodes[0].listwallets(), [wallets[1]]) + assert_equal(self.nodes[0].cli('-getinfo').send_cli()['balance'], amounts[1]) + + self.log.info("Test -getinfo -rpcwallet=remaining-non-default-wallet returns its balance") + assert_equal(self.nodes[0].cli('-getinfo').send_cli('-rpcwallet={}'.format(wallets[1]))['balance'], amounts[1]) + + self.log.info("Test -getinfo with -rpcwallet=unloaded wallet returns no balance") + assert 'balance' not in self.nodes[0].cli('-getinfo').send_cli('-rpcwallet={}'.format(wallets[2])).keys() else: self.log.info("*** Wallet not compiled; cli getwalletinfo and -getinfo wallet tests skipped") + self.nodes[0].generate(1) # maintain block parity with the wallet_compiled conditional branch self.log.info("Test -version with node stopped") self.stop_node(0) - cli_response = self.nodes[0].cli().send_cli('-version') + cli_response = self.nodes[0].cli('-version').send_cli() assert "{} RPC client version".format(self.config['environment']['PACKAGE_NAME']) in cli_response self.log.info("Test -rpcwait option successfully waits for RPC connection") @@ -85,7 +126,7 @@ class TestBitcoinCli(BitcoinTestFramework): self.nodes[0].wait_for_cookie_credentials() # ensure cookie file is available to avoid race condition blocks = self.nodes[0].cli('-rpcwait').send_cli('getblockcount') self.nodes[0].wait_for_rpc_connection() - assert_equal(blocks, BLOCKS) + assert_equal(blocks, BLOCKS + 1) if __name__ == '__main__': diff --git a/test/functional/mempool_reorg.py b/test/functional/mempool_reorg.py index 2d23343fd5..8e1f87e42c 100755 --- a/test/functional/mempool_reorg.py +++ b/test/functional/mempool_reorg.py @@ -16,6 +16,12 @@ from test_framework.util import assert_equal, assert_raises_rpc_error class MempoolCoinbaseTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 + self.extra_args = [ + [ + '-whitelist=noban@127.0.0.1', # immediate tx relay + ], + [] + ] def skip_test_if_missing_module(self): self.skip_if_no_wallet() @@ -38,17 +44,21 @@ class MempoolCoinbaseTest(BitcoinTestFramework): # 3. Indirect (coinbase and child both in chain) : spend_103 and spend_103_1 # Use invalidatblock to make all of the above coinbase spends invalid (immature coinbase), # and make sure the mempool code behaves correctly. - b = [ self.nodes[0].getblockhash(n) for n in range(101, 105) ] - coinbase_txids = [ self.nodes[0].getblock(h)['tx'][0] for h in b ] + b = [self.nodes[0].getblockhash(n) for n in range(101, 105)] + coinbase_txids = [self.nodes[0].getblock(h)['tx'][0] for h in b] spend_101_raw = create_raw_transaction(self.nodes[0], coinbase_txids[1], node1_address, amount=49.99) spend_102_raw = create_raw_transaction(self.nodes[0], coinbase_txids[2], node0_address, amount=49.99) spend_103_raw = create_raw_transaction(self.nodes[0], coinbase_txids[3], node0_address, amount=49.99) # Create a transaction which is time-locked to two blocks in the future - timelock_tx = self.nodes[0].createrawtransaction([{"txid": coinbase_txids[0], "vout": 0}], {node0_address: 49.99}) - # Set the time lock - timelock_tx = timelock_tx.replace("ffffffff", "11111191", 1) - timelock_tx = timelock_tx[:-8] + hex(self.nodes[0].getblockcount() + 2)[2:] + "000000" + timelock_tx = self.nodes[0].createrawtransaction( + inputs=[{ + "txid": coinbase_txids[0], + "vout": 0, + }], + outputs={node0_address: 49.99}, + locktime=self.nodes[0].getblockcount() + 2, + ) timelock_tx = self.nodes[0].signrawtransactionwithwallet(timelock_tx)["hex"] # This will raise an exception because the timelock transaction is too immature to spend assert_raises_rpc_error(-26, "non-final", self.nodes[0].sendrawtransaction, timelock_tx) @@ -67,6 +77,10 @@ class MempoolCoinbaseTest(BitcoinTestFramework): # Broadcast and mine 103_1: spend_103_1_id = self.nodes[0].sendrawtransaction(spend_103_1_raw) last_block = self.nodes[0].generate(1) + # Sync blocks, so that peer 1 gets the block before timelock_tx + # Otherwise, peer 1 would put the timelock_tx in recentRejects + self.sync_all() + # Time-locked transaction can now be spent timelock_tx_id = self.nodes[0].sendrawtransaction(timelock_tx) diff --git a/test/functional/p2p_compactblocks.py b/test/functional/p2p_compactblocks.py index 7905cf5018..66e6f8c424 100755 --- a/test/functional/p2p_compactblocks.py +++ b/test/functional/p2p_compactblocks.py @@ -378,8 +378,6 @@ class CompactBlocksTest(BitcoinTestFramework): # request for announce in ["inv", "header"]: block = self.build_block_on_tip(node, segwit=segwit) - with mininode_lock: - test_node.last_message.pop("getdata", None) if announce == "inv": test_node.send_message(msg_inv([CInv(2, block.sha256)])) @@ -387,10 +385,8 @@ class CompactBlocksTest(BitcoinTestFramework): test_node.send_header_for_blocks([block]) else: test_node.send_header_for_blocks([block]) - wait_until(lambda: "getdata" in test_node.last_message, timeout=30, lock=mininode_lock) - assert_equal(len(test_node.last_message["getdata"].inv), 1) + test_node.wait_for_getdata([block.sha256], timeout=30) assert_equal(test_node.last_message["getdata"].inv[0].type, 4) - assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) # Send back a compactblock message that omits the coinbase comp_block = HeaderAndShortIDs() @@ -567,10 +563,8 @@ class CompactBlocksTest(BitcoinTestFramework): assert_equal(int(node.getbestblockhash(), 16), block.hashPrevBlock) # We should receive a getdata request - wait_until(lambda: "getdata" in test_node.last_message, timeout=10, lock=mininode_lock) - assert_equal(len(test_node.last_message["getdata"].inv), 1) + test_node.wait_for_getdata([block.sha256], timeout=10) assert test_node.last_message["getdata"].inv[0].type == 2 or test_node.last_message["getdata"].inv[0].type == 2 | MSG_WITNESS_FLAG - assert_equal(test_node.last_message["getdata"].inv[0].hash, block.sha256) # Deliver the block if version == 2: diff --git a/test/functional/p2p_segwit.py b/test/functional/p2p_segwit.py index fcf61eea3e..dbdce6552a 100755 --- a/test/functional/p2p_segwit.py +++ b/test/functional/p2p_segwit.py @@ -862,13 +862,13 @@ class SegWitTest(BitcoinTestFramework): # We can't send over the p2p network, because this is too big to relay # TODO: repeat this test with a block that can be relayed - self.nodes[0].submitblock(block.serialize().hex()) + assert_equal('bad-witness-nonce-size', self.nodes[0].submitblock(block.serialize().hex())) assert self.nodes[0].getbestblockhash() != block.hash block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.pop() assert get_virtual_size(block) < MAX_BLOCK_BASE_SIZE - self.nodes[0].submitblock(block.serialize().hex()) + assert_equal(None, self.nodes[0].submitblock(block.serialize().hex())) assert self.nodes[0].getbestblockhash() == block.hash @@ -975,14 +975,14 @@ class SegWitTest(BitcoinTestFramework): add_witness_commitment(block, nonce=1) block.vtx[0].wit = CTxWitness() # drop the nonce block.solve() - self.nodes[0].submitblock(block.serialize().hex()) + assert_equal('bad-witness-merkle-match', self.nodes[0].submitblock(block.serialize().hex())) assert self.nodes[0].getbestblockhash() != block.hash # Now redo commitment with the standard nonce, but let bitcoind fill it in. add_witness_commitment(block, nonce=0) block.vtx[0].wit = CTxWitness() block.solve() - self.nodes[0].submitblock(block.serialize().hex()) + assert_equal(None, self.nodes[0].submitblock(block.serialize().hex())) assert_equal(self.nodes[0].getbestblockhash(), block.hash) # This time, add a tx with non-empty witness, but don't supply @@ -997,7 +997,7 @@ class SegWitTest(BitcoinTestFramework): block_2.vtx[0].vout.pop() block_2.vtx[0].wit = CTxWitness() - self.nodes[0].submitblock(block_2.serialize().hex()) + assert_equal('bad-txnmrklroot', self.nodes[0].submitblock(block_2.serialize().hex())) # Tip should not advance! assert self.nodes[0].getbestblockhash() != block_2.hash diff --git a/test/functional/test_framework/messages.py b/test/functional/test_framework/messages.py index 33fba1c69a..4855f62a8f 100755 --- a/test/functional/test_framework/messages.py +++ b/test/functional/test_framework/messages.py @@ -603,16 +603,16 @@ class CBlock(CBlockHeader): __slots__ = ("vtx",) def __init__(self, header=None): - super(CBlock, self).__init__(header) + super().__init__(header) self.vtx = [] def deserialize(self, f): - super(CBlock, self).deserialize(f) + super().deserialize(f) self.vtx = deser_vector(f, CTransaction) def serialize(self, with_witness=True): r = b"" - r += super(CBlock, self).serialize() + r += super().serialize() if with_witness: r += ser_vector(self.vtx, "serialize_with_witness") else: @@ -752,7 +752,7 @@ class P2PHeaderAndShortIDs: class P2PHeaderAndShortWitnessIDs(P2PHeaderAndShortIDs): __slots__ = () def serialize(self): - return super(P2PHeaderAndShortWitnessIDs, self).serialize(with_witness=True) + return super().serialize(with_witness=True) # Calculate the BIP 152-compact blocks shortid for a given transaction hash def calculate_shortid(k0, k1, tx_hash): diff --git a/test/functional/test_framework/script.py b/test/functional/test_framework/script.py index 016a2b4f0f..e475ed8596 100644 --- a/test/functional/test_framework/script.py +++ b/test/functional/test_framework/script.py @@ -97,7 +97,7 @@ class CScriptOp(int): return _opcode_instances[n] except IndexError: assert len(_opcode_instances) == n - _opcode_instances.append(super(CScriptOp, cls).__new__(cls, n)) + _opcode_instances.append(super().__new__(cls, n)) return _opcode_instances[n] # Populate opcode instance table @@ -372,7 +372,7 @@ class CScriptTruncatedPushDataError(CScriptInvalidError): """Invalid pushdata due to truncation""" def __init__(self, msg, data): self.data = data - super(CScriptTruncatedPushDataError, self).__init__(msg) + super().__init__(msg) # This is used, eg, for blockchain heights in coinbase scripts (bip34) @@ -458,14 +458,14 @@ class CScript(bytes): def __new__(cls, value=b''): if isinstance(value, bytes) or isinstance(value, bytearray): - return super(CScript, cls).__new__(cls, value) + return super().__new__(cls, value) else: def coerce_iterable(iterable): for instance in iterable: yield cls.__coerce_instance(instance) # Annoyingly on both python2 and python3 bytes.join() always # returns a bytes instance even when subclassed. - return super(CScript, cls).__new__(cls, b''.join(coerce_iterable(value))) + return super().__new__(cls, b''.join(coerce_iterable(value))) def raw_iter(self): """Raw iteration diff --git a/test/functional/test_framework/test_node.py b/test/functional/test_framework/test_node.py index de724407a0..c0075fd8ec 100755 --- a/test/functional/test_framework/test_node.py +++ b/test/functional/test_framework/test_node.py @@ -582,7 +582,7 @@ class TestNodeCLI(): if command is not None: p_args += [command] p_args += pos_args + named_args - self.log.debug("Running bitcoin-cli command: %s" % command) + self.log.debug("Running bitcoin-cli {}".format(p_args[2:])) process = subprocess.Popen(p_args, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) cli_stdout, cli_stderr = process.communicate(input=self.input) returncode = process.poll() diff --git a/test/functional/wallet_importmulti.py b/test/functional/wallet_importmulti.py index f152fcd1a4..bd4fcdabcf 100755 --- a/test/functional/wallet_importmulti.py +++ b/test/functional/wallet_importmulti.py @@ -32,6 +32,7 @@ from test_framework.wallet_util import ( test_address, ) + class ImportMultiTest(BitcoinTestFramework): def set_test_params(self): self.num_nodes = 2 @@ -51,7 +52,7 @@ class ImportMultiTest(BitcoinTestFramework): result = self.nodes[1].importmulti([req]) observed_warnings = [] if 'warnings' in result[0]: - observed_warnings = result[0]['warnings'] + observed_warnings = result[0]['warnings'] assert_equal("\n".join(sorted(warnings)), "\n".join(sorted(observed_warnings))) assert_equal(result[0]['success'], success) if error_code is not None: @@ -63,6 +64,7 @@ class ImportMultiTest(BitcoinTestFramework): self.nodes[0].generate(1) self.nodes[1].generate(1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] + self.nodes[1].syncwithvalidationinterfacequeue() node0_address1 = self.nodes[0].getaddressinfo(self.nodes[0].getnewaddress()) @@ -257,6 +259,7 @@ class ImportMultiTest(BitcoinTestFramework): self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) self.nodes[1].generate(1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] + self.nodes[1].syncwithvalidationinterfacequeue() self.log.info("Should import a p2sh") self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_addr}, @@ -277,6 +280,7 @@ class ImportMultiTest(BitcoinTestFramework): self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) self.nodes[1].generate(1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] + self.nodes[1].syncwithvalidationinterfacequeue() self.log.info("Should import a p2sh with respective redeem script") self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_addr}, @@ -297,6 +301,7 @@ class ImportMultiTest(BitcoinTestFramework): self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) self.nodes[1].generate(1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] + self.nodes[1].syncwithvalidationinterfacequeue() self.log.info("Should import a p2sh with respective redeem script and private keys") self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_addr}, @@ -322,6 +327,7 @@ class ImportMultiTest(BitcoinTestFramework): self.nodes[1].sendtoaddress(multisig.p2sh_addr, 10.00) self.nodes[1].generate(1) timestamp = self.nodes[1].getblock(self.nodes[1].getbestblockhash())['mediantime'] + self.nodes[1].syncwithvalidationinterfacequeue() self.log.info("Should import a p2sh with respective redeem script and private keys") self.test_importmulti({"scriptPubKey": {"address": multisig.p2sh_addr}, @@ -851,5 +857,6 @@ class ImportMultiTest(BitcoinTestFramework): addr = wrpc.getnewaddress('', 'bech32') assert_equal(addr, addresses[i]) + if __name__ == '__main__': ImportMultiTest().main() diff --git a/test/functional/wallet_txn_clone.py b/test/functional/wallet_txn_clone.py index 99559090ee..ad23206c90 100755 --- a/test/functional/wallet_txn_clone.py +++ b/test/functional/wallet_txn_clone.py @@ -29,7 +29,7 @@ class TxnMallTest(BitcoinTestFramework): def setup_network(self): # Start with split network: - super(TxnMallTest, self).setup_network() + super().setup_network() disconnect_nodes(self.nodes[1], 2) disconnect_nodes(self.nodes[2], 1) diff --git a/test/fuzz/test_runner.py b/test/fuzz/test_runner.py index 2455d3a3c3..e2454c4237 100755 --- a/test/fuzz/test_runner.py +++ b/test/fuzz/test_runner.py @@ -5,12 +5,13 @@ """Run fuzz test targets. """ +from concurrent.futures import ThreadPoolExecutor, as_completed import argparse import configparser +import logging import os -import sys import subprocess -import logging +import sys def main(): @@ -36,6 +37,12 @@ def main(): help="A comma-separated list of targets to exclude", ) parser.add_argument( + '--par', + type=int, + default=4, + help='How many targets to merge or execute in parallel.', + ) + parser.add_argument( 'seed_dir', help='The seed corpus to run on (must contain subfolders for each fuzz target).', ) @@ -124,25 +131,29 @@ def main(): logging.error("subprocess timed out: Currently only libFuzzer is supported") sys.exit(1) - if args.m_dir: - merge_inputs( + with ThreadPoolExecutor(max_workers=args.par) as fuzz_pool: + if args.m_dir: + merge_inputs( + fuzz_pool=fuzz_pool, + corpus=args.seed_dir, + test_list=test_list_selection, + build_dir=config["environment"]["BUILDDIR"], + merge_dir=args.m_dir, + ) + return + + run_once( + fuzz_pool=fuzz_pool, corpus=args.seed_dir, test_list=test_list_selection, build_dir=config["environment"]["BUILDDIR"], - merge_dir=args.m_dir, + use_valgrind=args.valgrind, ) - return - - run_once( - corpus=args.seed_dir, - test_list=test_list_selection, - build_dir=config["environment"]["BUILDDIR"], - use_valgrind=args.valgrind, - ) -def merge_inputs(*, corpus, test_list, build_dir, merge_dir): +def merge_inputs(*, fuzz_pool, corpus, test_list, build_dir, merge_dir): logging.info("Merge the inputs in the passed dir into the seed_dir. Passed dir {}".format(merge_dir)) + jobs = [] for t in test_list: args = [ os.path.join(build_dir, 'src', 'test', 'fuzz', t), @@ -153,12 +164,20 @@ def merge_inputs(*, corpus, test_list, build_dir, merge_dir): ] os.makedirs(os.path.join(corpus, t), exist_ok=True) os.makedirs(os.path.join(merge_dir, t), exist_ok=True) - logging.debug('Run {} with args {}'.format(t, args)) - output = subprocess.run(args, check=True, stderr=subprocess.PIPE, universal_newlines=True).stderr - logging.debug('Output: {}'.format(output)) + def job(t, args): + output = 'Run {} with args {}\n'.format(t, " ".join(args)) + output += subprocess.run(args, check=True, stderr=subprocess.PIPE, universal_newlines=True).stderr + logging.debug(output) + + jobs.append(fuzz_pool.submit(job, t, args)) + + for future in as_completed(jobs): + future.result() -def run_once(*, corpus, test_list, build_dir, use_valgrind): + +def run_once(*, fuzz_pool, corpus, test_list, build_dir, use_valgrind): + jobs = [] for t in test_list: corpus_path = os.path.join(corpus, t) os.makedirs(corpus_path, exist_ok=True) @@ -169,10 +188,18 @@ def run_once(*, corpus, test_list, build_dir, use_valgrind): ] if use_valgrind: args = ['valgrind', '--quiet', '--error-exitcode=1'] + args - logging.debug('Run {} with args {}'.format(t, args)) - result = subprocess.run(args, stderr=subprocess.PIPE, universal_newlines=True) - output = result.stderr - logging.debug('Output: {}'.format(output)) + + def job(t, args): + output = 'Run {} with args {}'.format(t, args) + result = subprocess.run(args, stderr=subprocess.PIPE, universal_newlines=True) + output += result.stderr + return output, result + + jobs.append(fuzz_pool.submit(job, t, args)) + + for future in as_completed(jobs): + output, result = future.result() + logging.debug(output) try: result.check_returncode() except subprocess.CalledProcessError as e: @@ -180,7 +207,7 @@ def run_once(*, corpus, test_list, build_dir, use_valgrind): logging.info(e.stdout) if e.stderr: logging.info(e.stderr) - logging.info("Target \"{}\" failed with exit code {}: {}".format(t, e.returncode, " ".join(args))) + logging.info("Target \"{}\" failed with exit code {}".format(" ".join(result.args), e.returncode)) sys.exit(1) |