diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/Makefile.test.include | 18 | ||||
-rw-r--r-- | src/bench/block_assemble.cpp | 9 | ||||
-rw-r--r-- | src/bench/duplicate_inputs.cpp | 8 | ||||
-rw-r--r-- | src/bench/mempool_eviction.cpp | 12 | ||||
-rw-r--r-- | src/bench/wallet_balance.cpp | 11 | ||||
-rw-r--r-- | src/net_processing.cpp | 27 | ||||
-rw-r--r-- | src/net_processing.h | 2 | ||||
-rw-r--r-- | src/node/transaction.cpp | 4 | ||||
-rw-r--r-- | src/random.cpp | 5 | ||||
-rw-r--r-- | src/random.h | 1 | ||||
-rw-r--r-- | src/rest.cpp | 2 | ||||
-rw-r--r-- | src/span.h | 19 | ||||
-rw-r--r-- | src/test/fuzz/fees.cpp | 3 | ||||
-rw-r--r-- | src/test/fuzz/integer.cpp | 11 | ||||
-rw-r--r-- | src/test/fuzz/kitchen_sink.cpp | 25 | ||||
-rw-r--r-- | src/test/fuzz/parse_hd_keypath.cpp | 10 | ||||
-rw-r--r-- | src/test/fuzz/string.cpp | 4 | ||||
-rw-r--r-- | src/test/fuzz/system.cpp | 123 | ||||
-rw-r--r-- | src/test/util/logging.cpp | 8 | ||||
-rw-r--r-- | src/test/util/logging.h | 14 | ||||
-rw-r--r-- | src/txmempool.cpp | 11 | ||||
-rw-r--r-- | src/txmempool.h | 18 | ||||
-rw-r--r-- | src/util/system.cpp | 4 | ||||
-rw-r--r-- | src/validation.cpp | 18 | ||||
-rw-r--r-- | src/wallet/test/wallet_tests.cpp | 114 | ||||
-rw-r--r-- | src/wallet/wallet.cpp | 3 |
26 files changed, 460 insertions, 24 deletions
diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 06dc59cf5c..48db60f086 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -49,6 +49,7 @@ FUZZ_TARGETS = \ test/fuzz/key \ test/fuzz/key_io \ test/fuzz/key_origin_info_deserialize \ + test/fuzz/kitchen_sink \ test/fuzz/locale \ test/fuzz/merkle_block_deserialize \ test/fuzz/merkleblock \ @@ -64,13 +65,12 @@ FUZZ_TARGETS = \ test/fuzz/parse_numbers \ test/fuzz/parse_script \ test/fuzz/parse_univalue \ - test/fuzz/prevector \ test/fuzz/partial_merkle_tree_deserialize \ test/fuzz/partially_signed_transaction_deserialize \ test/fuzz/pow \ test/fuzz/prefilled_transaction_deserialize \ + test/fuzz/prevector \ test/fuzz/primitives_transaction \ - test/fuzz/process_messages \ test/fuzz/process_message \ test/fuzz/process_message_addr \ test/fuzz/process_message_block \ @@ -96,6 +96,7 @@ FUZZ_TARGETS = \ test/fuzz/process_message_tx \ test/fuzz/process_message_verack \ test/fuzz/process_message_version \ + test/fuzz/process_messages \ test/fuzz/protocol \ test/fuzz/psbt \ test/fuzz/psbt_input_deserialize \ @@ -116,6 +117,7 @@ FUZZ_TARGETS = \ test/fuzz/string \ test/fuzz/strprintf \ test/fuzz/sub_net_deserialize \ + test/fuzz/system \ test/fuzz/timedata \ test/fuzz/transaction \ test/fuzz/tx_in \ @@ -567,6 +569,12 @@ test_fuzz_key_origin_info_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_key_origin_info_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_key_origin_info_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_kitchen_sink_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_kitchen_sink_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_kitchen_sink_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_kitchen_sink_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_kitchen_sink_SOURCES = test/fuzz/kitchen_sink.cpp + test_fuzz_locale_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_locale_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_locale_LDADD = $(FUZZ_SUITE_LD_COMMON) @@ -969,6 +977,12 @@ test_fuzz_sub_net_deserialize_LDADD = $(FUZZ_SUITE_LD_COMMON) test_fuzz_sub_net_deserialize_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) test_fuzz_sub_net_deserialize_SOURCES = test/fuzz/deserialize.cpp +test_fuzz_system_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) +test_fuzz_system_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) +test_fuzz_system_LDADD = $(FUZZ_SUITE_LD_COMMON) +test_fuzz_system_LDFLAGS = $(RELDFLAGS) $(AM_LDFLAGS) $(LIBTOOL_APP_LDFLAGS) +test_fuzz_system_SOURCES = test/fuzz/system.cpp + test_fuzz_timedata_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) test_fuzz_timedata_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) test_fuzz_timedata_LDADD = $(FUZZ_SUITE_LD_COMMON) diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 1a0084c915..268f67cada 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -16,7 +16,14 @@ static void AssembleBlock(benchmark::State& state) { - RegTestingSetup test_setup; + TestingSetup test_setup{ + CBaseChainParams::REGTEST, + /* extra_args */ { + "-nodebuglogfile", + "-nodebug", + }, + }; + const std::vector<unsigned char> op_true{OP_TRUE}; CScriptWitness witness; witness.stack.push_back(op_true); diff --git a/src/bench/duplicate_inputs.cpp b/src/bench/duplicate_inputs.cpp index 57673ccb84..e87f15042b 100644 --- a/src/bench/duplicate_inputs.cpp +++ b/src/bench/duplicate_inputs.cpp @@ -14,7 +14,13 @@ static void DuplicateInputs(benchmark::State& state) { - RegTestingSetup test_setup; + TestingSetup test_setup{ + CBaseChainParams::REGTEST, + /* extra_args */ { + "-nodebuglogfile", + "-nodebug", + }, + }; const CScript SCRIPT_PUB{CScript(OP_TRUE)}; diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index 7df024def6..69483f2914 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -16,8 +16,8 @@ static void AddTx(const CTransactionRef& tx, const CAmount& nFee, CTxMemPool& po unsigned int sigOpCost = 4; LockPoints lp; pool.addUnchecked(CTxMemPoolEntry( - tx, nFee, nTime, nHeight, - spendsCoinbase, sigOpCost, lp)); + tx, nFee, nTime, nHeight, + spendsCoinbase, sigOpCost, lp)); } // Right now this is only testing eviction performance in an extremely small @@ -25,7 +25,13 @@ static void AddTx(const CTransactionRef& tx, const CAmount& nFee, CTxMemPool& po // unique transactions for a more meaningful performance measurement. static void MempoolEviction(benchmark::State& state) { - RegTestingSetup test_setup; + TestingSetup test_setup{ + CBaseChainParams::REGTEST, + /* extra_args */ { + "-nodebuglogfile", + "-nodebug", + }, + }; CMutableTransaction tx1 = CMutableTransaction(); tx1.vin.resize(1); diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index 05d61fca22..810c344ab5 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -14,7 +14,14 @@ static void WalletBalance(benchmark::State& state, const bool set_dirty, const bool add_watchonly, const bool add_mine) { - RegTestingSetup test_setup; + TestingSetup test_setup{ + CBaseChainParams::REGTEST, + /* extra_args */ { + "-nodebuglogfile", + "-nodebug", + }, + }; + const auto& ADDRESS_WATCHONLY = ADDRESS_BCRT1_UNSPENDABLE; NodeContext node; @@ -25,7 +32,7 @@ static void WalletBalance(benchmark::State& state, const bool set_dirty, const b bool first_run; if (wallet.LoadWallet(first_run) != DBErrors::LOAD_OK) assert(false); } - auto handler = chain->handleNotifications({ &wallet, [](CWallet*) {} }); + auto handler = chain->handleNotifications({&wallet, [](CWallet*) {}}); const Optional<std::string> address_mine{add_mine ? Optional<std::string>{getnewaddress(wallet)} : nullopt}; if (add_watchonly) importaddress(wallet, ADDRESS_WATCHONLY); diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 26327ac6eb..50a8a8a882 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -810,6 +810,19 @@ void PeerLogicValidation::InitializeNode(CNode *pnode) { PushNodeVersion(pnode, connman, GetTime()); } +void PeerLogicValidation::ReattemptInitialBroadcast(CScheduler& scheduler) const +{ + std::set<uint256> unbroadcast_txids = m_mempool.GetUnbroadcastTxs(); + + for (const uint256& txid : unbroadcast_txids) { + RelayTransaction(txid, *connman); + } + + // schedule next run for 10-15 minutes in the future + const std::chrono::milliseconds delta = std::chrono::minutes{10} + GetRandMillis(std::chrono::minutes{5}); + scheduler.scheduleFromNow([&] { ReattemptInitialBroadcast(scheduler); }, delta); +} + void PeerLogicValidation::FinalizeNode(NodeId nodeid, bool& fUpdateConnectionTime) { fUpdateConnectionTime = false; LOCK(cs_main); @@ -1159,6 +1172,10 @@ PeerLogicValidation::PeerLogicValidation(CConnman* connmanIn, BanMan* banman, CS // timer. static_assert(EXTRA_PEER_CHECK_INTERVAL < STALE_CHECK_INTERVAL, "peer eviction timer should be less than stale tip check timer"); scheduler.scheduleEvery([this, consensusParams] { this->CheckForStaleTipAndEvictPeers(consensusParams); }, std::chrono::seconds{EXTRA_PEER_CHECK_INTERVAL}); + + // schedule next run for 10-15 minutes in the future + const std::chrono::milliseconds delta = std::chrono::minutes{10} + GetRandMillis(std::chrono::minutes{5}); + scheduler.scheduleFromNow([&] { ReattemptInitialBroadcast(scheduler); }, delta); } /** @@ -1587,7 +1604,7 @@ void static ProcessGetBlockData(CNode* pfrom, const CChainParams& chainparams, c } } -void static ProcessGetData(CNode* pfrom, const CChainParams& chainparams, CConnman* connman, const CTxMemPool& mempool, const std::atomic<bool>& interruptMsgProc) LOCKS_EXCLUDED(cs_main) +void static ProcessGetData(CNode* pfrom, const CChainParams& chainparams, CConnman* connman, CTxMemPool& mempool, const std::atomic<bool>& interruptMsgProc) LOCKS_EXCLUDED(cs_main) { AssertLockNotHeld(cs_main); @@ -1636,7 +1653,13 @@ void static ProcessGetData(CNode* pfrom, const CChainParams& chainparams, CConnm push = true; } } - if (!push) { + + if (push) { + // We interpret fulfilling a GETDATA for a transaction as a + // successful initial broadcast and remove it from our + // unbroadcast set. + mempool.RemoveUnbroadcastTx(inv.hash); + } else { vNotFound.push_back(inv); } } diff --git a/src/net_processing.h b/src/net_processing.h index 3d9bc65a3a..a85d5e7c70 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -76,6 +76,8 @@ public: void CheckForStaleTipAndEvictPeers(const Consensus::Params &consensusParams); /** If we have extra outbound peers, try to disconnect the one with the oldest block announcement */ void EvictExtraOutboundPeers(int64_t time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /** Retrieve unbroadcast transactions from the mempool and reattempt sending to peers */ + void ReattemptInitialBroadcast(CScheduler& scheduler) const; private: int64_t m_stale_tip_check_time; //!< Next time to check for stale tip diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index 201406ce3b..3841d8687d 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -78,6 +78,10 @@ TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef t } if (relay) { + // the mempool tracks locally submitted transactions to make a + // best-effort of initial broadcast + node.mempool->AddUnbroadcastTx(hashTx); + RelayTransaction(hashTx, *node.connman); } diff --git a/src/random.cpp b/src/random.cpp index 765239e1f5..b408b1e13e 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -592,6 +592,11 @@ std::chrono::microseconds GetRandMicros(std::chrono::microseconds duration_max) return std::chrono::microseconds{GetRand(duration_max.count())}; } +std::chrono::milliseconds GetRandMillis(std::chrono::milliseconds duration_max) noexcept +{ + return std::chrono::milliseconds{GetRand(duration_max.count())}; +} + int GetRandInt(int nMax) noexcept { return GetRand(nMax); diff --git a/src/random.h b/src/random.h index 0c5008685b..690125079b 100644 --- a/src/random.h +++ b/src/random.h @@ -69,6 +69,7 @@ void GetRandBytes(unsigned char* buf, int num) noexcept; uint64_t GetRand(uint64_t nMax) noexcept; std::chrono::microseconds GetRandMicros(std::chrono::microseconds duration_max) noexcept; +std::chrono::milliseconds GetRandMillis(std::chrono::milliseconds duration_max) noexcept; int GetRandInt(int nMax) noexcept; uint256 GetRandHash() noexcept; diff --git a/src/rest.cpp b/src/rest.cpp index b389bf2028..5f99e26bad 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -607,7 +607,7 @@ static bool rest_blockhash_by_height(HTTPRequest* req, std::string height_str; const RetFormat rf = ParseDataFormat(height_str, str_uri_part); - int32_t blockheight; + int32_t blockheight = -1; // Initialization done only to prevent valgrind false positive, see https://github.com/bitcoin/bitcoin/pull/18785 if (!ParseInt32(height_str, &blockheight) || blockheight < 0) { return RESTERR(req, HTTP_BAD_REQUEST, "Invalid height: " + SanitizeString(height_str)); } diff --git a/src/span.h b/src/span.h index 664ea8d4c6..73b37e35f3 100644 --- a/src/span.h +++ b/src/span.h @@ -25,6 +25,23 @@ public: constexpr Span(C* data, std::ptrdiff_t size) noexcept : m_data(data), m_size(size) {} constexpr Span(C* data, C* end) noexcept : m_data(data), m_size(end - data) {} + /** Implicit conversion of spans between compatible types. + * + * Specifically, if a pointer to an array of type O can be implicitly converted to a pointer to an array of type + * C, then permit implicit conversion of Span<O> to Span<C>. This matches the behavior of the corresponding + * C++20 std::span constructor. + * + * For example this means that a Span<T> can be converted into a Span<const T>. + */ + template <typename O, typename std::enable_if<std::is_convertible<O (*)[], C (*)[]>::value, int>::type = 0> + constexpr Span(const Span<O>& other) noexcept : m_data(other.m_data), m_size(other.m_size) {} + + /** Default copy constructor. */ + constexpr Span(const Span&) noexcept = default; + + /** Default assignment operator. */ + Span& operator=(const Span& other) noexcept = default; + constexpr C* data() const noexcept { return m_data; } constexpr C* begin() const noexcept { return m_data; } constexpr C* end() const noexcept { return m_data + m_size; } @@ -44,6 +61,8 @@ public: friend constexpr bool operator<=(const Span& a, const Span& b) noexcept { return !(b < a); } friend constexpr bool operator>(const Span& a, const Span& b) noexcept { return (b < a); } friend constexpr bool operator>=(const Span& a, const Span& b) noexcept { return !(a < b); } + + template <typename O> friend class Span; }; /** Create a span to a container exposing data() and size(). diff --git a/src/test/fuzz/fees.cpp b/src/test/fuzz/fees.cpp index 090994263e..f29acace23 100644 --- a/src/test/fuzz/fees.cpp +++ b/src/test/fuzz/fees.cpp @@ -8,6 +8,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <util/fees.h> #include <cstdint> #include <string> @@ -23,4 +24,6 @@ void test_one_input(const std::vector<uint8_t>& buffer) const CAmount rounded_fee = fee_filter_rounder.round(current_minimum_fee); assert(MoneyRange(rounded_fee)); } + const FeeReason fee_reason = fuzzed_data_provider.PickValueInArray({FeeReason::NONE, FeeReason::HALF_ESTIMATE, FeeReason::FULL_ESTIMATE, FeeReason::DOUBLE_ESTIMATE, FeeReason::CONSERVATIVE, FeeReason::MEMPOOL_MIN, FeeReason::PAYTXFEE, FeeReason::FALLBACK, FeeReason::REQUIRED}); + (void)StringForFeeReason(fee_reason); } diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index 9dbf0fcc90..d5b5ec3c9a 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -24,8 +24,8 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> -#include <time.h> #include <uint256.h> +#include <util/check.h> #include <util/moneystr.h> #include <util/strencodings.h> #include <util/string.h> @@ -35,6 +35,7 @@ #include <cassert> #include <chrono> +#include <ctime> #include <limits> #include <set> #include <vector> @@ -287,8 +288,12 @@ void test_one_input(const std::vector<uint8_t>& buffer) try { const uint64_t deserialized_u64 = ReadCompactSize(stream); assert(u64 == deserialized_u64 && stream.empty()); + } catch (const std::ios_base::failure&) { } - catch (const std::ios_base::failure&) { - } + } + + try { + CHECK_NONFATAL(b); + } catch (const NonFatalCheckError&) { } } diff --git a/src/test/fuzz/kitchen_sink.cpp b/src/test/fuzz/kitchen_sink.cpp new file mode 100644 index 0000000000..af6dc71322 --- /dev/null +++ b/src/test/fuzz/kitchen_sink.cpp @@ -0,0 +1,25 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <rpc/util.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <util/error.h> + +#include <cstdint> +#include <vector> + +// The fuzzing kitchen sink: Fuzzing harness for functions that need to be +// fuzzed but a.) don't belong in any existing fuzzing harness file, and +// b.) are not important enough to warrant their own fuzzing harness file. +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + + const TransactionError transaction_error = fuzzed_data_provider.PickValueInArray<TransactionError>({TransactionError::OK, TransactionError::MISSING_INPUTS, TransactionError::ALREADY_IN_CHAIN, TransactionError::P2P_DISABLED, TransactionError::MEMPOOL_REJECTED, TransactionError::MEMPOOL_ERROR, TransactionError::INVALID_PSBT, TransactionError::PSBT_MISMATCH, TransactionError::SIGHASH_MISMATCH, TransactionError::MAX_FEE_EXCEEDED}); + (void)JSONRPCTransactionError(transaction_error); + (void)RPCErrorFromTransactionError(transaction_error); + (void)TransactionErrorString(transaction_error); +} diff --git a/src/test/fuzz/parse_hd_keypath.cpp b/src/test/fuzz/parse_hd_keypath.cpp index 9a23f4b2d4..f668ca8c48 100644 --- a/src/test/fuzz/parse_hd_keypath.cpp +++ b/src/test/fuzz/parse_hd_keypath.cpp @@ -2,12 +2,22 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> #include <util/bip32.h> +#include <cstdint> +#include <vector> + void test_one_input(const std::vector<uint8_t>& buffer) { const std::string keypath_str(buffer.begin(), buffer.end()); std::vector<uint32_t> keypath; (void)ParseHDKeypath(keypath_str, keypath); + + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + const std::vector<uint32_t> random_keypath = ConsumeRandomLengthIntegralVector<uint32_t>(fuzzed_data_provider); + (void)FormatHDKeypath(random_keypath); + (void)WriteHDKeypath(random_keypath); } diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index 3de0cf8db7..49bee0e81f 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -115,4 +115,8 @@ void test_one_input(const std::vector<uint8_t>& buffer) assert(data_stream.empty()); assert(deserialized_string == random_string_1); } + { + int64_t amount_out; + (void)ParseFixedPoint(random_string_1, fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 1024), &amount_out); + } } diff --git a/src/test/fuzz/system.cpp b/src/test/fuzz/system.cpp new file mode 100644 index 0000000000..7f378c2b13 --- /dev/null +++ b/src/test/fuzz/system.cpp @@ -0,0 +1,123 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <util/system.h> + +#include <cstdint> +#include <string> +#include <vector> + +namespace { +std::string GetArgumentName(const std::string& name) +{ + size_t idx = name.find('='); + if (idx == std::string::npos) { + idx = name.size(); + } + return name.substr(0, idx); +} +} // namespace + +void test_one_input(const std::vector<uint8_t>& buffer) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + ArgsManager args_manager{}; + + if (fuzzed_data_provider.ConsumeBool()) { + SetupHelpOptions(args_manager); + } + + while (fuzzed_data_provider.ConsumeBool()) { + switch (fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 7)) { + case 0: { + args_manager.SelectConfigNetwork(fuzzed_data_provider.ConsumeRandomLengthString(16)); + break; + } + case 1: { + args_manager.SoftSetArg(fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeRandomLengthString(16)); + break; + } + case 2: { + args_manager.ForceSetArg(fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeRandomLengthString(16)); + break; + } + case 3: { + args_manager.SoftSetBoolArg(fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeBool()); + break; + } + case 4: { + const OptionsCategory options_category = fuzzed_data_provider.PickValueInArray<OptionsCategory>({OptionsCategory::OPTIONS, OptionsCategory::CONNECTION, OptionsCategory::WALLET, OptionsCategory::WALLET_DEBUG_TEST, OptionsCategory::ZMQ, OptionsCategory::DEBUG_TEST, OptionsCategory::CHAINPARAMS, OptionsCategory::NODE_RELAY, OptionsCategory::BLOCK_CREATION, OptionsCategory::RPC, OptionsCategory::GUI, OptionsCategory::COMMANDS, OptionsCategory::REGISTER_COMMANDS, OptionsCategory::HIDDEN}); + // Avoid hitting: + // util/system.cpp:425: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. + const std::string argument_name = GetArgumentName(fuzzed_data_provider.ConsumeRandomLengthString(16)); + if (args_manager.GetArgFlags(argument_name) != nullopt) { + break; + } + args_manager.AddArg(argument_name, fuzzed_data_provider.ConsumeRandomLengthString(16), fuzzed_data_provider.ConsumeIntegral<unsigned int>(), options_category); + break; + } + case 5: { + // Avoid hitting: + // util/system.cpp:425: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. + const std::vector<std::string> names = ConsumeRandomLengthStringVector(fuzzed_data_provider); + std::vector<std::string> hidden_arguments; + for (const std::string& name : names) { + const std::string hidden_argument = GetArgumentName(name); + if (args_manager.GetArgFlags(hidden_argument) != nullopt) { + continue; + } + if (std::find(hidden_arguments.begin(), hidden_arguments.end(), hidden_argument) != hidden_arguments.end()) { + continue; + } + hidden_arguments.push_back(hidden_argument); + } + args_manager.AddHiddenArgs(hidden_arguments); + break; + } + case 6: { + args_manager.ClearArgs(); + break; + } + case 7: { + const std::vector<std::string> random_arguments = ConsumeRandomLengthStringVector(fuzzed_data_provider); + std::vector<const char*> argv; + argv.resize(random_arguments.size()); + for (const std::string& random_argument : random_arguments) { + argv.push_back(random_argument.c_str()); + } + try { + std::string error; + (void)args_manager.ParseParameters(argv.size(), argv.data(), error); + } catch (const std::logic_error&) { + } + break; + } + } + } + + const std::string s1 = fuzzed_data_provider.ConsumeRandomLengthString(16); + const std::string s2 = fuzzed_data_provider.ConsumeRandomLengthString(16); + const int64_t i64 = fuzzed_data_provider.ConsumeIntegral<int64_t>(); + const bool b = fuzzed_data_provider.ConsumeBool(); + + (void)args_manager.GetArg(s1, i64); + (void)args_manager.GetArg(s1, s2); + (void)args_manager.GetArgFlags(s1); + (void)args_manager.GetArgs(s1); + (void)args_manager.GetBoolArg(s1, b); + try { + (void)args_manager.GetChainName(); + } catch (const std::runtime_error&) { + } + (void)args_manager.GetHelpMessage(); + (void)args_manager.GetUnrecognizedSections(); + (void)args_manager.GetUnsuitableSectionOnlyArgs(); + (void)args_manager.IsArgNegated(s1); + (void)args_manager.IsArgSet(s1); + + (void)HelpRequested(args_manager); +} diff --git a/src/test/util/logging.cpp b/src/test/util/logging.cpp index fe2e69104b..65a64f2384 100644 --- a/src/test/util/logging.cpp +++ b/src/test/util/logging.cpp @@ -11,13 +11,13 @@ #include <stdexcept> -DebugLogHelper::DebugLogHelper(std::string message) - : m_message{std::move(message)} +DebugLogHelper::DebugLogHelper(std::string message, MatchFn match) + : m_message{std::move(message)}, m_match(std::move(match)) { m_print_connection = LogInstance().PushBackCallback( [this](const std::string& s) { if (m_found) return; - m_found = s.find(m_message) != std::string::npos; + m_found = s.find(m_message) != std::string::npos && m_match(&s); }); noui_test_redirect(); } @@ -26,7 +26,7 @@ void DebugLogHelper::check_found() { noui_reconnect(); LogInstance().DeleteCallback(m_print_connection); - if (!m_found) { + if (!m_found && m_match(nullptr)) { throw std::runtime_error(strprintf("'%s' not found in debug log\n", m_message)); } } diff --git a/src/test/util/logging.h b/src/test/util/logging.h index 45ec44173c..1fcf7ca305 100644 --- a/src/test/util/logging.h +++ b/src/test/util/logging.h @@ -17,10 +17,22 @@ class DebugLogHelper bool m_found{false}; std::list<std::function<void(const std::string&)>>::iterator m_print_connection; + //! Custom match checking function. + //! + //! Invoked with pointers to lines containing matching strings, and with + //! null if check_found() is called without any successful match. + //! + //! Can return true to enable default DebugLogHelper behavior of: + //! (1) ending search after first successful match, and + //! (2) raising an error in check_found if no match was found + //! Can return false to do the opposite in either case. + using MatchFn = std::function<bool(const std::string* line)>; + MatchFn m_match; + void check_found(); public: - DebugLogHelper(std::string message); + DebugLogHelper(std::string message, MatchFn match = [](const std::string*){ return true; }); ~DebugLogHelper() { check_found(); } }; diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 102e50dc5c..c5c0208d8f 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -417,6 +417,8 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) for (const CTxIn& txin : it->GetTx().vin) mapNextTx.erase(txin.prevout); + RemoveUnbroadcastTx(hash, true /* add logging because unchecked */ ); + if (vTxHashes.size() > 1) { vTxHashes[it->vTxHashesIdx] = std::move(vTxHashes.back()); vTxHashes[it->vTxHashesIdx].second->vTxHashesIdx = it->vTxHashesIdx; @@ -919,6 +921,15 @@ size_t CTxMemPool::DynamicMemoryUsage() const { return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 12 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(mapLinks) + memusage::DynamicUsage(vTxHashes) + cachedInnerUsage; } +void CTxMemPool::RemoveUnbroadcastTx(const uint256& txid, const bool unchecked) { + LOCK(cs); + + if (m_unbroadcast_txids.erase(txid)) + { + LogPrint(BCLog::MEMPOOL, "Removed %i from set of unbroadcast txns%s\n", txid.GetHex(), (unchecked ? " before confirmation that txn was sent out" : "")); + } +} + void CTxMemPool::RemoveStaged(setEntries &stage, bool updateDescendants, MemPoolRemovalReason reason) { AssertLockHeld(cs); UpdateForRemoveFromMempool(stage, updateDescendants); diff --git a/src/txmempool.h b/src/txmempool.h index 3dae0a04c7..4bee78b8d6 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -549,6 +549,9 @@ private: std::vector<indexed_transaction_set::const_iterator> GetSortedDepthAndScore() const EXCLUSIVE_LOCKS_REQUIRED(cs); + /** track locally submitted transactions to periodically retry initial broadcast */ + std::set<uint256> m_unbroadcast_txids GUARDED_BY(cs); + public: indirectmap<COutPoint, const CTransaction*> mapNextTx GUARDED_BY(cs); std::map<uint256, CAmount> mapDeltas; @@ -698,6 +701,21 @@ public: size_t DynamicMemoryUsage() const; + /** Adds a transaction to the unbroadcast set */ + void AddUnbroadcastTx(const uint256& txid) { + LOCK(cs); + m_unbroadcast_txids.insert(txid); + } + + /** Removes a transaction from the unbroadcast set */ + void RemoveUnbroadcastTx(const uint256& txid, const bool unchecked = false); + + /** Returns transactions in unbroadcast set */ + const std::set<uint256> GetUnbroadcastTxs() const { + LOCK(cs); + return m_unbroadcast_txids; + } + private: /** UpdateForDescendants is used by UpdateTransactionsFromBlock to update * the descendants for a single transaction that has been added to the diff --git a/src/util/system.cpp b/src/util/system.cpp index 0790ea0d48..2013b416db 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -17,7 +17,7 @@ #endif #ifndef WIN32 -// for posix_fallocate +// for posix_fallocate, in configure.ac we check if it is present after this #ifdef __linux__ #ifdef _POSIX_C_SOURCE @@ -1019,7 +1019,7 @@ void AllocateFileRange(FILE *file, unsigned int offset, unsigned int length) { } ftruncate(fileno(file), static_cast<off_t>(offset) + length); #else - #if defined(__linux__) + #if defined(HAVE_POSIX_FALLOCATE) // Version using posix_fallocate off_t nEndPos = (off_t)offset + length; if (0 == posix_fallocate(fileno(file), 0, nEndPos)) return; diff --git a/src/validation.cpp b/src/validation.cpp index 25975e3e31..d18803b342 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -4998,6 +4998,7 @@ bool LoadMempool(CTxMemPool& pool) int64_t expired = 0; int64_t failed = 0; int64_t already_there = 0; + int64_t unbroadcast = 0; int64_t nNow = GetTime(); try { @@ -5051,12 +5052,21 @@ bool LoadMempool(CTxMemPool& pool) for (const auto& i : mapDeltas) { pool.PrioritiseTransaction(i.first, i.second); } + + std::set<uint256> unbroadcast_txids; + file >> unbroadcast_txids; + unbroadcast = unbroadcast_txids.size(); + + for (const auto& txid : unbroadcast_txids) { + pool.AddUnbroadcastTx(txid); + } + } catch (const std::exception& e) { LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); return false; } - LogPrintf("Imported mempool transactions from disk: %i succeeded, %i failed, %i expired, %i already there\n", count, failed, expired, already_there); + LogPrintf("Imported mempool transactions from disk: %i succeeded, %i failed, %i expired, %i already there, %i waiting for initial broadcast\n", count, failed, expired, already_there, unbroadcast); return true; } @@ -5066,6 +5076,7 @@ bool DumpMempool(const CTxMemPool& pool) std::map<uint256, CAmount> mapDeltas; std::vector<TxMempoolInfo> vinfo; + std::set<uint256> unbroadcast_txids; static Mutex dump_mutex; LOCK(dump_mutex); @@ -5076,6 +5087,7 @@ bool DumpMempool(const CTxMemPool& pool) mapDeltas[i.first] = i.second; } vinfo = pool.infoAll(); + unbroadcast_txids = pool.GetUnbroadcastTxs(); } int64_t mid = GetTimeMicros(); @@ -5100,6 +5112,10 @@ bool DumpMempool(const CTxMemPool& pool) } file << mapDeltas; + + LogPrintf("Writing %d unbroadcast transactions to disk.\n", unbroadcast_txids.size()); + file << unbroadcast_txids; + if (!FileCommit(file.Get())) throw std::runtime_error("FileCommit failed"); file.fclose(); diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 2bdeb89f3c..1215956bb4 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -4,6 +4,7 @@ #include <wallet/wallet.h> +#include <future> #include <memory> #include <stdint.h> #include <vector> @@ -12,6 +13,7 @@ #include <node/context.h> #include <policy/policy.h> #include <rpc/server.h> +#include <test/util/logging.h> #include <test/util/setup_common.h> #include <validation.h> #include <wallet/coincontrol.h> @@ -26,6 +28,36 @@ extern UniValue importwallet(const JSONRPCRequest& request); BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) +static std::shared_ptr<CWallet> TestLoadWallet(interfaces::Chain& chain) +{ + std::string error; + std::vector<std::string> warnings; + auto wallet = CWallet::CreateWalletFromFile(chain, WalletLocation(""), error, warnings); + wallet->postInitProcess(); + return wallet; +} + +static void TestUnloadWallet(std::shared_ptr<CWallet>&& wallet) +{ + SyncWithValidationInterfaceQueue(); + wallet->m_chain_notifications_handler.reset(); + UnloadWallet(std::move(wallet)); +} + +static CMutableTransaction TestSimpleSpend(const CTransaction& from, uint32_t index, const CKey& key, const CScript& pubkey) +{ + CMutableTransaction mtx; + mtx.vout.push_back({from.vout[index].nValue - DEFAULT_TRANSACTION_MAXFEE, pubkey}); + mtx.vin.push_back({CTxIn{from.GetHash(), index}}); + FillableSigningProvider keystore; + keystore.AddKey(key); + std::map<COutPoint, Coin> coins; + coins[mtx.vin[0].prevout].out = from.vout[index]; + std::map<int, std::string> input_errors; + BOOST_CHECK(SignTransaction(mtx, &keystore, coins, SIGHASH_ALL, input_errors)); + return mtx; +} + static void AddKey(CWallet& wallet, const CKey& key) { auto spk_man = wallet.GetOrCreateLegacyScriptPubKeyMan(); @@ -658,4 +690,86 @@ BOOST_FIXTURE_TEST_CASE(wallet_descriptor_test, BasicTestingSetup) BOOST_CHECK_EXCEPTION(vr >> w_desc, std::ios_base::failure, malformed_descriptor); } +//! Test CreateWalletFromFile function and its behavior handling potential race +//! conditions if it's called the same time an incoming transaction shows up in +//! the mempool or a new block. +//! +//! It isn't possible for a unit test to totally verify there aren't race +//! conditions without hooking into the implementation more, so this test just +//! verifies that new transactions are detected during loading without any +//! notifications at all, to infer that timing of notifications shouldn't +//! matter. The test could be extended to cover other scenarios in the future. +BOOST_FIXTURE_TEST_CASE(CreateWalletFromFile, TestChain100Setup) +{ + // Create new wallet with known key and unload it. + auto chain = interfaces::MakeChain(m_node); + auto wallet = TestLoadWallet(*chain); + CKey key; + key.MakeNewKey(true); + AddKey(*wallet, key); + TestUnloadWallet(std::move(wallet)); + + // Add log hook to detect AddToWallet events from rescans, blockConnected, + // and transactionAddedToMempool notifications + int addtx_count = 0; + DebugLogHelper addtx_counter("[default wallet] AddToWallet", [&](const std::string* s) { + if (s) ++addtx_count; + return false; + }); + + bool rescan_completed = false; + DebugLogHelper rescan_check("[default wallet] Rescan completed", [&](const std::string* s) { + if (s) { + // For now, just assert that cs_main is being held during the + // rescan, ensuring that a new block couldn't be connected + // that the wallet would miss. After + // https://github.com/bitcoin/bitcoin/pull/16426 when cs_main is no + // longer held, the test can be extended to append a new block here + // and check it's handled correctly. + AssertLockHeld(::cs_main); + rescan_completed = true; + } + return false; + }); + + // Block the queue to prevent the wallet receiving blockConnected and + // transactionAddedToMempool notifications, and create block and mempool + // transactions paying to the wallet + std::promise<void> promise; + CallFunctionInValidationInterfaceQueue([&promise] { + promise.get_future().wait(); + }); + std::string error; + m_coinbase_txns.push_back(CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + auto block_tx = TestSimpleSpend(*m_coinbase_txns[0], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); + m_coinbase_txns.push_back(CreateAndProcessBlock({block_tx}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())).vtx[0]); + auto mempool_tx = TestSimpleSpend(*m_coinbase_txns[1], 0, coinbaseKey, GetScriptForRawPubKey(key.GetPubKey())); + BOOST_CHECK(chain->broadcastTransaction(MakeTransactionRef(mempool_tx), DEFAULT_TRANSACTION_MAXFEE, false, error)); + + // Reload wallet and make sure new transactions are detected despite events + // being blocked + wallet = TestLoadWallet(*chain); + BOOST_CHECK(rescan_completed); + BOOST_CHECK_EQUAL(addtx_count, 2); + unsigned int block_tx_time, mempool_tx_time; + { + LOCK(wallet->cs_wallet); + block_tx_time = wallet->mapWallet.at(block_tx.GetHash()).nTimeReceived; + mempool_tx_time = wallet->mapWallet.at(mempool_tx.GetHash()).nTimeReceived; + } + + // Unblock notification queue and make sure stale blockConnected and + // transactionAddedToMempool events are processed + promise.set_value(); + SyncWithValidationInterfaceQueue(); + BOOST_CHECK_EQUAL(addtx_count, 4); + { + LOCK(wallet->cs_wallet); + BOOST_CHECK_EQUAL(block_tx_time, wallet->mapWallet.at(block_tx.GetHash()).nTimeReceived); + BOOST_CHECK_EQUAL(mempool_tx_time, wallet->mapWallet.at(mempool_tx.GetHash()).nTimeReceived); + } + + TestUnloadWallet(std::move(wallet)); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index b6f25de64e..6d46841cee 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1993,7 +1993,8 @@ void CWallet::ResendWalletTransactions() // that these are our transactions. if (GetTime() < nNextResend || !fBroadcastTransactions) return; bool fFirst = (nNextResend == 0); - nNextResend = GetTime() + GetRand(30 * 60); + // resend 12-36 hours from now, ~1 day on average. + nNextResend = GetTime() + (12 * 60 * 60) + GetRand(24 * 60 * 60); if (fFirst) return; // Only do it if there's been a new block since last time |