diff options
Diffstat (limited to 'src')
251 files changed, 6645 insertions, 3384 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index caeaee511f..e73245daeb 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -134,6 +134,7 @@ BITCOIN_CORE_H = \ coins.h \ common/bloom.h \ common/run_command.h \ + common/url.h \ compat/assumptions.h \ compat/byteswap.h \ compat/compat.h \ @@ -176,6 +177,7 @@ BITCOIN_CORE_H = \ kernel/checks.h \ kernel/coinstats.h \ kernel/context.h \ + kernel/mempool_entry.h \ kernel/mempool_limits.h \ kernel/mempool_options.h \ kernel/mempool_persist.h \ @@ -198,6 +200,7 @@ BITCOIN_CORE_H = \ node/blockstorage.h \ node/caches.h \ node/chainstate.h \ + node/chainstatemanager_args.h \ node/coin.h \ node/connection_types.h \ node/context.h \ @@ -209,6 +212,7 @@ BITCOIN_CORE_H = \ node/minisketchwrapper.h \ node/psbt.h \ node/transaction.h \ + node/txreconciliation.h \ node/utxo_snapshot.h \ node/validation_cache_args.h \ noui.h \ @@ -255,7 +259,6 @@ BITCOIN_CORE_H = \ support/events.h \ support/lockedpool.h \ sync.h \ - threadinterrupt.h \ threadsafety.h \ timedata.h \ torcontrol.h \ @@ -294,6 +297,7 @@ BITCOIN_CORE_H = \ util/syserror.h \ util/system.h \ util/thread.h \ + util/threadinterrupt.h \ util/threadnames.h \ util/time.h \ util/tokenpipe.h \ @@ -301,7 +305,6 @@ BITCOIN_CORE_H = \ util/translation.h \ util/types.h \ util/ui_change_type.h \ - util/url.h \ util/vector.h \ validation.h \ validationinterface.h \ @@ -345,11 +348,7 @@ obj/build.h: FORCE "$(abs_top_srcdir)" libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h - -# server: shared between bitcoind and bitcoin-qt -# Contains code accessing mempool and chain state that is meant to be separated -# from wallet and gui code (see node/README.md). Shared code should go in -# libbitcoin_common or libbitcoin_util libraries, instead. +# node # libbitcoin_node_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) libbitcoin_node_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_node_a_SOURCES = \ @@ -384,6 +383,7 @@ libbitcoin_node_a_SOURCES = \ node/blockstorage.cpp \ node/caches.cpp \ node/chainstate.cpp \ + node/chainstatemanager_args.cpp \ node/coin.cpp \ node/connection_types.cpp \ node/context.cpp \ @@ -396,6 +396,8 @@ libbitcoin_node_a_SOURCES = \ node/minisketchwrapper.cpp \ node/psbt.cpp \ node/transaction.cpp \ + node/txreconciliation.cpp \ + node/utxo_snapshot.cpp \ node/validation_cache_args.cpp \ noui.cpp \ policy/fees.cpp \ @@ -438,7 +440,9 @@ endif if !ENABLE_WALLET libbitcoin_node_a_SOURCES += dummywallet.cpp endif +# +# zmq # if ENABLE_ZMQ libbitcoin_zmq_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(ZMQ_CFLAGS) libbitcoin_zmq_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) @@ -449,10 +453,9 @@ libbitcoin_zmq_a_SOURCES = \ zmq/zmqrpc.cpp \ zmq/zmqutil.cpp endif +# - -# wallet: shared between bitcoind and bitcoin-qt, but only linked -# when wallet enabled +# wallet # libbitcoin_wallet_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) $(BDB_CPPFLAGS) $(SQLITE_CFLAGS) libbitcoin_wallet_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_wallet_a_SOURCES = \ @@ -491,14 +494,17 @@ endif if USE_BDB libbitcoin_wallet_a_SOURCES += wallet/bdb.cpp wallet/salvage.cpp endif +# +# wallet tool # libbitcoin_wallet_tool_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libbitcoin_wallet_tool_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_wallet_tool_a_SOURCES = \ wallet/wallettool.cpp \ $(BITCOIN_CORE_H) +# -# crypto primitives library +# crypto # crypto_libbitcoin_crypto_base_la_CPPFLAGS = $(AM_CPPFLAGS) # Specify -static in both CXXFLAGS and LDFLAGS so libtool will only build a @@ -578,8 +584,9 @@ crypto_libbitcoin_crypto_arm_shani_la_CPPFLAGS = $(AM_CPPFLAGS) crypto_libbitcoin_crypto_arm_shani_la_CXXFLAGS += $(ARM_SHANI_CXXFLAGS) crypto_libbitcoin_crypto_arm_shani_la_CPPFLAGS += -DENABLE_ARM_SHANI crypto_libbitcoin_crypto_arm_shani_la_SOURCES = crypto/sha256_arm_shani.cpp +# -# consensus: shared between all executables that validate any consensus rules. +# consensus # libbitcoin_consensus_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_consensus_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_consensus_a_SOURCES = \ @@ -615,8 +622,9 @@ libbitcoin_consensus_a_SOURCES = \ util/strencodings.cpp \ util/strencodings.h \ version.h +# -# common: shared between bitcoind, and bitcoin-qt and non-server tools +# common # libbitcoin_common_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libbitcoin_common_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_common_a_SOURCES = \ @@ -625,6 +633,7 @@ libbitcoin_common_a_SOURCES = \ chainparams.cpp \ coins.cpp \ common/bloom.cpp \ + common/interfaces.cpp \ common/run_command.cpp \ compressor.cpp \ core_read.cpp \ @@ -656,24 +665,26 @@ libbitcoin_common_a_SOURCES = \ warnings.cpp \ $(BITCOIN_CORE_H) -# util: shared between all executables. -libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) +if USE_LIBEVENT +libbitcoin_common_a_CPPFLAGS += $(EVENT_CFLAGS) +libbitcoin_common_a_SOURCES += common/url.cpp +endif +# + +# util # +libbitcoin_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_util_a_SOURCES = \ support/lockedpool.cpp \ chainparamsbase.cpp \ clientversion.cpp \ fs.cpp \ - interfaces/echo.cpp \ - interfaces/handler.cpp \ - interfaces/init.cpp \ logging.cpp \ random.cpp \ randomenv.cpp \ rpc/request.cpp \ support/cleanse.cpp \ sync.cpp \ - threadinterrupt.cpp \ util/asmap.cpp \ util/bip32.cpp \ util/bytevectorhash.cpp \ @@ -691,6 +702,7 @@ libbitcoin_util_a_SOURCES = \ util/readwritefile.cpp \ util/settings.cpp \ util/thread.cpp \ + util/threadinterrupt.cpp \ util/threadnames.cpp \ util/serfloat.cpp \ util/spanparsing.cpp \ @@ -700,12 +712,9 @@ libbitcoin_util_a_SOURCES = \ util/time.cpp \ util/tokenpipe.cpp \ $(BITCOIN_CORE_H) +# -if USE_LIBEVENT -libbitcoin_util_a_SOURCES += util/url.cpp -endif - -# cli: shared between bitcoin-cli and bitcoin-qt +# cli # libbitcoin_cli_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) libbitcoin_cli_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_cli_a_SOURCES = \ @@ -766,6 +775,7 @@ endif bitcoin_cli_LDADD = \ $(LIBBITCOIN_CLI) \ $(LIBUNIVALUE) \ + $(LIBBITCOIN_COMMON) \ $(LIBBITCOIN_UTIL) \ $(LIBBITCOIN_CRYPTO) @@ -902,6 +912,7 @@ libbitcoinkernel_la_SOURCES = \ node/blockstorage.cpp \ node/chainstate.cpp \ node/interface_ui.cpp \ + node/utxo_snapshot.cpp \ policy/feerate.cpp \ policy/fees.cpp \ policy/packages.cpp \ @@ -925,7 +936,6 @@ libbitcoinkernel_la_SOURCES = \ support/cleanse.cpp \ support/lockedpool.cpp \ sync.cpp \ - threadinterrupt.cpp \ txdb.cpp \ txmempool.cpp \ uint256.cpp \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index e1e2066877..f1e4e706a1 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -32,6 +32,7 @@ bench_bench_bitcoin_SOURCES = \ bench/examples.cpp \ bench/gcs_filter.cpp \ bench/hashpadding.cpp \ + bench/load_external.cpp \ bench/lockedpool.cpp \ bench/logging.cpp \ bench/mempool_eviction.cpp \ @@ -79,6 +80,7 @@ if ENABLE_WALLET bench_bench_bitcoin_SOURCES += bench/coin_selection.cpp bench_bench_bitcoin_SOURCES += bench/wallet_balance.cpp bench_bench_bitcoin_SOURCES += bench/wallet_loading.cpp +bench_bench_bitcoin_SOURCES += bench/wallet_create_tx.cpp bench_bench_bitcoin_LDADD += $(BDB_LIBS) $(SQLITE_LIBS) endif diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 253f64d2c3..74c30f1caf 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -66,6 +66,7 @@ BITCOIN_TESTS =\ test/addrman_tests.cpp \ test/allocator_tests.cpp \ test/amount_tests.cpp \ + test/argsman_tests.cpp \ test/arith_uint256_tests.cpp \ test/banman_tests.cpp \ test/base32_tests.cpp \ @@ -77,6 +78,7 @@ BITCOIN_TESTS =\ test/blockencodings_tests.cpp \ test/blockfilter_index_tests.cpp \ test/blockfilter_tests.cpp \ + test/blockmanager_tests.cpp \ test/bloom_tests.cpp \ test/bswap_tests.cpp \ test/checkqueue_tests.cpp \ @@ -147,6 +149,7 @@ BITCOIN_TESTS =\ test/transaction_tests.cpp \ test/txindex_tests.cpp \ test/txpackage_tests.cpp \ + test/txreconciliation_tests.cpp \ test/txrequest_tests.cpp \ test/txvalidation_tests.cpp \ test/txvalidationcache_tests.cpp \ @@ -196,8 +199,6 @@ FUZZ_WALLET_SRC += \ endif # USE_SQLITE BITCOIN_TEST_SUITE += \ - wallet/test/util.cpp \ - wallet/test/util.h \ wallet/test/wallet_test_fixture.cpp \ wallet/test/wallet_test_fixture.h \ wallet/test/init_test_fixture.cpp \ @@ -374,8 +375,8 @@ endif if TARGET_WINDOWS else if ENABLE_BENCH - @echo "Running bench/bench_bitcoin (one iteration sanity check)..." - $(BENCH_BINARY) --sanity-check > /dev/null + @echo "Running bench/bench_bitcoin (one iteration sanity check, only high priority)..." + $(BENCH_BINARY) -sanity-check -priority-level=high > /dev/null endif endif $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check diff --git a/src/Makefile.test_fuzz.include b/src/Makefile.test_fuzz.include index b35d713d57..aa9c052750 100644 --- a/src/Makefile.test_fuzz.include +++ b/src/Makefile.test_fuzz.include @@ -11,7 +11,8 @@ TEST_FUZZ_H = \ test/fuzz/fuzz.h \ test/fuzz/FuzzedDataProvider.h \ test/fuzz/util.h \ - test/fuzz/util/mempool.h + test/fuzz/util/mempool.h \ + test/fuzz/util/net.h libtest_fuzz_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libtest_fuzz_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) @@ -19,4 +20,5 @@ libtest_fuzz_a_SOURCES = \ test/fuzz/fuzz.cpp \ test/fuzz/util.cpp \ test/fuzz/util/mempool.cpp \ + test/fuzz/util/net.cpp \ $(TEST_FUZZ_H) diff --git a/src/Makefile.test_util.include b/src/Makefile.test_util.include index ada789f1b0..a4e8b3f842 100644 --- a/src/Makefile.test_util.include +++ b/src/Makefile.test_util.include @@ -5,20 +5,24 @@ LIBTEST_UTIL=libtest_util.a EXTRA_LIBRARIES += \ - $(LIBTEST_UTIL) + $(LIBTEST_UTIL) TEST_UTIL_H = \ - test/util/blockfilter.h \ - test/util/chainstate.h \ - test/util/logging.h \ - test/util/mining.h \ - test/util/net.h \ - test/util/script.h \ - test/util/setup_common.h \ - test/util/str.h \ - test/util/transaction_utils.h \ - test/util/validation.h \ - test/util/wallet.h + test/util/blockfilter.h \ + test/util/chainstate.h \ + test/util/logging.h \ + test/util/mining.h \ + test/util/net.h \ + test/util/script.h \ + test/util/setup_common.h \ + test/util/str.h \ + test/util/transaction_utils.h \ + test/util/txmempool.h \ + test/util/validation.h + +if ENABLE_WALLET +TEST_UTIL_H += wallet/test/util.h +endif # ENABLE_WALLET libtest_util_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libtest_util_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) @@ -31,6 +35,11 @@ libtest_util_a_SOURCES = \ test/util/setup_common.cpp \ test/util/str.cpp \ test/util/transaction_utils.cpp \ - test/util/validation.cpp \ - test/util/wallet.cpp \ - $(TEST_UTIL_H) + test/util/txmempool.cpp \ + test/util/validation.cpp + +if ENABLE_WALLET +libtest_util_a_SOURCES += wallet/test/util.cpp +endif # ENABLE_WALLET + +libtest_util_a_SOURCES += $(TEST_UTIL_H) diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index f14d1f89b6..019b133345 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -133,7 +133,7 @@ static void AddrManAddThenGood(benchmark::Bench& bench) }); } -BENCHMARK(AddrManAdd); -BENCHMARK(AddrManSelect); -BENCHMARK(AddrManGetAddr); -BENCHMARK(AddrManAddThenGood); +BENCHMARK(AddrManAdd, benchmark::PriorityLevel::HIGH); +BENCHMARK(AddrManSelect, benchmark::PriorityLevel::HIGH); +BENCHMARK(AddrManGetAddr, benchmark::PriorityLevel::HIGH); +BENCHMARK(AddrManAddThenGood, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/base58.cpp b/src/bench/base58.cpp index 6f6b4e3bfa..3d08b7201b 100644 --- a/src/bench/base58.cpp +++ b/src/bench/base58.cpp @@ -50,6 +50,6 @@ static void Base58Decode(benchmark::Bench& bench) } -BENCHMARK(Base58Encode); -BENCHMARK(Base58CheckEncode); -BENCHMARK(Base58Decode); +BENCHMARK(Base58Encode, benchmark::PriorityLevel::HIGH); +BENCHMARK(Base58CheckEncode, benchmark::PriorityLevel::HIGH); +BENCHMARK(Base58Decode, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/bech32.cpp b/src/bench/bech32.cpp index 0f89a8e2b5..1a166e7081 100644 --- a/src/bench/bech32.cpp +++ b/src/bench/bech32.cpp @@ -32,5 +32,5 @@ static void Bech32Decode(benchmark::Bench& bench) } -BENCHMARK(Bech32Encode); -BENCHMARK(Bech32Decode); +BENCHMARK(Bech32Encode, benchmark::PriorityLevel::HIGH); +BENCHMARK(Bech32Decode, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/bench.cpp b/src/bench/bench.cpp index 26975bb59d..1a3a006286 100644 --- a/src/bench/bench.cpp +++ b/src/bench/bench.cpp @@ -6,6 +6,7 @@ #include <fs.h> #include <test/util/setup_common.h> +#include <util/string.h> #include <chrono> #include <fstream> @@ -41,18 +42,42 @@ void GenerateTemplateResults(const std::vector<ankerl::nanobench::Result>& bench } // namespace -benchmark::BenchRunner::BenchmarkMap& benchmark::BenchRunner::benchmarks() +namespace benchmark { + +// map a label to one or multiple priority levels +std::map<std::string, uint8_t> map_label_priority = { + {"high", PriorityLevel::HIGH}, + {"low", PriorityLevel::LOW}, + {"all", 0xff} +}; + +std::string ListPriorities() { - static std::map<std::string, BenchFunction> benchmarks_map; + using item_t = std::pair<std::string, uint8_t>; + auto sort_by_priority = [](item_t a, item_t b){ return a.second < b.second; }; + std::set<item_t, decltype(sort_by_priority)> sorted_priorities(map_label_priority.begin(), map_label_priority.end(), sort_by_priority); + return Join(sorted_priorities, ',', [](const auto& entry){ return entry.first; }); +} + +uint8_t StringToPriority(const std::string& str) +{ + auto it = map_label_priority.find(str); + if (it == map_label_priority.end()) throw std::runtime_error(strprintf("Unknown priority level %s", str)); + return it->second; +} + +BenchRunner::BenchmarkMap& BenchRunner::benchmarks() +{ + static BenchmarkMap benchmarks_map; return benchmarks_map; } -benchmark::BenchRunner::BenchRunner(std::string name, benchmark::BenchFunction func) +BenchRunner::BenchRunner(std::string name, BenchFunction func, PriorityLevel level) { - benchmarks().insert(std::make_pair(name, func)); + benchmarks().insert(std::make_pair(name, std::make_pair(func, level))); } -void benchmark::BenchRunner::RunAll(const Args& args) +void BenchRunner::RunAll(const Args& args) { std::regex reFilter(args.regex_filter); std::smatch baseMatch; @@ -62,13 +87,19 @@ void benchmark::BenchRunner::RunAll(const Args& args) } std::vector<ankerl::nanobench::Result> benchmarkResults; - for (const auto& p : benchmarks()) { - if (!std::regex_match(p.first, baseMatch, reFilter)) { + for (const auto& [name, bench_func] : benchmarks()) { + const auto& [func, priority_level] = bench_func; + + if (!(priority_level & args.priority)) { + continue; + } + + if (!std::regex_match(name, baseMatch, reFilter)) { continue; } if (args.is_list_only) { - std::cout << p.first << std::endl; + std::cout << name << std::endl; continue; } @@ -76,7 +107,7 @@ void benchmark::BenchRunner::RunAll(const Args& args) if (args.sanity_check) { bench.epochs(1).epochIterations(1); } - bench.name(p.first); + bench.name(name); if (args.min_time > 0ms) { // convert to nanos before dividing to reduce rounding errors std::chrono::nanoseconds min_time_ns = args.min_time; @@ -84,11 +115,11 @@ void benchmark::BenchRunner::RunAll(const Args& args) } if (args.asymptote.empty()) { - p.second(bench); + func(bench); } else { for (auto n : args.asymptote) { bench.complexityN(n); - p.second(bench); + func(bench); } std::cout << bench.complexityBigO() << std::endl; } @@ -103,3 +134,5 @@ void benchmark::BenchRunner::RunAll(const Args& args) "{{/result}}"); GenerateTemplateResults(benchmarkResults, args.output_json, ankerl::nanobench::templates::json()); } + +} // namespace benchmark diff --git a/src/bench/bench.h b/src/bench/bench.h index 17535e4e81..63e1bf67e2 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -41,6 +41,16 @@ using ankerl::nanobench::Bench; typedef std::function<void(Bench&)> BenchFunction; +enum PriorityLevel : uint8_t +{ + LOW = 1 << 0, + HIGH = 1 << 2, +}; + +// List priority labels, comma-separated and sorted by increasing priority +std::string ListPriorities(); +uint8_t StringToPriority(const std::string& str); + struct Args { bool is_list_only; bool sanity_check; @@ -49,22 +59,24 @@ struct Args { fs::path output_csv; fs::path output_json; std::string regex_filter; + uint8_t priority; }; class BenchRunner { - typedef std::map<std::string, BenchFunction> BenchmarkMap; + // maps from "name" -> (function, priority_level) + typedef std::map<std::string, std::pair<BenchFunction, PriorityLevel>> BenchmarkMap; static BenchmarkMap& benchmarks(); public: - BenchRunner(std::string name, BenchFunction func); + BenchRunner(std::string name, BenchFunction func, PriorityLevel level); static void RunAll(const Args& args); }; } // namespace benchmark -// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo); -#define BENCHMARK(n) \ - benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n); +// BENCHMARK(foo) expands to: benchmark::BenchRunner bench_11foo("foo", foo, priority_level); +#define BENCHMARK(n, priority_level) \ + benchmark::BenchRunner PASTE2(bench_, PASTE2(__LINE__, n))(STRINGIZE(n), n, priority_level); #endif // BITCOIN_BENCH_BENCH_H diff --git a/src/bench/bench_bitcoin.cpp b/src/bench/bench_bitcoin.cpp index 1bb4d34db9..1ac8db19fd 100644 --- a/src/bench/bench_bitcoin.cpp +++ b/src/bench/bench_bitcoin.cpp @@ -18,6 +18,8 @@ static const char* DEFAULT_BENCH_FILTER = ".*"; static constexpr int64_t DEFAULT_MIN_TIME_MS{10}; +/** Priority level default value, run "all" priority levels */ +static const std::string DEFAULT_PRIORITY{"all"}; static void SetupBenchArgs(ArgsManager& argsman) { @@ -30,6 +32,8 @@ static void SetupBenchArgs(ArgsManager& argsman) argsman.AddArg("-output-csv=<output.csv>", "Generate CSV file with the most important benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-output-json=<output.json>", "Generate JSON file with all benchmark results", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-sanity-check", "Run benchmarks for only one iteration", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-priority-level=<l1,l2,l3>", strprintf("Run benchmarks of one or multiple priority level(s) (%s), default: '%s'", + benchmark::ListPriorities(), DEFAULT_PRIORITY), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); } // parses a comma separated list like "10,20,30,50" @@ -45,6 +49,14 @@ static std::vector<double> parseAsymptote(const std::string& str) { return numbers; } +static uint8_t parsePriorityLevel(const std::string& str) { + uint8_t levels{0}; + for (const auto& level: SplitString(str, ',')) { + levels |= benchmark::StringToPriority(level); + } + return levels; +} + int main(int argc, char** argv) { ArgsManager argsman; @@ -106,16 +118,22 @@ int main(int argc, char** argv) return EXIT_SUCCESS; } - benchmark::Args args; - args.asymptote = parseAsymptote(argsman.GetArg("-asymptote", "")); - args.is_list_only = argsman.GetBoolArg("-list", false); - args.min_time = std::chrono::milliseconds(argsman.GetIntArg("-min-time", DEFAULT_MIN_TIME_MS)); - args.output_csv = argsman.GetPathArg("-output-csv"); - args.output_json = argsman.GetPathArg("-output-json"); - args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER); - args.sanity_check = argsman.GetBoolArg("-sanity-check", false); + try { + benchmark::Args args; + args.asymptote = parseAsymptote(argsman.GetArg("-asymptote", "")); + args.is_list_only = argsman.GetBoolArg("-list", false); + args.min_time = std::chrono::milliseconds(argsman.GetIntArg("-min-time", DEFAULT_MIN_TIME_MS)); + args.output_csv = argsman.GetPathArg("-output-csv"); + args.output_json = argsman.GetPathArg("-output-json"); + args.regex_filter = argsman.GetArg("-filter", DEFAULT_BENCH_FILTER); + args.sanity_check = argsman.GetBoolArg("-sanity-check", false); + args.priority = parsePriorityLevel(argsman.GetArg("-priority-level", DEFAULT_PRIORITY)); - benchmark::BenchRunner::RunAll(args); + benchmark::BenchRunner::RunAll(args); - return EXIT_SUCCESS; + return EXIT_SUCCESS; + } catch (const std::exception& e) { + tfm::format(std::cerr, "Error: %s\n", e.what()); + return EXIT_FAILURE; + } } diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 4ed5397330..69258377d5 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -8,7 +8,6 @@ #include <test/util/mining.h> #include <test/util/script.h> #include <test/util/setup_common.h> -#include <test/util/wallet.h> #include <txmempool.h> #include <validation.h> @@ -47,4 +46,4 @@ static void AssembleBlock(benchmark::Bench& bench) }); } -BENCHMARK(AssembleBlock); +BENCHMARK(AssembleBlock, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/ccoins_caching.cpp b/src/bench/ccoins_caching.cpp index 22dfb7aa5b..5d55ed9332 100644 --- a/src/bench/ccoins_caching.cpp +++ b/src/bench/ccoins_caching.cpp @@ -51,4 +51,4 @@ static void CCoinsCaching(benchmark::Bench& bench) ECC_Stop(); } -BENCHMARK(CCoinsCaching); +BENCHMARK(CCoinsCaching, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/chacha20.cpp b/src/bench/chacha20.cpp index a6f4eec4ca..9584dd58bb 100644 --- a/src/bench/chacha20.cpp +++ b/src/bench/chacha20.cpp @@ -39,6 +39,6 @@ static void CHACHA20_1MB(benchmark::Bench& bench) CHACHA20(bench, BUFFER_SIZE_LARGE); } -BENCHMARK(CHACHA20_64BYTES); -BENCHMARK(CHACHA20_256BYTES); -BENCHMARK(CHACHA20_1MB); +BENCHMARK(CHACHA20_64BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_256BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_1MB, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/chacha_poly_aead.cpp b/src/bench/chacha_poly_aead.cpp index e994279a4d..15b3a4f310 100644 --- a/src/bench/chacha_poly_aead.cpp +++ b/src/bench/chacha_poly_aead.cpp @@ -115,12 +115,12 @@ static void HASH_1MB(benchmark::Bench& bench) HASH(bench, BUFFER_SIZE_LARGE); } -BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ONLY_ENCRYPT); -BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ONLY_ENCRYPT); -BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ONLY_ENCRYPT); -BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ENCRYPT_DECRYPT); -BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ENCRYPT_DECRYPT); -BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ENCRYPT_DECRYPT); -BENCHMARK(HASH_64BYTES); -BENCHMARK(HASH_256BYTES); -BENCHMARK(HASH_1MB); +BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ONLY_ENCRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_POLY1305_AEAD_64BYTES_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_POLY1305_AEAD_256BYTES_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(CHACHA20_POLY1305_AEAD_1MB_ENCRYPT_DECRYPT, benchmark::PriorityLevel::HIGH); +BENCHMARK(HASH_64BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(HASH_256BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(HASH_1MB, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 53aa470042..747279c161 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -50,5 +50,5 @@ static void DeserializeAndCheckBlockTest(benchmark::Bench& bench) }); } -BENCHMARK(DeserializeBlockTest); -BENCHMARK(DeserializeAndCheckBlockTest); +BENCHMARK(DeserializeBlockTest, benchmark::PriorityLevel::HIGH); +BENCHMARK(DeserializeAndCheckBlockTest, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index 602081fb9b..8517c9fee2 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -70,4 +70,4 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) queue.StopWorkerThreads(); ECC_Stop(); } -BENCHMARK(CCheckQueueSpeedPrevectorJob); +BENCHMARK(CCheckQueueSpeedPrevectorJob, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 6ada28115e..53d89039a7 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -121,5 +121,5 @@ static void BnBExhaustion(benchmark::Bench& bench) }); } -BENCHMARK(CoinSelection); -BENCHMARK(BnBExhaustion); +BENCHMARK(CoinSelection, benchmark::PriorityLevel::HIGH); +BENCHMARK(BnBExhaustion, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/crypto_hash.cpp b/src/bench/crypto_hash.cpp index d17ec503e7..162b02f344 100644 --- a/src/bench/crypto_hash.cpp +++ b/src/bench/crypto_hash.cpp @@ -150,19 +150,19 @@ static void MuHashPrecompute(benchmark::Bench& bench) }); } -BENCHMARK(RIPEMD160); -BENCHMARK(SHA1); -BENCHMARK(SHA256); -BENCHMARK(SHA512); -BENCHMARK(SHA3_256_1M); - -BENCHMARK(SHA256_32b); -BENCHMARK(SipHash_32b); -BENCHMARK(SHA256D64_1024); -BENCHMARK(FastRandom_32bit); -BENCHMARK(FastRandom_1bit); - -BENCHMARK(MuHash); -BENCHMARK(MuHashMul); -BENCHMARK(MuHashDiv); -BENCHMARK(MuHashPrecompute); +BENCHMARK(RIPEMD160, benchmark::PriorityLevel::HIGH); +BENCHMARK(SHA1, benchmark::PriorityLevel::HIGH); +BENCHMARK(SHA256, benchmark::PriorityLevel::HIGH); +BENCHMARK(SHA512, benchmark::PriorityLevel::HIGH); +BENCHMARK(SHA3_256_1M, benchmark::PriorityLevel::HIGH); + +BENCHMARK(SHA256_32b, benchmark::PriorityLevel::HIGH); +BENCHMARK(SipHash_32b, benchmark::PriorityLevel::HIGH); +BENCHMARK(SHA256D64_1024, benchmark::PriorityLevel::HIGH); +BENCHMARK(FastRandom_32bit, benchmark::PriorityLevel::HIGH); +BENCHMARK(FastRandom_1bit, benchmark::PriorityLevel::HIGH); + +BENCHMARK(MuHash, benchmark::PriorityLevel::HIGH); +BENCHMARK(MuHashMul, benchmark::PriorityLevel::HIGH); +BENCHMARK(MuHashDiv, benchmark::PriorityLevel::HIGH); +BENCHMARK(MuHashPrecompute, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/descriptors.cpp b/src/bench/descriptors.cpp index 5c868a8573..972a6ff953 100644 --- a/src/bench/descriptors.cpp +++ b/src/bench/descriptors.cpp @@ -4,6 +4,7 @@ #include <bench/bench.h> #include <key.h> +#include <pubkey.h> #include <script/descriptor.h> #include <script/standard.h> @@ -12,6 +13,9 @@ static void ExpandDescriptor(benchmark::Bench& bench) { + const ECCVerifyHandle verify_handle; + ECC_Start(); + const auto desc_str = "sh(wsh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232)))"; const std::pair<int64_t, int64_t> range = {0, 1000}; FlatSigningProvider provider; @@ -25,6 +29,8 @@ static void ExpandDescriptor(benchmark::Bench& bench) assert(success); } }); + + ECC_Stop(); } -BENCHMARK(ExpandDescriptor); +BENCHMARK(ExpandDescriptor, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/duplicate_inputs.cpp b/src/bench/duplicate_inputs.cpp index 02a2e689b1..559854ff48 100644 --- a/src/bench/duplicate_inputs.cpp +++ b/src/bench/duplicate_inputs.cpp @@ -62,4 +62,4 @@ static void DuplicateInputs(benchmark::Bench& bench) }); } -BENCHMARK(DuplicateInputs); +BENCHMARK(DuplicateInputs, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/examples.cpp b/src/bench/examples.cpp index 72a9922e94..abef69cc42 100644 --- a/src/bench/examples.cpp +++ b/src/bench/examples.cpp @@ -18,4 +18,4 @@ static void Trig(benchmark::Bench& bench) }); } -BENCHMARK(Trig); +BENCHMARK(Trig, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/gcs_filter.cpp b/src/bench/gcs_filter.cpp index 80babb213b..b795ebff39 100644 --- a/src/bench/gcs_filter.cpp +++ b/src/bench/gcs_filter.cpp @@ -81,8 +81,8 @@ static void GCSFilterMatch(benchmark::Bench& bench) filter.Match(GCSFilter::Element()); }); } -BENCHMARK(GCSBlockFilterGetHash); -BENCHMARK(GCSFilterConstruct); -BENCHMARK(GCSFilterDecode); -BENCHMARK(GCSFilterDecodeSkipCheck); -BENCHMARK(GCSFilterMatch); +BENCHMARK(GCSBlockFilterGetHash, benchmark::PriorityLevel::HIGH); +BENCHMARK(GCSFilterConstruct, benchmark::PriorityLevel::HIGH); +BENCHMARK(GCSFilterDecode, benchmark::PriorityLevel::HIGH); +BENCHMARK(GCSFilterDecodeSkipCheck, benchmark::PriorityLevel::HIGH); +BENCHMARK(GCSFilterMatch, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/hashpadding.cpp b/src/bench/hashpadding.cpp index 753c8c2881..ac5aeebe51 100644 --- a/src/bench/hashpadding.cpp +++ b/src/bench/hashpadding.cpp @@ -26,7 +26,7 @@ static void PrePadded(benchmark::Bench& bench) }); } -BENCHMARK(PrePadded); +BENCHMARK(PrePadded, benchmark::PriorityLevel::HIGH); static void RegularPadded(benchmark::Bench& bench) { @@ -44,4 +44,4 @@ static void RegularPadded(benchmark::Bench& bench) }); } -BENCHMARK(RegularPadded); +BENCHMARK(RegularPadded, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/load_external.cpp b/src/bench/load_external.cpp new file mode 100644 index 0000000000..be01b2a483 --- /dev/null +++ b/src/bench/load_external.cpp @@ -0,0 +1,63 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <bench/data.h> +#include <chainparams.h> +#include <test/util/setup_common.h> +#include <validation.h> + +/** + * The LoadExternalBlockFile() function is used during -reindex and -loadblock. + * + * Create a test file that's similar to a datadir/blocks/blk?????.dat file, + * It contains around 134 copies of the same block (typical size of real block files). + * For each block in the file, LoadExternalBlockFile() won't find its parent, + * and so will skip the block. (In the real system, it will re-read the block + * from disk later when it encounters its parent.) + * + * This benchmark measures the performance of deserializing the block (or just + * its header, beginning with PR 16981). + */ +static void LoadExternalBlockFile(benchmark::Bench& bench) +{ + const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(CBaseChainParams::MAIN)}; + + // Create a single block as in the blocks files (magic bytes, block size, + // block data) as a stream object. + const fs::path blkfile{testing_setup.get()->m_path_root / "blk.dat"}; + CDataStream ss(SER_DISK, 0); + auto params{testing_setup->m_node.chainman->GetParams()}; + ss << params.MessageStart(); + ss << static_cast<uint32_t>(benchmark::data::block413567.size()); + // We can't use the streaming serialization (ss << benchmark::data::block413567) + // because that first writes a compact size. + ss.write(MakeByteSpan(benchmark::data::block413567)); + + // Create the test file. + { + // "wb+" is "binary, O_RDWR | O_CREAT | O_TRUNC". + FILE* file{fsbridge::fopen(blkfile, "wb+")}; + // Make the test block file about 128 MB in length. + for (size_t i = 0; i < node::MAX_BLOCKFILE_SIZE / ss.size(); ++i) { + if (fwrite(ss.data(), 1, ss.size(), file) != ss.size()) { + throw std::runtime_error("write to test file failed\n"); + } + } + fclose(file); + } + + Chainstate& chainstate{testing_setup->m_node.chainman->ActiveChainstate()}; + std::multimap<uint256, FlatFilePos> blocks_with_unknown_parent; + FlatFilePos pos; + bench.run([&] { + // "rb" is "binary, O_RDONLY", positioned to the start of the file. + // The file will be closed by LoadExternalBlockFile(). + FILE* file{fsbridge::fopen(blkfile, "rb")}; + chainstate.LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent); + }); + fs::remove(blkfile); +} + +BENCHMARK(LoadExternalBlockFile, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/lockedpool.cpp b/src/bench/lockedpool.cpp index b6d8824aba..ac8262654c 100644 --- a/src/bench/lockedpool.cpp +++ b/src/bench/lockedpool.cpp @@ -39,4 +39,4 @@ static void BenchLockedPool(benchmark::Bench& bench) addr.clear(); } -BENCHMARK(BenchLockedPool); +BENCHMARK(BenchLockedPool, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/logging.cpp b/src/bench/logging.cpp index d28777df9e..49a9e59893 100644 --- a/src/bench/logging.cpp +++ b/src/bench/logging.cpp @@ -41,8 +41,8 @@ static void LoggingNoFile(benchmark::Bench& bench) }); } -BENCHMARK(LoggingYoThreadNames); -BENCHMARK(LoggingNoThreadNames); -BENCHMARK(LoggingYoCategory); -BENCHMARK(LoggingNoCategory); -BENCHMARK(LoggingNoFile); +BENCHMARK(LoggingYoThreadNames, benchmark::PriorityLevel::HIGH); +BENCHMARK(LoggingNoThreadNames, benchmark::PriorityLevel::HIGH); +BENCHMARK(LoggingYoCategory, benchmark::PriorityLevel::HIGH); +BENCHMARK(LoggingNoCategory, benchmark::PriorityLevel::HIGH); +BENCHMARK(LoggingNoFile, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index 60d991fab9..2b133f58f4 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> +#include <kernel/mempool_entry.h> #include <policy/policy.h> #include <test/util/setup_common.h> #include <txmempool.h> @@ -132,4 +133,4 @@ static void MempoolEviction(benchmark::Bench& bench) }); } -BENCHMARK(MempoolEviction); +BENCHMARK(MempoolEviction, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp index 725a6f8f5b..67564508d9 100644 --- a/src/bench/mempool_stress.cpp +++ b/src/bench/mempool_stress.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> +#include <kernel/mempool_entry.h> #include <policy/policy.h> #include <test/util/setup_common.h> #include <txmempool.h> @@ -114,5 +115,5 @@ static void MempoolCheck(benchmark::Bench& bench) }); } -BENCHMARK(ComplexMemPool); -BENCHMARK(MempoolCheck); +BENCHMARK(ComplexMemPool, benchmark::PriorityLevel::HIGH); +BENCHMARK(MempoolCheck, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/merkle_root.cpp b/src/bench/merkle_root.cpp index ba6629b9f0..4140d67bc7 100644 --- a/src/bench/merkle_root.cpp +++ b/src/bench/merkle_root.cpp @@ -23,4 +23,4 @@ static void MerkleRoot(benchmark::Bench& bench) }); } -BENCHMARK(MerkleRoot); +BENCHMARK(MerkleRoot, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/peer_eviction.cpp b/src/bench/peer_eviction.cpp index f05f5e8f64..d83342be93 100644 --- a/src/bench/peer_eviction.cpp +++ b/src/bench/peer_eviction.cpp @@ -40,7 +40,7 @@ static void EvictionProtection0Networks250Candidates(benchmark::Bench& bench) { EvictionProtectionCommon( bench, - 250 /* num_candidates */, + /*num_candidates=*/250, [](NodeEvictionCandidate& c) { c.m_connected = std::chrono::seconds{c.id}; c.m_network = NET_IPV4; @@ -51,7 +51,7 @@ static void EvictionProtection1Networks250Candidates(benchmark::Bench& bench) { EvictionProtectionCommon( bench, - 250 /* num_candidates */, + /*num_candidates=*/250, [](NodeEvictionCandidate& c) { c.m_connected = std::chrono::seconds{c.id}; c.m_is_local = false; @@ -67,7 +67,7 @@ static void EvictionProtection2Networks250Candidates(benchmark::Bench& bench) { EvictionProtectionCommon( bench, - 250 /* num_candidates */, + /*num_candidates=*/250, [](NodeEvictionCandidate& c) { c.m_connected = std::chrono::seconds{c.id}; c.m_is_local = false; @@ -85,7 +85,7 @@ static void EvictionProtection3Networks050Candidates(benchmark::Bench& bench) { EvictionProtectionCommon( bench, - 50 /* num_candidates */, + /*num_candidates=*/50, [](NodeEvictionCandidate& c) { c.m_connected = std::chrono::seconds{c.id}; c.m_is_local = (c.id == 28 || c.id == 47); // 2 localhost @@ -103,7 +103,7 @@ static void EvictionProtection3Networks100Candidates(benchmark::Bench& bench) { EvictionProtectionCommon( bench, - 100 /* num_candidates */, + /*num_candidates=*/100, [](NodeEvictionCandidate& c) { c.m_connected = std::chrono::seconds{c.id}; c.m_is_local = (c.id >= 55 && c.id < 60); // 5 localhost @@ -121,7 +121,7 @@ static void EvictionProtection3Networks250Candidates(benchmark::Bench& bench) { EvictionProtectionCommon( bench, - 250 /* num_candidates */, + /*num_candidates=*/250, [](NodeEvictionCandidate& c) { c.m_connected = std::chrono::seconds{c.id}; c.m_is_local = (c.id >= 140 && c.id < 160); // 20 localhost @@ -141,15 +141,15 @@ static void EvictionProtection3Networks250Candidates(benchmark::Bench& bench) // - 250 candidates is the number of peers reported by operators of busy nodes // No disadvantaged networks, with 250 eviction candidates. -BENCHMARK(EvictionProtection0Networks250Candidates); +BENCHMARK(EvictionProtection0Networks250Candidates, benchmark::PriorityLevel::HIGH); // 1 disadvantaged network (Tor) with 250 eviction candidates. -BENCHMARK(EvictionProtection1Networks250Candidates); +BENCHMARK(EvictionProtection1Networks250Candidates, benchmark::PriorityLevel::HIGH); // 2 disadvantaged networks (I2P, Tor) with 250 eviction candidates. -BENCHMARK(EvictionProtection2Networks250Candidates); +BENCHMARK(EvictionProtection2Networks250Candidates, benchmark::PriorityLevel::HIGH); // 3 disadvantaged networks (I2P/localhost/Tor) with 50/100/250 eviction candidates. -BENCHMARK(EvictionProtection3Networks050Candidates); -BENCHMARK(EvictionProtection3Networks100Candidates); -BENCHMARK(EvictionProtection3Networks250Candidates); +BENCHMARK(EvictionProtection3Networks050Candidates, benchmark::PriorityLevel::HIGH); +BENCHMARK(EvictionProtection3Networks100Candidates, benchmark::PriorityLevel::HIGH); +BENCHMARK(EvictionProtection3Networks250Candidates, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/poly1305.cpp b/src/bench/poly1305.cpp index cdef97c0ea..ad5a72ffde 100644 --- a/src/bench/poly1305.cpp +++ b/src/bench/poly1305.cpp @@ -36,6 +36,6 @@ static void POLY1305_1MB(benchmark::Bench& bench) POLY1305(bench, BUFFER_SIZE_LARGE); } -BENCHMARK(POLY1305_64BYTES); -BENCHMARK(POLY1305_256BYTES); -BENCHMARK(POLY1305_1MB); +BENCHMARK(POLY1305_64BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(POLY1305_256BYTES, benchmark::PriorityLevel::HIGH); +BENCHMARK(POLY1305_1MB, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/prevector.cpp b/src/bench/prevector.cpp index b3688bab1b..9e2e7d11c4 100644 --- a/src/bench/prevector.cpp +++ b/src/bench/prevector.cpp @@ -85,12 +85,12 @@ static void PrevectorDeserialize(benchmark::Bench& bench) { \ Prevector##name<nontrivial_t>(bench); \ } \ - BENCHMARK(Prevector##name##Nontrivial); \ + BENCHMARK(Prevector##name##Nontrivial, benchmark::PriorityLevel::HIGH); \ static void Prevector##name##Trivial(benchmark::Bench& bench) \ { \ Prevector##name<trivial_t>(bench); \ } \ - BENCHMARK(Prevector##name##Trivial); + BENCHMARK(Prevector##name##Trivial, benchmark::PriorityLevel::HIGH); PREVECTOR_TEST(Clear) PREVECTOR_TEST(Destructor) diff --git a/src/bench/rollingbloom.cpp b/src/bench/rollingbloom.cpp index 8f05e3bad0..865d99f9e8 100644 --- a/src/bench/rollingbloom.cpp +++ b/src/bench/rollingbloom.cpp @@ -32,5 +32,5 @@ static void RollingBloomReset(benchmark::Bench& bench) }); } -BENCHMARK(RollingBloom); -BENCHMARK(RollingBloomReset); +BENCHMARK(RollingBloom, benchmark::PriorityLevel::HIGH); +BENCHMARK(RollingBloomReset, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/rpc_blockchain.cpp b/src/bench/rpc_blockchain.cpp index e6fc8d21f4..5a178f308a 100644 --- a/src/bench/rpc_blockchain.cpp +++ b/src/bench/rpc_blockchain.cpp @@ -45,7 +45,7 @@ static void BlockToJsonVerbose(benchmark::Bench& bench) }); } -BENCHMARK(BlockToJsonVerbose); +BENCHMARK(BlockToJsonVerbose, benchmark::PriorityLevel::HIGH); static void BlockToJsonVerboseWrite(benchmark::Bench& bench) { @@ -57,4 +57,4 @@ static void BlockToJsonVerboseWrite(benchmark::Bench& bench) }); } -BENCHMARK(BlockToJsonVerboseWrite); +BENCHMARK(BlockToJsonVerboseWrite, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/rpc_mempool.cpp b/src/bench/rpc_mempool.cpp index 0e6fdae3d7..6bac6419e5 100644 --- a/src/bench/rpc_mempool.cpp +++ b/src/bench/rpc_mempool.cpp @@ -4,6 +4,7 @@ #include <bench/bench.h> #include <chainparamsbase.h> +#include <kernel/mempool_entry.h> #include <rpc/mempool.h> #include <test/util/setup_common.h> #include <txmempool.h> @@ -40,4 +41,4 @@ static void RpcMempool(benchmark::Bench& bench) }); } -BENCHMARK(RpcMempool); +BENCHMARK(RpcMempool, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/strencodings.cpp b/src/bench/strencodings.cpp index 69b3a83cbf..16d14278a7 100644 --- a/src/bench/strencodings.cpp +++ b/src/bench/strencodings.cpp @@ -15,4 +15,4 @@ static void HexStrBench(benchmark::Bench& bench) }); } -BENCHMARK(HexStrBench); +BENCHMARK(HexStrBench, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/util_time.cpp b/src/bench/util_time.cpp index afc733482e..8256b5f7ba 100644 --- a/src/bench/util_time.cpp +++ b/src/bench/util_time.cpp @@ -36,7 +36,7 @@ static void BenchTimeMillisSys(benchmark::Bench& bench) }); } -BENCHMARK(BenchTimeDeprecated); -BENCHMARK(BenchTimeMillis); -BENCHMARK(BenchTimeMillisSys); -BENCHMARK(BenchTimeMock); +BENCHMARK(BenchTimeDeprecated, benchmark::PriorityLevel::HIGH); +BENCHMARK(BenchTimeMillis, benchmark::PriorityLevel::HIGH); +BENCHMARK(BenchTimeMillisSys, benchmark::PriorityLevel::HIGH); +BENCHMARK(BenchTimeMock, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index 8e4708f260..f0e9db8ba1 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -96,5 +96,5 @@ static void VerifyNestedIfScript(benchmark::Bench& bench) }); } -BENCHMARK(VerifyScriptBench); -BENCHMARK(VerifyNestedIfScript); +BENCHMARK(VerifyScriptBench, benchmark::PriorityLevel::HIGH); +BENCHMARK(VerifyNestedIfScript, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index a5dd2f3bb1..5a52774a8e 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -7,7 +7,7 @@ #include <node/context.h> #include <test/util/mining.h> #include <test/util/setup_common.h> -#include <test/util/wallet.h> +#include <wallet/test/util.h> #include <validationinterface.h> #include <wallet/receive.h> #include <wallet/wallet.h> @@ -20,6 +20,8 @@ using wallet::DBErrors; using wallet::GetBalance; using wallet::WALLET_FLAG_DESCRIPTORS; +const std::string ADDRESS_BCRT1_UNSPENDABLE = "bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj"; + static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const bool add_mine) { const auto test_setup = MakeNoLogFileContext<const TestingSetup>(); @@ -57,7 +59,7 @@ static void WalletBalanceClean(benchmark::Bench& bench) { WalletBalance(bench, / static void WalletBalanceMine(benchmark::Bench& bench) { WalletBalance(bench, /*set_dirty=*/false, /*add_mine=*/true); } static void WalletBalanceWatch(benchmark::Bench& bench) { WalletBalance(bench, /*set_dirty=*/false, /*add_mine=*/false); } -BENCHMARK(WalletBalanceDirty); -BENCHMARK(WalletBalanceClean); -BENCHMARK(WalletBalanceMine); -BENCHMARK(WalletBalanceWatch); +BENCHMARK(WalletBalanceDirty, benchmark::PriorityLevel::HIGH); +BENCHMARK(WalletBalanceClean, benchmark::PriorityLevel::HIGH); +BENCHMARK(WalletBalanceMine, benchmark::PriorityLevel::HIGH); +BENCHMARK(WalletBalanceWatch, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/wallet_create_tx.cpp b/src/bench/wallet_create_tx.cpp new file mode 100644 index 0000000000..3cbf2c9008 --- /dev/null +++ b/src/bench/wallet_create_tx.cpp @@ -0,0 +1,143 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <chainparams.h> +#include <wallet/coincontrol.h> +#include <consensus/merkle.h> +#include <kernel/chain.h> +#include <node/context.h> +#include <test/util/setup_common.h> +#include <validation.h> +#include <wallet/spend.h> +#include <wallet/test/util.h> +#include <wallet/wallet.h> + +using wallet::CWallet; +using wallet::CreateMockWalletDatabase; +using wallet::DBErrors; +using wallet::WALLET_FLAG_DESCRIPTORS; + +struct TipBlock +{ + uint256 prev_block_hash; + int64_t prev_block_time; + int tip_height; +}; + +TipBlock getTip(const CChainParams& params, const node::NodeContext& context) +{ + auto tip = WITH_LOCK(::cs_main, return context.chainman->ActiveTip()); + return (tip) ? TipBlock{tip->GetBlockHash(), tip->GetBlockTime(), tip->nHeight} : + TipBlock{params.GenesisBlock().GetHash(), params.GenesisBlock().GetBlockTime(), 0}; +} + +void generateFakeBlock(const CChainParams& params, + const node::NodeContext& context, + CWallet& wallet, + const CScript& coinbase_out_script) +{ + TipBlock tip{getTip(params, context)}; + + // Create block + CBlock block; + CMutableTransaction coinbase_tx; + coinbase_tx.vin.resize(1); + coinbase_tx.vin[0].prevout.SetNull(); + coinbase_tx.vout.resize(2); + coinbase_tx.vout[0].scriptPubKey = coinbase_out_script; + coinbase_tx.vout[0].nValue = 49 * COIN; + coinbase_tx.vin[0].scriptSig = CScript() << ++tip.tip_height << OP_0; + coinbase_tx.vout[1].scriptPubKey = coinbase_out_script; // extra output + coinbase_tx.vout[1].nValue = 1 * COIN; + block.vtx = {MakeTransactionRef(std::move(coinbase_tx))}; + + block.nVersion = VERSIONBITS_LAST_OLD_BLOCK_VERSION; + block.hashPrevBlock = tip.prev_block_hash; + block.hashMerkleRoot = BlockMerkleRoot(block); + block.nTime = ++tip.prev_block_time; + block.nBits = params.GenesisBlock().nBits; + block.nNonce = 0; + + { + LOCK(::cs_main); + // Add it to the index + CBlockIndex* pindex{context.chainman->m_blockman.AddToBlockIndex(block, context.chainman->m_best_header)}; + // add it to the chain + context.chainman->ActiveChain().SetTip(*pindex); + } + + // notify wallet + const auto& pindex = WITH_LOCK(::cs_main, return context.chainman->ActiveChain().Tip()); + wallet.blockConnected(kernel::MakeBlockInfo(pindex, &block)); +} + +struct PreSelectInputs { + // How many coins from the wallet the process should select + int num_of_internal_inputs; + // future: this could have external inputs as well. +}; + +static void WalletCreateTx(benchmark::Bench& bench, const OutputType output_type, bool allow_other_inputs, std::optional<PreSelectInputs> preset_inputs) +{ + const auto test_setup = MakeNoLogFileContext<const TestingSetup>(); + + CWallet wallet{test_setup->m_node.chain.get(), "", gArgs, CreateMockWalletDatabase()}; + { + LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet.SetupDescriptorScriptPubKeyMans(); + if (wallet.LoadWallet() != DBErrors::LOAD_OK) assert(false); + } + + // Generate destinations + CScript dest = GetScriptForDestination(getNewDestination(wallet, output_type)); + + // Generate chain; each coinbase will have two outputs to fill-up the wallet + const auto& params = Params(); + unsigned int chain_size = 5000; // 5k blocks means 10k UTXO for the wallet (minus 200 due COINBASE_MATURITY) + for (unsigned int i = 0; i < chain_size; ++i) { + generateFakeBlock(params, test_setup->m_node, wallet, dest); + } + + // Check available balance + auto bal = wallet::GetAvailableBalance(wallet); // Cache + assert(bal == 50 * COIN * (chain_size - COINBASE_MATURITY)); + + wallet::CCoinControl coin_control; + coin_control.m_allow_other_inputs = allow_other_inputs; + + CAmount target = 0; + if (preset_inputs) { + // Select inputs, each has 49 BTC + wallet::CoinFilterParams filter_coins; + filter_coins.max_count = preset_inputs->num_of_internal_inputs; + const auto& res = WITH_LOCK(wallet.cs_wallet, + return wallet::AvailableCoins(wallet, /*coinControl=*/nullptr, /*feerate=*/std::nullopt, filter_coins)); + for (int i=0; i < preset_inputs->num_of_internal_inputs; i++) { + const auto& coin{res.coins.at(output_type)[i]}; + target += coin.txout.nValue; + coin_control.Select(coin.outpoint); + } + } + + // If automatic coin selection is enabled, add the value of another UTXO to the target + if (coin_control.m_allow_other_inputs) target += 50 * COIN; + std::vector<wallet::CRecipient> recipients = {{dest, target, true}}; + + bench.epochIterations(5).run([&] { + LOCK(wallet.cs_wallet); + const auto& tx_res = CreateTransaction(wallet, recipients, -1, coin_control); + assert(tx_res); + }); +} + +static void WalletCreateTxUseOnlyPresetInputs(benchmark::Bench& bench) { WalletCreateTx(bench, OutputType::BECH32, /*allow_other_inputs=*/false, + {{/*num_of_internal_inputs=*/4}}); } + +static void WalletCreateTxUsePresetInputsAndCoinSelection(benchmark::Bench& bench) { WalletCreateTx(bench, OutputType::BECH32, /*allow_other_inputs=*/true, + {{/*num_of_internal_inputs=*/4}}); } + +BENCHMARK(WalletCreateTxUseOnlyPresetInputs, benchmark::PriorityLevel::LOW) +BENCHMARK(WalletCreateTxUsePresetInputsAndCoinSelection, benchmark::PriorityLevel::LOW) diff --git a/src/bench/wallet_loading.cpp b/src/bench/wallet_loading.cpp index 27e4dd015d..2f7dc53b0c 100644 --- a/src/bench/wallet_loading.cpp +++ b/src/bench/wallet_loading.cpp @@ -7,7 +7,7 @@ #include <node/context.h> #include <test/util/mining.h> #include <test/util/setup_common.h> -#include <test/util/wallet.h> +#include <wallet/test/util.h> #include <util/translation.h> #include <validationinterface.h> #include <wallet/context.h> @@ -52,30 +52,6 @@ static void AddTx(CWallet& wallet) wallet.AddToWallet(MakeTransactionRef(mtx), TxStateInactive{}); } -static std::unique_ptr<WalletDatabase> DuplicateMockDatabase(WalletDatabase& database, DatabaseOptions& options) -{ - auto new_database = CreateMockWalletDatabase(options); - - // Get a cursor to the original database - auto batch = database.MakeBatch(); - batch->StartCursor(); - - // Get a batch for the new database - auto new_batch = new_database->MakeBatch(); - - // Read all records from the original database and write them to the new one - while (true) { - CDataStream key(SER_DISK, CLIENT_VERSION); - CDataStream value(SER_DISK, CLIENT_VERSION); - bool complete; - batch->ReadAtCursor(key, value, complete); - if (complete) break; - new_batch->Write(key, value); - } - - return new_database; -} - static void WalletLoading(benchmark::Bench& bench, bool legacy_wallet) { const auto test_setup = MakeNoLogFileContext<TestingSetup>(); @@ -118,10 +94,10 @@ static void WalletLoading(benchmark::Bench& bench, bool legacy_wallet) #ifdef USE_BDB static void WalletLoadingLegacy(benchmark::Bench& bench) { WalletLoading(bench, /*legacy_wallet=*/true); } -BENCHMARK(WalletLoadingLegacy); +BENCHMARK(WalletLoadingLegacy, benchmark::PriorityLevel::HIGH); #endif #ifdef USE_SQLITE static void WalletLoadingDescriptors(benchmark::Bench& bench) { WalletLoading(bench, /*legacy_wallet=*/false); } -BENCHMARK(WalletLoadingDescriptors); +BENCHMARK(WalletLoadingDescriptors, benchmark::PriorityLevel::HIGH); #endif diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index e64e2202ba..55fa3116ac 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -9,6 +9,7 @@ #include <chainparamsbase.h> #include <clientversion.h> +#include <common/url.h> #include <compat/compat.h> #include <compat/stdin.h> #include <policy/feerate.h> @@ -21,7 +22,6 @@ #include <util/strencodings.h> #include <util/system.h> #include <util/translation.h> -#include <util/url.h> #include <algorithm> #include <chrono> @@ -82,7 +82,7 @@ static void SetupCliArgs(ArgsManager& argsman) DEFAULT_NBLOCKS, DEFAULT_MAX_TRIES), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-addrinfo", "Get the number of addresses known to the node, per network and total, after filtering for quality and recency. The total number of addresses known to the node may be higher.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-getinfo", "Get general information from the remote server. Note that unlike server-side RPC calls, the results of -getinfo is the result of multiple non-atomic requests. Some entries in the result may represent results from different states (e.g. wallet balance may be as of a different block from the chain state reported)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-getinfo", "Get general information from the remote server. Note that unlike server-side RPC calls, the output of -getinfo is the result of multiple non-atomic requests. Some entries in the output may represent results from different states (e.g. wallet balance may be as of a different block from the chain state reported)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-netinfo", "Get network peer connection information from the remote server. An optional integer argument from 0 to 4 can be passed for different peers listings (default: 0). Pass \"help\" for detailed help documentation.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); SetupChainParamsBaseOptions(argsman); @@ -642,7 +642,7 @@ public: " send Time since last message sent to the peer, in seconds\n" " recv Time since last message received from the peer, in seconds\n" " txn Time since last novel transaction received from the peer and accepted into our mempool, in minutes\n" - " \"*\" - the peer requested we not relay transactions to it (relaytxes is false)\n" + " \"*\" - whether we relay transactions to this peer (relaytxes is false)\n" " blk Time since last novel block passing initial validity checks received from the peer, in minutes\n" " hb High-bandwidth BIP152 compact block relay\n" " \".\" (to) - we selected the peer as a high-bandwidth peer\n" diff --git a/src/bitcoin-util.cpp b/src/bitcoin-util.cpp index fb184c0486..ad908de3e1 100644 --- a/src/bitcoin-util.cpp +++ b/src/bitcoin-util.cpp @@ -82,13 +82,12 @@ static int AppInitUtil(ArgsManager& args, int argc, char* argv[]) return CONTINUE_EXECUTION; } -static void grind_task(uint32_t nBits, CBlockHeader& header_orig, uint32_t offset, uint32_t step, std::atomic<bool>& found) +static void grind_task(uint32_t nBits, CBlockHeader header, uint32_t offset, uint32_t step, std::atomic<bool>& found, uint32_t& proposed_nonce) { arith_uint256 target; bool neg, over; target.SetCompact(nBits, &neg, &over); if (target == 0 || neg || over) return; - CBlockHeader header = header_orig; // working copy header.nNonce = offset; uint32_t finish = std::numeric_limits<uint32_t>::max() - step; @@ -99,7 +98,7 @@ static void grind_task(uint32_t nBits, CBlockHeader& header_orig, uint32_t offse do { if (UintToArith256(header.GetHash()) <= target) { if (!found.exchange(true)) { - header_orig.nNonce = header.nNonce; + proposed_nonce = header.nNonce; } return; } @@ -123,16 +122,19 @@ static int Grind(const std::vector<std::string>& args, std::string& strPrint) uint32_t nBits = header.nBits; std::atomic<bool> found{false}; + uint32_t proposed_nonce{}; std::vector<std::thread> threads; int n_tasks = std::max(1u, std::thread::hardware_concurrency()); for (int i = 0; i < n_tasks; ++i) { - threads.emplace_back( grind_task, nBits, std::ref(header), i, n_tasks, std::ref(found) ); + threads.emplace_back(grind_task, nBits, header, i, n_tasks, std::ref(found), std::ref(proposed_nonce)); } for (auto& t : threads) { t.join(); } - if (!found) { + if (found) { + header.nNonce = proposed_nonce; + } else { strPrint = "Could not satisfy difficulty target"; return EXIT_FAILURE; } diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index d556300ee2..78c1a2060c 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -9,6 +9,7 @@ #include <chainparams.h> #include <chainparamsbase.h> #include <clientversion.h> +#include <common/url.h> #include <compat/compat.h> #include <interfaces/init.h> #include <key.h> @@ -17,7 +18,6 @@ #include <tinyformat.h> #include <util/system.h> #include <util/translation.h> -#include <util/url.h> #include <wallet/wallettool.h> #include <exception> diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index 9f81640ddb..d8d4e34e47 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -9,6 +9,7 @@ #include <chainparams.h> #include <clientversion.h> +#include <common/url.h> #include <compat/compat.h> #include <init.h> #include <interfaces/chain.h> @@ -25,7 +26,6 @@ #include <util/threadnames.h> #include <util/tokenpipe.h> #include <util/translation.h> -#include <util/url.h> #include <any> #include <functional> diff --git a/src/chain.h b/src/chain.h index 2d3b084b9b..e052b074a7 100644 --- a/src/chain.h +++ b/src/chain.h @@ -213,10 +213,6 @@ public: //! (memory only) Maximum nTime in the chain up to and including this block. unsigned int nTimeMax{0}; - CBlockIndex() - { - } - explicit CBlockIndex(const CBlockHeader& block) : nVersion{block.nVersion}, hashMerkleRoot{block.hashMerkleRoot}, @@ -355,6 +351,24 @@ public: //! Efficiently find an ancestor of this block. CBlockIndex* GetAncestor(int height); const CBlockIndex* GetAncestor(int height) const; + + CBlockIndex() = default; + ~CBlockIndex() = default; + +protected: + //! CBlockIndex should not allow public copy construction because equality + //! comparison via pointer is very common throughout the codebase, making + //! use of copy a footgun. Also, use of copies do not have the benefit + //! of simplifying lifetime considerations due to attributes like pprev and + //! pskip, which are at risk of becoming dangling pointers in a copied + //! instance. + //! + //! We declare these protected instead of simply deleting them so that + //! CDiskBlockIndex can reuse copy construction. + CBlockIndex(const CBlockIndex&) = default; + CBlockIndex& operator=(const CBlockIndex&) = delete; + CBlockIndex(CBlockIndex&&) = delete; + CBlockIndex& operator=(CBlockIndex&&) = delete; }; arith_uint256 GetBlockProof(const CBlockIndex& block); diff --git a/src/checkqueue.h b/src/checkqueue.h index bead6f0c6f..c4a64444e9 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -199,11 +199,12 @@ public: WITH_LOCK(m_mutex, m_request_stop = false); } + bool HasThreads() const { return !m_worker_threads.empty(); } + ~CCheckQueue() { assert(m_worker_threads.empty()); } - }; /** diff --git a/src/common/interfaces.cpp b/src/common/interfaces.cpp new file mode 100644 index 0000000000..b9b4f5dded --- /dev/null +++ b/src/common/interfaces.cpp @@ -0,0 +1,53 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <interfaces/echo.h> +#include <interfaces/handler.h> + +#include <boost/signals2/connection.hpp> +#include <memory> +#include <utility> + +namespace common { +namespace { +class CleanupHandler : public interfaces::Handler +{ +public: + explicit CleanupHandler(std::function<void()> cleanup) : m_cleanup(std::move(cleanup)) {} + ~CleanupHandler() override { if (!m_cleanup) return; m_cleanup(); m_cleanup = nullptr; } + void disconnect() override { if (!m_cleanup) return; m_cleanup(); m_cleanup = nullptr; } + std::function<void()> m_cleanup; +}; + +class SignalHandler : public interfaces::Handler +{ +public: + explicit SignalHandler(boost::signals2::connection connection) : m_connection(std::move(connection)) {} + + void disconnect() override { m_connection.disconnect(); } + + boost::signals2::scoped_connection m_connection; +}; + +class EchoImpl : public interfaces::Echo +{ +public: + std::string echo(const std::string& echo) override { return echo; } +}; +} // namespace +} // namespace common + +namespace interfaces { +std::unique_ptr<Handler> MakeCleanupHandler(std::function<void()> cleanup) +{ + return std::make_unique<common::CleanupHandler>(std::move(cleanup)); +} + +std::unique_ptr<Handler> MakeSignalHandler(boost::signals2::connection connection) +{ + return std::make_unique<common::SignalHandler>(std::move(connection)); +} + +std::unique_ptr<Echo> MakeEcho() { return std::make_unique<common::EchoImpl>(); } +} // namespace interfaces diff --git a/src/util/url.cpp b/src/common/url.cpp index ea9323e666..5200d55096 100644 --- a/src/util/url.cpp +++ b/src/common/url.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <util/url.h> +#include <common/url.h> #include <event2/http.h> diff --git a/src/util/url.h b/src/common/url.h index 5a7b11fa04..7bbd8b60de 100644 --- a/src/util/url.h +++ b/src/common/url.h @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_UTIL_URL_H -#define BITCOIN_UTIL_URL_H +#ifndef BITCOIN_COMMON_URL_H +#define BITCOIN_COMMON_URL_H #include <string> @@ -11,4 +11,4 @@ using UrlDecodeFn = std::string(const std::string& url_encoded); UrlDecodeFn urlDecode; extern UrlDecodeFn* const URL_DECODE; -#endif // BITCOIN_UTIL_URL_H +#endif // BITCOIN_COMMON_URL_H diff --git a/src/compat/compat.h b/src/compat/compat.h index a8e5552c0a..cc37797577 100644 --- a/src/compat/compat.h +++ b/src/compat/compat.h @@ -109,14 +109,6 @@ typedef char* sockopt_arg_type; #define USE_POLL #endif -bool static inline IsSelectableSocket(const SOCKET& s) { -#if defined(USE_POLL) || defined(WIN32) - return true; -#else - return (s < FD_SETSIZE); -#endif -} - // MSG_NOSIGNAL is not available on some platforms, if it doesn't exist define it as 0 #if !defined(MSG_NOSIGNAL) #define MSG_NOSIGNAL 0 diff --git a/src/crypto/sha512.cpp b/src/crypto/sha512.cpp index 85a7bbcb53..59b79609dd 100644 --- a/src/crypto/sha512.cpp +++ b/src/crypto/sha512.cpp @@ -30,7 +30,7 @@ void inline Round(uint64_t a, uint64_t b, uint64_t c, uint64_t& d, uint64_t e, u h = t1 + t2; } -/** Initialize SHA-256 state. */ +/** Initialize SHA-512 state. */ void inline Initialize(uint64_t* s) { s[0] = 0x6a09e667f3bcc908ull; diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index 4dbc839941..7f45e35aef 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -128,7 +128,7 @@ static leveldb::Options GetOptions(size_t nCacheSize) } CDBWrapper::CDBWrapper(const fs::path& path, size_t nCacheSize, bool fMemory, bool fWipe, bool obfuscate) - : m_name{fs::PathToString(path.stem())} + : m_name{fs::PathToString(path.stem())}, m_path{path}, m_is_memory{fMemory} { penv = nullptr; readoptions.verify_checksums = true; diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 665eaa0e98..1052da01d5 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -39,6 +39,10 @@ public: class CDBWrapper; +namespace dbwrapper { + using leveldb::DestroyDB; +} + /** These should be considered an implementation detail of the specific database. */ namespace dbwrapper_private { @@ -219,6 +223,12 @@ private: std::vector<unsigned char> CreateObfuscateKey() const; + //! path to filesystem storage + const fs::path m_path; + + //! whether or not the database resides in memory + bool m_is_memory; + public: /** * @param[in] path Location in the filesystem where leveldb data will be stored. @@ -268,6 +278,14 @@ public: return WriteBatch(batch, fSync); } + //! @returns filesystem path to the on-disk data. + std::optional<fs::path> StoragePath() { + if (m_is_memory) { + return {}; + } + return m_path; + } + template <typename K> bool Exists(const K& key) const { diff --git a/src/external_signer.cpp b/src/external_signer.cpp index 0e582629f7..f255834830 100644 --- a/src/external_signer.cpp +++ b/src/external_signer.cpp @@ -81,6 +81,9 @@ bool ExternalSigner::SignTransaction(PartiallySignedTransaction& psbtx, std::str for (const auto& entry : input.hd_keypaths) { if (parsed_m_fingerprint == MakeUCharSpan(entry.second.fingerprint)) return true; } + for (const auto& entry : input.m_tap_bip32_paths) { + if (parsed_m_fingerprint == MakeUCharSpan(entry.second.second.fingerprint)) return true; + } return false; }; diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 1a19555f76..6f84d5c83b 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -192,19 +192,16 @@ std::string RequestMethodString(HTTPRequest::RequestMethod m) switch (m) { case HTTPRequest::GET: return "GET"; - break; case HTTPRequest::POST: return "POST"; - break; case HTTPRequest::HEAD: return "HEAD"; - break; case HTTPRequest::PUT: return "PUT"; - break; - default: + case HTTPRequest::UNKNOWN: return "unknown"; - } + } // no default case, so the compiler can warn about missing cases + assert(false); } /** HTTP request callback */ @@ -626,19 +623,14 @@ HTTPRequest::RequestMethod HTTPRequest::GetRequestMethod() const switch (evhttp_request_get_command(req)) { case EVHTTP_REQ_GET: return GET; - break; case EVHTTP_REQ_POST: return POST; - break; case EVHTTP_REQ_HEAD: return HEAD; - break; case EVHTTP_REQ_PUT: return PUT; - break; default: return UNKNOWN; - break; } } diff --git a/src/i2p.cpp b/src/i2p.cpp index 28be8009dc..d54486ecff 100644 --- a/src/i2p.cpp +++ b/src/i2p.cpp @@ -18,6 +18,7 @@ #include <util/spanparsing.h> #include <util/strencodings.h> #include <util/system.h> +#include <util/threadinterrupt.h> #include <chrono> #include <memory> @@ -9,8 +9,8 @@ #include <fs.h> #include <netaddress.h> #include <sync.h> -#include <threadinterrupt.h> #include <util/sock.h> +#include <util/threadinterrupt.h> #include <memory> #include <optional> diff --git a/src/index/base.h b/src/index/base.h index 349178a535..54c59f7557 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -7,7 +7,7 @@ #include <dbwrapper.h> #include <interfaces/chain.h> -#include <threadinterrupt.h> +#include <util/threadinterrupt.h> #include <validationinterface.h> #include <string> diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index d3559b1b75..271e5bb1f6 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -144,17 +144,13 @@ bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block) } } - // TODO: Deduplicate BIP30 related code - bool is_bip30_block{(block.height == 91722 && block.hash == uint256S("0x00000000000271a2dc26e7667f8419f2e15416dc6955e5a6c6cdf3f2574dd08e")) || - (block.height == 91812 && block.hash == uint256S("0x00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f"))}; - // Add the new utxos created from the block assert(block.data); for (size_t i = 0; i < block.data->vtx.size(); ++i) { const auto& tx{block.data->vtx.at(i)}; // Skip duplicate txid coinbase transactions (BIP30). - if (is_bip30_block && tx->IsCoinBase()) { + if (IsBIP30Unspendable(*pindex) && tx->IsCoinBase()) { m_total_unspendable_amount += block_subsidy; m_total_unspendables_bip30 += block_subsidy; continue; diff --git a/src/init.cpp b/src/init.cpp index 25b40c6c6e..f31a45aa9f 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -40,11 +40,13 @@ #include <node/blockstorage.h> #include <node/caches.h> #include <node/chainstate.h> +#include <node/chainstatemanager_args.h> #include <node/context.h> #include <node/interface_ui.h> #include <node/mempool_args.h> #include <node/mempool_persist_args.h> #include <node/miner.h> +#include <node/txreconciliation.h> #include <node/validation_cache_args.h> #include <policy/feerate.h> #include <policy/fees.h> @@ -65,7 +67,6 @@ #include <torcontrol.h> #include <txdb.h> #include <txmempool.h> -#include <txorphanage.h> #include <util/asmap.h> #include <util/check.h> #include <util/moneystr.h> @@ -476,6 +477,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-onlynet=<net>", "Make automatic outbound connections only to network <net> (" + Join(GetNetworkNames(), ", ") + "). Inbound and manual connections are not affected by this option. It can be specified multiple times to allow multiple networks.", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerbloomfilters", strprintf("Support filtering of blocks and transaction with bloom filters (default: %u)", DEFAULT_PEERBLOOMFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-peerblockfilters", strprintf("Serve compact block filters to peers per BIP 157 (default: %u)", DEFAULT_PEERBLOCKFILTERS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-txreconciliation", strprintf("Enable transaction reconciliations per BIP 330 (default: %d)", DEFAULT_TXRECONCILIATION_ENABLE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::CONNECTION); // TODO: remove the sentence "Nodes not using ... incoming connections." once the changes from // https://github.com/bitcoin/bitcoin/pull/23542 have become widespread. argsman.AddArg("-port=<port>", strprintf("Listen for connections on <port>. Nodes not using the default ports (default: %u, testnet: %u, signet: %u, regtest: %u) are unlikely to get incoming connections. Not relevant for I2P (see doc/i2p.md).", defaultChainParams->GetDefaultPort(), testnetChainParams->GetDefaultPort(), signetChainParams->GetDefaultPort(), regtestChainParams->GetDefaultPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::CONNECTION); @@ -552,7 +554,10 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-capturemessages", "Capture all P2P messages to disk", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-mocktime=<n>", "Replace actual time with " + UNIX_EPOCH_TIME + " (default: 0)", ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-maxsigcachesize=<n>", strprintf("Limit sum of signature cache and script execution cache sizes to <n> MiB (default: %u)", DEFAULT_MAX_SIG_CACHE_BYTES >> 20), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); - argsman.AddArg("-maxtipage=<n>", strprintf("Maximum tip age in seconds to consider node in initial block download (default: %u)", DEFAULT_MAX_TIP_AGE), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); + argsman.AddArg("-maxtipage=<n>", + strprintf("Maximum tip age in seconds to consider node in initial block download (default: %u)", + Ticks<std::chrono::seconds>(DEFAULT_MAX_TIP_AGE)), + ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-printpriority", strprintf("Log transaction fee rate in " + CURRENCY_UNIT + "/kvB when mining blocks (default: %u)", DEFAULT_PRINTPRIORITY), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::DEBUG_TEST); argsman.AddArg("-uacomment=<cmt>", "Append comment to the user agent string", ArgsManager::ALLOW_ANY, OptionsCategory::DEBUG_TEST); @@ -928,21 +933,6 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb init::SetLoggingCategories(args); init::SetLoggingLevel(args); - fCheckBlockIndex = args.GetBoolArg("-checkblockindex", chainparams.DefaultConsistencyChecks()); - fCheckpointsEnabled = args.GetBoolArg("-checkpoints", DEFAULT_CHECKPOINTS_ENABLED); - - hashAssumeValid = uint256S(args.GetArg("-assumevalid", chainparams.GetConsensus().defaultAssumeValid.GetHex())); - - if (args.IsArgSet("-minimumchainwork")) { - const std::string minChainWorkStr = args.GetArg("-minimumchainwork", ""); - if (!IsHexNumber(minChainWorkStr)) { - return InitError(strprintf(Untranslated("Invalid non-hex (%s) minimum chain work value specified"), minChainWorkStr)); - } - nMinimumChainWork = UintToArith256(uint256S(minChainWorkStr)); - } else { - nMinimumChainWork = UintToArith256(chainparams.GetConsensus().nMinimumChainWork); - } - // block pruning; get the amount of disk space (in MiB) to allot for block & undo files int64_t nPruneArg = args.GetIntArg("-prune", 0); if (nPruneArg < 0) { @@ -993,8 +983,6 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb if (args.GetIntArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) > 1) return InitError(Untranslated("Unknown rpcserialversion requested.")); - nMaxTipAge = args.GetIntArg("-maxtipage", DEFAULT_MAX_TIP_AGE); - if (args.GetBoolArg("-reindex-chainstate", false)) { // indexes that must be deactivated to prevent index corruption, see #24630 if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) { @@ -1042,6 +1030,16 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb } #endif // USE_SYSCALL_SANDBOX + // Also report errors from parsing before daemonization + { + ChainstateManager::Options chainman_opts_dummy{ + .chainparams = chainparams, + }; + if (const auto error{ApplyArgsManOptions(args, chainman_opts_dummy)}) { + return InitError(*error); + } + } + return true; } @@ -1144,7 +1142,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) LogPrintf("Script verification uses %d additional threads\n", script_threads); if (script_threads >= 1) { - g_parallel_script_checks = true; StartScriptCheckWorkerThreads(script_threads); } @@ -1255,6 +1252,51 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // as they would never get updated. if (!ignores_incoming_txs) node.fee_estimator = std::make_unique<CBlockPolicyEstimator>(FeeestPath(args)); + // Check port numbers + for (const std::string port_option : { + "-port", + "-rpcport", + }) { + if (args.IsArgSet(port_option)) { + const std::string port = args.GetArg(port_option, ""); + uint16_t n; + if (!ParseUInt16(port, &n) || n == 0) { + return InitError(InvalidPortErrMsg(port_option, port)); + } + } + } + + for (const std::string port_option : { + "-i2psam", + "-onion", + "-proxy", + "-rpcbind", + "-torcontrol", + "-whitebind", + "-zmqpubhashblock", + "-zmqpubhashtx", + "-zmqpubrawblock", + "-zmqpubrawtx", + "-zmqpubsequence", + }) { + for (const std::string& socket_addr : args.GetArgs(port_option)) { + std::string host_out; + uint16_t port_out{0}; + if (!SplitHostPort(socket_addr, port_out, host_out)) { + return InitError(InvalidPortErrMsg(port_option, socket_addr)); + } + } + } + + for (const std::string& socket_addr : args.GetArgs("-bind")) { + std::string host_out; + uint16_t port_out{0}; + std::string bind_socket_addr = socket_addr.substr(0, socket_addr.rfind('=')); + if (!SplitHostPort(bind_socket_addr, port_out, host_out)) { + return InitError(InvalidPortErrMsg("-bind", socket_addr)); + } + } + // sanitize comments per BIP-0014, format user agent and check total size std::vector<std::string> uacomments; for (const std::string& cmt : args.GetArgs("-uacomment")) { @@ -1388,6 +1430,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) fReindex = args.GetBoolArg("-reindex", false); bool fReindexChainState = args.GetBoolArg("-reindex-chainstate", false); + ChainstateManager::Options chainman_opts{ + .chainparams = chainparams, + .adjusted_time_callback = GetAdjustedTime, + }; + Assert(!ApplyArgsManOptions(args, chainman_opts)); // no error can happen, already checked in AppInitParameterInteraction // cache size calculations CacheSizes cache_sizes = CalculateCacheSizes(args, g_enabled_filter_types.size()); @@ -1424,10 +1471,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) for (bool fLoaded = false; !fLoaded && !ShutdownRequested();) { node.mempool = std::make_unique<CTxMemPool>(mempool_opts); - const ChainstateManager::Options chainman_opts{ - .chainparams = chainparams, - .adjusted_time_callback = GetAdjustedTime, - }; node.chainman = std::make_unique<ChainstateManager>(chainman_opts); ChainstateManager& chainman = *node.chainman; @@ -1528,7 +1571,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) { - g_coin_stats_index = std::make_unique<CoinStatsIndex>(interfaces::MakeChain(node), /* cache size */ 0, false, fReindex); + g_coin_stats_index = std::make_unique<CoinStatsIndex>(interfaces::MakeChain(node), /*cache_size=*/0, false, fReindex); if (!g_coin_stats_index->Start()) { return false; } @@ -1569,6 +1612,24 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) return false; } + int chain_active_height = WITH_LOCK(cs_main, return chainman.ActiveChain().Height()); + + // On first startup, warn on low block storage space + if (!fReindex && !fReindexChainState && chain_active_height <= 1) { + uint64_t additional_bytes_needed = fPruneMode ? nPruneTarget + : chainparams.AssumedBlockchainSize() * 1024 * 1024 * 1024; + + if (!CheckDiskSpace(args.GetBlocksDirPath(), additional_bytes_needed)) { + InitWarning(strprintf(_( + "Disk space for %s may not accommodate the block files. " \ + "Approximately %u GB of data will be stored in this directory." + ), + fs::quoted(fs::PathToString(args.GetBlocksDirPath())), + chainparams.AssumedBlockchainSize() + )); + } + } + // Either install a handler to notify us when genesis activates, or set fHaveGenesis directly. // No locking, as this happens before any background thread is started. boost::signals2::connection block_notify_genesis_wait_connection; @@ -1618,8 +1679,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // ********************************************************* Step 12: start node - int chain_active_height; - //// debug print { LOCK(cs_main); @@ -1663,7 +1722,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) const auto BadPortWarning = [](const char* prefix, uint16_t port) { return strprintf(_("%s request to listen on port %u. This port is considered \"bad\" and " - "thus it is unlikely that any Bitcoin Core peers connect to it. See " + "thus it is unlikely that any peer will connect to it. See " "doc/p2p-bad-ports.md for details and a full list."), prefix, port); diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index 5fc0e540a9..7a3d88b18f 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_INTERFACES_CHAIN_H #define BITCOIN_INTERFACES_CHAIN_H +#include <blockfilter.h> #include <primitives/transaction.h> // For CTransactionRef #include <util/settings.h> // For util::SettingsValue @@ -143,6 +144,13 @@ public: //! or one of its ancestors. virtual std::optional<int> findLocatorFork(const CBlockLocator& locator) = 0; + //! Returns whether a block filter index is available. + virtual bool hasBlockFilterIndex(BlockFilterType filter_type) = 0; + + //! Returns whether any of the elements match the block via a BIP 157 block filter + //! or std::nullopt if the block filter for this block couldn't be found. + virtual std::optional<bool> blockFilterMatchesAny(BlockFilterType filter_type, const uint256& block_hash, const GCSFilter::ElementSet& filter_set) = 0; + //! Return whether node has the block and optionally return block metadata //! or contents. virtual bool findBlock(const uint256& hash, const FoundBlock& block={}) = 0; diff --git a/src/interfaces/echo.cpp b/src/interfaces/echo.cpp deleted file mode 100644 index 9bbb42217b..0000000000 --- a/src/interfaces/echo.cpp +++ /dev/null @@ -1,18 +0,0 @@ -// Copyright (c) 2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <interfaces/echo.h> - -#include <memory> - -namespace interfaces { -namespace { -class EchoImpl : public Echo -{ -public: - std::string echo(const std::string& echo) override { return echo; } -}; -} // namespace -std::unique_ptr<Echo> MakeEcho() { return std::make_unique<EchoImpl>(); } -} // namespace interfaces diff --git a/src/interfaces/handler.cpp b/src/interfaces/handler.cpp deleted file mode 100644 index adb7031cbc..0000000000 --- a/src/interfaces/handler.cpp +++ /dev/null @@ -1,45 +0,0 @@ -// Copyright (c) 2018-2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <interfaces/handler.h> - - -#include <boost/signals2/connection.hpp> -#include <utility> - -namespace interfaces { -namespace { - -class HandlerImpl : public Handler -{ -public: - explicit HandlerImpl(boost::signals2::connection connection) : m_connection(std::move(connection)) {} - - void disconnect() override { m_connection.disconnect(); } - - boost::signals2::scoped_connection m_connection; -}; - -class CleanupHandler : public Handler -{ -public: - explicit CleanupHandler(std::function<void()> cleanup) : m_cleanup(std::move(cleanup)) {} - ~CleanupHandler() override { if (!m_cleanup) return; m_cleanup(); m_cleanup = nullptr; } - void disconnect() override { if (!m_cleanup) return; m_cleanup(); m_cleanup = nullptr; } - std::function<void()> m_cleanup; -}; - -} // namespace - -std::unique_ptr<Handler> MakeHandler(boost::signals2::connection connection) -{ - return std::make_unique<HandlerImpl>(std::move(connection)); -} - -std::unique_ptr<Handler> MakeHandler(std::function<void()> cleanup) -{ - return std::make_unique<CleanupHandler>(std::move(cleanup)); -} - -} // namespace interfaces diff --git a/src/interfaces/handler.h b/src/interfaces/handler.h index 11baf9dd65..f46f5e04a5 100644 --- a/src/interfaces/handler.h +++ b/src/interfaces/handler.h @@ -29,10 +29,10 @@ public: }; //! Return handler wrapping a boost signal connection. -std::unique_ptr<Handler> MakeHandler(boost::signals2::connection connection); +std::unique_ptr<Handler> MakeSignalHandler(boost::signals2::connection connection); //! Return handler wrapping a cleanup function. -std::unique_ptr<Handler> MakeHandler(std::function<void()> cleanup); +std::unique_ptr<Handler> MakeCleanupHandler(std::function<void()> cleanup); } // namespace interfaces diff --git a/src/interfaces/init.cpp b/src/interfaces/init.cpp deleted file mode 100644 index f0f8aa5fed..0000000000 --- a/src/interfaces/init.cpp +++ /dev/null @@ -1,17 +0,0 @@ -// Copyright (c) 2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <interfaces/chain.h> -#include <interfaces/echo.h> -#include <interfaces/init.h> -#include <interfaces/node.h> -#include <interfaces/wallet.h> - -namespace interfaces { -std::unique_ptr<Node> Init::makeNode() { return {}; } -std::unique_ptr<Chain> Init::makeChain() { return {}; } -std::unique_ptr<WalletLoader> Init::makeWalletLoader(Chain& chain) { return {}; } -std::unique_ptr<Echo> Init::makeEcho() { return {}; } -Ipc* Init::ipc() { return nullptr; } -} // namespace interfaces diff --git a/src/interfaces/init.h b/src/interfaces/init.h index 2153076366..5b8f61640e 100644 --- a/src/interfaces/init.h +++ b/src/interfaces/init.h @@ -5,6 +5,11 @@ #ifndef BITCOIN_INTERFACES_INIT_H #define BITCOIN_INTERFACES_INIT_H +#include <interfaces/chain.h> +#include <interfaces/echo.h> +#include <interfaces/node.h> +#include <interfaces/wallet.h> + #include <memory> namespace node { @@ -12,11 +17,7 @@ struct NodeContext; } // namespace node namespace interfaces { -class Chain; -class Echo; class Ipc; -class Node; -class WalletLoader; //! Initial interface created when a process is first started, and used to give //! and get access to other interfaces (Node, Chain, Wallet, etc). @@ -29,11 +30,11 @@ class Init { public: virtual ~Init() = default; - virtual std::unique_ptr<Node> makeNode(); - virtual std::unique_ptr<Chain> makeChain(); - virtual std::unique_ptr<WalletLoader> makeWalletLoader(Chain& chain); - virtual std::unique_ptr<Echo> makeEcho(); - virtual Ipc* ipc(); + virtual std::unique_ptr<Node> makeNode() { return nullptr; } + virtual std::unique_ptr<Chain> makeChain() { return nullptr; } + virtual std::unique_ptr<WalletLoader> makeWalletLoader(Chain& chain) { return nullptr; } + virtual std::unique_ptr<Echo> makeEcho() { return nullptr; } + virtual Ipc* ipc() { return nullptr; } }; //! Return implementation of Init interface for the node process. If the argv diff --git a/src/kernel/chainstatemanager_opts.h b/src/kernel/chainstatemanager_opts.h index 520d0e8e75..226bb6031e 100644 --- a/src/kernel/chainstatemanager_opts.h +++ b/src/kernel/chainstatemanager_opts.h @@ -5,13 +5,19 @@ #ifndef BITCOIN_KERNEL_CHAINSTATEMANAGER_OPTS_H #define BITCOIN_KERNEL_CHAINSTATEMANAGER_OPTS_H +#include <arith_uint256.h> +#include <uint256.h> #include <util/time.h> #include <cstdint> #include <functional> +#include <optional> class CChainParams; +static constexpr bool DEFAULT_CHECKPOINTS_ENABLED{true}; +static constexpr auto DEFAULT_MAX_TIP_AGE{24h}; + namespace kernel { /** @@ -22,6 +28,14 @@ namespace kernel { struct ChainstateManagerOpts { const CChainParams& chainparams; const std::function<NodeClock::time_point()> adjusted_time_callback{nullptr}; + std::optional<bool> check_block_index{}; + bool checkpoints_enabled{DEFAULT_CHECKPOINTS_ENABLED}; + //! If set, it will override the minimum work we will assume exists on some valid chain. + std::optional<arith_uint256> minimum_chain_work{}; + //! If set, it will override the block hash whose ancestors we will assume to have valid scripts without checking them. + std::optional<uint256> assumed_valid_block{}; + //! If the tip is older than this, the node is considered to be in initial block download. + std::chrono::seconds max_tip_age{DEFAULT_MAX_TIP_AGE}; }; } // namespace kernel diff --git a/src/kernel/mempool_entry.h b/src/kernel/mempool_entry.h new file mode 100644 index 0000000000..e1ba4296ef --- /dev/null +++ b/src/kernel/mempool_entry.h @@ -0,0 +1,174 @@ +// Copyright (c) 2009-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_KERNEL_MEMPOOL_ENTRY_H +#define BITCOIN_KERNEL_MEMPOOL_ENTRY_H + +#include <consensus/amount.h> +#include <consensus/validation.h> +#include <core_memusage.h> +#include <policy/policy.h> +#include <policy/settings.h> +#include <primitives/transaction.h> +#include <util/epochguard.h> +#include <util/overflow.h> + +#include <chrono> +#include <functional> +#include <memory> +#include <set> +#include <stddef.h> +#include <stdint.h> + +class CBlockIndex; + +struct LockPoints { + // Will be set to the blockchain height and median time past + // values that would be necessary to satisfy all relative locktime + // constraints (BIP68) of this tx given our view of block chain history + int height{0}; + int64_t time{0}; + // As long as the current chain descends from the highest height block + // containing one of the inputs used in the calculation, then the cached + // values are still valid even after a reorg. + CBlockIndex* maxInputBlock{nullptr}; +}; + +struct CompareIteratorByHash { + // SFINAE for T where T is either a pointer type (e.g., a txiter) or a reference_wrapper<T> + // (e.g. a wrapped CTxMemPoolEntry&) + template <typename T> + bool operator()(const std::reference_wrapper<T>& a, const std::reference_wrapper<T>& b) const + { + return a.get().GetTx().GetHash() < b.get().GetTx().GetHash(); + } + template <typename T> + bool operator()(const T& a, const T& b) const + { + return a->GetTx().GetHash() < b->GetTx().GetHash(); + } +}; + +/** \class CTxMemPoolEntry + * + * CTxMemPoolEntry stores data about the corresponding transaction, as well + * as data about all in-mempool transactions that depend on the transaction + * ("descendant" transactions). + * + * When a new entry is added to the mempool, we update the descendant state + * (nCountWithDescendants, nSizeWithDescendants, and nModFeesWithDescendants) for + * all ancestors of the newly added transaction. + * + */ + +class CTxMemPoolEntry +{ +public: + typedef std::reference_wrapper<const CTxMemPoolEntry> CTxMemPoolEntryRef; + // two aliases, should the types ever diverge + typedef std::set<CTxMemPoolEntryRef, CompareIteratorByHash> Parents; + typedef std::set<CTxMemPoolEntryRef, CompareIteratorByHash> Children; + +private: + const CTransactionRef tx; + mutable Parents m_parents; + mutable Children m_children; + const CAmount nFee; //!< Cached to avoid expensive parent-transaction lookups + const size_t nTxWeight; //!< ... and avoid recomputing tx weight (also used for GetTxSize()) + const size_t nUsageSize; //!< ... and total memory usage + const int64_t nTime; //!< Local time when entering the mempool + const unsigned int entryHeight; //!< Chain height when entering the mempool + const bool spendsCoinbase; //!< keep track of transactions that spend a coinbase + const int64_t sigOpCost; //!< Total sigop cost + CAmount m_modified_fee; //!< Used for determining the priority of the transaction for mining in a block + LockPoints lockPoints; //!< Track the height and time at which tx was final + + // Information about descendants of this transaction that are in the + // mempool; if we remove this transaction we must remove all of these + // descendants as well. + uint64_t nCountWithDescendants{1}; //!< number of descendant transactions + uint64_t nSizeWithDescendants; //!< ... and size + CAmount nModFeesWithDescendants; //!< ... and total fees (all including us) + + // Analogous statistics for ancestor transactions + uint64_t nCountWithAncestors{1}; + uint64_t nSizeWithAncestors; + CAmount nModFeesWithAncestors; + int64_t nSigOpCostWithAncestors; + +public: + CTxMemPoolEntry(const CTransactionRef& tx, CAmount fee, + int64_t time, unsigned int entry_height, + bool spends_coinbase, + int64_t sigops_cost, LockPoints lp) + : tx{tx}, + nFee{fee}, + nTxWeight(GetTransactionWeight(*tx)), + nUsageSize{RecursiveDynamicUsage(tx)}, + nTime{time}, + entryHeight{entry_height}, + spendsCoinbase{spends_coinbase}, + sigOpCost{sigops_cost}, + m_modified_fee{nFee}, + lockPoints{lp}, + nSizeWithDescendants{GetTxSize()}, + nModFeesWithDescendants{nFee}, + nSizeWithAncestors{GetTxSize()}, + nModFeesWithAncestors{nFee}, + nSigOpCostWithAncestors{sigOpCost} {} + + const CTransaction& GetTx() const { return *this->tx; } + CTransactionRef GetSharedTx() const { return this->tx; } + const CAmount& GetFee() const { return nFee; } + size_t GetTxSize() const + { + return GetVirtualTransactionSize(nTxWeight, sigOpCost, ::nBytesPerSigOp); + } + size_t GetTxWeight() const { return nTxWeight; } + std::chrono::seconds GetTime() const { return std::chrono::seconds{nTime}; } + unsigned int GetHeight() const { return entryHeight; } + int64_t GetSigOpCost() const { return sigOpCost; } + CAmount GetModifiedFee() const { return m_modified_fee; } + size_t DynamicMemoryUsage() const { return nUsageSize; } + const LockPoints& GetLockPoints() const { return lockPoints; } + + // Adjusts the descendant state. + void UpdateDescendantState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount); + // Adjusts the ancestor state + void UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps); + // Updates the modified fees with descendants/ancestors. + void UpdateModifiedFee(CAmount fee_diff) + { + nModFeesWithDescendants = SaturatingAdd(nModFeesWithDescendants, fee_diff); + nModFeesWithAncestors = SaturatingAdd(nModFeesWithAncestors, fee_diff); + m_modified_fee = SaturatingAdd(m_modified_fee, fee_diff); + } + + // Update the LockPoints after a reorg + void UpdateLockPoints(const LockPoints& lp) + { + lockPoints = lp; + } + + uint64_t GetCountWithDescendants() const { return nCountWithDescendants; } + uint64_t GetSizeWithDescendants() const { return nSizeWithDescendants; } + CAmount GetModFeesWithDescendants() const { return nModFeesWithDescendants; } + + bool GetSpendsCoinbase() const { return spendsCoinbase; } + + uint64_t GetCountWithAncestors() const { return nCountWithAncestors; } + uint64_t GetSizeWithAncestors() const { return nSizeWithAncestors; } + CAmount GetModFeesWithAncestors() const { return nModFeesWithAncestors; } + int64_t GetSigOpCostWithAncestors() const { return nSigOpCostWithAncestors; } + + const Parents& GetMemPoolParentsConst() const { return m_parents; } + const Children& GetMemPoolChildrenConst() const { return m_children; } + Parents& GetMemPoolParents() const { return m_parents; } + Children& GetMemPoolChildren() const { return m_children; } + + mutable size_t vTxHashesIdx; //!< Index in mempool's vTxHashes + mutable Epoch::Marker m_epoch_marker; //!< epoch when last touched, useful for graph algorithms +}; + +#endif // BITCOIN_KERNEL_MEMPOOL_ENTRY_H diff --git a/src/logging.cpp b/src/logging.cpp index a3f1d39be5..ed0c2a56a5 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -180,6 +180,8 @@ const CLogCategoryDesc LogCategories[] = #endif {BCLog::UTIL, "util"}, {BCLog::BLOCKSTORE, "blockstorage"}, + {BCLog::TXRECONCILIATION, "txreconciliation"}, + {BCLog::SCAN, "scan"}, {BCLog::ALL, "1"}, {BCLog::ALL, "all"}, }; @@ -280,6 +282,10 @@ std::string LogCategoryToStr(BCLog::LogFlags category) return "util"; case BCLog::LogFlags::BLOCKSTORE: return "blockstorage"; + case BCLog::LogFlags::TXRECONCILIATION: + return "txreconciliation"; + case BCLog::LogFlags::SCAN: + return "scan"; case BCLog::LogFlags::ALL: return "all"; } diff --git a/src/logging.h b/src/logging.h index fe91ee43a5..14a0f08f8d 100644 --- a/src/logging.h +++ b/src/logging.h @@ -66,6 +66,8 @@ namespace BCLog { #endif UTIL = (1 << 25), BLOCKSTORE = (1 << 26), + TXRECONCILIATION = (1 << 27), + SCAN = (1 << 28), ALL = ~(uint32_t)0, }; enum class Level { diff --git a/src/mapport.cpp b/src/mapport.cpp index 6262e51879..975ec4da6a 100644 --- a/src/mapport.cpp +++ b/src/mapport.cpp @@ -13,10 +13,10 @@ #include <net.h> #include <netaddress.h> #include <netbase.h> -#include <threadinterrupt.h> #include <util/syscall_sandbox.h> #include <util/system.h> #include <util/thread.h> +#include <util/threadinterrupt.h> #ifdef USE_NATPMP #include <compat/compat.h> diff --git a/src/minisketch/configure.ac b/src/minisketch/configure.ac index 9dc66e7fd2..83910448a2 100644 --- a/src/minisketch/configure.ac +++ b/src/minisketch/configure.ac @@ -124,9 +124,6 @@ if test "x$use_ccache" != "xno"; then fi AC_MSG_RESULT($use_ccache) fi -if test "x$use_ccache" = "xyes"; then - AX_CHECK_COMPILE_FLAG([-Qunused-arguments],[NOWARN_CXXFLAGS="$NOWARN_CXXFLAGS -Qunused-arguments"],,[[$CXXFLAG_WERROR]]) -fi VERIFY_DEFINES=-DMINISKETCH_VERIFY RELEASE_DEFINES= diff --git a/src/minisketch/src/bench.cpp b/src/minisketch/src/bench.cpp index f55944a448..dc44379fdb 100644 --- a/src/minisketch/src/bench.cpp +++ b/src/minisketch/src/bench.cpp @@ -62,13 +62,11 @@ int main(int argc, char** argv) { if (!states[0]) { printf(" -\t"); } else { - double total = 0.0; for (auto& state : states) { auto start = std::chrono::steady_clock::now(); minisketch_decode(state, 2 * syndromes, roots.data()); auto stop = std::chrono::steady_clock::now(); std::chrono::duration<double> dur(stop - start); - total += dur.count(); benches.push_back(dur.count()); } std::sort(benches.begin(), benches.end()); @@ -98,7 +96,6 @@ int main(int argc, char** argv) { if (!states[0]) { printf(" -\t"); } else { - double total = 0.0; for (auto& state : states) { auto start = std::chrono::steady_clock::now(); for (auto val : data) { @@ -106,7 +103,6 @@ int main(int argc, char** argv) { } auto stop = std::chrono::steady_clock::now(); std::chrono::duration<double> dur(stop - start); - total += dur.count(); benches.push_back(dur.count()); } std::sort(benches.begin(), benches.end()); diff --git a/src/minisketch/src/int_utils.h b/src/minisketch/src/int_utils.h index 62b2c38a29..d21ba56f33 100644 --- a/src/minisketch/src/int_utils.h +++ b/src/minisketch/src/int_utils.h @@ -129,17 +129,7 @@ constexpr inline I Mask() { return ((I((I(-1)) << (std::numeric_limits<I>::digit /** Compute the smallest power of two that is larger than val. */ template<typename I> static inline int CountBits(I val, int max) { -#ifdef HAVE_CLZ - (void)max; - if (val == 0) return 0; - if (std::numeric_limits<unsigned>::digits >= std::numeric_limits<I>::digits) { - return std::numeric_limits<unsigned>::digits - __builtin_clz(val); - } else if (std::numeric_limits<unsigned long>::digits >= std::numeric_limits<I>::digits) { - return std::numeric_limits<unsigned long>::digits - __builtin_clzl(val); - } else { - return std::numeric_limits<unsigned long long>::digits - __builtin_clzll(val); - } -#elif _MSC_VER +#ifdef _MSC_VER (void)max; unsigned long index; unsigned char ret; @@ -149,7 +139,17 @@ static inline int CountBits(I val, int max) { ret = _BitScanReverse64(&index, val); } if (!ret) return 0; - return index; + return index + 1; +#elif HAVE_CLZ + (void)max; + if (val == 0) return 0; + if (std::numeric_limits<unsigned>::digits >= std::numeric_limits<I>::digits) { + return std::numeric_limits<unsigned>::digits - __builtin_clz(val); + } else if (std::numeric_limits<unsigned long>::digits >= std::numeric_limits<I>::digits) { + return std::numeric_limits<unsigned long>::digits - __builtin_clzl(val); + } else { + return std::numeric_limits<unsigned long long>::digits - __builtin_clzll(val); + } #else while (max && (val >> (max - 1) == 0)) --max; return max; diff --git a/src/minisketch/src/test.cpp b/src/minisketch/src/test.cpp index 417937ea5f..85b9e9e396 100644 --- a/src/minisketch/src/test.cpp +++ b/src/minisketch/src/test.cpp @@ -9,6 +9,7 @@ #include <limits> #include <random> #include <stdexcept> +#include <string> #include <vector> #include "../include/minisketch.h" diff --git a/src/net.cpp b/src/net.cpp index 0736f3ef1b..374e93a2bd 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -31,6 +31,7 @@ #include <util/syscall_sandbox.h> #include <util/system.h> #include <util/thread.h> +#include <util/threadinterrupt.h> #include <util/trace.h> #include <util/translation.h> @@ -970,8 +971,7 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, return; } - if (!IsSelectableSocket(sock->Get())) - { + if (!sock->IsSelectable()) { LogPrintf("connection from %s dropped: non-selectable socket\n", addr.ToString()); return; } @@ -24,10 +24,10 @@ #include <span.h> #include <streams.h> #include <sync.h> -#include <threadinterrupt.h> #include <uint256.h> #include <util/check.h> #include <util/sock.h> +#include <util/threadinterrupt.h> #include <atomic> #include <condition_variable> @@ -489,10 +489,8 @@ public: /** Whether this peer provides all services that we want. Used for eviction decisions */ std::atomic_bool m_has_all_wanted_services{false}; - /** Whether we should relay transactions to this peer (their version - * message did not include fRelay=false and this is not a block-relay-only - * connection). This only changes from false to true. It will never change - * back to false. Used only in inbound eviction logic. */ + /** Whether we should relay transactions to this peer. This only changes + * from false to true. It will never change back to false. */ std::atomic_bool m_relays_txs{false}; /** Whether this peer has loaded a bloom filter. Used only in inbound diff --git a/src/net_permissions.h b/src/net_permissions.h index 662464083c..c9d5ec2989 100644 --- a/src/net_permissions.h +++ b/src/net_permissions.h @@ -35,7 +35,8 @@ enum class NetPermissionFlags : uint32_t { // unlimited amounts of addrs. Addr = (1U << 7), - // True if the user did not specifically set fine grained permissions + // True if the user did not specifically set fine-grained permissions with + // the -whitebind or -whitelist configuration options. Implicit = (1U << 31), All = BloomFilter | ForceRelay | Relay | NoBan | Mempool | Download | Addr, }; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index eca6263392..4dfd77c6cf 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -16,10 +16,12 @@ #include <hash.h> #include <headerssync.h> #include <index/blockfilterindex.h> +#include <kernel/mempool_entry.h> #include <merkleblock.h> #include <netbase.h> #include <netmessagemaker.h> #include <node/blockstorage.h> +#include <node/txreconciliation.h> #include <policy/fees.h> #include <policy/policy.h> #include <policy/settings.h> @@ -271,12 +273,7 @@ struct Peer { struct TxRelay { mutable RecursiveMutex m_bloom_filter_mutex; - /** Whether the peer wishes to receive transaction announcements. - * - * This is initially set based on the fRelay flag in the received - * `version` message. If initially set to false, it can only be flipped - * to true if we have offered the peer NODE_BLOOM services and it sends - * us a `filterload` or `filterclear` message. See BIP37. */ + /** Whether we relay transactions to this peer. */ bool m_relay_txs GUARDED_BY(m_bloom_filter_mutex){false}; /** A bloom filter for which transactions to announce to the peer. See BIP37. */ std::unique_ptr<CBloomFilter> m_bloom_filter PT_GUARDED_BY(m_bloom_filter_mutex) GUARDED_BY(m_bloom_filter_mutex){nullptr}; @@ -298,7 +295,7 @@ struct Peer { std::atomic<std::chrono::seconds> m_last_mempool_req{0s}; /** The next time after which we will send an `inv` message containing * transaction announcements to this peer. */ - std::chrono::microseconds m_next_inv_send_time GUARDED_BY(NetEventsInterface::g_msgproc_mutex){0}; + std::chrono::microseconds m_next_inv_send_time GUARDED_BY(m_tx_inventory_mutex){0}; /** Minimum fee rate with which to filter transaction announcements to this node. See BIP133. */ std::atomic<CAmount> m_fee_filter_received{0}; @@ -368,9 +365,6 @@ struct Peer { /** Total number of addresses that were processed (excludes rate-limited ones). */ std::atomic<uint64_t> m_addr_processed{0}; - /** Set of txids to reconsider once their parent transactions have been accepted **/ - std::set<uint256> m_orphan_work_set GUARDED_BY(g_cs_orphans); - /** Whether we've sent this peer a getheaders in response to an inv prior to initial-headers-sync completing */ bool m_inv_triggered_getheaders_before_sync GUARDED_BY(NetEventsInterface::g_msgproc_mutex){false}; @@ -399,9 +393,7 @@ struct Peer { private: Mutex m_tx_relay_mutex; - /** Transaction relay data. Will be a nullptr if we're not relaying - * transactions with this peer (e.g. if it's a block-relay-only peer or - * the peer has sent us fRelay=false with bloom filters disabled). */ + /** Transaction relay data. May be a nullptr. */ std::unique_ptr<TxRelay> m_tx_relay GUARDED_BY(m_tx_relay_mutex); }; @@ -589,8 +581,17 @@ private: */ bool MaybeDiscourageAndDisconnect(CNode& pnode, Peer& peer); - void ProcessOrphanTx(std::set<uint256>& orphan_work_set) EXCLUSIVE_LOCKS_REQUIRED(cs_main, g_cs_orphans) - EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); + /** + * Reconsider orphan transactions after a parent has been accepted to the mempool. + * + * @peer[in] peer The peer whose orphan transactions we will reconsider. Generally only one + * orphan will be reconsidered on each call of this function. This set + * may be added to if accepting an orphan causes its children to be + * reconsidered. + * @return True if there are still orphans in this peer's work set. + */ + bool ProcessOrphanTx(Peer& peer) + EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, g_msgproc_mutex, cs_main); /** Process a single headers message from a peer. * * @param[in] pfrom CNode of the peer @@ -642,9 +643,8 @@ private: * @param[in] chain_start_header Where these headers connect in our index. * @param[in,out] headers The headers to be processed. * - * @return True if chain was low work and a headers sync was - * initiated (and headers will be empty after calling); false - * otherwise. + * @return True if chain was low work (headers will be empty after + * calling); false otherwise. */ bool TryLowWorkHeadersSync(Peer& peer, CNode& pfrom, const CBlockIndex* chain_start_header, @@ -661,9 +661,9 @@ private: */ bool MaybeSendGetHeaders(CNode& pfrom, const CBlockLocator& locator, Peer& peer) EXCLUSIVE_LOCKS_REQUIRED(g_msgproc_mutex); /** Potentially fetch blocks from this peer upon receipt of a new headers tip */ - void HeadersDirectFetchBlocks(CNode& pfrom, const Peer& peer, const CBlockIndex* pindexLast); + void HeadersDirectFetchBlocks(CNode& pfrom, const Peer& peer, const CBlockIndex& last_header); /** Update peer state based on received headers message */ - void UpdatePeerStateForReceivedHeaders(CNode& pfrom, const CBlockIndex *pindexLast, bool received_new_header, bool may_have_more_headers); + void UpdatePeerStateForReceivedHeaders(CNode& pfrom, const CBlockIndex& last_header, bool received_new_header, bool may_have_more_headers); void SendBlockTransactions(CNode& pfrom, Peer& peer, const CBlock& block, const BlockTransactionsRequest& req); @@ -708,6 +708,7 @@ private: ChainstateManager& m_chainman; CTxMemPool& m_mempool; TxRequestTracker m_txrequest GUARDED_BY(::cs_main); + std::unique_ptr<TxReconciliationTracker> m_txreconciliation; /** The height of the best chain */ std::atomic<int> m_best_height{-1}; @@ -924,14 +925,14 @@ private: /** Storage for orphan information */ TxOrphanage m_orphanage; - void AddToCompactExtraTransactions(const CTransactionRef& tx) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans); + void AddToCompactExtraTransactions(const CTransactionRef& tx) EXCLUSIVE_LOCKS_REQUIRED(g_msgproc_mutex); /** Orphan/conflicted/etc transactions that are kept for compact block reconstruction. * The last -blockreconstructionextratxn/DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN of * these are kept in a ring buffer */ - std::vector<std::pair<uint256, CTransactionRef>> vExtraTxnForCompact GUARDED_BY(g_cs_orphans); + std::vector<std::pair<uint256, CTransactionRef>> vExtraTxnForCompact GUARDED_BY(g_msgproc_mutex); /** Offset into vExtraTxnForCompact to insert the next tx */ - size_t vExtraTxnForCompactIt GUARDED_BY(g_cs_orphans) = 0; + size_t vExtraTxnForCompactIt GUARDED_BY(g_msgproc_mutex) = 0; /** Check whether the last unknown block a peer advertised is not yet known. */ void ProcessBlockAvailability(NodeId nodeid) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -1294,7 +1295,7 @@ void PeerManagerImpl::FindNextBlocksToDownload(const Peer& peer, unsigned int co // Make sure pindexBestKnownBlock is up to date, we'll need it. ProcessBlockAvailability(peer.m_id); - if (state->pindexBestKnownBlock == nullptr || state->pindexBestKnownBlock->nChainWork < m_chainman.ActiveChain().Tip()->nChainWork || state->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { + if (state->pindexBestKnownBlock == nullptr || state->pindexBestKnownBlock->nChainWork < m_chainman.ActiveChain().Tip()->nChainWork || state->pindexBestKnownBlock->nChainWork < m_chainman.MinimumChainWork()) { // This peer has nothing interesting. return; } @@ -1383,7 +1384,7 @@ void PeerManagerImpl::PushNodeVersion(CNode& pnode, const Peer& peer) CService addr_you = addr.IsRoutable() && !IsProxy(addr) && addr.IsAddrV1Compatible() ? addr : CService(); uint64_t your_services{addr.nServices}; - const bool tx_relay = !m_ignore_incoming_txs && !pnode.IsBlockOnlyConn() && !pnode.IsFeelerConn(); + const bool tx_relay{!RejectIncomingTxs(pnode)}; m_connman.PushMessage(&pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, my_services, nTime, your_services, addr_you, // Together the pre-version-31402 serialization of CAddress "addrYou" (without nTime) my_services, CService(), // Together the pre-version-31402 serialization of CAddress "addrMe" (without nTime) @@ -1495,8 +1496,9 @@ void PeerManagerImpl::FinalizeNode(const CNode& node) for (const QueuedBlock& entry : state->vBlocksInFlight) { mapBlocksInFlight.erase(entry.pindex->GetBlockHash()); } - WITH_LOCK(g_cs_orphans, m_orphanage.EraseForPeer(nodeid)); + m_orphanage.EraseForPeer(nodeid); m_txrequest.DisconnectedPeer(nodeid); + if (m_txreconciliation) m_txreconciliation->ForgetPeer(nodeid); m_num_preferred_download_peers -= state->fPreferredDownload; m_peers_downloading_from -= (state->nBlocksInFlight != 0); assert(m_peers_downloading_from >= 0); @@ -1741,6 +1743,8 @@ std::optional<std::string> PeerManagerImpl::FetchBlock(NodeId peer_id, const CBl LOCK(cs_main); // Mark block as in-flight unless it already is (for this peer). + // If the peer does not send us a block, vBlocksInFlight remains non-empty, + // causing us to timeout and disconnect. // If a block was already in-flight for a different peer, its BLOCKTXN // response will be dropped. if (!BlockRequested(peer_id, block_index)) return "Already requested from this peer"; @@ -1781,6 +1785,11 @@ PeerManagerImpl::PeerManagerImpl(CConnman& connman, AddrMan& addrman, m_mempool(pool), m_ignore_incoming_txs(ignore_incoming_txs) { + // While Erlay support is incomplete, it must be enabled explicitly via -txreconciliation. + // This argument can go away after Erlay support is complete. + if (gArgs.GetBoolArg("-txreconciliation", DEFAULT_TXRECONCILIATION_ENABLE)) { + m_txreconciliation = std::make_unique<TxReconciliationTracker>(TXRECONCILIATION_VERSION); + } } void PeerManagerImpl::StartScheduledTasks(CScheduler& scheduler) @@ -2010,8 +2019,15 @@ void PeerManagerImpl::RelayTransaction(const uint256& txid, const uint256& wtxid auto tx_relay = peer.GetTxRelay(); if (!tx_relay) continue; - const uint256& hash{peer.m_wtxid_relay ? wtxid : txid}; LOCK(tx_relay->m_tx_inventory_mutex); + // Only queue transactions for announcement once the version handshake + // is completed. The time of arrival for these transactions is + // otherwise at risk of leaking to a spy, if the spy is able to + // distinguish transactions received during the handshake from the rest + // in the announcement. + if (tx_relay->m_next_inv_send_time == 0s) continue; + + const uint256& hash{peer.m_wtxid_relay ? wtxid : txid}; if (!tx_relay->m_tx_inventory_known_filter.contains(hash)) { tx_relay->m_tx_inventory_to_send.insert(hash); } @@ -2284,9 +2300,9 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic std::vector<uint256> parent_ids_to_add; { LOCK(m_mempool.cs); - auto txiter = m_mempool.GetIter(tx->GetHash()); - if (txiter) { - const CTxMemPoolEntry::Parents& parents = (*txiter)->GetMemPoolParentsConst(); + auto tx_iter = m_mempool.GetIter(tx->GetHash()); + if (tx_iter) { + const CTxMemPoolEntry::Parents& parents = (*tx_iter)->GetMemPoolParentsConst(); parent_ids_to_add.reserve(parents.size()); for (const CTxMemPoolEntry& parent : parents) { if (parent.GetTime() > now - UNCONDITIONAL_RELAY_DELAY) { @@ -2389,7 +2405,7 @@ arith_uint256 PeerManagerImpl::GetAntiDoSWorkThreshold() // near our tip. near_chaintip_work = tip->nChainWork - std::min<arith_uint256>(144*GetBlockProof(*tip), tip->nChainWork); } - return std::max(near_chaintip_work, arith_uint256(nMinimumChainWork)); + return std::max(near_chaintip_work, m_chainman.MinimumChainWork()); } /** @@ -2560,16 +2576,20 @@ bool PeerManagerImpl::TryLowWorkHeadersSync(Peer& peer, CNode& pfrom, const CBlo peer.m_headers_sync.reset(new HeadersSyncState(peer.m_id, m_chainparams.GetConsensus(), chain_start_header, minimum_chain_work)); - // Now a HeadersSyncState object for tracking this synchronization is created, - // process the headers using it as normal. - return IsContinuationOfLowWorkHeadersSync(peer, pfrom, headers); + // Now a HeadersSyncState object for tracking this synchronization + // is created, process the headers using it as normal. Failures are + // handled inside of IsContinuationOfLowWorkHeadersSync. + (void)IsContinuationOfLowWorkHeadersSync(peer, pfrom, headers); } else { LogPrint(BCLog::NET, "Ignoring low-work chain (height=%u) from peer=%d\n", chain_start_header->nHeight + headers.size(), pfrom.GetId()); - // Since this is a low-work headers chain, no further processing is required. - headers = {}; - return true; } + + // The peer has not yet given us a chain that meets our work threshold, + // so we want to prevent further processing of the headers in any case. + headers = {}; + return true; } + return false; } @@ -2602,22 +2622,21 @@ bool PeerManagerImpl::MaybeSendGetHeaders(CNode& pfrom, const CBlockLocator& loc } /* - * Given a new headers tip ending in pindexLast, potentially request blocks towards that tip. + * Given a new headers tip ending in last_header, potentially request blocks towards that tip. * We require that the given tip have at least as much work as our tip, and for * our current tip to be "close to synced" (see CanDirectFetch()). */ -void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const Peer& peer, const CBlockIndex* pindexLast) +void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const Peer& peer, const CBlockIndex& last_header) { const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); LOCK(cs_main); CNodeState *nodestate = State(pfrom.GetId()); - if (CanDirectFetch() && pindexLast->IsValid(BLOCK_VALID_TREE) && m_chainman.ActiveChain().Tip()->nChainWork <= pindexLast->nChainWork) { - + if (CanDirectFetch() && last_header.IsValid(BLOCK_VALID_TREE) && m_chainman.ActiveChain().Tip()->nChainWork <= last_header.nChainWork) { std::vector<const CBlockIndex*> vToFetch; - const CBlockIndex *pindexWalk = pindexLast; - // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. + const CBlockIndex* pindexWalk{&last_header}; + // Calculate all the blocks we'd need to switch to last_header, up to a limit. while (pindexWalk && !m_chainman.ActiveChain().Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && !IsBlockRequested(pindexWalk->GetBlockHash()) && @@ -2633,8 +2652,8 @@ void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const Peer& peer, c // direct fetch and rely on parallel download instead. if (!m_chainman.ActiveChain().Contains(pindexWalk)) { LogPrint(BCLog::NET, "Large reorg, won't direct fetch to %s (%d)\n", - pindexLast->GetBlockHash().ToString(), - pindexLast->nHeight); + last_header.GetBlockHash().ToString(), + last_header.nHeight); } else { std::vector<CInv> vGetData; // Download as much as possible, from earliest to latest. @@ -2651,14 +2670,15 @@ void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const Peer& peer, c } if (vGetData.size() > 1) { LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", - pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); + last_header.GetBlockHash().ToString(), + last_header.nHeight); } if (vGetData.size() > 0) { if (!m_ignore_incoming_txs && nodestate->m_provides_cmpctblocks && vGetData.size() == 1 && mapBlocksInFlight.size() == 1 && - pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { + last_header.pprev->IsValid(BLOCK_VALID_CHAIN)) { // In any case, we want to download using a compact block, not a regular one vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); } @@ -2669,12 +2689,12 @@ void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const Peer& peer, c } /** - * Given receipt of headers from a peer ending in pindexLast, along with + * Given receipt of headers from a peer ending in last_header, along with * whether that header was new and whether the headers message was full, * update the state we keep for the peer. */ void PeerManagerImpl::UpdatePeerStateForReceivedHeaders(CNode& pfrom, - const CBlockIndex *pindexLast, bool received_new_header, bool may_have_more_headers) + const CBlockIndex& last_header, bool received_new_header, bool may_have_more_headers) { LOCK(cs_main); CNodeState *nodestate = State(pfrom.GetId()); @@ -2683,14 +2703,13 @@ void PeerManagerImpl::UpdatePeerStateForReceivedHeaders(CNode& pfrom, } nodestate->nUnconnectingHeaders = 0; - assert(pindexLast); - UpdateBlockAvailability(pfrom.GetId(), pindexLast->GetBlockHash()); + UpdateBlockAvailability(pfrom.GetId(), last_header.GetBlockHash()); // From here, pindexBestKnownBlock should be guaranteed to be non-null, // because it is set in UpdateBlockAvailability. Some nullptr checks // are still present, however, as belt-and-suspenders. - if (received_new_header && pindexLast->nChainWork > m_chainman.ActiveChain().Tip()->nChainWork) { + if (received_new_header && last_header.nChainWork > m_chainman.ActiveChain().Tip()->nChainWork) { nodestate->m_last_block_announcement = GetTime(); } @@ -2699,14 +2718,14 @@ void PeerManagerImpl::UpdatePeerStateForReceivedHeaders(CNode& pfrom, if (m_chainman.ActiveChainstate().IsInitialBlockDownload() && !may_have_more_headers) { // If the peer has no more headers to give us, then we know we have // their tip. - if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { + if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < m_chainman.MinimumChainWork()) { // This peer has too little work on their headers chain to help // us sync -- disconnect if it is an outbound disconnection // candidate. - // Note: We compare their tip to nMinimumChainWork (rather than + // Note: We compare their tip to the minimum chain work (rather than // m_chainman.ActiveChain().Tip()) because we won't start block download // until we have a headers chain that has at least - // nMinimumChainWork, even if a peer has a chain past our tip, + // the minimum chain work, even if a peer has a chain past our tip, // as an anti-DoS measure. if (pfrom.IsOutboundOrBlockRelayConn()) { LogPrintf("Disconnecting outbound peer %d -- headers chain has insufficient work\n", pfrom.GetId()); @@ -2856,7 +2875,7 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, return; } } - Assume(pindexLast); + assert(pindexLast); // Consider fetching more headers if we are not using our headers-sync mechanism. if (nCount == MAX_HEADERS_RESULTS && !have_headers_sync) { @@ -2867,41 +2886,32 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, } } - UpdatePeerStateForReceivedHeaders(pfrom, pindexLast, received_new_header, nCount == MAX_HEADERS_RESULTS); + UpdatePeerStateForReceivedHeaders(pfrom, *pindexLast, received_new_header, nCount == MAX_HEADERS_RESULTS); // Consider immediately downloading blocks. - HeadersDirectFetchBlocks(pfrom, peer, pindexLast); + HeadersDirectFetchBlocks(pfrom, peer, *pindexLast); return; } -/** - * Reconsider orphan transactions after a parent has been accepted to the mempool. - * - * @param[in,out] orphan_work_set The set of orphan transactions to reconsider. Generally only one - * orphan will be reconsidered on each call of this function. This set - * may be added to if accepting an orphan causes its children to be - * reconsidered. - */ -void PeerManagerImpl::ProcessOrphanTx(std::set<uint256>& orphan_work_set) +bool PeerManagerImpl::ProcessOrphanTx(Peer& peer) { + AssertLockHeld(g_msgproc_mutex); AssertLockHeld(cs_main); - AssertLockHeld(g_cs_orphans); - while (!orphan_work_set.empty()) { - const uint256 orphanHash = *orphan_work_set.begin(); - orphan_work_set.erase(orphan_work_set.begin()); - - const auto [porphanTx, from_peer] = m_orphanage.GetTx(orphanHash); - if (porphanTx == nullptr) continue; + CTransactionRef porphanTx = nullptr; + NodeId from_peer = -1; + bool more = false; + while (CTransactionRef porphanTx = m_orphanage.GetTxToReconsider(peer.m_id, from_peer, more)) { const MempoolAcceptResult result = m_chainman.ProcessTransaction(porphanTx); const TxValidationState& state = result.m_state; + const uint256& orphanHash = porphanTx->GetHash(); if (result.m_result_type == MempoolAcceptResult::ResultType::VALID) { LogPrint(BCLog::MEMPOOL, " accepted orphan tx %s\n", orphanHash.ToString()); RelayTransaction(orphanHash, porphanTx->GetWitnessHash()); - m_orphanage.AddChildrenToWorkSet(*porphanTx, orphan_work_set); + m_orphanage.AddChildrenToWorkSet(*porphanTx, peer.m_id); m_orphanage.EraseTx(orphanHash); for (const CTransactionRef& removedTx : result.m_replaced_transactions.value()) { AddToCompactExtraTransactions(removedTx); @@ -2952,6 +2962,8 @@ void PeerManagerImpl::ProcessOrphanTx(std::set<uint256>& orphan_work_set) break; } } + + return more; } bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& node, Peer& peer, @@ -3241,8 +3253,6 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::SENDADDRV2)); } - m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::VERACK)); - pfrom.m_has_all_wanted_services = HasAllDesirableServiceFlags(nServices); peer->m_their_services = nServices; pfrom.SetAddrLocal(addrMe); @@ -3252,11 +3262,14 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } peer->m_starting_height = starting_height; - // We only initialize the m_tx_relay data structure if: - // - this isn't an outbound block-relay-only connection; and - // - fRelay=true or we're offering NODE_BLOOM to this peer - // (NODE_BLOOM means that the peer may turn on tx relay later) + // Only initialize the Peer::TxRelay m_relay_txs data structure if: + // - this isn't an outbound block-relay-only connection, and + // - this isn't an outbound feeler connection, and + // - fRelay=true (the peer wishes to receive transaction announcements) + // or we're offering NODE_BLOOM to this peer. NODE_BLOOM means that + // the peer may turn on transaction relay later. if (!pfrom.IsBlockOnlyConn() && + !pfrom.IsFeelerConn() && (fRelay || (peer->m_our_services & NODE_BLOOM))) { auto* const tx_relay = peer->SetTxRelay(); { @@ -3266,6 +3279,22 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (fRelay) pfrom.m_relays_txs = true; } + if (greatest_common_version >= WTXID_RELAY_VERSION && m_txreconciliation) { + // Per BIP-330, we announce txreconciliation support if: + // - protocol version per the peer's VERSION message supports WTXID_RELAY; + // - transaction relay is supported per the peer's VERSION message (see m_relays_txs); + // - this is not a block-relay-only connection and not a feeler (see m_relays_txs); + // - this is not an addr fetch connection; + // - we are not in -blocksonly mode. + if (pfrom.m_relays_txs && !pfrom.IsAddrFetchConn() && !m_ignore_incoming_txs) { + const uint64_t recon_salt = m_txreconciliation->PreRegisterPeer(pfrom.GetId()); + m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::SENDTXRCNCL, + TXRECONCILIATION_VERSION, recon_salt)); + } + } + + m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::VERACK)); + // Potentially mark this peer as a preferred download peer. { LOCK(cs_main); @@ -3274,39 +3303,20 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, m_num_preferred_download_peers += state->fPreferredDownload; } - // Self advertisement & GETADDR logic - if (!pfrom.IsInboundConn() && SetupAddressRelay(pfrom, *peer)) { - // For outbound peers, we try to relay our address (so that other - // nodes can try to find us more quickly, as we have no guarantee - // that an outbound peer is even aware of how to reach us) and do a - // one-time address fetch (to help populate/update our addrman). If - // we're starting up for the first time, our addrman may be pretty - // empty and no one will know who we are, so these mechanisms are - // important to help us connect to the network. - // + // Attempt to initialize address relay for outbound peers and use result + // to decide whether to send GETADDR, so that we don't send it to + // inbound or outbound block-relay-only peers. + bool send_getaddr{false}; + if (!pfrom.IsInboundConn()) { + send_getaddr = SetupAddressRelay(pfrom, *peer); + } + if (send_getaddr) { + // Do a one-time address fetch to help populate/update our addrman. + // If we're starting up for the first time, our addrman may be pretty + // empty, so this mechanism is important to help us connect to the network. // We skip this for block-relay-only peers. We want to avoid // potentially leaking addr information and we do not want to // indicate to the peer that we will participate in addr relay. - if (fListen && !m_chainman.ActiveChainstate().IsInitialBlockDownload()) - { - CAddress addr{GetLocalAddress(pfrom.addr), peer->m_our_services, Now<NodeSeconds>()}; - FastRandomContext insecure_rand; - if (addr.IsRoutable()) - { - LogPrint(BCLog::NET, "ProcessMessages: advertising address %s\n", addr.ToString()); - PushAddress(*peer, addr, insecure_rand); - } else if (IsPeerAddrLocalGood(&pfrom)) { - // Override just the address with whatever the peer sees us as. - // Leave the port in addr as it was returned by GetLocalAddress() - // above, as this is an outbound connection and the peer cannot - // observe our listening port. - addr.SetIP(addrMe); - LogPrint(BCLog::NET, "ProcessMessages: advertising address %s\n", addr.ToString()); - PushAddress(*peer, addr, insecure_rand); - } - } - - // Get recent addresses m_connman.PushMessage(&pfrom, CNetMsgMaker(greatest_common_version).Make(NetMsgType::GETADDR)); peer->m_getaddr_sent = true; // When requesting a getaddr, accept an additional MAX_ADDR_TO_SEND addresses in response @@ -3393,6 +3403,30 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // they may wish to request compact blocks from us m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, /*high_bandwidth=*/false, /*version=*/CMPCTBLOCKS_VERSION)); } + + if (m_txreconciliation) { + if (!peer->m_wtxid_relay || !m_txreconciliation->IsPeerRegistered(pfrom.GetId())) { + // We could have optimistically pre-registered/registered the peer. In that case, + // we should forget about the reconciliation state here if this wasn't followed + // by WTXIDRELAY (since WTXIDRELAY can't be announced later). + m_txreconciliation->ForgetPeer(pfrom.GetId()); + } + } + + if (auto tx_relay = peer->GetTxRelay()) { + // `TxRelay::m_tx_inventory_to_send` must be empty before the + // version handshake is completed as + // `TxRelay::m_next_inv_send_time` is first initialised in + // `SendMessages` after the verack is received. Any transactions + // received during the version handshake would otherwise + // immediately be advertised without random delay, potentially + // leaking the time of arrival to a spy. + Assume(WITH_LOCK( + tx_relay->m_tx_inventory_mutex, + return tx_relay->m_tx_inventory_to_send.empty() && + tx_relay->m_next_inv_send_time == 0s)); + } + pfrom.fSuccessfullyConnected = true; return; } @@ -3456,6 +3490,61 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } + // Received from a peer demonstrating readiness to announce transactions via reconciliations. + // This feature negotiation must happen between VERSION and VERACK to avoid relay problems + // from switching announcement protocols after the connection is up. + if (msg_type == NetMsgType::SENDTXRCNCL) { + if (!m_txreconciliation) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "sendtxrcncl from peer=%d ignored, as our node does not have txreconciliation enabled\n", pfrom.GetId()); + return; + } + + if (pfrom.fSuccessfullyConnected) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "sendtxrcncl received after verack from peer=%d; disconnecting\n", pfrom.GetId()); + pfrom.fDisconnect = true; + return; + } + + // Peer must not offer us reconciliations if we specified no tx relay support in VERSION. + if (RejectIncomingTxs(pfrom)) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "sendtxrcncl received from peer=%d to which we indicated no tx relay; disconnecting\n", pfrom.GetId()); + pfrom.fDisconnect = true; + return; + } + + // Peer must not offer us reconciliations if they specified no tx relay support in VERSION. + // This flag might also be false in other cases, but the RejectIncomingTxs check above + // eliminates them, so that this flag fully represents what we are looking for. + if (!pfrom.m_relays_txs) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "sendtxrcncl received from peer=%d which indicated no tx relay to us; disconnecting\n", pfrom.GetId()); + pfrom.fDisconnect = true; + return; + } + + uint32_t peer_txreconcl_version; + uint64_t remote_salt; + vRecv >> peer_txreconcl_version >> remote_salt; + + const ReconciliationRegisterResult result = m_txreconciliation->RegisterPeer(pfrom.GetId(), pfrom.IsInboundConn(), + peer_txreconcl_version, remote_salt); + switch (result) { + case ReconciliationRegisterResult::NOT_FOUND: + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "Ignore unexpected txreconciliation signal from peer=%d\n", pfrom.GetId()); + break; + case ReconciliationRegisterResult::SUCCESS: + break; + case ReconciliationRegisterResult::ALREADY_REGISTERED: + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "txreconciliation protocol violation from peer=%d (sendtxrcncl received from already registered peer); disconnecting\n", pfrom.GetId()); + pfrom.fDisconnect = true; + return; + case ReconciliationRegisterResult::PROTOCOL_VIOLATION: + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "txreconciliation protocol violation from peer=%d; disconnecting\n", pfrom.GetId()); + pfrom.fDisconnect = true; + return; + } + return; + } + if (!pfrom.fSuccessfullyConnected) { LogPrint(BCLog::NET, "Unsupported message \"%s\" prior to verack from peer=%d\n", SanitizeString(msg_type), pfrom.GetId()); return; @@ -3810,12 +3899,12 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Note that if we were to be on a chain that forks from the checkpointed // chain, then serving those headers to a peer that has seen the // checkpointed chain would cause that peer to disconnect us. Requiring - // that our chainwork exceed nMinimumChainWork is a protection against + // that our chainwork exceed the minimum chain work is a protection against // being fed a bogus chain when we started up for the first time and // getting partitioned off the honest network for serving that chain to // others. if (m_chainman.ActiveTip() == nullptr || - (m_chainman.ActiveTip()->nChainWork < nMinimumChainWork && !pfrom.HasPermission(NetPermissionFlags::Download))) { + (m_chainman.ActiveTip()->nChainWork < m_chainman.MinimumChainWork() && !pfrom.HasPermission(NetPermissionFlags::Download))) { LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because active chain has too little work; sending empty response\n", pfrom.GetId()); // Just respond with an empty headers message, to tell the peer to // go away but not treat us as unresponsive. @@ -3903,7 +3992,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, AddKnownTx(*peer, txid); } - LOCK2(cs_main, g_cs_orphans); + LOCK(cs_main); m_txrequest.ReceivedResponse(pfrom.GetId(), txid); if (tx.HasWitness()) m_txrequest.ReceivedResponse(pfrom.GetId(), wtxid); @@ -3944,7 +4033,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, m_txrequest.ForgetTxHash(tx.GetHash()); m_txrequest.ForgetTxHash(tx.GetWitnessHash()); RelayTransaction(tx.GetHash(), tx.GetWitnessHash()); - m_orphanage.AddChildrenToWorkSet(tx, peer->m_orphan_work_set); + m_orphanage.AddChildrenToWorkSet(tx, peer->m_id); pfrom.m_last_tx_time = GetTime<std::chrono::seconds>(); @@ -3958,7 +4047,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } // Recursively process any orphan transactions that depended on this one - ProcessOrphanTx(peer->m_orphan_work_set); + ProcessOrphanTx(*peer); } else if (state.GetResult() == TxValidationResult::TX_MISSING_INPUTS) { @@ -4139,7 +4228,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, bool fBlockReconstructed = false; { - LOCK2(cs_main, g_cs_orphans); + LOCK(cs_main); // If AcceptBlockHeader returned true, it set pindex assert(pindex); UpdateBlockAvailability(pfrom.GetId(), pindex->GetBlockHash()); @@ -4279,7 +4368,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // (eg disk space). Because we only try to reconstruct blocks when // we're close to caught up (via the CanDirectFetch() requirement // above, combined with the behavior of not requesting blocks until - // we have a chain with at least nMinimumChainWork), and we ignore + // we have a chain with at least the minimum chain work), and we ignore // compact blocks with less work than our tip, it is safe to treat // reconstructed compact blocks as having been requested. ProcessBlock(pfrom, pblock, /*force_processing=*/true, /*min_pow_checked=*/true); @@ -4767,16 +4856,17 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt } } + bool has_more_orphans; { - LOCK2(cs_main, g_cs_orphans); - if (!peer->m_orphan_work_set.empty()) { - ProcessOrphanTx(peer->m_orphan_work_set); - } + LOCK(cs_main); + has_more_orphans = ProcessOrphanTx(*peer); } if (pfrom->fDisconnect) return false; + if (has_more_orphans) return true; + // this maintains the order of responses // and prevents m_getdata_requests to grow unbounded { @@ -4784,11 +4874,6 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt if (!peer->m_getdata_requests.empty()) return true; } - { - LOCK(g_cs_orphans); - if (!peer->m_orphan_work_set.empty()) return true; - } - // Don't bother if send buffer is too full to respond anyway if (pfrom->fPauseSend) return false; @@ -5145,7 +5230,7 @@ void PeerManagerImpl::MaybeSendSendHeaders(CNode& node, Peer& peer) LOCK(cs_main); CNodeState &state = *State(node.GetId()); if (state.pindexBestKnownBlock != nullptr && - state.pindexBestKnownBlock->nChainWork > nMinimumChainWork) { + state.pindexBestKnownBlock->nChainWork > m_chainman.MinimumChainWork()) { // Tell our peer we prefer to receive headers rather than inv's // We send this to non-NODE NETWORK peers as well, because even // non-NODE NETWORK peers can announce blocks (such as pruning @@ -5224,6 +5309,7 @@ bool PeerManagerImpl::RejectIncomingTxs(const CNode& peer) const { // block-relay-only peers may never send txs to us if (peer.IsBlockOnlyConn()) return true; + if (peer.IsFeelerConn()) return true; // In -blocksonly mode, peers need the 'relay' permission to send txs to us if (m_ignore_incoming_txs && !peer.HasPermission(NetPermissionFlags::Relay)) return true; return false; @@ -5237,8 +5323,9 @@ bool PeerManagerImpl::SetupAddressRelay(const CNode& node, Peer& peer) if (node.IsBlockOnlyConn()) return false; if (!peer.m_addr_relay_enabled.exchange(true)) { - // First addr message we have received from the peer, initialize - // m_addr_known + // During version message processing (non-block-relay-only outbound peers) + // or on first addr-related message we have received (inbound peers), initialize + // m_addr_known. peer.m_addr_known = std::make_unique<CRollingBloomFilter>(5000, 0.001); } diff --git a/src/netaddress.cpp b/src/netaddress.cpp index ca148bfa51..eabab3dd99 100644 --- a/src/netaddress.cpp +++ b/src/netaddress.cpp @@ -588,7 +588,7 @@ static std::string IPv6ToString(Span<const uint8_t> a, uint32_t scope_id) return r; } -static std::string OnionToString(Span<const uint8_t> addr) +std::string OnionToString(Span<const uint8_t> addr) { uint8_t checksum[torv3::CHECKSUM_LEN]; torv3::Checksum(addr, checksum); diff --git a/src/netaddress.h b/src/netaddress.h index e52beb783d..11086eaee0 100644 --- a/src/netaddress.h +++ b/src/netaddress.h @@ -111,6 +111,8 @@ static constexpr size_t ADDR_INTERNAL_SIZE = 10; /// SAM 3.1 and earlier do not support specifying ports and force the port to 0. static constexpr uint16_t I2P_SAM31_PORT{0}; +std::string OnionToString(Span<const uint8_t> addr); + /** * Network address. */ diff --git a/src/netbase.cpp b/src/netbase.cpp index a5f7bda875..8169b40ea6 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -304,8 +304,7 @@ enum class IntrRecvError { * read. * * @see This function can be interrupted by calling InterruptSocks5(bool). - * Sockets can be made non-blocking with SetSocketNonBlocking(const - * SOCKET&). + * Sockets can be made non-blocking with Sock::SetNonBlocking(). */ static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, int timeout, const Sock& sock) { @@ -503,7 +502,7 @@ std::unique_ptr<Sock> CreateSockTCP(const CService& address_family) // Ensure that waiting for I/O on this socket won't result in undefined // behavior. - if (!IsSelectableSocket(sock->Get())) { + if (!sock->IsSelectable()) { LogPrintf("Cannot create connection: non-selectable socket created (fd >= FD_SETSIZE ?)\n"); return nullptr; } @@ -525,7 +524,7 @@ std::unique_ptr<Sock> CreateSockTCP(const CService& address_family) } // Set the non-blocking option on the socket. - if (!SetSocketNonBlocking(sock->Get())) { + if (!sock->SetNonBlocking()) { LogPrintf("Error setting socket to non-blocking: %s\n", NetworkErrorString(WSAGetLastError())); return nullptr; } @@ -717,21 +716,6 @@ bool LookupSubNet(const std::string& subnet_str, CSubNet& subnet_out) return false; } -bool SetSocketNonBlocking(const SOCKET& hSocket) -{ -#ifdef WIN32 - u_long nOne = 1; - if (ioctlsocket(hSocket, FIONBIO, &nOne) == SOCKET_ERROR) { -#else - int fFlags = fcntl(hSocket, F_GETFL, 0); - if (fcntl(hSocket, F_SETFL, fFlags | O_NONBLOCK) == SOCKET_ERROR) { -#endif - return false; - } - - return true; -} - void InterruptSocks5(bool interrupt) { interruptSocks5Recv = interrupt; diff --git a/src/netbase.h b/src/netbase.h index fadc8b418e..f7816f5d1d 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -221,8 +221,6 @@ bool ConnectSocketDirectly(const CService &addrConnect, const Sock& sock, int nT */ bool ConnectThroughProxy(const Proxy& proxy, const std::string& strDest, uint16_t port, const Sock& sock, int nTimeout, bool& outProxyConnectionFailed); -/** Enable non-blocking mode for a socket */ -bool SetSocketNonBlocking(const SOCKET& hSocket); void InterruptSocks5(bool interrupt); /** diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 57f81e6bb6..04d46f4361 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -524,6 +524,16 @@ void BlockManager::FlushUndoFile(int block_file, bool finalize) void BlockManager::FlushBlockFile(bool fFinalize, bool finalize_undo) { LOCK(cs_LastBlockFile); + + if (m_blockfile_info.size() < 1) { + // Return if we haven't loaded any blockfiles yet. This happens during + // chainstate init, when we call ChainstateManager::MaybeRebalanceCaches() (which + // then calls FlushStateToDisk()), resulting in a call to this function before we + // have populated `m_blockfile_info` via LoadBlockIndexDB(). + return; + } + assert(static_cast<int>(m_blockfile_info.size()) > m_last_blockfile); + FlatFilePos block_pos_old(m_last_blockfile, m_blockfile_info[m_last_blockfile].nSize); if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) { AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); @@ -789,19 +799,24 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, c return true; } -/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp) { unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); FlatFilePos blockPos; - if (dbp != nullptr) { + const auto position_known {dbp != nullptr}; + if (position_known) { blockPos = *dbp; + } else { + // when known, blockPos.nPos points at the offset of the block data in the blk file. that already accounts for + // the serialization header present in the file (the 4 magic message start bytes + the 4 length bytes = 8 bytes = BLOCK_SERIALIZATION_HEADER_SIZE). + // we add BLOCK_SERIALIZATION_HEADER_SIZE only for new blocks since they will have the serialization header added when written to disk. + nBlockSize += static_cast<unsigned int>(BLOCK_SERIALIZATION_HEADER_SIZE); } - if (!FindBlockPos(blockPos, nBlockSize + 8, nHeight, active_chain, block.GetBlockTime(), dbp != nullptr)) { + if (!FindBlockPos(blockPos, nBlockSize, nHeight, active_chain, block.GetBlockTime(), position_known)) { error("%s: FindBlockPos failed", __func__); return FlatFilePos(); } - if (dbp == nullptr) { + if (!position_known) { if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) { AbortNode("Failed to write block"); return FlatFilePos(); diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 37d74ed102..29501c1959 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -44,6 +44,9 @@ static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB /** The maximum size of a blk?????.dat file (since 0.8) */ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB +/** Size of header written by WriteBlockToDisk before a serialized CBlock */ +static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = CMessageHeader::MESSAGE_START_SIZE + sizeof(unsigned int); + extern std::atomic_bool fImporting; extern std::atomic_bool fReindex; /** Pruning-related variables and constants */ @@ -171,6 +174,7 @@ public: bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /** Store block on disk. If dbp is not nullptr, then it provides the known position of the block within a block file on disk. */ FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp); /** Calculate the amount of disk space the block & undo files currently use */ diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index 3f1d6dd743..60acb614b4 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -4,9 +4,11 @@ #include <node/chainstate.h> +#include <arith_uint256.h> #include <chain.h> #include <coins.h> #include <consensus/params.h> +#include <logging.h> #include <node/blockstorage.h> #include <node/caches.h> #include <sync.h> @@ -21,6 +23,7 @@ #include <algorithm> #include <atomic> #include <cassert> +#include <limits> #include <memory> #include <vector> @@ -32,13 +35,13 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize return options.reindex || options.reindex_chainstate || chainstate->CoinsTip().GetBestBlock().IsNull(); }; - if (!hashAssumeValid.IsNull()) { - LogPrintf("Assuming ancestors of block %s have valid signatures.\n", hashAssumeValid.GetHex()); + if (!chainman.AssumedValidBlock().IsNull()) { + LogPrintf("Assuming ancestors of block %s have valid signatures.\n", chainman.AssumedValidBlock().GetHex()); } else { LogPrintf("Validating signatures for all blocks.\n"); } - LogPrintf("Setting nMinimumChainWork=%s\n", nMinimumChainWork.GetHex()); - if (nMinimumChainWork < UintToArith256(chainman.GetConsensus().nMinimumChainWork)) { + LogPrintf("Setting nMinimumChainWork=%s\n", chainman.MinimumChainWork().GetHex()); + if (chainman.MinimumChainWork() < UintToArith256(chainman.GetConsensus().nMinimumChainWork)) { LogPrintf("Warning: nMinimumChainWork set below default value of %s\n", chainman.GetConsensus().nMinimumChainWork.GetHex()); } if (nPruneTarget == std::numeric_limits<uint64_t>::max()) { @@ -48,10 +51,15 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize } LOCK(cs_main); - chainman.InitializeChainstate(options.mempool); chainman.m_total_coinstip_cache = cache_sizes.coins; chainman.m_total_coinsdb_cache = cache_sizes.coins_db; + // Load the fully validated chainstate. + chainman.InitializeChainstate(options.mempool); + + // Load a chain created from a UTXO snapshot, if any exist. + chainman.DetectSnapshotChainstate(options.mempool); + auto& pblocktree{chainman.m_blockman.m_block_tree_db}; // new CBlockTreeDB tries to delete the existing file, which // fails if it's still open from the previous loop. Close it first: @@ -98,12 +106,20 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize return {ChainstateLoadStatus::FAILURE, _("Error initializing block database")}; } + // Conservative value which is arbitrarily chosen, as it will ultimately be changed + // by a call to `chainman.MaybeRebalanceCaches()`. We just need to make sure + // that the sum of the two caches (40%) does not exceed the allowable amount + // during this temporary initialization state. + double init_cache_fraction = 0.2; + // At this point we're either in reindex or we've loaded a useful // block tree into BlockIndex()! for (Chainstate* chainstate : chainman.GetAll()) { + LogPrintf("Initializing chainstate %s\n", chainstate->ToString()); + chainstate->InitCoinsDB( - /*cache_size_bytes=*/cache_sizes.coins_db, + /*cache_size_bytes=*/chainman.m_total_coinsdb_cache * init_cache_fraction, /*in_memory=*/options.coins_db_in_memory, /*should_wipe=*/options.reindex || options.reindex_chainstate); @@ -125,7 +141,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize } // The on-disk coinsdb is now in a good state, create the cache - chainstate->InitCoinsCache(cache_sizes.coins); + chainstate->InitCoinsCache(chainman.m_total_coinstip_cache * init_cache_fraction); assert(chainstate->CanFlushToDisk()); if (!is_coinsview_empty(chainstate)) { @@ -146,6 +162,11 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize }; } + // Now that chainstates are loaded and we're able to flush to + // disk, rebalance the coins caches to desired levels based + // on the condition of each chainstate. + chainman.MaybeRebalanceCaches(); + return {ChainstateLoadStatus::SUCCESS, {}}; } diff --git a/src/node/chainstatemanager_args.cpp b/src/node/chainstatemanager_args.cpp new file mode 100644 index 0000000000..b0d929626b --- /dev/null +++ b/src/node/chainstatemanager_args.cpp @@ -0,0 +1,39 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/chainstatemanager_args.h> + +#include <arith_uint256.h> +#include <tinyformat.h> +#include <uint256.h> +#include <util/strencodings.h> +#include <util/system.h> +#include <util/translation.h> +#include <validation.h> + +#include <chrono> +#include <optional> +#include <string> + +namespace node { +std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts) +{ + if (auto value{args.GetBoolArg("-checkblockindex")}) opts.check_block_index = *value; + + if (auto value{args.GetBoolArg("-checkpoints")}) opts.checkpoints_enabled = *value; + + if (auto value{args.GetArg("-minimumchainwork")}) { + if (!IsHexNumber(*value)) { + return strprintf(Untranslated("Invalid non-hex (%s) minimum chain work value specified"), *value); + } + opts.minimum_chain_work = UintToArith256(uint256S(*value)); + } + + if (auto value{args.GetArg("-assumevalid")}) opts.assumed_valid_block = uint256S(*value); + + if (auto value{args.GetIntArg("-maxtipage")}) opts.max_tip_age = std::chrono::seconds{*value}; + + return std::nullopt; +} +} // namespace node diff --git a/src/node/chainstatemanager_args.h b/src/node/chainstatemanager_args.h new file mode 100644 index 0000000000..6c46b998f2 --- /dev/null +++ b/src/node/chainstatemanager_args.h @@ -0,0 +1,19 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_CHAINSTATEMANAGER_ARGS_H +#define BITCOIN_NODE_CHAINSTATEMANAGER_ARGS_H + +#include <validation.h> + +#include <optional> + +class ArgsManager; +struct bilingual_str; + +namespace node { +std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts); +} // namespace node + +#endif // BITCOIN_NODE_CHAINSTATEMANAGER_ARGS_H diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 8a0011a629..7a15f3649b 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -4,26 +4,29 @@ #include <addrdb.h> #include <banman.h> +#include <blockfilter.h> #include <chain.h> #include <chainparams.h> #include <deploymentstatus.h> #include <external_signer.h> +#include <index/blockfilterindex.h> #include <init.h> #include <interfaces/chain.h> #include <interfaces/handler.h> #include <interfaces/node.h> #include <interfaces/wallet.h> +#include <kernel/chain.h> +#include <kernel/mempool_entry.h> #include <mapport.h> #include <net.h> #include <net_processing.h> #include <netaddress.h> #include <netbase.h> #include <node/blockstorage.h> -#include <kernel/chain.h> #include <node/coin.h> #include <node/context.h> -#include <node/transaction.h> #include <node/interface_ui.h> +#include <node/transaction.h> #include <policy/feerate.h> #include <policy/fees.h> #include <policy/policy.h> @@ -61,7 +64,7 @@ using interfaces::BlockTip; using interfaces::Chain; using interfaces::FoundBlock; using interfaces::Handler; -using interfaces::MakeHandler; +using interfaces::MakeSignalHandler; using interfaces::Node; using interfaces::WalletLoader; @@ -333,50 +336,50 @@ public: } std::unique_ptr<Handler> handleInitMessage(InitMessageFn fn) override { - return MakeHandler(::uiInterface.InitMessage_connect(fn)); + return MakeSignalHandler(::uiInterface.InitMessage_connect(fn)); } std::unique_ptr<Handler> handleMessageBox(MessageBoxFn fn) override { - return MakeHandler(::uiInterface.ThreadSafeMessageBox_connect(fn)); + return MakeSignalHandler(::uiInterface.ThreadSafeMessageBox_connect(fn)); } std::unique_ptr<Handler> handleQuestion(QuestionFn fn) override { - return MakeHandler(::uiInterface.ThreadSafeQuestion_connect(fn)); + return MakeSignalHandler(::uiInterface.ThreadSafeQuestion_connect(fn)); } std::unique_ptr<Handler> handleShowProgress(ShowProgressFn fn) override { - return MakeHandler(::uiInterface.ShowProgress_connect(fn)); + return MakeSignalHandler(::uiInterface.ShowProgress_connect(fn)); } std::unique_ptr<Handler> handleInitWallet(InitWalletFn fn) override { - return MakeHandler(::uiInterface.InitWallet_connect(fn)); + return MakeSignalHandler(::uiInterface.InitWallet_connect(fn)); } std::unique_ptr<Handler> handleNotifyNumConnectionsChanged(NotifyNumConnectionsChangedFn fn) override { - return MakeHandler(::uiInterface.NotifyNumConnectionsChanged_connect(fn)); + return MakeSignalHandler(::uiInterface.NotifyNumConnectionsChanged_connect(fn)); } std::unique_ptr<Handler> handleNotifyNetworkActiveChanged(NotifyNetworkActiveChangedFn fn) override { - return MakeHandler(::uiInterface.NotifyNetworkActiveChanged_connect(fn)); + return MakeSignalHandler(::uiInterface.NotifyNetworkActiveChanged_connect(fn)); } std::unique_ptr<Handler> handleNotifyAlertChanged(NotifyAlertChangedFn fn) override { - return MakeHandler(::uiInterface.NotifyAlertChanged_connect(fn)); + return MakeSignalHandler(::uiInterface.NotifyAlertChanged_connect(fn)); } std::unique_ptr<Handler> handleBannedListChanged(BannedListChangedFn fn) override { - return MakeHandler(::uiInterface.BannedListChanged_connect(fn)); + return MakeSignalHandler(::uiInterface.BannedListChanged_connect(fn)); } std::unique_ptr<Handler> handleNotifyBlockTip(NotifyBlockTipFn fn) override { - return MakeHandler(::uiInterface.NotifyBlockTip_connect([fn](SynchronizationState sync_state, const CBlockIndex* block) { + return MakeSignalHandler(::uiInterface.NotifyBlockTip_connect([fn](SynchronizationState sync_state, const CBlockIndex* block) { fn(sync_state, BlockTip{block->nHeight, block->GetBlockTime(), block->GetBlockHash()}, GuessVerificationProgress(Params().TxData(), block)); })); } std::unique_ptr<Handler> handleNotifyHeaderTip(NotifyHeaderTipFn fn) override { - return MakeHandler( + return MakeSignalHandler( ::uiInterface.NotifyHeaderTip_connect([fn](SynchronizationState sync_state, int64_t height, int64_t timestamp, bool presync) { fn(sync_state, BlockTip{(int)height, timestamp, uint256{}}, presync); })); @@ -536,6 +539,20 @@ public: } return std::nullopt; } + bool hasBlockFilterIndex(BlockFilterType filter_type) override + { + return GetBlockFilterIndex(filter_type) != nullptr; + } + std::optional<bool> blockFilterMatchesAny(BlockFilterType filter_type, const uint256& block_hash, const GCSFilter::ElementSet& filter_set) override + { + const BlockFilterIndex* block_filter_index{GetBlockFilterIndex(filter_type)}; + if (!block_filter_index) return std::nullopt; + + BlockFilter filter; + const CBlockIndex* index{WITH_LOCK(::cs_main, return chainman().m_blockman.LookupBlockIndex(block_hash))}; + if (index == nullptr || !block_filter_index->LookupFilter(index, filter)) return std::nullopt; + return filter.GetFilter().MatchAny(filter_set); + } bool findBlock(const uint256& hash, const FoundBlock& block) override { WAIT_LOCK(cs_main, lock); diff --git a/src/node/psbt.cpp b/src/node/psbt.cpp index 57162cd679..ca3fc0955d 100644 --- a/src/node/psbt.cpp +++ b/src/node/psbt.cpp @@ -59,7 +59,7 @@ PSBTAnalysis AnalyzePSBT(PartiallySignedTransaction psbtx) } // Check if it is final - if (!utxo.IsNull() && !PSBTInputSigned(input)) { + if (!PSBTInputSignedAndVerified(psbtx, i, &txdata)) { input_analysis.is_final = false; // Figure out what is missing diff --git a/src/node/txreconciliation.cpp b/src/node/txreconciliation.cpp new file mode 100644 index 0000000000..ed04a78cec --- /dev/null +++ b/src/node/txreconciliation.cpp @@ -0,0 +1,169 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/txreconciliation.h> + +#include <util/check.h> +#include <util/system.h> + +#include <unordered_map> +#include <variant> + + +namespace { + +/** Static salt component used to compute short txids for sketch construction, see BIP-330. */ +const std::string RECON_STATIC_SALT = "Tx Relay Salting"; +const HashWriter RECON_SALT_HASHER = TaggedHash(RECON_STATIC_SALT); + +/** + * Salt (specified by BIP-330) constructed from contributions from both peers. It is used + * to compute transaction short IDs, which are then used to construct a sketch representing a set + * of transactions we want to announce to the peer. + */ +uint256 ComputeSalt(uint64_t salt1, uint64_t salt2) +{ + // According to BIP-330, salts should be combined in ascending order. + return (HashWriter(RECON_SALT_HASHER) << std::min(salt1, salt2) << std::max(salt1, salt2)).GetSHA256(); +} + +/** + * Keeps track of txreconciliation-related per-peer state. + */ +class TxReconciliationState +{ +public: + /** + * TODO: This field is public to ignore -Wunused-private-field. Make private once used in + * the following commits. + * + * Reconciliation protocol assumes using one role consistently: either a reconciliation + * initiator (requesting sketches), or responder (sending sketches). This defines our role, + * based on the direction of the p2p connection. + * + */ + bool m_we_initiate; + + /** + * TODO: These fields are public to ignore -Wunused-private-field. Make private once used in + * the following commits. + * + * These values are used to salt short IDs, which is necessary for transaction reconciliations. + */ + uint64_t m_k0, m_k1; + + TxReconciliationState(bool we_initiate, uint64_t k0, uint64_t k1) : m_we_initiate(we_initiate), m_k0(k0), m_k1(k1) {} +}; + +} // namespace + +/** Actual implementation for TxReconciliationTracker's data structure. */ +class TxReconciliationTracker::Impl +{ +private: + mutable Mutex m_txreconciliation_mutex; + + // Local protocol version + uint32_t m_recon_version; + + /** + * Keeps track of txreconciliation states of eligible peers. + * For pre-registered peers, the locally generated salt is stored. + * For registered peers, the locally generated salt is forgotten, and the state (including + * "full" salt) is stored instead. + */ + std::unordered_map<NodeId, std::variant<uint64_t, TxReconciliationState>> m_states GUARDED_BY(m_txreconciliation_mutex); + +public: + explicit Impl(uint32_t recon_version) : m_recon_version(recon_version) {} + + uint64_t PreRegisterPeer(NodeId peer_id) EXCLUSIVE_LOCKS_REQUIRED(!m_txreconciliation_mutex) + { + AssertLockNotHeld(m_txreconciliation_mutex); + LOCK(m_txreconciliation_mutex); + + LogPrintLevel(BCLog::TXRECONCILIATION, BCLog::Level::Debug, "Pre-register peer=%d\n", peer_id); + const uint64_t local_salt{GetRand(UINT64_MAX)}; + + // We do this exactly once per peer (which are unique by NodeId, see GetNewNodeId) so it's + // safe to assume we don't have this record yet. + Assume(m_states.emplace(peer_id, local_salt).second); + return local_salt; + } + + ReconciliationRegisterResult RegisterPeer(NodeId peer_id, bool is_peer_inbound, uint32_t peer_recon_version, + uint64_t remote_salt) EXCLUSIVE_LOCKS_REQUIRED(!m_txreconciliation_mutex) + { + AssertLockNotHeld(m_txreconciliation_mutex); + LOCK(m_txreconciliation_mutex); + auto recon_state = m_states.find(peer_id); + + if (recon_state == m_states.end()) return ReconciliationRegisterResult::NOT_FOUND; + + if (std::holds_alternative<TxReconciliationState>(recon_state->second)) { + return ReconciliationRegisterResult::ALREADY_REGISTERED; + } + + uint64_t local_salt = *std::get_if<uint64_t>(&recon_state->second); + + // If the peer supports the version which is lower than ours, we downgrade to the version + // it supports. For now, this only guarantees that nodes with future reconciliation + // versions have the choice of reconciling with this current version. However, they also + // have the choice to refuse supporting reconciliations if the common version is not + // satisfactory (e.g. too low). + const uint32_t recon_version{std::min(peer_recon_version, m_recon_version)}; + // v1 is the lowest version, so suggesting something below must be a protocol violation. + if (recon_version < 1) return ReconciliationRegisterResult::PROTOCOL_VIOLATION; + + LogPrintLevel(BCLog::TXRECONCILIATION, BCLog::Level::Debug, "Register peer=%d (inbound=%i)\n", + peer_id, is_peer_inbound); + + const uint256 full_salt{ComputeSalt(local_salt, remote_salt)}; + recon_state->second = TxReconciliationState(!is_peer_inbound, full_salt.GetUint64(0), full_salt.GetUint64(1)); + return ReconciliationRegisterResult::SUCCESS; + } + + void ForgetPeer(NodeId peer_id) EXCLUSIVE_LOCKS_REQUIRED(!m_txreconciliation_mutex) + { + AssertLockNotHeld(m_txreconciliation_mutex); + LOCK(m_txreconciliation_mutex); + if (m_states.erase(peer_id)) { + LogPrintLevel(BCLog::TXRECONCILIATION, BCLog::Level::Debug, "Forget txreconciliation state of peer=%d\n", peer_id); + } + } + + bool IsPeerRegistered(NodeId peer_id) const EXCLUSIVE_LOCKS_REQUIRED(!m_txreconciliation_mutex) + { + AssertLockNotHeld(m_txreconciliation_mutex); + LOCK(m_txreconciliation_mutex); + auto recon_state = m_states.find(peer_id); + return (recon_state != m_states.end() && + std::holds_alternative<TxReconciliationState>(recon_state->second)); + } +}; + +TxReconciliationTracker::TxReconciliationTracker(uint32_t recon_version) : m_impl{std::make_unique<TxReconciliationTracker::Impl>(recon_version)} {} + +TxReconciliationTracker::~TxReconciliationTracker() = default; + +uint64_t TxReconciliationTracker::PreRegisterPeer(NodeId peer_id) +{ + return m_impl->PreRegisterPeer(peer_id); +} + +ReconciliationRegisterResult TxReconciliationTracker::RegisterPeer(NodeId peer_id, bool is_peer_inbound, + uint32_t peer_recon_version, uint64_t remote_salt) +{ + return m_impl->RegisterPeer(peer_id, is_peer_inbound, peer_recon_version, remote_salt); +} + +void TxReconciliationTracker::ForgetPeer(NodeId peer_id) +{ + m_impl->ForgetPeer(peer_id); +} + +bool TxReconciliationTracker::IsPeerRegistered(NodeId peer_id) const +{ + return m_impl->IsPeerRegistered(peer_id); +} diff --git a/src/node/txreconciliation.h b/src/node/txreconciliation.h new file mode 100644 index 0000000000..4591dd5df7 --- /dev/null +++ b/src/node/txreconciliation.h @@ -0,0 +1,91 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_TXRECONCILIATION_H +#define BITCOIN_NODE_TXRECONCILIATION_H + +#include <net.h> +#include <sync.h> + +#include <memory> +#include <tuple> + +/** Whether transaction reconciliation protocol should be enabled by default. */ +static constexpr bool DEFAULT_TXRECONCILIATION_ENABLE{false}; +/** Supported transaction reconciliation protocol version */ +static constexpr uint32_t TXRECONCILIATION_VERSION{1}; + +enum class ReconciliationRegisterResult { + NOT_FOUND, + SUCCESS, + ALREADY_REGISTERED, + PROTOCOL_VIOLATION, +}; + +/** + * Transaction reconciliation is a way for nodes to efficiently announce transactions. + * This object keeps track of all txreconciliation-related communications with the peers. + * The high-level protocol is: + * 0. Txreconciliation protocol handshake. + * 1. Once we receive a new transaction, add it to the set instead of announcing immediately. + * 2. At regular intervals, a txreconciliation initiator requests a sketch from a peer, where a + * sketch is a compressed representation of short form IDs of the transactions in their set. + * 3. Once the initiator received a sketch from the peer, the initiator computes a local sketch, + * and combines the two sketches to attempt finding the difference in *sets*. + * 4a. If the difference was not larger than estimated, see SUCCESS below. + * 4b. If the difference was larger than estimated, initial txreconciliation fails. The initiator + * requests a larger sketch via an extension round (allowed only once). + * - If extension succeeds (a larger sketch is sufficient), see SUCCESS below. + * - If extension fails (a larger sketch is insufficient), see FAILURE below. + * + * SUCCESS. The initiator knows full symmetrical difference and can request what the initiator is + * missing and announce to the peer what the peer is missing. + * + * FAILURE. The initiator notifies the peer about the failure and announces all transactions from + * the corresponding set. Once the peer received the failure notification, the peer + * announces all transactions from their set. + + * This is a modification of the Erlay protocol (https://arxiv.org/abs/1905.10518) with two + * changes (sketch extensions instead of bisections, and an extra INV exchange round), both + * are motivated in BIP-330. + */ +class TxReconciliationTracker +{ +private: + class Impl; + const std::unique_ptr<Impl> m_impl; + +public: + explicit TxReconciliationTracker(uint32_t recon_version); + ~TxReconciliationTracker(); + + /** + * Step 0. Generates initial part of the state (salt) required to reconcile txs with the peer. + * The salt is used for short ID computation required for txreconciliation. + * The function returns the salt. + * A peer can't participate in future txreconciliations without this call. + * This function must be called only once per peer. + */ + uint64_t PreRegisterPeer(NodeId peer_id); + + /** + * Step 0. Once the peer agreed to reconcile txs with us, generate the state required to track + * ongoing reconciliations. Must be called only after pre-registering the peer and only once. + */ + ReconciliationRegisterResult RegisterPeer(NodeId peer_id, bool is_peer_inbound, + uint32_t peer_recon_version, uint64_t remote_salt); + + /** + * Attempts to forget txreconciliation-related state of the peer (if we previously stored any). + * After this, we won't be able to reconcile transactions with the peer. + */ + void ForgetPeer(NodeId peer_id); + + /** + * Check if a peer is registered to reconcile transactions with us. + */ + bool IsPeerRegistered(NodeId peer_id) const; +}; + +#endif // BITCOIN_NODE_TXRECONCILIATION_H diff --git a/src/node/utxo_snapshot.cpp b/src/node/utxo_snapshot.cpp new file mode 100644 index 0000000000..bab1b75211 --- /dev/null +++ b/src/node/utxo_snapshot.cpp @@ -0,0 +1,91 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/utxo_snapshot.h> + +#include <fs.h> +#include <logging.h> +#include <streams.h> +#include <uint256.h> +#include <util/system.h> +#include <validation.h> + +#include <cstdio> +#include <optional> + +namespace node { + +bool WriteSnapshotBaseBlockhash(Chainstate& snapshot_chainstate) +{ + AssertLockHeld(::cs_main); + assert(snapshot_chainstate.m_from_snapshot_blockhash); + + const std::optional<fs::path> chaindir = snapshot_chainstate.CoinsDB().StoragePath(); + assert(chaindir); // Sanity check that chainstate isn't in-memory. + const fs::path write_to = *chaindir / node::SNAPSHOT_BLOCKHASH_FILENAME; + + FILE* file{fsbridge::fopen(write_to, "wb")}; + AutoFile afile{file}; + if (afile.IsNull()) { + LogPrintf("[snapshot] failed to open base blockhash file for writing: %s\n", + fs::PathToString(write_to)); + return false; + } + afile << *snapshot_chainstate.m_from_snapshot_blockhash; + + if (afile.fclose() != 0) { + LogPrintf("[snapshot] failed to close base blockhash file %s after writing\n", + fs::PathToString(write_to)); + return false; + } + return true; +} + +std::optional<uint256> ReadSnapshotBaseBlockhash(fs::path chaindir) +{ + if (!fs::exists(chaindir)) { + LogPrintf("[snapshot] cannot read base blockhash: no chainstate dir " /* Continued */ + "exists at path %s\n", fs::PathToString(chaindir)); + return std::nullopt; + } + const fs::path read_from = chaindir / node::SNAPSHOT_BLOCKHASH_FILENAME; + const std::string read_from_str = fs::PathToString(read_from); + + if (!fs::exists(read_from)) { + LogPrintf("[snapshot] snapshot chainstate dir is malformed! no base blockhash file " /* Continued */ + "exists at path %s. Try deleting %s and calling loadtxoutset again?\n", + fs::PathToString(chaindir), read_from_str); + return std::nullopt; + } + + uint256 base_blockhash; + FILE* file{fsbridge::fopen(read_from, "rb")}; + AutoFile afile{file}; + if (afile.IsNull()) { + LogPrintf("[snapshot] failed to open base blockhash file for reading: %s\n", + read_from_str); + return std::nullopt; + } + afile >> base_blockhash; + + if (std::fgetc(afile.Get()) != EOF) { + LogPrintf("[snapshot] warning: unexpected trailing data in %s\n", read_from_str); + } else if (std::ferror(afile.Get())) { + LogPrintf("[snapshot] warning: i/o error reading %s\n", read_from_str); + } + return base_blockhash; +} + +std::optional<fs::path> FindSnapshotChainstateDir() +{ + fs::path possible_dir = + gArgs.GetDataDirNet() / fs::u8path(strprintf("chainstate%s", SNAPSHOT_CHAINSTATE_SUFFIX)); + + if (fs::exists(possible_dir)) { + return possible_dir; + } + return std::nullopt; +} + +} // namespace node diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h index 9dd6f06997..c94521792f 100644 --- a/src/node/utxo_snapshot.h +++ b/src/node/utxo_snapshot.h @@ -6,8 +6,14 @@ #ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H #define BITCOIN_NODE_UTXO_SNAPSHOT_H +#include <fs.h> #include <uint256.h> #include <serialize.h> +#include <validation.h> + +#include <optional> + +extern RecursiveMutex cs_main; namespace node { //! Metadata describing a serialized version of a UTXO set from which an @@ -33,6 +39,33 @@ public: SERIALIZE_METHODS(SnapshotMetadata, obj) { READWRITE(obj.m_base_blockhash, obj.m_coins_count); } }; + +//! The file in the snapshot chainstate dir which stores the base blockhash. This is +//! needed to reconstruct snapshot chainstates on init. +//! +//! Because we only allow loading a single snapshot at a time, there will only be one +//! chainstate directory with this filename present within it. +const fs::path SNAPSHOT_BLOCKHASH_FILENAME{"base_blockhash"}; + +//! Write out the blockhash of the snapshot base block that was used to construct +//! this chainstate. This value is read in during subsequent initializations and +//! used to reconstruct snapshot-based chainstates. +bool WriteSnapshotBaseBlockhash(Chainstate& snapshot_chainstate) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + +//! Read the blockhash of the snapshot base block that was used to construct the +//! chainstate. +std::optional<uint256> ReadSnapshotBaseBlockhash(fs::path chaindir) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + +//! Suffix appended to the chainstate (leveldb) dir when created based upon +//! a snapshot. +constexpr std::string_view SNAPSHOT_CHAINSTATE_SUFFIX = "_snapshot"; + + +//! Return a path to the snapshot-based chainstate dir, if one exists. +std::optional<fs::path> FindSnapshotChainstateDir(); + } // namespace node #endif // BITCOIN_NODE_UTXO_SNAPSHOT_H diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 22defb91a9..1cd9624000 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -8,6 +8,7 @@ #include <clientversion.h> #include <consensus/amount.h> #include <fs.h> +#include <kernel/mempool_entry.h> #include <logging.h> #include <policy/feerate.h> #include <primitives/transaction.h> @@ -16,7 +17,6 @@ #include <streams.h> #include <sync.h> #include <tinyformat.h> -#include <txmempool.h> #include <uint256.h> #include <util/serfloat.h> #include <util/system.h> @@ -1011,20 +1011,37 @@ void CBlockPolicyEstimator::FlushUnconfirmed() LogPrint(BCLog::ESTIMATEFEE, "Recorded %u unconfirmed txs from mempool in %gs\n", num_entries, Ticks<SecondsDouble>(endclear - startclear)); } -FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) +static std::set<double> MakeFeeSet(const CFeeRate& min_incremental_fee, + double max_filter_fee_rate, + double fee_filter_spacing) { - CAmount minFeeLimit = std::max(CAmount(1), minIncrementalFee.GetFeePerK() / 2); - feeset.insert(0); - for (double bucketBoundary = minFeeLimit; bucketBoundary <= MAX_FILTER_FEERATE; bucketBoundary *= FEE_FILTER_SPACING) { - feeset.insert(bucketBoundary); + std::set<double> fee_set; + + const CAmount min_fee_limit{std::max(CAmount(1), min_incremental_fee.GetFeePerK() / 2)}; + fee_set.insert(0); + for (double bucket_boundary = min_fee_limit; + bucket_boundary <= max_filter_fee_rate; + bucket_boundary *= fee_filter_spacing) { + + fee_set.insert(bucket_boundary); } + + return fee_set; +} + +FeeFilterRounder::FeeFilterRounder(const CFeeRate& minIncrementalFee) + : m_fee_set{MakeFeeSet(minIncrementalFee, MAX_FILTER_FEERATE, FEE_FILTER_SPACING)} +{ } CAmount FeeFilterRounder::round(CAmount currentMinFee) { - std::set<double>::iterator it = feeset.lower_bound(currentMinFee); - if ((it != feeset.begin() && insecure_rand.rand32() % 3 != 0) || it == feeset.end()) { - it--; + AssertLockNotHeld(m_insecure_rand_mutex); + std::set<double>::iterator it = m_fee_set.lower_bound(currentMinFee); + if (it == m_fee_set.end() || + (it != m_fee_set.begin() && + WITH_LOCK(m_insecure_rand_mutex, return insecure_rand.rand32()) % 3 != 0)) { + --it; } return static_cast<CAmount>(*it); } diff --git a/src/policy/fees.h b/src/policy/fees.h index e4628bf853..204c4f2118 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -299,14 +299,15 @@ private: public: /** Create new FeeFilterRounder */ - explicit FeeFilterRounder(const CFeeRate& minIncrementalFee); + explicit FeeFilterRounder(const CFeeRate& min_incremental_fee); - /** Quantize a minimum fee for privacy purpose before broadcast. Not thread-safe due to use of FastRandomContext */ - CAmount round(CAmount currentMinFee); + /** Quantize a minimum fee for privacy purpose before broadcast. */ + CAmount round(CAmount currentMinFee) EXCLUSIVE_LOCKS_REQUIRED(!m_insecure_rand_mutex); private: - std::set<double> feeset; - FastRandomContext insecure_rand; + const std::set<double> m_fee_set; + Mutex m_insecure_rand_mutex; + FastRandomContext insecure_rand GUARDED_BY(m_insecure_rand_mutex); }; #endif // BITCOIN_POLICY_FEES_H diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index 55f47f485b..3a347b41ed 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -5,6 +5,7 @@ #include <policy/rbf.h> #include <consensus/amount.h> +#include <kernel/mempool_entry.h> #include <policy/feerate.h> #include <primitives/transaction.h> #include <sync.h> diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index f496ea022e..6b4a6335a1 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -17,6 +17,7 @@ #include <ios> #include <limits> #include <memory> +#include <numeric> #include <string> #include <tuple> #include <utility> @@ -280,6 +281,12 @@ inline void SerializeTransaction(const TxType& tx, Stream& s) { s << tx.nLockTime; } +template<typename TxType> +inline CAmount CalculateOutputValue(const TxType& tx) +{ + return std::accumulate(tx.vout.cbegin(), tx.vout.cend(), CAmount{0}, [](CAmount sum, const auto& txout) { return sum + txout.nValue; }); +} + /** The basic transaction that is broadcasted on the network and contained in * blocks. A transaction can contain multiple inputs and outputs. diff --git a/src/protocol.cpp b/src/protocol.cpp index bdd1cc2aff..23c68b335b 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -44,6 +44,7 @@ const char *CFHEADERS="cfheaders"; const char *GETCFCHECKPT="getcfcheckpt"; const char *CFCHECKPT="cfcheckpt"; const char *WTXIDRELAY="wtxidrelay"; +const char *SENDTXRCNCL="sendtxrcncl"; } // namespace NetMsgType /** All known message types. Keep this in the same order as the list of @@ -84,6 +85,7 @@ const static std::string allNetMessageTypes[] = { NetMsgType::GETCFCHECKPT, NetMsgType::CFCHECKPT, NetMsgType::WTXIDRELAY, + NetMsgType::SENDTXRCNCL, }; const static std::vector<std::string> allNetMessageTypesVec(std::begin(allNetMessageTypes), std::end(allNetMessageTypes)); diff --git a/src/protocol.h b/src/protocol.h index b85dc0d820..51fabf8da0 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -258,6 +258,12 @@ extern const char* CFCHECKPT; * @since protocol version 70016 as described by BIP 339. */ extern const char* WTXIDRELAY; +/** + * Contains a 4-byte version number and an 8-byte salt. + * The salt is used to compute short txids needed for efficient + * txreconciliation, as described by BIP 330. + */ +extern const char* SENDTXRCNCL; }; // namespace NetMsgType /* Get a vector of all valid message types (see above) */ diff --git a/src/psbt.cpp b/src/psbt.cpp index 36fec74bc9..461987c503 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -4,6 +4,7 @@ #include <psbt.h> +#include <policy/policy.h> #include <util/check.h> #include <util/strencodings.h> @@ -218,8 +219,14 @@ void PSBTOutput::FillSignatureData(SignatureData& sigdata) const for (const auto& key_pair : hd_keypaths) { sigdata.misc_pubkeys.emplace(key_pair.first.GetID(), key_pair); } - if (m_tap_tree.has_value() && m_tap_internal_key.IsFullyValid()) { - TaprootSpendData spenddata = m_tap_tree->GetSpendData(); + if (!m_tap_tree.empty() && m_tap_internal_key.IsFullyValid()) { + TaprootBuilder builder; + for (const auto& [depth, leaf_ver, script] : m_tap_tree) { + builder.Add((int)depth, script, (int)leaf_ver, /*track=*/true); + } + assert(builder.IsComplete()); + builder.Finalize(m_tap_internal_key); + TaprootSpendData spenddata = builder.GetSpendData(); sigdata.tr_spenddata.internal_key = m_tap_internal_key; sigdata.tr_spenddata.Merge(spenddata); @@ -243,8 +250,8 @@ void PSBTOutput::FromSignatureData(const SignatureData& sigdata) if (!sigdata.tr_spenddata.internal_key.IsNull()) { m_tap_internal_key = sigdata.tr_spenddata.internal_key; } - if (sigdata.tr_builder.has_value()) { - m_tap_tree = sigdata.tr_builder; + if (sigdata.tr_builder.has_value() && sigdata.tr_builder->HasScripts()) { + m_tap_tree = sigdata.tr_builder->GetTreeTuples(); } for (const auto& [pubkey, leaf_origin] : sigdata.taproot_misc_pubkeys) { m_tap_bip32_paths.emplace(pubkey, leaf_origin); @@ -265,13 +272,43 @@ void PSBTOutput::Merge(const PSBTOutput& output) if (redeem_script.empty() && !output.redeem_script.empty()) redeem_script = output.redeem_script; if (witness_script.empty() && !output.witness_script.empty()) witness_script = output.witness_script; if (m_tap_internal_key.IsNull() && !output.m_tap_internal_key.IsNull()) m_tap_internal_key = output.m_tap_internal_key; - if (m_tap_tree.has_value() && !output.m_tap_tree.has_value()) m_tap_tree = output.m_tap_tree; + if (m_tap_tree.empty() && !output.m_tap_tree.empty()) m_tap_tree = output.m_tap_tree; } + bool PSBTInputSigned(const PSBTInput& input) { return !input.final_script_sig.empty() || !input.final_script_witness.IsNull(); } +bool PSBTInputSignedAndVerified(const PartiallySignedTransaction psbt, unsigned int input_index, const PrecomputedTransactionData* txdata) +{ + CTxOut utxo; + assert(psbt.inputs.size() >= input_index); + const PSBTInput& input = psbt.inputs[input_index]; + + if (input.non_witness_utxo) { + // If we're taking our information from a non-witness UTXO, verify that it matches the prevout. + COutPoint prevout = psbt.tx->vin[input_index].prevout; + if (prevout.n >= input.non_witness_utxo->vout.size()) { + return false; + } + if (input.non_witness_utxo->GetHash() != prevout.hash) { + return false; + } + utxo = input.non_witness_utxo->vout[prevout.n]; + } else if (!input.witness_utxo.IsNull()) { + utxo = input.witness_utxo; + } else { + return false; + } + + if (txdata) { + return VerifyScript(input.final_script_sig, utxo.scriptPubKey, &input.final_script_witness, STANDARD_SCRIPT_VERIFY_FLAGS, MutableTransactionSignatureChecker{&(*psbt.tx), input_index, utxo.nValue, *txdata, MissingDataBehavior::FAIL}); + } else { + return VerifyScript(input.final_script_sig, utxo.scriptPubKey, &input.final_script_witness, STANDARD_SCRIPT_VERIFY_FLAGS, MutableTransactionSignatureChecker{&(*psbt.tx), input_index, utxo.nValue, MissingDataBehavior::FAIL}); + } +} + size_t CountPSBTUnsignedInputs(const PartiallySignedTransaction& psbt) { size_t count = 0; for (const auto& input : psbt.inputs) { @@ -325,7 +362,7 @@ bool SignPSBTInput(const SigningProvider& provider, PartiallySignedTransaction& PSBTInput& input = psbt.inputs.at(index); const CMutableTransaction& tx = *psbt.tx; - if (PSBTInputSigned(input)) { + if (PSBTInputSignedAndVerified(psbt, index, txdata)) { return true; } diff --git a/src/psbt.h b/src/psbt.h index d5c67802c7..fc9ac71eee 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -178,7 +178,7 @@ void SerializeHDKeypath(Stream& s, KeyOriginInfo hd_keypath) template<typename Stream> void SerializeHDKeypaths(Stream& s, const std::map<CPubKey, KeyOriginInfo>& hd_keypaths, CompactSizeWriter type) { - for (auto keypath_pair : hd_keypaths) { + for (const auto& keypath_pair : hd_keypaths) { if (!keypath_pair.first.IsValid()) { throw std::ios_base::failure("Invalid CPubKey being serialized"); } @@ -713,7 +713,7 @@ struct PSBTOutput CScript witness_script; std::map<CPubKey, KeyOriginInfo> hd_keypaths; XOnlyPubKey m_tap_internal_key; - std::optional<TaprootBuilder> m_tap_tree; + std::vector<std::tuple<uint8_t, uint8_t, CScript>> m_tap_tree; std::map<XOnlyPubKey, std::pair<std::set<uint256>, KeyOriginInfo>> m_tap_bip32_paths; std::map<std::vector<unsigned char>, std::vector<unsigned char>> unknown; std::set<PSBTProprietary> m_proprietary; @@ -754,15 +754,11 @@ struct PSBTOutput } // Write taproot tree - if (m_tap_tree.has_value()) { + if (!m_tap_tree.empty()) { SerializeToVector(s, PSBT_OUT_TAP_TREE); std::vector<unsigned char> value; CVectorWriter s_value(s.GetType(), s.GetVersion(), value, 0); - const auto& tuples = m_tap_tree->GetTreeTuples(); - for (const auto& tuple : tuples) { - uint8_t depth = std::get<0>(tuple); - uint8_t leaf_ver = std::get<1>(tuple); - CScript script = std::get<2>(tuple); + for (const auto& [depth, leaf_ver, script] : m_tap_tree) { s_value << depth; s_value << leaf_ver; s_value << script; @@ -858,10 +854,13 @@ struct PSBTOutput } else if (key.size() != 1) { throw std::ios_base::failure("Output Taproot tree key is more than one byte type"); } - m_tap_tree.emplace(); std::vector<unsigned char> tree_v; s >> tree_v; SpanReader s_tree(s.GetType(), s.GetVersion(), tree_v); + if (s_tree.empty()) { + throw std::ios_base::failure("Output Taproot tree must not be empty"); + } + TaprootBuilder builder; while (!s_tree.empty()) { uint8_t depth; uint8_t leaf_ver; @@ -875,9 +874,10 @@ struct PSBTOutput if ((leaf_ver & ~TAPROOT_LEAF_MASK) != 0) { throw std::ios_base::failure("Output Taproot tree has a leaf with an invalid leaf version"); } - m_tap_tree->Add((int)depth, script, (int)leaf_ver, true /* track */); + m_tap_tree.push_back(std::make_tuple(depth, leaf_ver, script)); + builder.Add((int)depth, script, (int)leaf_ver, /*track=*/true); } - if (!m_tap_tree->IsComplete()) { + if (!builder.IsComplete()) { throw std::ios_base::failure("Output Taproot tree is malformed"); } break; @@ -931,11 +931,6 @@ struct PSBTOutput } } - // Finalize m_tap_tree so that all of the computed things are computed - if (m_tap_tree.has_value() && m_tap_tree->IsComplete() && m_tap_internal_key.IsFullyValid()) { - m_tap_tree->Finalize(m_tap_internal_key); - } - if (!found_sep) { throw std::ios_base::failure("Separator is missing at the end of an output map"); } @@ -1223,9 +1218,12 @@ std::string PSBTRoleName(PSBTRole role); /** Compute a PrecomputedTransactionData object from a psbt. */ PrecomputedTransactionData PrecomputePSBTData(const PartiallySignedTransaction& psbt); -/** Checks whether a PSBTInput is already signed. */ +/** Checks whether a PSBTInput is already signed by checking for non-null finalized fields. */ bool PSBTInputSigned(const PSBTInput& input); +/** Checks whether a PSBTInput is already signed by doing script verification using final fields. */ +bool PSBTInputSignedAndVerified(const PartiallySignedTransaction psbt, unsigned int input_index, const PrecomputedTransactionData* txdata); + /** Signs a PSBTInput, verifying that all provided data matches what is being signed. * * txdata should be the output of PrecomputePSBTData (which can be shared across diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index cc01e4d54a..1bd7795a38 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -311,11 +311,7 @@ void BitcoinApplication::createSplashScreen(const NetworkStyle *networkStyle) { assert(!m_splash); m_splash = new SplashScreen(networkStyle); - // We don't hold a direct pointer to the splash screen after creation, but the splash - // screen will take care of deleting itself when finish() happens. m_splash->show(); - connect(this, &BitcoinApplication::splashFinished, m_splash, &SplashScreen::finish); - connect(this, &BitcoinApplication::requestedShutdown, m_splash, &QWidget::close); } void BitcoinApplication::createNode(interfaces::Init& init) @@ -373,6 +369,9 @@ void BitcoinApplication::requestShutdown() w->hide(); } + delete m_splash; + m_splash = nullptr; + // Show a simple window indicating shutdown status // Do this first as some of the steps may take some time below, // for example the RPC console may still be executing a command. @@ -412,10 +411,13 @@ void BitcoinApplication::requestShutdown() void BitcoinApplication::initializeResult(bool success, interfaces::BlockAndHeaderTipInfo tip_info) { qDebug() << __func__ << ": Initialization result: " << success; + // Set exit result. returnValue = success ? EXIT_SUCCESS : EXIT_FAILURE; - if(success) - { + if(success) { + delete m_splash; + m_splash = nullptr; + // Log this only after AppInitMain finishes, as then logging setup is guaranteed complete qInfo() << "Platform customization:" << platformStyle->getName(); clientModel = new ClientModel(node(), optionsModel); @@ -438,7 +440,6 @@ void BitcoinApplication::initializeResult(bool success, interfaces::BlockAndHead } else { window->showMinimized(); } - Q_EMIT splashFinished(); Q_EMIT windowShown(window); #ifdef ENABLE_WALLET @@ -455,7 +456,6 @@ void BitcoinApplication::initializeResult(bool success, interfaces::BlockAndHead #endif pollShutdownTimer->start(SHUTDOWN_POLLING_DELAY); } else { - Q_EMIT splashFinished(); // Make sure splash screen doesn't stick around during shutdown requestShutdown(); } } diff --git a/src/qt/bitcoin.h b/src/qt/bitcoin.h index 9ad37ca6c9..4f9eed5f49 100644 --- a/src/qt/bitcoin.h +++ b/src/qt/bitcoin.h @@ -89,7 +89,6 @@ public Q_SLOTS: Q_SIGNALS: void requestedInitialize(); void requestedShutdown(); - void splashFinished(); void windowShown(BitcoinGUI* window); protected: diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index 894a401e56..2bd38103ed 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -410,7 +410,7 @@ void BitcoinGUI::createActions() connect(action, &QAction::triggered, [this, path] { auto activity = new OpenWalletActivity(m_wallet_controller, this); - connect(activity, &OpenWalletActivity::opened, this, &BitcoinGUI::setCurrentWallet); + connect(activity, &OpenWalletActivity::opened, this, &BitcoinGUI::setCurrentWallet, Qt::QueuedConnection); activity->open(path); }); } @@ -512,7 +512,7 @@ void BitcoinGUI::createMenuBar() connect(minimize_action, &QAction::triggered, [] { QApplication::activeWindow()->showMinimized(); }); - connect(qApp, &QApplication::focusWindowChanged, [minimize_action] (QWindow* window) { + connect(qApp, &QApplication::focusWindowChanged, this, [minimize_action] (QWindow* window) { minimize_action->setEnabled(window != nullptr && (window->flags() & Qt::Dialog) != Qt::Dialog && window->windowState() != Qt::WindowMinimized); }); @@ -527,7 +527,7 @@ void BitcoinGUI::createMenuBar() } }); - connect(qApp, &QApplication::focusWindowChanged, [zoom_action] (QWindow* window) { + connect(qApp, &QApplication::focusWindowChanged, this, [zoom_action] (QWindow* window) { zoom_action->setEnabled(window != nullptr); }); #endif diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index bd9a90a890..3bedbf29d8 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -477,7 +477,7 @@ void CoinControlDialog::updateLabels(CCoinControl& m_coin_control, WalletModel * nBytes -= 34; // Fee - nPayFee = model->wallet().getMinimumFee(nBytes, m_coin_control, nullptr /* returned_target */, nullptr /* reason */); + nPayFee = model->wallet().getMinimumFee(nBytes, m_coin_control, /*returned_target=*/nullptr, /*reason=*/nullptr); if (nPayAmount > 0) { diff --git a/src/qt/forms/debugwindow.ui b/src/qt/forms/debugwindow.ui index ead977296a..f1b66341d1 100644 --- a/src/qt/forms/debugwindow.ui +++ b/src/qt/forms/debugwindow.ui @@ -1185,10 +1185,10 @@ <item row="6" column="0"> <widget class="QLabel" name="peerRelayTxesLabel"> <property name="toolTip"> - <string>Whether the peer requested us to relay transactions.</string> + <string>Whether we relay transactions to this peer.</string> </property> <property name="text"> - <string>Wants Tx Relay</string> + <string>Transaction Relay</string> </property> </widget> </item> diff --git a/src/qt/forms/optionsdialog.ui b/src/qt/forms/optionsdialog.ui index 582f02132a..6be47e94fe 100644 --- a/src/qt/forms/optionsdialog.ui +++ b/src/qt/forms/optionsdialog.ui @@ -286,7 +286,7 @@ <item> <widget class="QLineEdit" name="externalSignerPath"> <property name="toolTip"> - <string>Full path to a Bitcoin Core compatible script (e.g. C:\Downloads\hwi.exe or /Users/you/Downloads/hwi.py). Beware: malware can steal your coins!</string> + <string>Full path to a %1 compatible script (e.g. C:\Downloads\hwi.exe or /Users/you/Downloads/hwi.py). Beware: malware can steal your coins!</string> </property> </widget> </item> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index b9f0be41e3..6e88b57e08 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -615,9 +615,10 @@ bool SetStartOnSystemStartup(bool fAutoStart) else { char pszExePath[MAX_PATH+1]; - ssize_t r = readlink("/proc/self/exe", pszExePath, sizeof(pszExePath) - 1); - if (r == -1) + ssize_t r = readlink("/proc/self/exe", pszExePath, sizeof(pszExePath)); + if (r == -1 || r > MAX_PATH) { return false; + } pszExePath[r] = '\0'; fs::create_directories(GetAutostartDir()); diff --git a/src/qt/main.cpp b/src/qt/main.cpp index e8f39584ad..45131a1cf5 100644 --- a/src/qt/main.cpp +++ b/src/qt/main.cpp @@ -4,9 +4,9 @@ #include <qt/bitcoin.h> +#include <common/url.h> #include <compat/compat.h> #include <util/translation.h> -#include <util/url.h> #include <QCoreApplication> diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 2b6711ca40..7c91eac37e 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -91,7 +91,9 @@ OptionsDialog::OptionsDialog(QWidget *parent, bool enableWallet) : ui->thirdPartyTxUrls->setVisible(false); } -#ifndef ENABLE_EXTERNAL_SIGNER +#ifdef ENABLE_EXTERNAL_SIGNER + ui->externalSignerPath->setToolTip(ui->externalSignerPath->toolTip().arg(PACKAGE_NAME)); +#else //: "External signing" means using devices such as hardware wallets. ui->externalSignerPath->setToolTip(tr("Compiled without external signing support (required for external signing)")); ui->externalSignerPath->setEnabled(false); diff --git a/src/qt/psbtoperationsdialog.cpp b/src/qt/psbtoperationsdialog.cpp index 333766ce21..afffae0784 100644 --- a/src/qt/psbtoperationsdialog.cpp +++ b/src/qt/psbtoperationsdialog.cpp @@ -56,7 +56,7 @@ void PSBTOperationsDialog::openWithPSBT(PartiallySignedTransaction psbtx) bool complete = FinalizePSBT(psbtx); // Make sure all existing signatures are fully combined before checking for completeness. if (m_wallet_model) { size_t n_could_sign; - TransactionError err = m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, false /* sign */, true /* bip32derivs */, &n_could_sign, m_transaction_data, complete); + TransactionError err = m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, &n_could_sign, m_transaction_data, complete); if (err != TransactionError::OK) { showStatus(tr("Failed to load transaction: %1") .arg(QString::fromStdString(TransactionErrorString(err).translated)), @@ -80,7 +80,7 @@ void PSBTOperationsDialog::signTransaction() WalletModel::UnlockContext ctx(m_wallet_model->requestUnlock()); - TransactionError err = m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, true /* sign */, true /* bip32derivs */, &n_signed, m_transaction_data, complete); + TransactionError err = m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/true, /*bip32derivs=*/true, &n_signed, m_transaction_data, complete); if (err != TransactionError::OK) { showStatus(tr("Failed to sign transaction: %1") @@ -245,7 +245,7 @@ size_t PSBTOperationsDialog::couldSignInputs(const PartiallySignedTransaction &p size_t n_signed; bool complete; - TransactionError err = m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, false /* sign */, false /* bip32derivs */, &n_signed, m_transaction_data, complete); + TransactionError err = m_wallet_model->wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/false, &n_signed, m_transaction_data, complete); if (err != TransactionError::OK) { return 0; diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 295450d6b7..a07686ab2b 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -1178,8 +1178,12 @@ void RPCConsole::updateDetailWidget() ui->peerPingTime->setText(GUIUtil::formatPingTime(stats->nodeStats.m_last_ping_time)); ui->peerMinPing->setText(GUIUtil::formatPingTime(stats->nodeStats.m_min_ping_time)); ui->timeoffset->setText(GUIUtil::formatTimeOffset(stats->nodeStats.nTimeOffset)); - ui->peerVersion->setText(QString::number(stats->nodeStats.nVersion)); - ui->peerSubversion->setText(QString::fromStdString(stats->nodeStats.cleanSubVer)); + if (stats->nodeStats.nVersion) { + ui->peerVersion->setText(QString::number(stats->nodeStats.nVersion)); + } + if (!stats->nodeStats.cleanSubVer.empty()) { + ui->peerSubversion->setText(QString::fromStdString(stats->nodeStats.cleanSubVer)); + } ui->peerConnectionType->setText(GUIUtil::ConnectionTypeToQString(stats->nodeStats.m_conn_type, /*prepend_direction=*/true)); ui->peerNetwork->setText(GUIUtil::NetworkToQString(stats->nodeStats.m_network)); if (stats->nodeStats.m_permission_flags == NetPermissionFlags::None) { diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index 53c352b393..57094fc857 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -289,7 +289,9 @@ bool SendCoinsDialog::PrepareSendText(QString& question_string, QString& informa updateCoinControlState(); - prepareStatus = model->prepareTransaction(*m_current_transaction, *m_coin_control); + CCoinControl coin_control = *m_coin_control; + coin_control.m_allow_other_inputs = !coin_control.HasSelected(); // future, could introduce a checkbox to customize this value. + prepareStatus = model->prepareTransaction(*m_current_transaction, coin_control); // process prepareStatus and on error generate message shown to user processSendCoinsReturn(prepareStatus, diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index bf04a6dd5f..e1692aeec7 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -161,16 +161,6 @@ bool SplashScreen::eventFilter(QObject * obj, QEvent * ev) { return QObject::eventFilter(obj, ev); } -void SplashScreen::finish() -{ - /* If the window is minimized, hide() will be ignored. */ - /* Make sure we de-minimize the splashscreen window before hiding */ - if (isMinimized()) - showNormal(); - hide(); - deleteLater(); // No more need for this -} - static void InitMessage(SplashScreen *splash, const std::string &message) { bool invoked = QMetaObject::invokeMethod(splash, "showMessage", diff --git a/src/qt/splashscreen.h b/src/qt/splashscreen.h index c14fc521a7..0f7fb6bef8 100644 --- a/src/qt/splashscreen.h +++ b/src/qt/splashscreen.h @@ -37,9 +37,6 @@ protected: void closeEvent(QCloseEvent *event) override; public Q_SLOTS: - /** Hide the splash screen window and schedule the splash screen object for deletion */ - void finish(); - /** Show message and progress */ void showMessage(const QString &message, int alignment, const QColor &color); diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index 6fc7a52435..a5cef9a1bb 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -72,7 +72,7 @@ void AppTests::appTests() qRegisterMetaType<interfaces::BlockAndHeaderTipInfo>("interfaces::BlockAndHeaderTipInfo"); m_app.parameterSetup(); - QVERIFY(m_app.createOptionsModel(true /* reset settings */)); + QVERIFY(m_app.createOptionsModel(/*resetSettings=*/true)); QScopedPointer<const NetworkStyle> style(NetworkStyle::instantiate(Params().NetworkIDString())); m_app.setupPlatformStyle(); m_app.createWindow(style.data()); diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index 846fa519ee..3f582e7cf6 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -79,8 +79,6 @@ int main(int argc, char* argv[]) setenv("QT_QPA_PLATFORM", "minimal", 0 /* overwrite */); #endif - // Don't remove this, it's needed to access - // QApplication:: and QCoreApplication:: in the tests BitcoinApplication app; app.setApplicationName("Bitcoin-Qt-test"); app.createNode(*init); diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index b71dfb0e9f..c32525b607 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -210,17 +210,17 @@ void TestGUI(interfaces::Node& node) // Send two transactions, and verify they are added to transaction list. TransactionTableModel* transactionTableModel = walletModel.getTransactionTableModel(); QCOMPARE(transactionTableModel->rowCount({}), 105); - uint256 txid1 = SendCoins(*wallet.get(), sendCoinsDialog, PKHash(), 5 * COIN, false /* rbf */); - uint256 txid2 = SendCoins(*wallet.get(), sendCoinsDialog, PKHash(), 10 * COIN, true /* rbf */); + uint256 txid1 = SendCoins(*wallet.get(), sendCoinsDialog, PKHash(), 5 * COIN, /*rbf=*/false); + uint256 txid2 = SendCoins(*wallet.get(), sendCoinsDialog, PKHash(), 10 * COIN, /*rbf=*/true); QCOMPARE(transactionTableModel->rowCount({}), 107); QVERIFY(FindTx(*transactionTableModel, txid1).isValid()); QVERIFY(FindTx(*transactionTableModel, txid2).isValid()); // Call bumpfee. Test disabled, canceled, enabled, then failing cases. - BumpFee(transactionView, txid1, true /* expect disabled */, "not BIP 125 replaceable" /* expected error */, false /* cancel */); - BumpFee(transactionView, txid2, false /* expect disabled */, {} /* expected error */, true /* cancel */); - BumpFee(transactionView, txid2, false /* expect disabled */, {} /* expected error */, false /* cancel */); - BumpFee(transactionView, txid2, true /* expect disabled */, "already bumped" /* expected error */, false /* cancel */); + BumpFee(transactionView, txid1, /*expectDisabled=*/true, /*expectError=*/"not BIP 125 replaceable", /*cancel=*/false); + BumpFee(transactionView, txid2, /*expectDisabled=*/false, /*expectError=*/{}, /*cancel=*/true); + BumpFee(transactionView, txid2, /*expectDisabled=*/false, /*expectError=*/{}, /*cancel=*/false); + BumpFee(transactionView, txid2, /*expectDisabled=*/true, /*expectError=*/"already bumped", /*cancel=*/false); // Check current balance on OverviewPage OverviewPage overviewPage(platformStyle.get()); diff --git a/src/qt/transactionrecord.h b/src/qt/transactionrecord.h index d8748d7dc9..fe9c85279e 100644 --- a/src/qt/transactionrecord.h +++ b/src/qt/transactionrecord.h @@ -76,12 +76,12 @@ public: static const int RecommendedNumConfirmations = 6; TransactionRecord(): - hash(), time(0), type(Other), address(""), debit(0), credit(0), idx(0) + hash(), time(0), type(Other), debit(0), credit(0), idx(0) { } TransactionRecord(uint256 _hash, qint64 _time): - hash(_hash), time(_time), type(Other), address(""), debit(0), + hash(_hash), time(_time), type(Other), debit(0), credit(0), idx(0) { } diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index b7432a0d77..b1c722388c 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -421,9 +421,6 @@ void TransactionView::abandonTx() // Abandon the wallet transaction over the walletModel model->wallet().abandonTransaction(hash); - - // Update the table - model->getTransactionTableModel()->updateTransaction(hashQStr, CT_UPDATED, false); } void TransactionView::bumpFee([[maybe_unused]] bool checked) diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index 8dc97e66a2..adc7793916 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -212,7 +212,7 @@ void WalletFrame::gotoLoadPSBT(bool from_clipboard) return; } std::ifstream in{filename.toLocal8Bit().data(), std::ios::binary}; - data.assign(std::istream_iterator<unsigned char>{in}, {}); + data.assign(std::istreambuf_iterator<char>{in}, {}); // Some psbt files may be base64 strings in the file rather than binary data std::string b64_str{data.begin(), data.end()}; diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index c6f3f5b00c..32c0402d45 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -217,7 +217,7 @@ WalletModel::SendCoinsReturn WalletModel::prepareTransaction(WalletModelTransact int nChangePosRet = -1; auto& newTx = transaction.getWtx(); - const auto& res = m_wallet->createTransaction(vecSend, coinControl, !wallet().privateKeysDisabled() /* sign */, nChangePosRet, nFeeRequired); + const auto& res = m_wallet->createTransaction(vecSend, coinControl, /*sign=*/!wallet().privateKeysDisabled(), nChangePosRet, nFeeRequired); newTx = res ? *res : nullptr; transaction.setTransactionFee(nFeeRequired); if (fSubtractFeeFromAmount && newTx) @@ -258,7 +258,7 @@ void WalletModel::sendCoins(WalletModelTransaction& transaction) } auto& newTx = transaction.getWtx(); - wallet().commitTransaction(newTx, {} /* mapValue */, std::move(vOrderForm)); + wallet().commitTransaction(newTx, /*value_map=*/{}, std::move(vOrderForm)); CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << *newTx; @@ -522,7 +522,9 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) questionString.append(tr("Warning: This may pay the additional fee by reducing change outputs or adding inputs, when necessary. It may add a new change output if one does not already exist. These changes may potentially leak privacy.")); } - auto confirmationDialog = new SendConfirmationDialog(tr("Confirm fee bump"), questionString, "", "", SEND_CONFIRM_DELAY, !m_wallet->privateKeysDisabled(), getOptionsModel()->getEnablePSBTControls(), nullptr); + const bool enable_send{!wallet().privateKeysDisabled() || wallet().hasExternalSigner()}; + const bool always_show_unsigned{getOptionsModel()->getEnablePSBTControls()}; + auto confirmationDialog = new SendConfirmationDialog(tr("Confirm fee bump"), questionString, "", "", SEND_CONFIRM_DELAY, enable_send, always_show_unsigned, nullptr); confirmationDialog->setAttribute(Qt::WA_DeleteOnClose); // TODO: Replace QDialog::exec() with safer QDialog::show(). const auto retval = static_cast<QMessageBox::StandardButton>(confirmationDialog->exec()); @@ -540,9 +542,10 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) // Short-circuit if we are returning a bumped transaction PSBT to clipboard if (retval == QMessageBox::Save) { + // "Create Unsigned" clicked PartiallySignedTransaction psbtx(mtx); bool complete = false; - const TransactionError err = wallet().fillPSBT(SIGHASH_ALL, false /* sign */, true /* bip32derivs */, nullptr, psbtx, complete); + const TransactionError err = wallet().fillPSBT(SIGHASH_ALL, /*sign=*/false, /*bip32derivs=*/true, nullptr, psbtx, complete); if (err != TransactionError::OK || complete) { QMessageBox::critical(nullptr, tr("Fee bump error"), tr("Can't draft transaction.")); return false; @@ -555,7 +558,7 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) return true; } - assert(!m_wallet->privateKeysDisabled()); + assert(!m_wallet->privateKeysDisabled() || wallet().hasExternalSigner()); // sign bumped transaction if (!m_wallet->signBumpTransaction(mtx)) { diff --git a/src/rest.cpp b/src/rest.cpp index 7f00db2222..033e93468e 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -305,8 +305,10 @@ static bool rest_block(const std::any& context, if (chainman.m_blockman.IsBlockPruned(pblockindex)) return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not available (pruned data)"); - if (!ReadBlockFromDisk(block, pblockindex, chainman.GetParams().GetConsensus())) - return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); + } + + if (!ReadBlockFromDisk(block, pblockindex, chainman.GetParams().GetConsensus())) { + return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } switch (rf) { @@ -590,6 +592,48 @@ static bool rest_chaininfo(const std::any& context, HTTPRequest* req, const std: } } + +RPCHelpMan getdeploymentinfo(); + +static bool rest_deploymentinfo(const std::any& context, HTTPRequest* req, const std::string& str_uri_part) +{ + if (!CheckWarmup(req)) return false; + + std::string hash_str; + const RESTResponseFormat rf = ParseDataFormat(hash_str, str_uri_part); + + switch (rf) { + case RESTResponseFormat::JSON: { + JSONRPCRequest jsonRequest; + jsonRequest.context = context; + jsonRequest.params = UniValue(UniValue::VARR); + + if (!hash_str.empty()) { + uint256 hash; + if (!ParseHashStr(hash_str, hash)) { + return RESTERR(req, HTTP_BAD_REQUEST, "Invalid hash: " + hash_str); + } + + const ChainstateManager* chainman = GetChainman(context, req); + if (!chainman) return false; + if (!WITH_LOCK(::cs_main, return chainman->m_blockman.LookupBlockIndex(ParseHashV(hash_str, "blockhash")))) { + return RESTERR(req, HTTP_BAD_REQUEST, "Block not found"); + } + + jsonRequest.params.pushKV("blockhash", hash_str); + } + + req->WriteHeader("Content-Type", "application/json"); + req->WriteReply(HTTP_OK, getdeploymentinfo().HandleRequest(jsonRequest).write() + "\n"); + return true; + } + default: { + return RESTERR(req, HTTP_NOT_FOUND, "output format not found (available: json)"); + } + } + +} + static bool rest_mempool(const std::any& context, HTTPRequest* req, const std::string& str_uri_part) { if (!CheckWarmup(req)) @@ -939,6 +983,8 @@ static const struct { {"/rest/mempool/", rest_mempool}, {"/rest/headers/", rest_headers}, {"/rest/getutxos", rest_getutxos}, + {"/rest/deploymentinfo/", rest_deploymentinfo}, + {"/rest/deploymentinfo", rest_deploymentinfo}, {"/rest/blockhashbyheight/", rest_blockhash_by_height}, }; diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 93d11c8276..784fb64d36 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -181,7 +181,8 @@ UniValue blockToJSON(BlockManager& blockman, const CBlock& block, const CBlockIn case TxVerbosity::SHOW_DETAILS: case TxVerbosity::SHOW_DETAILS_AND_PREVOUT: CBlockUndo blockUndo; - const bool have_undo{WITH_LOCK(::cs_main, return !blockman.IsBlockPruned(blockindex) && UndoReadFromDisk(blockUndo, blockindex))}; + const bool is_not_pruned{WITH_LOCK(::cs_main, return !blockman.IsBlockPruned(blockindex))}; + const bool have_undo{is_not_pruned && UndoReadFromDisk(blockUndo, blockindex)}; for (size_t i = 0; i < block.vtx.size(); ++i) { const CTransactionRef& tx = block.vtx.at(i); @@ -428,7 +429,9 @@ static RPCHelpMan getblockfrompeer() "getblockfrompeer", "Attempt to fetch block from a given peer.\n\n" "We must have the header for this block, e.g. using submitheader.\n" - "Subsequent calls for the same block and a new peer will cause the response from the previous peer to be ignored.\n\n" + "Subsequent calls for the same block and a new peer will cause the response from the previous peer to be ignored.\n" + "Peers generally ignore requests for a stale block that they never fully verified, or one that is more than a month old.\n" + "When a peer does not respond with a block, we will disconnect.\n\n" "Returns an empty JSON object if the request was successfully scheduled.", { {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The block hash to try to fetch"}, @@ -459,6 +462,12 @@ static RPCHelpMan getblockfrompeer() throw JSONRPCError(RPC_MISC_ERROR, "Block header missing"); } + // Fetching blocks before the node has syncing past their height can prevent block files from + // being pruned, so we avoid it if the node is in prune mode. + if (node::fPruneMode && index->nHeight > WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip()->nHeight)) { + throw JSONRPCError(RPC_MISC_ERROR, "In prune mode, only blocks that the node has already synced previously can be fetched from a peer"); + } + const bool block_has_data = WITH_LOCK(::cs_main, return index->nStatus & BLOCK_HAVE_DATA); if (block_has_data) { throw JSONRPCError(RPC_MISC_ERROR, "Block already downloaded"); @@ -571,30 +580,38 @@ static RPCHelpMan getblockheader() }; } -static CBlock GetBlockChecked(BlockManager& blockman, const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) +static CBlock GetBlockChecked(BlockManager& blockman, const CBlockIndex* pblockindex) { - AssertLockHeld(::cs_main); CBlock block; - if (blockman.IsBlockPruned(pblockindex)) { - throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)"); + { + LOCK(cs_main); + if (blockman.IsBlockPruned(pblockindex)) { + throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)"); + } } if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) { // Block not found on disk. This could be because we have the block // header in our index but not yet have the block or did not accept the - // block. + // block. Or if the block was pruned right after we released the lock above. throw JSONRPCError(RPC_MISC_ERROR, "Block not found on disk"); } return block; } -static CBlockUndo GetUndoChecked(BlockManager& blockman, const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +static CBlockUndo GetUndoChecked(BlockManager& blockman, const CBlockIndex* pblockindex) { - AssertLockHeld(::cs_main); CBlockUndo blockUndo; - if (blockman.IsBlockPruned(pblockindex)) { - throw JSONRPCError(RPC_MISC_ERROR, "Undo data not available (pruned data)"); + + // The Genesis block does not have undo data + if (pblockindex->nHeight == 0) return blockUndo; + + { + LOCK(cs_main); + if (blockman.IsBlockPruned(pblockindex)) { + throw JSONRPCError(RPC_MISC_ERROR, "Undo data not available (pruned data)"); + } } if (!UndoReadFromDisk(blockUndo, pblockindex)) { @@ -709,7 +726,6 @@ static RPCHelpMan getblock() } } - CBlock block; const CBlockIndex* pblockindex; const CBlockIndex* tip; ChainstateManager& chainman = EnsureAnyChainman(request.context); @@ -721,10 +737,10 @@ static RPCHelpMan getblock() if (!pblockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } - - block = GetBlockChecked(chainman.m_blockman, pblockindex); } + const CBlock block{GetBlockChecked(chainman.m_blockman, pblockindex)}; + if (verbosity <= 0) { CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); @@ -1307,7 +1323,7 @@ UniValue DeploymentInfo(const CBlockIndex* blockindex, const ChainstateManager& } } // anon namespace -static RPCHelpMan getdeploymentinfo() +RPCHelpMan getdeploymentinfo() { return RPCHelpMan{"getdeploymentinfo", "Returns an object containing various state info regarding deployments of consensus changes.", @@ -1771,8 +1787,10 @@ static RPCHelpMan getblockstats() {RPCResult::Type::NUM, "total_weight", /*optional=*/true, "Total weight of all non-coinbase transactions"}, {RPCResult::Type::NUM, "totalfee", /*optional=*/true, "The fee total"}, {RPCResult::Type::NUM, "txs", /*optional=*/true, "The number of transactions (including coinbase)"}, - {RPCResult::Type::NUM, "utxo_increase", /*optional=*/true, "The increase/decrease in the number of unspent outputs"}, + {RPCResult::Type::NUM, "utxo_increase", /*optional=*/true, "The increase/decrease in the number of unspent outputs (not discounting op_return and similar)"}, {RPCResult::Type::NUM, "utxo_size_inc", /*optional=*/true, "The increase/decrease in size for the utxo index (not discounting op_return and similar)"}, + {RPCResult::Type::NUM, "utxo_increase_actual", /*optional=*/true, "The increase/decrease in the number of unspent outputs, not counting unspendables"}, + {RPCResult::Type::NUM, "utxo_size_inc_actual", /*optional=*/true, "The increase/decrease in size for the utxo index, not counting unspendables"}, }}, RPCExamples{ HelpExampleCli("getblockstats", R"('"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09"' '["minfeerate","avgfeerate"]')") + @@ -1783,7 +1801,6 @@ static RPCHelpMan getblockstats() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { ChainstateManager& chainman = EnsureAnyChainman(request.context); - LOCK(cs_main); const CBlockIndex& pindex{*CHECK_NONFATAL(ParseHashOrHeight(request.params[0], chainman))}; std::set<std::string> stats; @@ -1803,7 +1820,7 @@ static RPCHelpMan getblockstats() const bool do_medianfee = do_all || stats.count("medianfee") != 0; const bool do_feerate_percentiles = do_all || stats.count("feerate_percentiles") != 0; const bool loop_inputs = do_all || do_medianfee || do_feerate_percentiles || - SetHasKeys(stats, "utxo_size_inc", "totalfee", "avgfee", "avgfeerate", "minfee", "maxfee", "minfeerate", "maxfeerate"); + SetHasKeys(stats, "utxo_increase", "utxo_increase_actual", "utxo_size_inc", "utxo_size_inc_actual", "totalfee", "avgfee", "avgfeerate", "minfee", "maxfee", "minfeerate", "maxfeerate"); const bool loop_outputs = do_all || loop_inputs || stats.count("total_out"); const bool do_calculate_size = do_mediantxsize || SetHasKeys(stats, "total_size", "avgtxsize", "mintxsize", "maxtxsize", "swtotal_size"); @@ -1825,7 +1842,9 @@ static RPCHelpMan getblockstats() int64_t swtxs = 0; int64_t total_size = 0; int64_t total_weight = 0; + int64_t utxos = 0; int64_t utxo_size_inc = 0; + int64_t utxo_size_inc_actual = 0; std::vector<CAmount> fee_array; std::vector<std::pair<CAmount, int64_t>> feerate_array; std::vector<int64_t> txsize_array; @@ -1838,7 +1857,18 @@ static RPCHelpMan getblockstats() if (loop_outputs) { for (const CTxOut& out : tx->vout) { tx_total_out += out.nValue; - utxo_size_inc += GetSerializeSize(out, PROTOCOL_VERSION) + PER_UTXO_OVERHEAD; + + size_t out_size = GetSerializeSize(out, PROTOCOL_VERSION) + PER_UTXO_OVERHEAD; + utxo_size_inc += out_size; + + // The Genesis block and the repeated BIP30 block coinbases don't change the UTXO + // set counts, so they have to be excluded from the statistics + if (pindex.nHeight == 0 || (IsBIP30Repeat(pindex) && tx->IsCoinBase())) continue; + // Skip unspendable outputs since they are not included in the UTXO set + if (out.scriptPubKey.IsUnspendable()) continue; + + ++utxos; + utxo_size_inc_actual += out_size; } } @@ -1880,7 +1910,9 @@ static RPCHelpMan getblockstats() const CTxOut& prevoutput = coin.out; tx_total_in += prevoutput.nValue; - utxo_size_inc -= GetSerializeSize(prevoutput, PROTOCOL_VERSION) + PER_UTXO_OVERHEAD; + size_t prevout_size = GetSerializeSize(prevoutput, PROTOCOL_VERSION) + PER_UTXO_OVERHEAD; + utxo_size_inc -= prevout_size; + utxo_size_inc_actual -= prevout_size; } CAmount txfee = tx_total_in - tx_total_out; @@ -1940,6 +1972,8 @@ static RPCHelpMan getblockstats() ret_all.pushKV("txs", (int64_t)block.vtx.size()); ret_all.pushKV("utxo_increase", outputs - inputs); ret_all.pushKV("utxo_size_inc", utxo_size_inc); + ret_all.pushKV("utxo_increase_actual", utxos - inputs); + ret_all.pushKV("utxo_size_inc_actual", utxo_size_inc_actual); if (do_all) { return ret_all; @@ -2019,6 +2053,40 @@ public: } }; +static const auto scan_action_arg_desc = RPCArg{ + "action", RPCArg::Type::STR, RPCArg::Optional::NO, "The action to execute\n" + "\"start\" for starting a scan\n" + "\"abort\" for aborting the current scan (returns true when abort was successful)\n" + "\"status\" for progress report (in %) of the current scan" +}; + +static const auto scan_objects_arg_desc = RPCArg{ + "scanobjects", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "Array of scan objects. Required for \"start\" action\n" + "Every scan object is either a string descriptor or an object:", + { + {"descriptor", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "An output descriptor"}, + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "An object with output descriptor and metadata", + { + {"desc", RPCArg::Type::STR, RPCArg::Optional::NO, "An output descriptor"}, + {"range", RPCArg::Type::RANGE, RPCArg::Default{1000}, "The range of HD chain indexes to explore (either end or [begin,end])"}, + }}, + }, + RPCArgOptions{.oneline_description="[scanobjects,...]"}, +}; + +static const auto scan_result_abort = RPCResult{ + "when action=='abort'", RPCResult::Type::BOOL, "success", + "True if scan will be aborted (not necessarily before this RPC returns), or false if there is no scan to abort" +}; +static const auto scan_result_status_none = RPCResult{ + "when action=='status' and no scan is in progress - possibly already completed", RPCResult::Type::NONE, "", "" +}; +static const auto scan_result_status_some = RPCResult{ + "when action=='status' and a scan is currently in progress", RPCResult::Type::OBJ, "", "", + {{RPCResult::Type::NUM, "progress", "Approximate percent complete"},} +}; + + static RPCHelpMan scantxoutset() { // scriptPubKey corresponding to mainnet address 12cbQLTFMXRnSzktFkuoG3eHoMeFtpTu3S @@ -2038,21 +2106,8 @@ static RPCHelpMan scantxoutset() "In the latter case, a range needs to be specified by below if different from 1000.\n" "For more information on output descriptors, see the documentation in the doc/descriptors.md file.\n", { - {"action", RPCArg::Type::STR, RPCArg::Optional::NO, "The action to execute\n" - "\"start\" for starting a scan\n" - "\"abort\" for aborting the current scan (returns true when abort was successful)\n" - "\"status\" for progress report (in %) of the current scan"}, - {"scanobjects", RPCArg::Type::ARR, RPCArg::Optional::OMITTED, "Array of scan objects. Required for \"start\" action\n" - "Every scan object is either a string descriptor or an object:", - { - {"descriptor", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "An output descriptor"}, - {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "An object with output descriptor and metadata", - { - {"desc", RPCArg::Type::STR, RPCArg::Optional::NO, "An output descriptor"}, - {"range", RPCArg::Type::RANGE, RPCArg::Default{1000}, "The range of HD chain indexes to explore (either end or [begin,end])"}, - }}, - }, - RPCArgOptions{.oneline_description="[scanobjects,...]"}}, + scan_action_arg_desc, + scan_objects_arg_desc, }, { RPCResult{"when action=='start'; only returns after scan completes", RPCResult::Type::OBJ, "", "", { @@ -2069,17 +2124,15 @@ static RPCHelpMan scantxoutset() {RPCResult::Type::STR_HEX, "scriptPubKey", "The script key"}, {RPCResult::Type::STR, "desc", "A specialized descriptor for the matched scriptPubKey"}, {RPCResult::Type::STR_AMOUNT, "amount", "The total amount in " + CURRENCY_UNIT + " of the unspent output"}, + {RPCResult::Type::BOOL, "coinbase", "Whether this is a coinbase output"}, {RPCResult::Type::NUM, "height", "Height of the unspent transaction output"}, }}, }}, {RPCResult::Type::STR_AMOUNT, "total_amount", "The total amount of all found unspent outputs in " + CURRENCY_UNIT}, }}, - RPCResult{"when action=='abort'", RPCResult::Type::BOOL, "success", "True if scan will be aborted (not necessarily before this RPC returns), or false if there is no scan to abort"}, - RPCResult{"when action=='status' and a scan is currently in progress", RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::NUM, "progress", "Approximate percent complete"}, - }}, - RPCResult{"when action=='status' and no scan is in progress - possibly already completed", RPCResult::Type::NONE, "", ""}, + scan_result_abort, + scan_result_status_some, + scan_result_status_none, }, RPCExamples{ HelpExampleCli("scantxoutset", "start \'[\"" + EXAMPLE_DESCRIPTOR_RAW + "\"]\'") + @@ -2172,6 +2225,7 @@ static RPCHelpMan scantxoutset() unspent.pushKV("scriptPubKey", HexStr(txo.scriptPubKey)); unspent.pushKV("desc", descriptors[txo.scriptPubKey]); unspent.pushKV("amount", ValueFromAmount(txo.nValue)); + unspent.pushKV("coinbase", coin.IsCoinBase()); unspent.pushKV("height", (int32_t)coin.nHeight); unspents.push_back(unspent); @@ -2179,13 +2233,211 @@ static RPCHelpMan scantxoutset() result.pushKV("unspents", unspents); result.pushKV("total_amount", ValueFromAmount(total_in)); } else { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid command"); + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid action '%s'", request.params[0].get_str())); } return result; }, }; } +/** RAII object to prevent concurrency issue when scanning blockfilters */ +static std::atomic<int> g_scanfilter_progress; +static std::atomic<int> g_scanfilter_progress_height; +static std::atomic<bool> g_scanfilter_in_progress; +static std::atomic<bool> g_scanfilter_should_abort_scan; +class BlockFiltersScanReserver +{ +private: + bool m_could_reserve{false}; +public: + explicit BlockFiltersScanReserver() = default; + + bool reserve() { + CHECK_NONFATAL(!m_could_reserve); + if (g_scanfilter_in_progress.exchange(true)) { + return false; + } + m_could_reserve = true; + return true; + } + + ~BlockFiltersScanReserver() { + if (m_could_reserve) { + g_scanfilter_in_progress = false; + } + } +}; + +static RPCHelpMan scanblocks() +{ + return RPCHelpMan{"scanblocks", + "\nReturn relevant blockhashes for given descriptors.\n" + "This call may take several minutes. Make sure to use no RPC timeout (bitcoin-cli -rpcclienttimeout=0)", + { + scan_action_arg_desc, + scan_objects_arg_desc, + RPCArg{"start_height", RPCArg::Type::NUM, RPCArg::Default{0}, "Height to start to scan from"}, + RPCArg{"stop_height", RPCArg::Type::NUM, RPCArg::DefaultHint{"chain tip"}, "Height to stop to scan"}, + RPCArg{"filtertype", RPCArg::Type::STR, RPCArg::Default{BlockFilterTypeName(BlockFilterType::BASIC)}, "The type name of the filter"} + }, + { + scan_result_status_none, + RPCResult{"When action=='start'; only returns after scan completes", RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::NUM, "from_height", "The height we started the scan from"}, + {RPCResult::Type::NUM, "to_height", "The height we ended the scan at"}, + {RPCResult::Type::ARR, "relevant_blocks", "Blocks that may have matched a scanobject.", { + {RPCResult::Type::STR_HEX, "blockhash", "A relevant blockhash"}, + }}, + }}, + RPCResult{"when action=='status' and a scan is currently in progress", RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::NUM, "progress", "Approximate percent complete"}, + {RPCResult::Type::NUM, "current_height", "Height of the block currently being scanned"}, + }, + }, + scan_result_abort, + }, + RPCExamples{ + HelpExampleCli("scanblocks", "start '[\"addr(bcrt1q4u4nsgk6ug0sqz7r3rj9tykjxrsl0yy4d0wwte)\"]' 300000") + + HelpExampleCli("scanblocks", "start '[\"addr(bcrt1q4u4nsgk6ug0sqz7r3rj9tykjxrsl0yy4d0wwte)\"]' 100 150 basic") + + HelpExampleCli("scanblocks", "status") + + HelpExampleRpc("scanblocks", "\"start\", [\"addr(bcrt1q4u4nsgk6ug0sqz7r3rj9tykjxrsl0yy4d0wwte)\"], 300000") + + HelpExampleRpc("scanblocks", "\"start\", [\"addr(bcrt1q4u4nsgk6ug0sqz7r3rj9tykjxrsl0yy4d0wwte)\"], 100, 150, \"basic\"") + + HelpExampleRpc("scanblocks", "\"status\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + UniValue ret(UniValue::VOBJ); + if (request.params[0].get_str() == "status") { + BlockFiltersScanReserver reserver; + if (reserver.reserve()) { + // no scan in progress + return NullUniValue; + } + ret.pushKV("progress", g_scanfilter_progress.load()); + ret.pushKV("current_height", g_scanfilter_progress_height.load()); + return ret; + } else if (request.params[0].get_str() == "abort") { + BlockFiltersScanReserver reserver; + if (reserver.reserve()) { + // reserve was possible which means no scan was running + return false; + } + // set the abort flag + g_scanfilter_should_abort_scan = true; + return true; + } + else if (request.params[0].get_str() == "start") { + BlockFiltersScanReserver reserver; + if (!reserver.reserve()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Scan already in progress, use action \"abort\" or \"status\""); + } + const std::string filtertype_name{request.params[4].isNull() ? "basic" : request.params[4].get_str()}; + + BlockFilterType filtertype; + if (!BlockFilterTypeByName(filtertype_name, filtertype)) { + throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Unknown filtertype"); + } + + BlockFilterIndex* index = GetBlockFilterIndex(filtertype); + if (!index) { + throw JSONRPCError(RPC_MISC_ERROR, "Index is not enabled for filtertype " + filtertype_name); + } + + NodeContext& node = EnsureAnyNodeContext(request.context); + ChainstateManager& chainman = EnsureChainman(node); + + // set the start-height + const CBlockIndex* block = nullptr; + const CBlockIndex* stop_block = nullptr; + { + LOCK(cs_main); + CChain& active_chain = chainman.ActiveChain(); + block = active_chain.Genesis(); + stop_block = active_chain.Tip(); + if (!request.params[2].isNull()) { + block = active_chain[request.params[2].getInt<int>()]; + if (!block) { + throw JSONRPCError(RPC_MISC_ERROR, "Invalid start_height"); + } + } + if (!request.params[3].isNull()) { + stop_block = active_chain[request.params[3].getInt<int>()]; + if (!stop_block || stop_block->nHeight < block->nHeight) { + throw JSONRPCError(RPC_MISC_ERROR, "Invalid stop_height"); + } + } + } + CHECK_NONFATAL(block); + + // loop through the scan objects, add scripts to the needle_set + GCSFilter::ElementSet needle_set; + for (const UniValue& scanobject : request.params[1].get_array().getValues()) { + FlatSigningProvider provider; + std::vector<CScript> scripts = EvalDescriptorStringOrObject(scanobject, provider); + for (const CScript& script : scripts) { + needle_set.emplace(script.begin(), script.end()); + } + } + UniValue blocks(UniValue::VARR); + const int amount_per_chunk = 10000; + const CBlockIndex* start_index = block; // for remembering the start of a blockfilter range + std::vector<BlockFilter> filters; + const CBlockIndex* start_block = block; // for progress reporting + const int total_blocks_to_process = stop_block->nHeight - start_block->nHeight; + + g_scanfilter_should_abort_scan = false; + g_scanfilter_progress = 0; + g_scanfilter_progress_height = start_block->nHeight; + + while (block) { + node.rpc_interruption_point(); // allow a clean shutdown + if (g_scanfilter_should_abort_scan) { + LogPrintf("scanblocks RPC aborted at height %d.\n", block->nHeight); + break; + } + const CBlockIndex* next = nullptr; + { + LOCK(cs_main); + CChain& active_chain = chainman.ActiveChain(); + next = active_chain.Next(block); + if (block == stop_block) next = nullptr; + } + if (start_index->nHeight + amount_per_chunk == block->nHeight || next == nullptr) { + LogPrint(BCLog::RPC, "Fetching blockfilters from height %d to height %d.\n", start_index->nHeight, block->nHeight); + if (index->LookupFilterRange(start_index->nHeight, block, filters)) { + for (const BlockFilter& filter : filters) { + // compare the elements-set with each filter + if (filter.GetFilter().MatchAny(needle_set)) { + blocks.push_back(filter.GetBlockHash().GetHex()); + LogPrint(BCLog::RPC, "scanblocks: found match in %s\n", filter.GetBlockHash().GetHex()); + } + } + } + start_index = block; + + // update progress + int blocks_processed = block->nHeight - start_block->nHeight; + if (total_blocks_to_process > 0) { // avoid division by zero + g_scanfilter_progress = (int)(100.0 / total_blocks_to_process * blocks_processed); + } else { + g_scanfilter_progress = 100; + } + g_scanfilter_progress_height = block->nHeight; + } + block = next; + } + ret.pushKV("from_height", start_block->nHeight); + ret.pushKV("to_height", g_scanfilter_progress_height.load()); + ret.pushKV("relevant_blocks", blocks); + } + else { + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Invalid action '%s'", request.params[0].get_str())); + } + return ret; +}, + }; +} + static RPCHelpMan getblockfilter() { return RPCHelpMan{"getblockfilter", @@ -2421,6 +2673,7 @@ void RegisterBlockchainRPCCommands(CRPCTable& t) {"blockchain", &verifychain}, {"blockchain", &preciousblock}, {"blockchain", &scantxoutset}, + {"blockchain", &scanblocks}, {"blockchain", &getblockfilter}, {"hidden", &invalidateblock}, {"hidden", &reconsiderblock}, diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index 612dbbdacf..a795296f35 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -83,6 +83,9 @@ static const CRPCConvertParam vRPCConvertParams[] = { "sendmany", 8, "fee_rate"}, { "sendmany", 9, "verbose" }, { "deriveaddresses", 1, "range" }, + { "scanblocks", 1, "scanobjects" }, + { "scanblocks", 2, "start_height" }, + { "scanblocks", 3, "stop_height" }, { "scantxoutset", 1, "scanobjects" }, { "addmultisigaddress", 0, "nrequired" }, { "addmultisigaddress", 1, "keys" }, @@ -99,6 +102,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getchaintxstats", 0, "nblocks" }, { "gettransaction", 1, "include_watchonly" }, { "gettransaction", 2, "verbose" }, + { "getrawtransaction", 1, "verbosity" }, { "getrawtransaction", 1, "verbose" }, { "createrawtransaction", 0, "inputs" }, { "createrawtransaction", 1, "outputs" }, @@ -274,16 +278,21 @@ UniValue RPCConvertValues(const std::string &strMethod, const std::vector<std::s UniValue RPCConvertNamedValues(const std::string &strMethod, const std::vector<std::string> &strParams) { UniValue params(UniValue::VOBJ); + UniValue positional_args{UniValue::VARR}; for (const std::string &s: strParams) { size_t pos = s.find('='); if (pos == std::string::npos) { - throw(std::runtime_error("No '=' in named argument '"+s+"', this needs to be present for every argument (even if it is empty)")); + positional_args.push_back(rpcCvtTable.convert(strMethod, positional_args.size()) ? ParseNonRFCJSONValue(s) : s); + continue; } std::string name = s.substr(0, pos); std::string value = s.substr(pos+1); + // Intentionally overwrite earlier named values with later ones as a + // convenience for scripts and command line users that want to merge + // options. if (!rpcCvtTable.convert(strMethod, name)) { // insert string value directly params.pushKV(name, value); @@ -293,5 +302,12 @@ UniValue RPCConvertNamedValues(const std::string &strMethod, const std::vector<s } } + if (!positional_args.empty()) { + // Use __pushKV instead of pushKV to avoid overwriting an explicit + // "args" value with an implicit one. Let the RPC server handle the + // request as given. + params.__pushKV("args", positional_args); + } + return params; } diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 706d783942..7a0c361ae0 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -10,6 +10,7 @@ #include <chainparams.h> #include <core_io.h> #include <fs.h> +#include <kernel/mempool_entry.h> #include <node/mempool_persist_args.h> #include <policy/rbf.h> #include <policy/settings.h> diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index d701a180ab..ba4236aeef 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -114,7 +114,7 @@ static RPCHelpMan getpeerinfo() { {RPCResult::Type::STR, "SERVICE_NAME", "the service name if it is recognised"} }}, - {RPCResult::Type::BOOL, "relaytxes", /*optional=*/true, "Whether peer has asked us to relay transactions to it"}, + {RPCResult::Type::BOOL, "relaytxes", /*optional=*/true, "Whether we relay transactions to this peer"}, {RPCResult::Type::NUM_TIME, "lastsend", "The " + UNIX_EPOCH_TIME + " of the last send"}, {RPCResult::Type::NUM_TIME, "lastrecv", "The " + UNIX_EPOCH_TIME + " of the last receive"}, {RPCResult::Type::NUM_TIME, "last_transaction", "The " + UNIX_EPOCH_TIME + " of the last valid transaction received from this peer"}, @@ -123,9 +123,9 @@ static RPCHelpMan getpeerinfo() {RPCResult::Type::NUM, "bytesrecv", "The total bytes received"}, {RPCResult::Type::NUM_TIME, "conntime", "The " + UNIX_EPOCH_TIME + " of the connection"}, {RPCResult::Type::NUM, "timeoffset", "The time offset in seconds"}, - {RPCResult::Type::NUM, "pingtime", /*optional=*/true, "ping time (if available)"}, - {RPCResult::Type::NUM, "minping", /*optional=*/true, "minimum observed ping time (if any at all)"}, - {RPCResult::Type::NUM, "pingwait", /*optional=*/true, "ping wait (if non-zero)"}, + {RPCResult::Type::NUM, "pingtime", /*optional=*/true, "The last ping time in milliseconds (ms), if any"}, + {RPCResult::Type::NUM, "minping", /*optional=*/true, "The minimum observed ping time in milliseconds (ms), if any"}, + {RPCResult::Type::NUM, "pingwait", /*optional=*/true, "The duration in milliseconds (ms) of an outstanding ping (if non-zero)"}, {RPCResult::Type::NUM, "version", "The peer version, such as 70001"}, {RPCResult::Type::STR, "subver", "The string version"}, {RPCResult::Type::BOOL, "inbound", "Inbound (true) or Outbound (false)"}, @@ -146,7 +146,7 @@ static RPCHelpMan getpeerinfo() { {RPCResult::Type::STR, "permission_type", Join(NET_PERMISSIONS_DOC, ",\n") + ".\n"}, }}, - {RPCResult::Type::NUM, "minfeefilter", /*optional=*/true, "The minimum fee rate for transactions this peer accepts"}, + {RPCResult::Type::NUM, "minfeefilter", "The minimum fee rate for transactions this peer accepts"}, {RPCResult::Type::OBJ_DYN, "bytessent_per_msg", "", { {RPCResult::Type::NUM, "msg", "The total bytes sent aggregated by message type\n" @@ -185,6 +185,14 @@ static RPCHelpMan getpeerinfo() UniValue obj(UniValue::VOBJ); CNodeStateStats statestats; bool fStateStats = peerman.GetNodeStateStats(stats.nodeid, statestats); + // GetNodeStateStats() requires the existence of a CNodeState and a Peer object + // to succeed for this peer. These are created at connection initialisation and + // exist for the duration of the connection - except if there is a race where the + // peer got disconnected in between the GetNodeStats() and the GetNodeStateStats() + // calls. In this case, the peer doesn't need to be reported here. + if (!fStateStats) { + continue; + } obj.pushKV("id", stats.nodeid); obj.pushKV("addr", stats.m_addr_name); if (stats.addrBind.IsValid()) { @@ -200,6 +208,9 @@ static RPCHelpMan getpeerinfo() ServiceFlags services{fStateStats ? statestats.their_services : ServiceFlags::NODE_NONE}; obj.pushKV("services", strprintf("%016x", services)); obj.pushKV("servicesnames", GetServicesNames(services)); + if (fStateStats) { + obj.pushKV("relaytxes", statestats.m_relay_txs); + } obj.pushKV("lastsend", count_seconds(stats.m_last_send)); obj.pushKV("lastrecv", count_seconds(stats.m_last_recv)); obj.pushKV("last_transaction", count_seconds(stats.m_last_tx_time)); @@ -235,8 +246,6 @@ static RPCHelpMan getpeerinfo() heights.push_back(height); } obj.pushKV("inflight", heights); - obj.pushKV("relaytxes", statestats.m_relay_txs); - obj.pushKV("minfeefilter", ValueFromAmount(statestats.m_fee_filter_received)); obj.pushKV("addr_relay_enabled", statestats.m_addr_relay_enabled); obj.pushKV("addr_processed", statestats.m_addr_processed); obj.pushKV("addr_rate_limited", statestats.m_addr_rate_limited); @@ -246,6 +255,7 @@ static RPCHelpMan getpeerinfo() permissions.push_back(permission); } obj.pushKV("permissions", permissions); + obj.pushKV("minfeefilter", fStateStats ? ValueFromAmount(statestats.m_fee_filter_received) : 0); UniValue sendPerMsgType(UniValue::VOBJ); for (const auto& i : stats.mapSendBytesPerMsgType) { @@ -729,9 +739,7 @@ static RPCHelpMan setban() if (!request.params[2].isNull()) banTime = request.params[2].getInt<int64_t>(); - bool absolute = false; - if (request.params[3].isTrue()) - absolute = true; + const bool absolute{request.params[3].isNull() ? false : request.params[3].get_bool()}; if (isSubnet) { node.banman->Ban(subNet, banTime, absolute); @@ -940,7 +948,7 @@ static RPCHelpMan addpeeraddress() const std::string& addr_string{request.params[0].get_str()}; const auto port{request.params[1].getInt<uint16_t>()}; - const bool tried{request.params[2].isTrue()}; + const bool tried{request.params[2].isNull() ? false : request.params[2].get_bool()}; UniValue obj(UniValue::VOBJ); CNetAddr net_addr; diff --git a/src/rpc/output_script.cpp b/src/rpc/output_script.cpp index 744f809814..2ac6d6d76f 100644 --- a/src/rpc/output_script.cpp +++ b/src/rpc/output_script.cpp @@ -222,10 +222,11 @@ static RPCHelpMan deriveaddresses() return RPCHelpMan{"deriveaddresses", {"\nDerives one or more addresses corresponding to an output descriptor.\n" "Examples of output descriptors are:\n" - " pkh(<pubkey>) P2PKH outputs for the given pubkey\n" - " wpkh(<pubkey>) Native segwit P2PKH outputs for the given pubkey\n" - " sh(multi(<n>,<pubkey>,<pubkey>,...)) P2SH-multisig outputs for the given threshold and pubkeys\n" - " raw(<hex script>) Outputs whose scriptPubKey equals the specified hex scripts\n" + " pkh(<pubkey>) P2PKH outputs for the given pubkey\n" + " wpkh(<pubkey>) Native segwit P2PKH outputs for the given pubkey\n" + " sh(multi(<n>,<pubkey>,<pubkey>,...)) P2SH-multisig outputs for the given threshold and pubkeys\n" + " raw(<hex script>) Outputs whose scriptPubKey equals the specified hex scripts\n" + " tr(<pubkey>,multi_a(<n>,<pubkey>,<pubkey>,...)) P2TR-multisig outputs for the given threshold and pubkeys\n" "\nIn the above, <pubkey> either refers to a fixed public key in hexadecimal notation, or to an xpub/xprv optionally followed by one\n" "or more path elements separated by \"/\", where \"h\" represents a hardened child key.\n" "For more information on output descriptors, see the documentation in the doc/descriptors.md file.\n"}, @@ -273,7 +274,7 @@ static RPCHelpMan deriveaddresses() UniValue addresses(UniValue::VARR); - for (int i = range_begin; i <= range_end; ++i) { + for (int64_t i = range_begin; i <= range_end; ++i) { FlatSigningProvider provider; std::vector<CScript> scripts; if (!desc->Expand(i, key_provider, scripts, provider)) { diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index f365de7d0c..3fe8b5f5ed 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -32,6 +32,7 @@ #include <script/signingprovider.h> #include <script/standard.h> #include <uint256.h> +#include <undo.h> #include <util/bip32.h> #include <util/check.h> #include <util/strencodings.h> @@ -50,15 +51,17 @@ using node::FindCoins; using node::GetTransaction; using node::NodeContext; using node::PSBTAnalysis; +using node::ReadBlockFromDisk; +using node::UndoReadFromDisk; -static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry, Chainstate& active_chainstate) +static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry, Chainstate& active_chainstate, const CTxUndo* txundo = nullptr, TxVerbosity verbosity = TxVerbosity::SHOW_TXID) { // Call into TxToUniv() in bitcoin-common to decode the transaction hex. // // Blockchain contextual information (confirmations and blocktime) is not // available to code in bitcoin-common, so we query them here and push the // data into the returned UniValue. - TxToUniv(tx, /*block_hash=*/uint256(), entry, /*include_hex=*/true, RPCSerializationFlags()); + TxToUniv(tx, /*block_hash=*/uint256(), entry, /*include_hex=*/true, RPCSerializationFlags(), txundo, verbosity); if (!hashBlock.IsNull()) { LOCK(cs_main); @@ -77,6 +80,17 @@ static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& } } +static std::vector<RPCResult> ScriptPubKeyDoc() { + return + { + {RPCResult::Type::STR, "asm", "Disassembly of the public key script"}, + {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, + {RPCResult::Type::STR_HEX, "hex", "The raw public key script bytes, hex-encoded"}, + {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, + {RPCResult::Type::STR, "type", "The type (one of: " + GetAllOutputTypes() + ")"}, + }; +} + static std::vector<RPCResult> DecodeTxDoc(const std::string& txid_field_doc) { return { @@ -112,14 +126,7 @@ static std::vector<RPCResult> DecodeTxDoc(const std::string& txid_field_doc) { {RPCResult::Type::STR_AMOUNT, "value", "The value in " + CURRENCY_UNIT}, {RPCResult::Type::NUM, "n", "index"}, - {RPCResult::Type::OBJ, "scriptPubKey", "", - { - {RPCResult::Type::STR, "asm", "Disassembly of the public key script"}, - {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, - {RPCResult::Type::STR_HEX, "hex", "The raw public key script bytes, hex-encoded"}, - {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, - {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, - }}, + {RPCResult::Type::OBJ, "scriptPubKey", "", ScriptPubKeyDoc()}, }}, }}, }; @@ -166,26 +173,26 @@ static RPCHelpMan getrawtransaction() { return RPCHelpMan{ "getrawtransaction", - "Return the raw transaction data.\n" - "\nBy default, this call only returns a transaction if it is in the mempool. If -txindex is enabled\n" + "By default, this call only returns a transaction if it is in the mempool. If -txindex is enabled\n" "and no blockhash argument is passed, it will return the transaction if it is in the mempool or any block.\n" "If a blockhash argument is passed, it will return the transaction if\n" - "the specified block is available and the transaction is in that block.\n" - "\nHint: Use gettransaction for wallet transactions.\n" + "the specified block is available and the transaction is in that block.\n\n" + "Hint: Use gettransaction for wallet transactions.\n\n" - "\nIf verbose is 'true', returns an Object with information about 'txid'.\n" - "If verbose is 'false' or omitted, returns a string that is serialized, hex-encoded data for 'txid'.", + "If verbosity is 0 or omitted, returns the serialized transaction as a hex-encoded string.\n" + "If verbosity is 1, returns a JSON Object with information about the transaction.\n" + "If verbosity is 2, returns a JSON Object with information about the transaction, including fee and prevout information.", { {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The transaction id"}, - {"verbose", RPCArg::Type::BOOL, RPCArg::Default{false}, "If false, return a string, otherwise return a json object"}, + {"verbosity|verbose", RPCArg::Type::NUM, RPCArg::Default{0}, "0 for hex-encoded data, 1 for a JSON object, and 2 for JSON object with fee and prevout"}, {"blockhash", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED_NAMED_ARG, "The block in which to look for the transaction"}, }, { - RPCResult{"if verbose is not set or set to false", - RPCResult::Type::STR, "data", "The serialized, hex-encoded data for 'txid'" + RPCResult{"if verbosity is not set or set to 0", + RPCResult::Type::STR, "data", "The serialized transaction as a hex-encoded string for 'txid'" }, - RPCResult{"if verbose is set to true", + RPCResult{"if verbosity is set to 1", RPCResult::Type::OBJ, "", "", Cat<std::vector<RPCResult>>( { @@ -198,20 +205,40 @@ static RPCHelpMan getrawtransaction() }, DecodeTxDoc(/*txid_field_doc=*/"The transaction id (same as provided)")), }, + RPCResult{"for verbosity = 2", + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::ELISION, "", "Same output as verbosity = 1"}, + {RPCResult::Type::NUM, "fee", /*optional=*/true, "transaction fee in " + CURRENCY_UNIT + ", omitted if block undo data is not available"}, + {RPCResult::Type::ARR, "vin", "", + { + {RPCResult::Type::OBJ, "", "utxo being spent, omitted if block undo data is not available", + { + {RPCResult::Type::ELISION, "", "Same output as verbosity = 1"}, + {RPCResult::Type::OBJ, "prevout", /*optional=*/true, "Only if undo information is available)", + { + {RPCResult::Type::BOOL, "generated", "Coinbase or not"}, + {RPCResult::Type::NUM, "height", "The height of the prevout"}, + {RPCResult::Type::STR_AMOUNT, "value", "The value in " + CURRENCY_UNIT}, + {RPCResult::Type::OBJ, "scriptPubKey", "", ScriptPubKeyDoc()}, + }}, + }}, + }}, + }}, }, RPCExamples{ HelpExampleCli("getrawtransaction", "\"mytxid\"") - + HelpExampleCli("getrawtransaction", "\"mytxid\" true") - + HelpExampleRpc("getrawtransaction", "\"mytxid\", true") - + HelpExampleCli("getrawtransaction", "\"mytxid\" false \"myblockhash\"") - + HelpExampleCli("getrawtransaction", "\"mytxid\" true \"myblockhash\"") + + HelpExampleCli("getrawtransaction", "\"mytxid\" 1") + + HelpExampleRpc("getrawtransaction", "\"mytxid\", 1") + + HelpExampleCli("getrawtransaction", "\"mytxid\" 0 \"myblockhash\"") + + HelpExampleCli("getrawtransaction", "\"mytxid\" 1 \"myblockhash\"") + + HelpExampleCli("getrawtransaction", "\"mytxid\" 2 \"myblockhash\"") }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { const NodeContext& node = EnsureAnyNodeContext(request.context); ChainstateManager& chainman = EnsureChainman(node); - bool in_active_chain = true; uint256 hash = ParseHashV(request.params[0], "parameter 1"); const CBlockIndex* blockindex = nullptr; @@ -220,10 +247,14 @@ static RPCHelpMan getrawtransaction() throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "The genesis block coinbase is not considered an ordinary transaction and cannot be retrieved"); } - // Accept either a bool (true) or a num (>=1) to indicate verbose output. - bool fVerbose = false; + // Accept either a bool (true) or a num (>=0) to indicate verbosity. + int verbosity{0}; if (!request.params[1].isNull()) { - fVerbose = request.params[1].isNum() ? (request.params[1].getInt<int>() != 0) : request.params[1].get_bool(); + if (request.params[1].isBool()) { + verbosity = request.params[1].get_bool(); + } else { + verbosity = request.params[1].getInt<int>(); + } } if (!request.params[2].isNull()) { @@ -234,7 +265,6 @@ static RPCHelpMan getrawtransaction() if (!blockindex) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block hash not found"); } - in_active_chain = chainman.ActiveChain().Contains(blockindex); } bool f_txindex_ready = false; @@ -262,13 +292,43 @@ static RPCHelpMan getrawtransaction() throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, errmsg + ". Use gettransaction for wallet transactions."); } - if (!fVerbose) { + if (verbosity <= 0) { return EncodeHexTx(*tx, RPCSerializationFlags()); } UniValue result(UniValue::VOBJ); - if (blockindex) result.pushKV("in_active_chain", in_active_chain); - TxToJSON(*tx, hash_block, result, chainman.ActiveChainstate()); + if (blockindex) { + LOCK(cs_main); + result.pushKV("in_active_chain", chainman.ActiveChain().Contains(blockindex)); + } + // If request is verbosity >= 1 but no blockhash was given, then look up the blockindex + if (request.params[2].isNull()) { + LOCK(cs_main); + blockindex = chainman.m_blockman.LookupBlockIndex(hash_block); + } + if (verbosity == 1) { + TxToJSON(*tx, hash_block, result, chainman.ActiveChainstate()); + return result; + } + + CBlockUndo blockUndo; + CBlock block; + const bool is_block_pruned{WITH_LOCK(cs_main, return chainman.m_blockman.IsBlockPruned(blockindex))}; + + if (tx->IsCoinBase() || + !blockindex || is_block_pruned || + !(UndoReadFromDisk(blockUndo, blockindex) && ReadBlockFromDisk(block, blockindex, Params().GetConsensus()))) { + TxToJSON(*tx, hash_block, result, chainman.ActiveChainstate()); + return result; + } + + CTxUndo* undoTX {nullptr}; + auto it = std::find_if(block.vtx.begin(), block.vtx.end(), [tx](CTransactionRef t){ return *t == *tx; }); + if (it != block.vtx.end()) { + // -1 as blockundo does not have coinbase tx + undoTX = &blockUndo.vtxundo.at(it - block.vtx.begin() - 1); + } + TxToJSON(*tx, hash_block, result, chainman.ActiveChainstate(), undoTX, TxVerbosity::SHOW_DETAILS_AND_PREVOUT); return result; }, }; @@ -304,7 +364,7 @@ static RPCHelpMan createrawtransaction() std::optional<bool> rbf; if (!request.params[3].isNull()) { - rbf = request.params[3].isTrue(); + rbf = request.params[3].get_bool(); } CMutableTransaction rawTx = ConstructTransaction(request.params[0], request.params[1], request.params[2], rbf); @@ -1241,13 +1301,9 @@ static RPCHelpMan decodepsbt() } // Taproot tree - if (output.m_tap_tree.has_value()) { + if (!output.m_tap_tree.empty()) { UniValue tree(UniValue::VARR); - const auto& tuples = output.m_tap_tree->GetTreeTuples(); - for (const auto& tuple : tuples) { - uint8_t depth = std::get<0>(tuple); - uint8_t leaf_ver = std::get<1>(tuple); - CScript script = std::get<2>(tuple); + for (const auto& [depth, leaf_ver, script] : output.m_tap_tree) { UniValue elem(UniValue::VOBJ); elem.pushKV("depth", (int)depth); elem.pushKV("leaf_ver", (int)leaf_ver); @@ -1453,7 +1509,7 @@ static RPCHelpMan createpsbt() std::optional<bool> rbf; if (!request.params[3].isNull()) { - rbf = request.params[3].isTrue(); + rbf = request.params[3].get_bool(); } CMutableTransaction rawTx = ConstructTransaction(request.params[0], request.params[1], request.params[2], rbf); diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 1d7bd2eb94..a026b7adfa 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -103,7 +103,7 @@ std::string CRPCTable::help(const std::string& strCommand, const JSONRPCRequest& { UniValue unused_result; if (setDone.insert(pcmd->unique_id).second) - pcmd->actor(jreq, unused_result, true /* last_handler */); + pcmd->actor(jreq, unused_result, /*last_handler=*/true); } catch (const std::exception& e) { @@ -399,10 +399,21 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c const std::vector<UniValue>& values = in.params.getValues(); std::unordered_map<std::string, const UniValue*> argsIn; for (size_t i=0; i<keys.size(); ++i) { - argsIn[keys[i]] = &values[i]; + auto [_, inserted] = argsIn.emplace(keys[i], &values[i]); + if (!inserted) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter " + keys[i] + " specified multiple times"); + } } - // Process expected parameters. + // Process expected parameters. If any parameters were left unspecified in + // the request before a parameter that was specified, null values need to be + // inserted at the unspecifed parameter positions, and the "hole" variable + // below tracks the number of null values that need to be inserted. + // The "initial_hole_size" variable stores the size of the initial hole, + // i.e. how many initial positional arguments were left unspecified. This is + // used after the for-loop to add initial positional arguments from the + // "args" parameter, if present. int hole = 0; + int initial_hole_size = 0; for (const std::string &argNamePattern: argNames) { std::vector<std::string> vargNames = SplitString(argNamePattern, '|'); auto fr = argsIn.end(); @@ -424,6 +435,24 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c argsIn.erase(fr); } else { hole += 1; + if (out.params.empty()) initial_hole_size = hole; + } + } + // If leftover "args" param was found, use it as a source of positional + // arguments and add named arguments after. This is a convenience for + // clients that want to pass a combination of named and positional + // arguments as described in doc/JSON-RPC-interface.md#parameter-passing + auto positional_args{argsIn.extract("args")}; + if (positional_args && positional_args.mapped()->isArray()) { + const bool has_named_arguments{initial_hole_size < (int)argNames.size()}; + if (initial_hole_size < (int)positional_args.mapped()->size() && has_named_arguments) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter " + argNames[initial_hole_size] + " specified twice both as positional and named argument"); + } + // Assign positional_args to out.params and append named_args after. + UniValue named_args{std::move(out.params)}; + out.params = *positional_args.mapped(); + for (size_t i{out.params.size()}; i < named_args.size(); ++i) { + out.params.push_back(named_args[i]); } } // If there are still arguments in the argsIn map, this is an error. diff --git a/src/rpc/txoutproof.cpp b/src/rpc/txoutproof.cpp index cd8b49bfe1..8c5468634d 100644 --- a/src/rpc/txoutproof.cpp +++ b/src/rpc/txoutproof.cpp @@ -84,13 +84,13 @@ static RPCHelpMan gettxoutproof() g_txindex->BlockUntilSyncedToCurrentChain(); } - LOCK(cs_main); - if (pblockindex == nullptr) { const CTransactionRef tx = GetTransaction(/*block_index=*/nullptr, /*mempool=*/nullptr, *setTxids.begin(), chainman.GetConsensus(), hashBlock); if (!tx || hashBlock.IsNull()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not yet in block"); } + + LOCK(cs_main); pblockindex = chainman.m_blockman.LookupBlockIndex(hashBlock); if (!pblockindex) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Transaction index corrupt"); diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 3e98e89791..dd5739faf7 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -65,11 +65,8 @@ void RPCTypeCheckObj(const UniValue& o, if (!fAllowNull && v.isNull()) throw JSONRPCError(RPC_TYPE_ERROR, strprintf("Missing %s", t.first)); - if (!(t.second.typeAny || v.type() == t.second.type || (fAllowNull && v.isNull()))) { - std::string err = strprintf("Expected type %s for %s, got %s", - uvTypeName(t.second.type), t.first, uvTypeName(v.type())); - throw JSONRPCError(RPC_TYPE_ERROR, err); - } + if (!(t.second.typeAny || v.type() == t.second.type || (fAllowNull && v.isNull()))) + throw JSONRPCError(RPC_TYPE_ERROR, strprintf("JSON value of type %s for field %s is not of expected type %s", uvTypeName(v.type()), t.first, uvTypeName(t.second.type))); } if (fStrict) diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 5da0d076d8..0d74a661a5 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -146,6 +146,16 @@ static bool CreateSig(const BaseSignatureCreator& creator, SignatureData& sigdat static bool CreateTaprootScriptSig(const BaseSignatureCreator& creator, SignatureData& sigdata, const SigningProvider& provider, std::vector<unsigned char>& sig_out, const XOnlyPubKey& pubkey, const uint256& leaf_hash, SigVersion sigversion) { + KeyOriginInfo info; + if (provider.GetKeyOriginByXOnly(pubkey, info)) { + auto it = sigdata.taproot_misc_pubkeys.find(pubkey); + if (it == sigdata.taproot_misc_pubkeys.end()) { + sigdata.taproot_misc_pubkeys.emplace(pubkey, std::make_pair(std::set<uint256>({leaf_hash}), info)); + } else { + it->second.first.insert(leaf_hash); + } + } + auto lookup_key = std::make_pair(pubkey, leaf_hash); auto it = sigdata.taproot_script_sigs.find(lookup_key); if (it != sigdata.taproot_script_sigs.end()) { @@ -170,17 +180,6 @@ static bool SignTaprootScript(const SigningProvider& provider, const BaseSignatu // <xonly pubkey> OP_CHECKSIG if (script.size() == 34 && script[33] == OP_CHECKSIG && script[0] == 0x20) { XOnlyPubKey pubkey{Span{script}.subspan(1, 32)}; - - KeyOriginInfo info; - if (provider.GetKeyOriginByXOnly(pubkey, info)) { - auto it = sigdata.taproot_misc_pubkeys.find(pubkey); - if (it == sigdata.taproot_misc_pubkeys.end()) { - sigdata.taproot_misc_pubkeys.emplace(pubkey, std::make_pair(std::set<uint256>({leaf_hash}), info)); - } else { - it->second.first.insert(leaf_hash); - } - } - std::vector<unsigned char> sig; if (CreateTaprootScriptSig(creator, sigdata, provider, sig, pubkey, leaf_hash, sigversion)) { result = Vector(std::move(sig)); diff --git a/src/script/standard.h b/src/script/standard.h index 1e6769782a..966a52b2c7 100644 --- a/src/script/standard.h +++ b/src/script/standard.h @@ -315,6 +315,8 @@ public: TaprootSpendData GetSpendData() const; /** Returns a vector of tuples representing the depth, leaf version, and script */ std::vector<std::tuple<uint8_t, uint8_t, CScript>> GetTreeTuples() const; + /** Returns true if there are any tapscripts */ + bool HasScripts() const { return !m_branch.empty(); } }; /** Given a TaprootSpendData and the output key, reconstruct its script tree. diff --git a/src/streams.h b/src/streams.h index ce1ba26d45..84b12f65aa 100644 --- a/src/streams.h +++ b/src/streams.h @@ -487,12 +487,14 @@ public: AutoFile(const AutoFile&) = delete; AutoFile& operator=(const AutoFile&) = delete; - void fclose() + int fclose() { + int retval{0}; if (file) { - ::fclose(file); + retval = ::fclose(file); file = nullptr; } + return retval; } /** Get wrapped FILE* with transfer of ownership. @@ -610,7 +612,6 @@ private: uint64_t nRewind; //!< how many bytes we guarantee to rewind std::vector<std::byte> vchBuf; //!< the buffer -protected: //! read data from the source to fill the buffer bool Fill() { unsigned int pos = nSrcPos % vchBuf.size(); @@ -628,6 +629,28 @@ protected: return true; } + //! Advance the stream's read pointer (m_read_pos) by up to 'length' bytes, + //! filling the buffer from the file so that at least one byte is available. + //! Return a pointer to the available buffer data and the number of bytes + //! (which may be less than the requested length) that may be accessed + //! beginning at that pointer. + std::pair<std::byte*, size_t> AdvanceStream(size_t length) + { + assert(m_read_pos <= nSrcPos); + if (m_read_pos + length > nReadLimit) { + throw std::ios_base::failure("Attempt to position past buffer limit"); + } + // If there are no bytes available, read from the file. + if (m_read_pos == nSrcPos && length > 0) Fill(); + + size_t buffer_offset{static_cast<size_t>(m_read_pos % vchBuf.size())}; + size_t buffer_available{static_cast<size_t>(vchBuf.size() - buffer_offset)}; + size_t bytes_until_source_pos{static_cast<size_t>(nSrcPos - m_read_pos)}; + size_t advance{std::min({length, buffer_available, bytes_until_source_pos})}; + m_read_pos += advance; + return std::make_pair(&vchBuf[buffer_offset], advance); + } + public: CBufferedFile(FILE* fileIn, uint64_t nBufSize, uint64_t nRewindIn, int nTypeIn, int nVersionIn) : nType(nTypeIn), nVersion(nVersionIn), nSrcPos(0), m_read_pos(0), nReadLimit(std::numeric_limits<uint64_t>::max()), nRewind(nRewindIn), vchBuf(nBufSize, std::byte{0}) @@ -665,24 +688,21 @@ public: //! read a number of bytes void read(Span<std::byte> dst) { - if (dst.size() + m_read_pos > nReadLimit) { - throw std::ios_base::failure("Read attempted past buffer limit"); - } while (dst.size() > 0) { - if (m_read_pos == nSrcPos) - Fill(); - unsigned int pos = m_read_pos % vchBuf.size(); - size_t nNow = dst.size(); - if (nNow + pos > vchBuf.size()) - nNow = vchBuf.size() - pos; - if (nNow + m_read_pos > nSrcPos) - nNow = nSrcPos - m_read_pos; - memcpy(dst.data(), &vchBuf[pos], nNow); - m_read_pos += nNow; - dst = dst.subspan(nNow); + auto [buffer_pointer, length]{AdvanceStream(dst.size())}; + memcpy(dst.data(), buffer_pointer, length); + dst = dst.subspan(length); } } + //! Move the read position ahead in the stream to the given position. + //! Use SetPos() to back up in the stream, not SkipTo(). + void SkipTo(const uint64_t file_pos) + { + assert(file_pos >= m_read_pos); + while (m_read_pos < file_pos) AdvanceStream(file_pos - m_read_pos); + } + //! return the current reading position uint64_t GetPos() const { return m_read_pos; diff --git a/src/sync.h b/src/sync.h index a9d0091bd2..1f4e191214 100644 --- a/src/sync.h +++ b/src/sync.h @@ -111,7 +111,7 @@ public: return PARENT::try_lock(); } - using UniqueLock = std::unique_lock<PARENT>; + using unique_lock = std::unique_lock<PARENT>; #ifdef __clang__ //! For negative capabilities in the Clang Thread Safety Analysis. //! A negative requirement uses the EXCLUSIVE_LOCKS_REQUIRED attribute, in conjunction @@ -147,11 +147,13 @@ inline void AssertLockNotHeldInline(const char* name, const char* file, int line inline void AssertLockNotHeldInline(const char* name, const char* file, int line, GlobalMutex* cs) LOCKS_EXCLUDED(cs) { AssertLockNotHeldInternal(name, file, line, cs); } #define AssertLockNotHeld(cs) AssertLockNotHeldInline(#cs, __FILE__, __LINE__, &cs) -/** Wrapper around std::unique_lock style lock for Mutex. */ -template <typename Mutex, typename Base = typename Mutex::UniqueLock> -class SCOPED_LOCKABLE UniqueLock : public Base +/** Wrapper around std::unique_lock style lock for MutexType. */ +template <typename MutexType> +class SCOPED_LOCKABLE UniqueLock : public MutexType::unique_lock { private: + using Base = typename MutexType::unique_lock; + void Enter(const char* pszName, const char* pszFile, int nLine) { EnterCritical(pszName, pszFile, nLine, Base::mutex()); @@ -173,7 +175,7 @@ private: } public: - UniqueLock(Mutex& mutexIn, const char* pszName, const char* pszFile, int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(mutexIn) : Base(mutexIn, std::defer_lock) + UniqueLock(MutexType& mutexIn, const char* pszName, const char* pszFile, int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(mutexIn) : Base(mutexIn, std::defer_lock) { if (fTry) TryEnter(pszName, pszFile, nLine); @@ -181,7 +183,7 @@ public: Enter(pszName, pszFile, nLine); } - UniqueLock(Mutex* pmutexIn, const char* pszName, const char* pszFile, int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(pmutexIn) + UniqueLock(MutexType* pmutexIn, const char* pszName, const char* pszFile, int nLine, bool fTry = false) EXCLUSIVE_LOCK_FUNCTION(pmutexIn) { if (!pmutexIn) return; @@ -241,29 +243,24 @@ public: #define REVERSE_LOCK(g) typename std::decay<decltype(g)>::type::reverse_lock UNIQUE_NAME(revlock)(g, #g, __FILE__, __LINE__) -template<typename MutexArg> -using DebugLock = UniqueLock<typename std::remove_reference<typename std::remove_pointer<MutexArg>::type>::type>; - // When locking a Mutex, require negative capability to ensure the lock // is not already held inline Mutex& MaybeCheckNotHeld(Mutex& cs) EXCLUSIVE_LOCKS_REQUIRED(!cs) LOCK_RETURNED(cs) { return cs; } inline Mutex* MaybeCheckNotHeld(Mutex* cs) EXCLUSIVE_LOCKS_REQUIRED(!cs) LOCK_RETURNED(cs) { return cs; } -// When locking a GlobalMutex, just check it is not locked in the surrounding scope -inline GlobalMutex& MaybeCheckNotHeld(GlobalMutex& cs) LOCKS_EXCLUDED(cs) LOCK_RETURNED(cs) { return cs; } -inline GlobalMutex* MaybeCheckNotHeld(GlobalMutex* cs) LOCKS_EXCLUDED(cs) LOCK_RETURNED(cs) { return cs; } - -// When locking a RecursiveMutex, it's okay to already hold the lock -// but check that it is not known to be locked in the surrounding scope anyway -inline RecursiveMutex& MaybeCheckNotHeld(RecursiveMutex& cs) LOCKS_EXCLUDED(cs) LOCK_RETURNED(cs) { return cs; } -inline RecursiveMutex* MaybeCheckNotHeld(RecursiveMutex* cs) LOCKS_EXCLUDED(cs) LOCK_RETURNED(cs) { return cs; } +// When locking a GlobalMutex or RecursiveMutex, just check it is not +// locked in the surrounding scope. +template <typename MutexType> +inline MutexType& MaybeCheckNotHeld(MutexType& m) LOCKS_EXCLUDED(m) LOCK_RETURNED(m) { return m; } +template <typename MutexType> +inline MutexType* MaybeCheckNotHeld(MutexType* m) LOCKS_EXCLUDED(m) LOCK_RETURNED(m) { return m; } -#define LOCK(cs) DebugLock<decltype(cs)> UNIQUE_NAME(criticalblock)(MaybeCheckNotHeld(cs), #cs, __FILE__, __LINE__) +#define LOCK(cs) UniqueLock UNIQUE_NAME(criticalblock)(MaybeCheckNotHeld(cs), #cs, __FILE__, __LINE__) #define LOCK2(cs1, cs2) \ - DebugLock<decltype(cs1)> criticalblock1(MaybeCheckNotHeld(cs1), #cs1, __FILE__, __LINE__); \ - DebugLock<decltype(cs2)> criticalblock2(MaybeCheckNotHeld(cs2), #cs2, __FILE__, __LINE__) -#define TRY_LOCK(cs, name) DebugLock<decltype(cs)> name(MaybeCheckNotHeld(cs), #cs, __FILE__, __LINE__, true) -#define WAIT_LOCK(cs, name) DebugLock<decltype(cs)> name(MaybeCheckNotHeld(cs), #cs, __FILE__, __LINE__) + UniqueLock criticalblock1(MaybeCheckNotHeld(cs1), #cs1, __FILE__, __LINE__); \ + UniqueLock criticalblock2(MaybeCheckNotHeld(cs2), #cs2, __FILE__, __LINE__) +#define TRY_LOCK(cs, name) UniqueLock name(MaybeCheckNotHeld(cs), #cs, __FILE__, __LINE__, true) +#define WAIT_LOCK(cs, name) UniqueLock name(MaybeCheckNotHeld(cs), #cs, __FILE__, __LINE__) #define ENTER_CRITICAL_SECTION(cs) \ { \ diff --git a/src/test/argsman_tests.cpp b/src/test/argsman_tests.cpp new file mode 100644 index 0000000000..d00876bc70 --- /dev/null +++ b/src/test/argsman_tests.cpp @@ -0,0 +1,1043 @@ +// Copyright (c) 2011-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/system.h> +#include <fs.h> +#include <sync.h> +#include <test/util/logging.h> +#include <test/util/setup_common.h> +#include <test/util/str.h> +#include <util/strencodings.h> +#include <univalue.h> + +#include <array> +#include <optional> +#include <cstdint> +#include <cstring> +#include <vector> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(argsman_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(util_datadir) +{ + // Use local args variable instead of m_args to avoid making assumptions about test setup + ArgsManager args; + args.ForceSetArg("-datadir", fs::PathToString(m_path_root)); + + const fs::path dd_norm = args.GetDataDirBase(); + + args.ForceSetArg("-datadir", fs::PathToString(dd_norm) + "/"); + args.ClearPathCache(); + BOOST_CHECK_EQUAL(dd_norm, args.GetDataDirBase()); + + args.ForceSetArg("-datadir", fs::PathToString(dd_norm) + "/."); + args.ClearPathCache(); + BOOST_CHECK_EQUAL(dd_norm, args.GetDataDirBase()); + + args.ForceSetArg("-datadir", fs::PathToString(dd_norm) + "/./"); + args.ClearPathCache(); + BOOST_CHECK_EQUAL(dd_norm, args.GetDataDirBase()); + + args.ForceSetArg("-datadir", fs::PathToString(dd_norm) + "/.//"); + args.ClearPathCache(); + BOOST_CHECK_EQUAL(dd_norm, args.GetDataDirBase()); +} + +struct TestArgsManager : public ArgsManager +{ + TestArgsManager() { m_network_only_args.clear(); } + void ReadConfigString(const std::string str_config) + { + std::istringstream streamConfig(str_config); + { + LOCK(cs_args); + m_settings.ro_config.clear(); + m_config_sections.clear(); + } + std::string error; + BOOST_REQUIRE(ReadConfigStream(streamConfig, "", error)); + } + void SetNetworkOnlyArg(const std::string arg) + { + LOCK(cs_args); + m_network_only_args.insert(arg); + } + void SetupArgs(const std::vector<std::pair<std::string, unsigned int>>& args) + { + for (const auto& arg : args) { + AddArg(arg.first, "", arg.second, OptionsCategory::OPTIONS); + } + } + using ArgsManager::GetSetting; + using ArgsManager::GetSettingsList; + using ArgsManager::ReadConfigStream; + using ArgsManager::cs_args; + using ArgsManager::m_network; + using ArgsManager::m_settings; +}; + +//! Test GetSetting and GetArg type coercion, negation, and default value handling. +class CheckValueTest : public TestChain100Setup +{ +public: + struct Expect { + util::SettingsValue setting; + bool default_string = false; + bool default_int = false; + bool default_bool = false; + const char* string_value = nullptr; + std::optional<int64_t> int_value; + std::optional<bool> bool_value; + std::optional<std::vector<std::string>> list_value; + const char* error = nullptr; + + explicit Expect(util::SettingsValue s) : setting(std::move(s)) {} + Expect& DefaultString() { default_string = true; return *this; } + Expect& DefaultInt() { default_int = true; return *this; } + Expect& DefaultBool() { default_bool = true; return *this; } + Expect& String(const char* s) { string_value = s; return *this; } + Expect& Int(int64_t i) { int_value = i; return *this; } + Expect& Bool(bool b) { bool_value = b; return *this; } + Expect& List(std::vector<std::string> m) { list_value = std::move(m); return *this; } + Expect& Error(const char* e) { error = e; return *this; } + }; + + void CheckValue(unsigned int flags, const char* arg, const Expect& expect) + { + TestArgsManager test; + test.SetupArgs({{"-value", flags}}); + const char* argv[] = {"ignored", arg}; + std::string error; + bool success = test.ParseParameters(arg ? 2 : 1, (char**)argv, error); + + BOOST_CHECK_EQUAL(test.GetSetting("-value").write(), expect.setting.write()); + auto settings_list = test.GetSettingsList("-value"); + if (expect.setting.isNull() || expect.setting.isFalse()) { + BOOST_CHECK_EQUAL(settings_list.size(), 0U); + } else { + BOOST_CHECK_EQUAL(settings_list.size(), 1U); + BOOST_CHECK_EQUAL(settings_list[0].write(), expect.setting.write()); + } + + if (expect.error) { + BOOST_CHECK(!success); + BOOST_CHECK_NE(error.find(expect.error), std::string::npos); + } else { + BOOST_CHECK(success); + BOOST_CHECK_EQUAL(error, ""); + } + + if (expect.default_string) { + BOOST_CHECK_EQUAL(test.GetArg("-value", "zzzzz"), "zzzzz"); + } else if (expect.string_value) { + BOOST_CHECK_EQUAL(test.GetArg("-value", "zzzzz"), expect.string_value); + } else { + BOOST_CHECK(!success); + } + + if (expect.default_int) { + BOOST_CHECK_EQUAL(test.GetIntArg("-value", 99999), 99999); + } else if (expect.int_value) { + BOOST_CHECK_EQUAL(test.GetIntArg("-value", 99999), *expect.int_value); + } else { + BOOST_CHECK(!success); + } + + if (expect.default_bool) { + BOOST_CHECK_EQUAL(test.GetBoolArg("-value", false), false); + BOOST_CHECK_EQUAL(test.GetBoolArg("-value", true), true); + } else if (expect.bool_value) { + BOOST_CHECK_EQUAL(test.GetBoolArg("-value", false), *expect.bool_value); + BOOST_CHECK_EQUAL(test.GetBoolArg("-value", true), *expect.bool_value); + } else { + BOOST_CHECK(!success); + } + + if (expect.list_value) { + auto l = test.GetArgs("-value"); + BOOST_CHECK_EQUAL_COLLECTIONS(l.begin(), l.end(), expect.list_value->begin(), expect.list_value->end()); + } else { + BOOST_CHECK(!success); + } + } +}; + +BOOST_FIXTURE_TEST_CASE(util_CheckValue, CheckValueTest) +{ + using M = ArgsManager; + + CheckValue(M::ALLOW_ANY, nullptr, Expect{{}}.DefaultString().DefaultInt().DefaultBool().List({})); + CheckValue(M::ALLOW_ANY, "-novalue", Expect{false}.String("0").Int(0).Bool(false).List({})); + CheckValue(M::ALLOW_ANY, "-novalue=", Expect{false}.String("0").Int(0).Bool(false).List({})); + CheckValue(M::ALLOW_ANY, "-novalue=0", Expect{true}.String("1").Int(1).Bool(true).List({"1"})); + CheckValue(M::ALLOW_ANY, "-novalue=1", Expect{false}.String("0").Int(0).Bool(false).List({})); + CheckValue(M::ALLOW_ANY, "-novalue=2", Expect{false}.String("0").Int(0).Bool(false).List({})); + CheckValue(M::ALLOW_ANY, "-novalue=abc", Expect{true}.String("1").Int(1).Bool(true).List({"1"})); + CheckValue(M::ALLOW_ANY, "-value", Expect{""}.String("").Int(0).Bool(true).List({""})); + CheckValue(M::ALLOW_ANY, "-value=", Expect{""}.String("").Int(0).Bool(true).List({""})); + CheckValue(M::ALLOW_ANY, "-value=0", Expect{"0"}.String("0").Int(0).Bool(false).List({"0"})); + CheckValue(M::ALLOW_ANY, "-value=1", Expect{"1"}.String("1").Int(1).Bool(true).List({"1"})); + CheckValue(M::ALLOW_ANY, "-value=2", Expect{"2"}.String("2").Int(2).Bool(true).List({"2"})); + CheckValue(M::ALLOW_ANY, "-value=abc", Expect{"abc"}.String("abc").Int(0).Bool(false).List({"abc"})); +} + +struct NoIncludeConfTest { + std::string Parse(const char* arg) + { + TestArgsManager test; + test.SetupArgs({{"-includeconf", ArgsManager::ALLOW_ANY}}); + std::array argv{"ignored", arg}; + std::string error; + (void)test.ParseParameters(argv.size(), argv.data(), error); + return error; + } +}; + +BOOST_FIXTURE_TEST_CASE(util_NoIncludeConf, NoIncludeConfTest) +{ + BOOST_CHECK_EQUAL(Parse("-noincludeconf"), ""); + BOOST_CHECK_EQUAL(Parse("-includeconf"), "-includeconf cannot be used from commandline; -includeconf=\"\""); + BOOST_CHECK_EQUAL(Parse("-includeconf=file"), "-includeconf cannot be used from commandline; -includeconf=\"file\""); +} + +BOOST_AUTO_TEST_CASE(util_ParseParameters) +{ + TestArgsManager testArgs; + const auto a = std::make_pair("-a", ArgsManager::ALLOW_ANY); + const auto b = std::make_pair("-b", ArgsManager::ALLOW_ANY); + const auto ccc = std::make_pair("-ccc", ArgsManager::ALLOW_ANY); + const auto d = std::make_pair("-d", ArgsManager::ALLOW_ANY); + + const char *argv_test[] = {"-ignored", "-a", "-b", "-ccc=argument", "-ccc=multiple", "f", "-d=e"}; + + std::string error; + LOCK(testArgs.cs_args); + testArgs.SetupArgs({a, b, ccc, d}); + BOOST_CHECK(testArgs.ParseParameters(0, (char**)argv_test, error)); + BOOST_CHECK(testArgs.m_settings.command_line_options.empty() && testArgs.m_settings.ro_config.empty()); + + BOOST_CHECK(testArgs.ParseParameters(1, (char**)argv_test, error)); + BOOST_CHECK(testArgs.m_settings.command_line_options.empty() && testArgs.m_settings.ro_config.empty()); + + BOOST_CHECK(testArgs.ParseParameters(7, (char**)argv_test, error)); + // expectation: -ignored is ignored (program name argument), + // -a, -b and -ccc end up in map, -d ignored because it is after + // a non-option argument (non-GNU option parsing) + BOOST_CHECK(testArgs.m_settings.command_line_options.size() == 3 && testArgs.m_settings.ro_config.empty()); + BOOST_CHECK(testArgs.IsArgSet("-a") && testArgs.IsArgSet("-b") && testArgs.IsArgSet("-ccc") + && !testArgs.IsArgSet("f") && !testArgs.IsArgSet("-d")); + BOOST_CHECK(testArgs.m_settings.command_line_options.count("a") && testArgs.m_settings.command_line_options.count("b") && testArgs.m_settings.command_line_options.count("ccc") + && !testArgs.m_settings.command_line_options.count("f") && !testArgs.m_settings.command_line_options.count("d")); + + BOOST_CHECK(testArgs.m_settings.command_line_options["a"].size() == 1); + BOOST_CHECK(testArgs.m_settings.command_line_options["a"].front().get_str() == ""); + BOOST_CHECK(testArgs.m_settings.command_line_options["ccc"].size() == 2); + BOOST_CHECK(testArgs.m_settings.command_line_options["ccc"].front().get_str() == "argument"); + BOOST_CHECK(testArgs.m_settings.command_line_options["ccc"].back().get_str() == "multiple"); + BOOST_CHECK(testArgs.GetArgs("-ccc").size() == 2); +} + +BOOST_AUTO_TEST_CASE(util_ParseInvalidParameters) +{ + TestArgsManager test; + test.SetupArgs({{"-registered", ArgsManager::ALLOW_ANY}}); + + const char* argv[] = {"ignored", "-registered"}; + std::string error; + BOOST_CHECK(test.ParseParameters(2, (char**)argv, error)); + BOOST_CHECK_EQUAL(error, ""); + + argv[1] = "-unregistered"; + BOOST_CHECK(!test.ParseParameters(2, (char**)argv, error)); + BOOST_CHECK_EQUAL(error, "Invalid parameter -unregistered"); + + // Make sure registered parameters prefixed with a chain name trigger errors. + // (Previously, they were accepted and ignored.) + argv[1] = "-test.registered"; + BOOST_CHECK(!test.ParseParameters(2, (char**)argv, error)); + BOOST_CHECK_EQUAL(error, "Invalid parameter -test.registered"); +} + +static void TestParse(const std::string& str, bool expected_bool, int64_t expected_int) +{ + TestArgsManager test; + test.SetupArgs({{"-value", ArgsManager::ALLOW_ANY}}); + std::string arg = "-value=" + str; + const char* argv[] = {"ignored", arg.c_str()}; + std::string error; + BOOST_CHECK(test.ParseParameters(2, (char**)argv, error)); + BOOST_CHECK_EQUAL(test.GetBoolArg("-value", false), expected_bool); + BOOST_CHECK_EQUAL(test.GetBoolArg("-value", true), expected_bool); + BOOST_CHECK_EQUAL(test.GetIntArg("-value", 99998), expected_int); + BOOST_CHECK_EQUAL(test.GetIntArg("-value", 99999), expected_int); +} + +// Test bool and int parsing. +BOOST_AUTO_TEST_CASE(util_ArgParsing) +{ + // Some of these cases could be ambiguous or surprising to users, and might + // be worth triggering errors or warnings in the future. But for now basic + // test coverage is useful to avoid breaking backwards compatibility + // unintentionally. + TestParse("", true, 0); + TestParse(" ", false, 0); + TestParse("0", false, 0); + TestParse("0 ", false, 0); + TestParse(" 0", false, 0); + TestParse("+0", false, 0); + TestParse("-0", false, 0); + TestParse("5", true, 5); + TestParse("5 ", true, 5); + TestParse(" 5", true, 5); + TestParse("+5", true, 5); + TestParse("-5", true, -5); + TestParse("0 5", false, 0); + TestParse("5 0", true, 5); + TestParse("050", true, 50); + TestParse("0.", false, 0); + TestParse("5.", true, 5); + TestParse("0.0", false, 0); + TestParse("0.5", false, 0); + TestParse("5.0", true, 5); + TestParse("5.5", true, 5); + TestParse("x", false, 0); + TestParse("x0", false, 0); + TestParse("x5", false, 0); + TestParse("0x", false, 0); + TestParse("5x", true, 5); + TestParse("0x5", false, 0); + TestParse("false", false, 0); + TestParse("true", false, 0); + TestParse("yes", false, 0); + TestParse("no", false, 0); +} + +BOOST_AUTO_TEST_CASE(util_GetBoolArg) +{ + TestArgsManager testArgs; + const auto a = std::make_pair("-a", ArgsManager::ALLOW_ANY); + const auto b = std::make_pair("-b", ArgsManager::ALLOW_ANY); + const auto c = std::make_pair("-c", ArgsManager::ALLOW_ANY); + const auto d = std::make_pair("-d", ArgsManager::ALLOW_ANY); + const auto e = std::make_pair("-e", ArgsManager::ALLOW_ANY); + const auto f = std::make_pair("-f", ArgsManager::ALLOW_ANY); + + const char *argv_test[] = { + "ignored", "-a", "-nob", "-c=0", "-d=1", "-e=false", "-f=true"}; + std::string error; + LOCK(testArgs.cs_args); + testArgs.SetupArgs({a, b, c, d, e, f}); + BOOST_CHECK(testArgs.ParseParameters(7, (char**)argv_test, error)); + + // Each letter should be set. + for (const char opt : "abcdef") + BOOST_CHECK(testArgs.IsArgSet({'-', opt}) || !opt); + + // Nothing else should be in the map + BOOST_CHECK(testArgs.m_settings.command_line_options.size() == 6 && + testArgs.m_settings.ro_config.empty()); + + // The -no prefix should get stripped on the way in. + BOOST_CHECK(!testArgs.IsArgSet("-nob")); + + // The -b option is flagged as negated, and nothing else is + BOOST_CHECK(testArgs.IsArgNegated("-b")); + BOOST_CHECK(!testArgs.IsArgNegated("-a")); + + // Check expected values. + BOOST_CHECK(testArgs.GetBoolArg("-a", false) == true); + BOOST_CHECK(testArgs.GetBoolArg("-b", true) == false); + BOOST_CHECK(testArgs.GetBoolArg("-c", true) == false); + BOOST_CHECK(testArgs.GetBoolArg("-d", false) == true); + BOOST_CHECK(testArgs.GetBoolArg("-e", true) == false); + BOOST_CHECK(testArgs.GetBoolArg("-f", true) == false); +} + +BOOST_AUTO_TEST_CASE(util_GetBoolArgEdgeCases) +{ + // Test some awful edge cases that hopefully no user will ever exercise. + TestArgsManager testArgs; + + // Params test + const auto foo = std::make_pair("-foo", ArgsManager::ALLOW_ANY); + const auto bar = std::make_pair("-bar", ArgsManager::ALLOW_ANY); + const char *argv_test[] = {"ignored", "-nofoo", "-foo", "-nobar=0"}; + testArgs.SetupArgs({foo, bar}); + std::string error; + BOOST_CHECK(testArgs.ParseParameters(4, (char**)argv_test, error)); + + // This was passed twice, second one overrides the negative setting. + BOOST_CHECK(!testArgs.IsArgNegated("-foo")); + BOOST_CHECK(testArgs.GetArg("-foo", "xxx") == ""); + + // A double negative is a positive, and not marked as negated. + BOOST_CHECK(!testArgs.IsArgNegated("-bar")); + BOOST_CHECK(testArgs.GetArg("-bar", "xxx") == "1"); + + // Config test + const char *conf_test = "nofoo=1\nfoo=1\nnobar=0\n"; + BOOST_CHECK(testArgs.ParseParameters(1, (char**)argv_test, error)); + testArgs.ReadConfigString(conf_test); + + // This was passed twice, second one overrides the negative setting, + // and the value. + BOOST_CHECK(!testArgs.IsArgNegated("-foo")); + BOOST_CHECK(testArgs.GetArg("-foo", "xxx") == "1"); + + // A double negative is a positive, and does not count as negated. + BOOST_CHECK(!testArgs.IsArgNegated("-bar")); + BOOST_CHECK(testArgs.GetArg("-bar", "xxx") == "1"); + + // Combined test + const char *combo_test_args[] = {"ignored", "-nofoo", "-bar"}; + const char *combo_test_conf = "foo=1\nnobar=1\n"; + BOOST_CHECK(testArgs.ParseParameters(3, (char**)combo_test_args, error)); + testArgs.ReadConfigString(combo_test_conf); + + // Command line overrides, but doesn't erase old setting + BOOST_CHECK(testArgs.IsArgNegated("-foo")); + BOOST_CHECK(testArgs.GetArg("-foo", "xxx") == "0"); + BOOST_CHECK(testArgs.GetArgs("-foo").size() == 0); + + // Command line overrides, but doesn't erase old setting + BOOST_CHECK(!testArgs.IsArgNegated("-bar")); + BOOST_CHECK(testArgs.GetArg("-bar", "xxx") == ""); + BOOST_CHECK(testArgs.GetArgs("-bar").size() == 1 + && testArgs.GetArgs("-bar").front() == ""); +} + +BOOST_AUTO_TEST_CASE(util_ReadConfigStream) +{ + const char *str_config = + "a=\n" + "b=1\n" + "ccc=argument\n" + "ccc=multiple\n" + "d=e\n" + "nofff=1\n" + "noggg=0\n" + "h=1\n" + "noh=1\n" + "noi=1\n" + "i=1\n" + "sec1.ccc=extend1\n" + "\n" + "[sec1]\n" + "ccc=extend2\n" + "d=eee\n" + "h=1\n" + "[sec2]\n" + "ccc=extend3\n" + "iii=2\n"; + + TestArgsManager test_args; + LOCK(test_args.cs_args); + const auto a = std::make_pair("-a", ArgsManager::ALLOW_ANY); + const auto b = std::make_pair("-b", ArgsManager::ALLOW_ANY); + const auto ccc = std::make_pair("-ccc", ArgsManager::ALLOW_ANY); + const auto d = std::make_pair("-d", ArgsManager::ALLOW_ANY); + const auto e = std::make_pair("-e", ArgsManager::ALLOW_ANY); + const auto fff = std::make_pair("-fff", ArgsManager::ALLOW_ANY); + const auto ggg = std::make_pair("-ggg", ArgsManager::ALLOW_ANY); + const auto h = std::make_pair("-h", ArgsManager::ALLOW_ANY); + const auto i = std::make_pair("-i", ArgsManager::ALLOW_ANY); + const auto iii = std::make_pair("-iii", ArgsManager::ALLOW_ANY); + test_args.SetupArgs({a, b, ccc, d, e, fff, ggg, h, i, iii}); + + test_args.ReadConfigString(str_config); + // expectation: a, b, ccc, d, fff, ggg, h, i end up in map + // so do sec1.ccc, sec1.d, sec1.h, sec2.ccc, sec2.iii + + BOOST_CHECK(test_args.m_settings.command_line_options.empty()); + BOOST_CHECK(test_args.m_settings.ro_config.size() == 3); + BOOST_CHECK(test_args.m_settings.ro_config[""].size() == 8); + BOOST_CHECK(test_args.m_settings.ro_config["sec1"].size() == 3); + BOOST_CHECK(test_args.m_settings.ro_config["sec2"].size() == 2); + + BOOST_CHECK(test_args.m_settings.ro_config[""].count("a")); + BOOST_CHECK(test_args.m_settings.ro_config[""].count("b")); + BOOST_CHECK(test_args.m_settings.ro_config[""].count("ccc")); + BOOST_CHECK(test_args.m_settings.ro_config[""].count("d")); + BOOST_CHECK(test_args.m_settings.ro_config[""].count("fff")); + BOOST_CHECK(test_args.m_settings.ro_config[""].count("ggg")); + BOOST_CHECK(test_args.m_settings.ro_config[""].count("h")); + BOOST_CHECK(test_args.m_settings.ro_config[""].count("i")); + BOOST_CHECK(test_args.m_settings.ro_config["sec1"].count("ccc")); + BOOST_CHECK(test_args.m_settings.ro_config["sec1"].count("h")); + BOOST_CHECK(test_args.m_settings.ro_config["sec2"].count("ccc")); + BOOST_CHECK(test_args.m_settings.ro_config["sec2"].count("iii")); + + BOOST_CHECK(test_args.IsArgSet("-a")); + BOOST_CHECK(test_args.IsArgSet("-b")); + BOOST_CHECK(test_args.IsArgSet("-ccc")); + BOOST_CHECK(test_args.IsArgSet("-d")); + BOOST_CHECK(test_args.IsArgSet("-fff")); + BOOST_CHECK(test_args.IsArgSet("-ggg")); + BOOST_CHECK(test_args.IsArgSet("-h")); + BOOST_CHECK(test_args.IsArgSet("-i")); + BOOST_CHECK(!test_args.IsArgSet("-zzz")); + BOOST_CHECK(!test_args.IsArgSet("-iii")); + + BOOST_CHECK_EQUAL(test_args.GetArg("-a", "xxx"), ""); + BOOST_CHECK_EQUAL(test_args.GetArg("-b", "xxx"), "1"); + BOOST_CHECK_EQUAL(test_args.GetArg("-ccc", "xxx"), "argument"); + BOOST_CHECK_EQUAL(test_args.GetArg("-d", "xxx"), "e"); + BOOST_CHECK_EQUAL(test_args.GetArg("-fff", "xxx"), "0"); + BOOST_CHECK_EQUAL(test_args.GetArg("-ggg", "xxx"), "1"); + BOOST_CHECK_EQUAL(test_args.GetArg("-h", "xxx"), "0"); + BOOST_CHECK_EQUAL(test_args.GetArg("-i", "xxx"), "1"); + BOOST_CHECK_EQUAL(test_args.GetArg("-zzz", "xxx"), "xxx"); + BOOST_CHECK_EQUAL(test_args.GetArg("-iii", "xxx"), "xxx"); + + for (const bool def : {false, true}) { + BOOST_CHECK(test_args.GetBoolArg("-a", def)); + BOOST_CHECK(test_args.GetBoolArg("-b", def)); + BOOST_CHECK(!test_args.GetBoolArg("-ccc", def)); + BOOST_CHECK(!test_args.GetBoolArg("-d", def)); + BOOST_CHECK(!test_args.GetBoolArg("-fff", def)); + BOOST_CHECK(test_args.GetBoolArg("-ggg", def)); + BOOST_CHECK(!test_args.GetBoolArg("-h", def)); + BOOST_CHECK(test_args.GetBoolArg("-i", def)); + BOOST_CHECK(test_args.GetBoolArg("-zzz", def) == def); + BOOST_CHECK(test_args.GetBoolArg("-iii", def) == def); + } + + BOOST_CHECK(test_args.GetArgs("-a").size() == 1 + && test_args.GetArgs("-a").front() == ""); + BOOST_CHECK(test_args.GetArgs("-b").size() == 1 + && test_args.GetArgs("-b").front() == "1"); + BOOST_CHECK(test_args.GetArgs("-ccc").size() == 2 + && test_args.GetArgs("-ccc").front() == "argument" + && test_args.GetArgs("-ccc").back() == "multiple"); + BOOST_CHECK(test_args.GetArgs("-fff").size() == 0); + BOOST_CHECK(test_args.GetArgs("-nofff").size() == 0); + BOOST_CHECK(test_args.GetArgs("-ggg").size() == 1 + && test_args.GetArgs("-ggg").front() == "1"); + BOOST_CHECK(test_args.GetArgs("-noggg").size() == 0); + BOOST_CHECK(test_args.GetArgs("-h").size() == 0); + BOOST_CHECK(test_args.GetArgs("-noh").size() == 0); + BOOST_CHECK(test_args.GetArgs("-i").size() == 1 + && test_args.GetArgs("-i").front() == "1"); + BOOST_CHECK(test_args.GetArgs("-noi").size() == 0); + BOOST_CHECK(test_args.GetArgs("-zzz").size() == 0); + + BOOST_CHECK(!test_args.IsArgNegated("-a")); + BOOST_CHECK(!test_args.IsArgNegated("-b")); + BOOST_CHECK(!test_args.IsArgNegated("-ccc")); + BOOST_CHECK(!test_args.IsArgNegated("-d")); + BOOST_CHECK(test_args.IsArgNegated("-fff")); + BOOST_CHECK(!test_args.IsArgNegated("-ggg")); + BOOST_CHECK(test_args.IsArgNegated("-h")); // last setting takes precedence + BOOST_CHECK(!test_args.IsArgNegated("-i")); // last setting takes precedence + BOOST_CHECK(!test_args.IsArgNegated("-zzz")); + + // Test sections work + test_args.SelectConfigNetwork("sec1"); + + // same as original + BOOST_CHECK_EQUAL(test_args.GetArg("-a", "xxx"), ""); + BOOST_CHECK_EQUAL(test_args.GetArg("-b", "xxx"), "1"); + BOOST_CHECK_EQUAL(test_args.GetArg("-fff", "xxx"), "0"); + BOOST_CHECK_EQUAL(test_args.GetArg("-ggg", "xxx"), "1"); + BOOST_CHECK_EQUAL(test_args.GetArg("-zzz", "xxx"), "xxx"); + BOOST_CHECK_EQUAL(test_args.GetArg("-iii", "xxx"), "xxx"); + // d is overridden + BOOST_CHECK(test_args.GetArg("-d", "xxx") == "eee"); + // section-specific setting + BOOST_CHECK(test_args.GetArg("-h", "xxx") == "1"); + // section takes priority for multiple values + BOOST_CHECK(test_args.GetArg("-ccc", "xxx") == "extend1"); + // check multiple values works + const std::vector<std::string> sec1_ccc_expected = {"extend1","extend2","argument","multiple"}; + const auto& sec1_ccc_res = test_args.GetArgs("-ccc"); + BOOST_CHECK_EQUAL_COLLECTIONS(sec1_ccc_res.begin(), sec1_ccc_res.end(), sec1_ccc_expected.begin(), sec1_ccc_expected.end()); + + test_args.SelectConfigNetwork("sec2"); + + // same as original + BOOST_CHECK(test_args.GetArg("-a", "xxx") == ""); + BOOST_CHECK(test_args.GetArg("-b", "xxx") == "1"); + BOOST_CHECK(test_args.GetArg("-d", "xxx") == "e"); + BOOST_CHECK(test_args.GetArg("-fff", "xxx") == "0"); + BOOST_CHECK(test_args.GetArg("-ggg", "xxx") == "1"); + BOOST_CHECK(test_args.GetArg("-zzz", "xxx") == "xxx"); + BOOST_CHECK(test_args.GetArg("-h", "xxx") == "0"); + // section-specific setting + BOOST_CHECK(test_args.GetArg("-iii", "xxx") == "2"); + // section takes priority for multiple values + BOOST_CHECK(test_args.GetArg("-ccc", "xxx") == "extend3"); + // check multiple values works + const std::vector<std::string> sec2_ccc_expected = {"extend3","argument","multiple"}; + const auto& sec2_ccc_res = test_args.GetArgs("-ccc"); + BOOST_CHECK_EQUAL_COLLECTIONS(sec2_ccc_res.begin(), sec2_ccc_res.end(), sec2_ccc_expected.begin(), sec2_ccc_expected.end()); + + // Test section only options + + test_args.SetNetworkOnlyArg("-d"); + test_args.SetNetworkOnlyArg("-ccc"); + test_args.SetNetworkOnlyArg("-h"); + + test_args.SelectConfigNetwork(CBaseChainParams::MAIN); + BOOST_CHECK(test_args.GetArg("-d", "xxx") == "e"); + BOOST_CHECK(test_args.GetArgs("-ccc").size() == 2); + BOOST_CHECK(test_args.GetArg("-h", "xxx") == "0"); + + test_args.SelectConfigNetwork("sec1"); + BOOST_CHECK(test_args.GetArg("-d", "xxx") == "eee"); + BOOST_CHECK(test_args.GetArgs("-d").size() == 1); + BOOST_CHECK(test_args.GetArgs("-ccc").size() == 2); + BOOST_CHECK(test_args.GetArg("-h", "xxx") == "1"); + + test_args.SelectConfigNetwork("sec2"); + BOOST_CHECK(test_args.GetArg("-d", "xxx") == "xxx"); + BOOST_CHECK(test_args.GetArgs("-d").size() == 0); + BOOST_CHECK(test_args.GetArgs("-ccc").size() == 1); + BOOST_CHECK(test_args.GetArg("-h", "xxx") == "0"); +} + +BOOST_AUTO_TEST_CASE(util_GetArg) +{ + TestArgsManager testArgs; + LOCK(testArgs.cs_args); + testArgs.m_settings.command_line_options.clear(); + testArgs.m_settings.command_line_options["strtest1"] = {"string..."}; + // strtest2 undefined on purpose + testArgs.m_settings.command_line_options["inttest1"] = {"12345"}; + testArgs.m_settings.command_line_options["inttest2"] = {"81985529216486895"}; + // inttest3 undefined on purpose + testArgs.m_settings.command_line_options["booltest1"] = {""}; + // booltest2 undefined on purpose + testArgs.m_settings.command_line_options["booltest3"] = {"0"}; + testArgs.m_settings.command_line_options["booltest4"] = {"1"}; + + // priorities + testArgs.m_settings.command_line_options["pritest1"] = {"a", "b"}; + testArgs.m_settings.ro_config[""]["pritest2"] = {"a", "b"}; + testArgs.m_settings.command_line_options["pritest3"] = {"a"}; + testArgs.m_settings.ro_config[""]["pritest3"] = {"b"}; + testArgs.m_settings.command_line_options["pritest4"] = {"a","b"}; + testArgs.m_settings.ro_config[""]["pritest4"] = {"c","d"}; + + BOOST_CHECK_EQUAL(testArgs.GetArg("strtest1", "default"), "string..."); + BOOST_CHECK_EQUAL(testArgs.GetArg("strtest2", "default"), "default"); + BOOST_CHECK_EQUAL(testArgs.GetIntArg("inttest1", -1), 12345); + BOOST_CHECK_EQUAL(testArgs.GetIntArg("inttest2", -1), 81985529216486895LL); + BOOST_CHECK_EQUAL(testArgs.GetIntArg("inttest3", -1), -1); + BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest1", false), true); + BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest2", false), false); + BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest3", false), false); + BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest4", false), true); + + BOOST_CHECK_EQUAL(testArgs.GetArg("pritest1", "default"), "b"); + BOOST_CHECK_EQUAL(testArgs.GetArg("pritest2", "default"), "a"); + BOOST_CHECK_EQUAL(testArgs.GetArg("pritest3", "default"), "a"); + BOOST_CHECK_EQUAL(testArgs.GetArg("pritest4", "default"), "b"); +} + +BOOST_AUTO_TEST_CASE(util_GetChainName) +{ + TestArgsManager test_args; + const auto testnet = std::make_pair("-testnet", ArgsManager::ALLOW_ANY); + const auto regtest = std::make_pair("-regtest", ArgsManager::ALLOW_ANY); + test_args.SetupArgs({testnet, regtest}); + + const char* argv_testnet[] = {"cmd", "-testnet"}; + const char* argv_regtest[] = {"cmd", "-regtest"}; + const char* argv_test_no_reg[] = {"cmd", "-testnet", "-noregtest"}; + const char* argv_both[] = {"cmd", "-testnet", "-regtest"}; + + // equivalent to "-testnet" + // regtest in testnet section is ignored + const char* testnetconf = "testnet=1\nregtest=0\n[test]\nregtest=1"; + std::string error; + + BOOST_CHECK(test_args.ParseParameters(0, (char**)argv_testnet, error)); + BOOST_CHECK_EQUAL(test_args.GetChainName(), "main"); + + BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_testnet, error)); + BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + + BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_regtest, error)); + BOOST_CHECK_EQUAL(test_args.GetChainName(), "regtest"); + + BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_test_no_reg, error)); + BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + + BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_both, error)); + BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); + + BOOST_CHECK(test_args.ParseParameters(0, (char**)argv_testnet, error)); + test_args.ReadConfigString(testnetconf); + BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + + BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_testnet, error)); + test_args.ReadConfigString(testnetconf); + BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + + BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_regtest, error)); + test_args.ReadConfigString(testnetconf); + BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); + + BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_test_no_reg, error)); + test_args.ReadConfigString(testnetconf); + BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + + BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_both, error)); + test_args.ReadConfigString(testnetconf); + BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); + + // check setting the network to test (and thus making + // [test] regtest=1 potentially relevant) doesn't break things + test_args.SelectConfigNetwork("test"); + + BOOST_CHECK(test_args.ParseParameters(0, (char**)argv_testnet, error)); + test_args.ReadConfigString(testnetconf); + BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + + BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_testnet, error)); + test_args.ReadConfigString(testnetconf); + BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + + BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_regtest, error)); + test_args.ReadConfigString(testnetconf); + BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); + + BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_test_no_reg, error)); + test_args.ReadConfigString(testnetconf); + BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + + BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_both, error)); + test_args.ReadConfigString(testnetconf); + BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); +} + +// Test different ways settings can be merged, and verify results. This test can +// be used to confirm that updates to settings code don't change behavior +// unintentionally. +// +// The test covers: +// +// - Combining different setting actions. Possible actions are: configuring a +// setting, negating a setting (adding "-no" prefix), and configuring/negating +// settings in a network section (adding "main." or "test." prefixes). +// +// - Combining settings from command line arguments and a config file. +// +// - Combining SoftSet and ForceSet calls. +// +// - Testing "main" and "test" network values to make sure settings from network +// sections are applied and to check for mainnet-specific behaviors like +// inheriting settings from the default section. +// +// - Testing network-specific settings like "-wallet", that may be ignored +// outside a network section, and non-network specific settings like "-server" +// that aren't sensitive to the network. +// +struct ArgsMergeTestingSetup : public BasicTestingSetup { + //! Max number of actions to sequence together. Can decrease this when + //! debugging to make test results easier to understand. + static constexpr int MAX_ACTIONS = 3; + + enum Action { NONE, SET, NEGATE, SECTION_SET, SECTION_NEGATE }; + using ActionList = Action[MAX_ACTIONS]; + + //! Enumerate all possible test configurations. + template <typename Fn> + void ForEachMergeSetup(Fn&& fn) + { + ActionList arg_actions = {}; + // command_line_options do not have sections. Only iterate over SET and NEGATE + ForEachNoDup(arg_actions, SET, NEGATE, [&] { + ActionList conf_actions = {}; + ForEachNoDup(conf_actions, SET, SECTION_NEGATE, [&] { + for (bool soft_set : {false, true}) { + for (bool force_set : {false, true}) { + for (const std::string& section : {CBaseChainParams::MAIN, CBaseChainParams::TESTNET, CBaseChainParams::SIGNET}) { + for (const std::string& network : {CBaseChainParams::MAIN, CBaseChainParams::TESTNET, CBaseChainParams::SIGNET}) { + for (bool net_specific : {false, true}) { + fn(arg_actions, conf_actions, soft_set, force_set, section, network, net_specific); + } + } + } + } + } + }); + }); + } + + //! Translate actions into a list of <key>=<value> setting strings. + std::vector<std::string> GetValues(const ActionList& actions, + const std::string& section, + const std::string& name, + const std::string& value_prefix) + { + std::vector<std::string> values; + int suffix = 0; + for (Action action : actions) { + if (action == NONE) break; + std::string prefix; + if (action == SECTION_SET || action == SECTION_NEGATE) prefix = section + "."; + if (action == SET || action == SECTION_SET) { + for (int i = 0; i < 2; ++i) { + values.push_back(prefix + name + "=" + value_prefix + ToString(++suffix)); + } + } + if (action == NEGATE || action == SECTION_NEGATE) { + values.push_back(prefix + "no" + name + "=1"); + } + } + return values; + } +}; + +// Regression test covering different ways config settings can be merged. The +// test parses and merges settings, representing the results as strings that get +// compared against an expected hash. To debug, the result strings can be dumped +// to a file (see comments below). +BOOST_FIXTURE_TEST_CASE(util_ArgsMerge, ArgsMergeTestingSetup) +{ + CHash256 out_sha; + FILE* out_file = nullptr; + if (const char* out_path = getenv("ARGS_MERGE_TEST_OUT")) { + out_file = fsbridge::fopen(out_path, "w"); + if (!out_file) throw std::system_error(errno, std::generic_category(), "fopen failed"); + } + + ForEachMergeSetup([&](const ActionList& arg_actions, const ActionList& conf_actions, bool soft_set, bool force_set, + const std::string& section, const std::string& network, bool net_specific) { + TestArgsManager parser; + LOCK(parser.cs_args); + + std::string desc = "net="; + desc += network; + parser.m_network = network; + + const std::string& name = net_specific ? "wallet" : "server"; + const std::string key = "-" + name; + parser.AddArg(key, name, ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + if (net_specific) parser.SetNetworkOnlyArg(key); + + auto args = GetValues(arg_actions, section, name, "a"); + std::vector<const char*> argv = {"ignored"}; + for (auto& arg : args) { + arg.insert(0, "-"); + desc += " "; + desc += arg; + argv.push_back(arg.c_str()); + } + std::string error; + BOOST_CHECK(parser.ParseParameters(argv.size(), argv.data(), error)); + BOOST_CHECK_EQUAL(error, ""); + + std::string conf; + for (auto& conf_val : GetValues(conf_actions, section, name, "c")) { + desc += " "; + desc += conf_val; + conf += conf_val; + conf += "\n"; + } + std::istringstream conf_stream(conf); + BOOST_CHECK(parser.ReadConfigStream(conf_stream, "filepath", error)); + BOOST_CHECK_EQUAL(error, ""); + + if (soft_set) { + desc += " soft"; + parser.SoftSetArg(key, "soft1"); + parser.SoftSetArg(key, "soft2"); + } + + if (force_set) { + desc += " force"; + parser.ForceSetArg(key, "force1"); + parser.ForceSetArg(key, "force2"); + } + + desc += " || "; + + if (!parser.IsArgSet(key)) { + desc += "unset"; + BOOST_CHECK(!parser.IsArgNegated(key)); + BOOST_CHECK_EQUAL(parser.GetArg(key, "default"), "default"); + BOOST_CHECK(parser.GetArgs(key).empty()); + } else if (parser.IsArgNegated(key)) { + desc += "negated"; + BOOST_CHECK_EQUAL(parser.GetArg(key, "default"), "0"); + BOOST_CHECK(parser.GetArgs(key).empty()); + } else { + desc += parser.GetArg(key, "default"); + desc += " |"; + for (const auto& arg : parser.GetArgs(key)) { + desc += " "; + desc += arg; + } + } + + std::set<std::string> ignored = parser.GetUnsuitableSectionOnlyArgs(); + if (!ignored.empty()) { + desc += " | ignored"; + for (const auto& arg : ignored) { + desc += " "; + desc += arg; + } + } + + desc += "\n"; + + out_sha.Write(MakeUCharSpan(desc)); + if (out_file) { + BOOST_REQUIRE(fwrite(desc.data(), 1, desc.size(), out_file) == desc.size()); + } + }); + + if (out_file) { + if (fclose(out_file)) throw std::system_error(errno, std::generic_category(), "fclose failed"); + out_file = nullptr; + } + + unsigned char out_sha_bytes[CSHA256::OUTPUT_SIZE]; + out_sha.Finalize(out_sha_bytes); + std::string out_sha_hex = HexStr(out_sha_bytes); + + // If check below fails, should manually dump the results with: + // + // ARGS_MERGE_TEST_OUT=results.txt ./test_bitcoin --run_test=util_tests/util_ArgsMerge + // + // And verify diff against previous results to make sure the changes are expected. + // + // Results file is formatted like: + // + // <input> || <IsArgSet/IsArgNegated/GetArg output> | <GetArgs output> | <GetUnsuitable output> + BOOST_CHECK_EQUAL(out_sha_hex, "d1e436c1cd510d0ec44d5205d4b4e3bee6387d316e0075c58206cb16603f3d82"); +} + +// Similar test as above, but for ArgsManager::GetChainName function. +struct ChainMergeTestingSetup : public BasicTestingSetup { + static constexpr int MAX_ACTIONS = 2; + + enum Action { NONE, ENABLE_TEST, DISABLE_TEST, NEGATE_TEST, ENABLE_REG, DISABLE_REG, NEGATE_REG }; + using ActionList = Action[MAX_ACTIONS]; + + //! Enumerate all possible test configurations. + template <typename Fn> + void ForEachMergeSetup(Fn&& fn) + { + ActionList arg_actions = {}; + ForEachNoDup(arg_actions, ENABLE_TEST, NEGATE_REG, [&] { + ActionList conf_actions = {}; + ForEachNoDup(conf_actions, ENABLE_TEST, NEGATE_REG, [&] { fn(arg_actions, conf_actions); }); + }); + } +}; + +BOOST_FIXTURE_TEST_CASE(util_ChainMerge, ChainMergeTestingSetup) +{ + CHash256 out_sha; + FILE* out_file = nullptr; + if (const char* out_path = getenv("CHAIN_MERGE_TEST_OUT")) { + out_file = fsbridge::fopen(out_path, "w"); + if (!out_file) throw std::system_error(errno, std::generic_category(), "fopen failed"); + } + + ForEachMergeSetup([&](const ActionList& arg_actions, const ActionList& conf_actions) { + TestArgsManager parser; + LOCK(parser.cs_args); + parser.AddArg("-regtest", "regtest", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + parser.AddArg("-testnet", "testnet", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + + auto arg = [](Action action) { return action == ENABLE_TEST ? "-testnet=1" : + action == DISABLE_TEST ? "-testnet=0" : + action == NEGATE_TEST ? "-notestnet=1" : + action == ENABLE_REG ? "-regtest=1" : + action == DISABLE_REG ? "-regtest=0" : + action == NEGATE_REG ? "-noregtest=1" : nullptr; }; + + std::string desc; + std::vector<const char*> argv = {"ignored"}; + for (Action action : arg_actions) { + const char* argstr = arg(action); + if (!argstr) break; + argv.push_back(argstr); + desc += " "; + desc += argv.back(); + } + std::string error; + BOOST_CHECK(parser.ParseParameters(argv.size(), argv.data(), error)); + BOOST_CHECK_EQUAL(error, ""); + + std::string conf; + for (Action action : conf_actions) { + const char* argstr = arg(action); + if (!argstr) break; + desc += " "; + desc += argstr + 1; + conf += argstr + 1; + conf += "\n"; + } + std::istringstream conf_stream(conf); + BOOST_CHECK(parser.ReadConfigStream(conf_stream, "filepath", error)); + BOOST_CHECK_EQUAL(error, ""); + + desc += " || "; + try { + desc += parser.GetChainName(); + } catch (const std::runtime_error& e) { + desc += "error: "; + desc += e.what(); + } + desc += "\n"; + + out_sha.Write(MakeUCharSpan(desc)); + if (out_file) { + BOOST_REQUIRE(fwrite(desc.data(), 1, desc.size(), out_file) == desc.size()); + } + }); + + if (out_file) { + if (fclose(out_file)) throw std::system_error(errno, std::generic_category(), "fclose failed"); + out_file = nullptr; + } + + unsigned char out_sha_bytes[CSHA256::OUTPUT_SIZE]; + out_sha.Finalize(out_sha_bytes); + std::string out_sha_hex = HexStr(out_sha_bytes); + + // If check below fails, should manually dump the results with: + // + // CHAIN_MERGE_TEST_OUT=results.txt ./test_bitcoin --run_test=util_tests/util_ChainMerge + // + // And verify diff against previous results to make sure the changes are expected. + // + // Results file is formatted like: + // + // <input> || <output> + BOOST_CHECK_EQUAL(out_sha_hex, "f263493e300023b6509963887444c41386f44b63bc30047eb8402e8c1144854c"); +} + +BOOST_AUTO_TEST_CASE(util_ReadWriteSettings) +{ + // Test writing setting. + TestArgsManager args1; + args1.ForceSetArg("-datadir", fs::PathToString(m_path_root)); + args1.LockSettings([&](util::Settings& settings) { settings.rw_settings["name"] = "value"; }); + args1.WriteSettingsFile(); + + // Test reading setting. + TestArgsManager args2; + args2.ForceSetArg("-datadir", fs::PathToString(m_path_root)); + args2.ReadSettingsFile(); + args2.LockSettings([&](util::Settings& settings) { BOOST_CHECK_EQUAL(settings.rw_settings["name"].get_str(), "value"); }); + + // Test error logging, and remove previously written setting. + { + ASSERT_DEBUG_LOG("Failed renaming settings file"); + fs::remove(args1.GetDataDirBase() / "settings.json"); + fs::create_directory(args1.GetDataDirBase() / "settings.json"); + args2.WriteSettingsFile(); + fs::remove(args1.GetDataDirBase() / "settings.json"); + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 78b82b9b20..1c13c0a909 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -7,6 +7,7 @@ #include <consensus/merkle.h> #include <pow.h> #include <streams.h> +#include <test/util/txmempool.h> #include <test/util/setup_common.h> diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp new file mode 100644 index 0000000000..dd7c32cc46 --- /dev/null +++ b/src/test/blockmanager_tests.cpp @@ -0,0 +1,42 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <node/blockstorage.h> +#include <node/context.h> +#include <validation.h> + +#include <boost/test/unit_test.hpp> +#include <test/util/setup_common.h> + +using node::BlockManager; +using node::BLOCK_SERIALIZATION_HEADER_SIZE; + +// use BasicTestingSetup here for the data directory configuration, setup, and cleanup +BOOST_FIXTURE_TEST_SUITE(blockmanager_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos) +{ + const auto params {CreateChainParams(ArgsManager{}, CBaseChainParams::MAIN)}; + BlockManager blockman {}; + CChain chain {}; + // simulate adding a genesis block normally + BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, *params, nullptr).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); + // simulate what happens during reindex + // simulate a well-formed genesis block being found at offset 8 in the blk00000.dat file + // the block is found at offset 8 because there is an 8 byte serialization header + // consisting of 4 magic bytes + 4 length bytes before each block in a well-formed blk file. + FlatFilePos pos{0, BLOCK_SERIALIZATION_HEADER_SIZE}; + BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, *params, &pos).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); + // now simulate what happens after reindex for the first new block processed + // the actual block contents don't matter, just that it's a block. + // verify that the write position is at offset 0x12d. + // this is a check to make sure that https://github.com/bitcoin/bitcoin/issues/21379 does not recur + // 8 bytes (for serialization header) + 285 (for serialized genesis block) = 293 + // add another 8 bytes for the second block's serialization header and we get 293 + 8 = 301 + FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1, chain, *params, nullptr)}; + BOOST_CHECK_EQUAL(actual.nPos, BLOCK_SERIALIZATION_HEADER_SIZE + ::GetSerializeSize(params->GenesisBlock(), CLIENT_VERSION) + BLOCK_SERIALIZATION_HEADER_SIZE); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/coinstatsindex_tests.cpp b/src/test/coinstatsindex_tests.cpp index 2a6a777cfe..8a2b0792fd 100644 --- a/src/test/coinstatsindex_tests.cpp +++ b/src/test/coinstatsindex_tests.cpp @@ -76,10 +76,16 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup) BOOST_CHECK(block_index != new_block_index); + // It is not safe to stop and destroy the index until it finishes handling + // the last BlockConnected notification. The BlockUntilSyncedToCurrentChain() + // call above is sufficient to ensure this, but the + // SyncWithValidationInterfaceQueue() call below is also needed to ensure + // TSAN always sees the test thread waiting for the notification thread, and + // avoid potential false positive reports. + SyncWithValidationInterfaceQueue(); + // Shutdown sequence (c.f. Shutdown() in init.cpp) coin_stats_index.Stop(); - - // Rest of shutdown sequence and destructors happen in ~TestingSetup() } // Test shutdown between BlockConnected and ChainStateFlushed notifications, diff --git a/src/test/fuzz/addrman.cpp b/src/test/fuzz/addrman.cpp index 7668940cbc..f3c29cd6b8 100644 --- a/src/test/fuzz/addrman.cpp +++ b/src/test/fuzz/addrman.cpp @@ -11,6 +11,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/fuzz/util/net.h> #include <test/util/setup_common.h> #include <time.h> #include <util/asmap.h> diff --git a/src/test/fuzz/banman.cpp b/src/test/fuzz/banman.cpp index b2969ecdc0..d10f4586b4 100644 --- a/src/test/fuzz/banman.cpp +++ b/src/test/fuzz/banman.cpp @@ -8,6 +8,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/fuzz/util/net.h> #include <test/util/setup_common.h> #include <util/readwritefile.h> #include <util/system.h> diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index 4406779015..e8b10a0ad0 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -11,6 +11,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/fuzz/util/net.h> #include <test/util/setup_common.h> #include <util/system.h> #include <util/translation.h> diff --git a/src/test/fuzz/i2p.cpp b/src/test/fuzz/i2p.cpp index fb6d23aca5..72b7f9e334 100644 --- a/src/test/fuzz/i2p.cpp +++ b/src/test/fuzz/i2p.cpp @@ -8,9 +8,10 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/fuzz/util/net.h> #include <test/util/setup_common.h> -#include <threadinterrupt.h> #include <util/system.h> +#include <util/threadinterrupt.h> void initialize_i2p() { diff --git a/src/test/fuzz/net.cpp b/src/test/fuzz/net.cpp index 741810f6a2..9f7c87dcd5 100644 --- a/src/test/fuzz/net.cpp +++ b/src/test/fuzz/net.cpp @@ -12,6 +12,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/fuzz/util/net.h> #include <test/util/net.h> #include <test/util/setup_common.h> #include <util/asmap.h> diff --git a/src/test/fuzz/net_permissions.cpp b/src/test/fuzz/net_permissions.cpp index e62fe0328e..21a6640ef4 100644 --- a/src/test/fuzz/net_permissions.cpp +++ b/src/test/fuzz/net_permissions.cpp @@ -6,6 +6,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/fuzz/util/net.h> #include <util/translation.h> #include <cassert> diff --git a/src/test/fuzz/netaddress.cpp b/src/test/fuzz/netaddress.cpp index 35e6688c61..2022f16a48 100644 --- a/src/test/fuzz/netaddress.cpp +++ b/src/test/fuzz/netaddress.cpp @@ -5,7 +5,7 @@ #include <netaddress.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> -#include <test/fuzz/util.h> +#include <test/fuzz/util/net.h> #include <cassert> #include <cstdint> diff --git a/src/test/fuzz/netbase_dns_lookup.cpp b/src/test/fuzz/netbase_dns_lookup.cpp index 31ea31744a..39d4935126 100644 --- a/src/test/fuzz/netbase_dns_lookup.cpp +++ b/src/test/fuzz/netbase_dns_lookup.cpp @@ -6,7 +6,7 @@ #include <netbase.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> -#include <test/fuzz/util.h> +#include <test/fuzz/util/net.h> #include <cstdint> #include <string> diff --git a/src/test/fuzz/node_eviction.cpp b/src/test/fuzz/node_eviction.cpp index e27b254580..0f204babfa 100644 --- a/src/test/fuzz/node_eviction.cpp +++ b/src/test/fuzz/node_eviction.cpp @@ -7,6 +7,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/fuzz/util/net.h> #include <algorithm> #include <cassert> diff --git a/src/test/fuzz/policy_estimator.cpp b/src/test/fuzz/policy_estimator.cpp index a3d57dbdd5..17c340695f 100644 --- a/src/test/fuzz/policy_estimator.cpp +++ b/src/test/fuzz/policy_estimator.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <kernel/mempool_entry.h> #include <policy/fees.h> #include <policy/fees_args.h> #include <primitives/transaction.h> diff --git a/src/test/fuzz/pow.cpp b/src/test/fuzz/pow.cpp index eba03da773..59eb6df878 100644 --- a/src/test/fuzz/pow.cpp +++ b/src/test/fuzz/pow.cpp @@ -9,6 +9,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <util/check.h> #include <util/overflow.h> #include <cstdint> @@ -25,7 +26,7 @@ FUZZ_TARGET_INIT(pow, initialize_pow) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const Consensus::Params& consensus_params = Params().GetConsensus(); - std::vector<CBlockIndex> blocks; + std::vector<std::unique_ptr<CBlockIndex>> blocks; const uint32_t fixed_time = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); const uint32_t fixed_bits = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); LIMITED_WHILE(fuzzed_data_provider.remaining_bytes() > 0, 10000) { @@ -33,9 +34,10 @@ FUZZ_TARGET_INIT(pow, initialize_pow) if (!block_header) { continue; } - CBlockIndex current_block{*block_header}; + CBlockIndex& current_block{ + *blocks.emplace_back(std::make_unique<CBlockIndex>(*block_header))}; { - CBlockIndex* previous_block = blocks.empty() ? nullptr : &PickValue(fuzzed_data_provider, blocks); + CBlockIndex* previous_block = blocks.empty() ? nullptr : PickValue(fuzzed_data_provider, blocks).get(); const int current_height = (previous_block != nullptr && previous_block->nHeight != std::numeric_limits<int>::max()) ? previous_block->nHeight + 1 : 0; if (fuzzed_data_provider.ConsumeBool()) { current_block.pprev = previous_block; @@ -57,7 +59,6 @@ FUZZ_TARGET_INIT(pow, initialize_pow) } else { current_block.nChainWork = ConsumeArithUInt256(fuzzed_data_provider); } - blocks.push_back(current_block); } { (void)GetBlockProof(current_block); @@ -67,9 +68,9 @@ FUZZ_TARGET_INIT(pow, initialize_pow) } } { - const CBlockIndex* to = &PickValue(fuzzed_data_provider, blocks); - const CBlockIndex* from = &PickValue(fuzzed_data_provider, blocks); - const CBlockIndex* tip = &PickValue(fuzzed_data_provider, blocks); + const auto& to = PickValue(fuzzed_data_provider, blocks); + const auto& from = PickValue(fuzzed_data_provider, blocks); + const auto& tip = PickValue(fuzzed_data_provider, blocks); try { (void)GetBlockProofEquivalentTime(*to, *from, *tip, consensus_params); } catch (const uint_error&) { diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index f6a35da4fc..f6000535b3 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -14,11 +14,11 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/fuzz/util/net.h> #include <test/util/mining.h> #include <test/util/net.h> #include <test/util/setup_common.h> #include <test/util/validation.h> -#include <txorphanage.h> #include <validationinterface.h> #include <version.h> @@ -130,6 +130,7 @@ FUZZ_TARGET_MSG(pong); FUZZ_TARGET_MSG(sendaddrv2); FUZZ_TARGET_MSG(sendcmpct); FUZZ_TARGET_MSG(sendheaders); +FUZZ_TARGET_MSG(sendtxrcncl); FUZZ_TARGET_MSG(tx); FUZZ_TARGET_MSG(verack); FUZZ_TARGET_MSG(version); diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index 1df1717ec3..41831fd176 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -9,11 +9,11 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/fuzz/util/net.h> #include <test/util/mining.h> #include <test/util/net.h> #include <test/util/setup_common.h> #include <test/util/validation.h> -#include <txorphanage.h> #include <validation.h> #include <validationinterface.h> diff --git a/src/test/fuzz/rbf.cpp b/src/test/fuzz/rbf.cpp index e06e57d919..678fc7a5aa 100644 --- a/src/test/fuzz/rbf.cpp +++ b/src/test/fuzz/rbf.cpp @@ -11,6 +11,7 @@ #include <test/fuzz/util.h> #include <test/fuzz/util/mempool.h> #include <test/util/setup_common.h> +#include <test/util/txmempool.h> #include <txmempool.h> #include <cstdint> diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 26913a41d2..f32046e69f 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -151,6 +151,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ "preciousblock", "pruneblockchain", "reconsiderblock", + "scanblocks", "scantxoutset", "sendrawtransaction", "setmocktime", diff --git a/src/test/fuzz/socks5.cpp b/src/test/fuzz/socks5.cpp index c3a6eed089..15f479b009 100644 --- a/src/test/fuzz/socks5.cpp +++ b/src/test/fuzz/socks5.cpp @@ -7,6 +7,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <test/fuzz/util/net.h> #include <test/util/setup_common.h> #include <cstdint> diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index 94399faf04..f6373351d8 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -4,6 +4,7 @@ #include <blockfilter.h> #include <clientversion.h> +#include <common/url.h> #include <logging.h> #include <netaddress.h> #include <netbase.h> @@ -27,7 +28,6 @@ #include <util/string.h> #include <util/system.h> #include <util/translation.h> -#include <util/url.h> #include <version.h> #include <cstdint> diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index a34e501fcc..46ca8e47e9 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -13,6 +13,7 @@ #include <test/util/mining.h> #include <test/util/script.h> #include <test/util/setup_common.h> +#include <test/util/txmempool.h> #include <util/rbf.h> #include <validation.h> #include <validationinterface.h> diff --git a/src/test/fuzz/txorphan.cpp b/src/test/fuzz/txorphan.cpp index 55060f31cf..02743051e8 100644 --- a/src/test/fuzz/txorphan.cpp +++ b/src/test/fuzz/txorphan.cpp @@ -36,7 +36,6 @@ FUZZ_TARGET_INIT(txorphan, initialize_orphanage) SetMockTime(ConsumeTime(fuzzed_data_provider)); TxOrphanage orphanage; - std::set<uint256> orphan_work_set; std::vector<COutPoint> outpoints; // initial outpoints used to construct transactions later for (uint8_t i = 0; i < 4; i++) { @@ -86,15 +85,19 @@ FUZZ_TARGET_INIT(txorphan, initialize_orphanage) CallOneOf( fuzzed_data_provider, [&] { - LOCK(g_cs_orphans); - orphanage.AddChildrenToWorkSet(*tx, orphan_work_set); + orphanage.AddChildrenToWorkSet(*tx, peer_id); }, [&] { - bool have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); { - LOCK(g_cs_orphans); - bool get_tx = orphanage.GetTx(tx->GetHash()).first != nullptr; - Assert(have_tx == get_tx); + NodeId originator; + bool more = true; + CTransactionRef ref = orphanage.GetTxToReconsider(peer_id, originator, more); + if (!ref) { + Assert(!more); + } else { + bool have_tx = orphanage.HaveTx(GenTxid::Txid(ref->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(ref->GetHash())); + Assert(have_tx); + } } }, [&] { @@ -102,14 +105,12 @@ FUZZ_TARGET_INIT(txorphan, initialize_orphanage) // AddTx should return false if tx is too big or already have it // tx weight is unknown, we only check when tx is already in orphanage { - LOCK(g_cs_orphans); bool add_tx = orphanage.AddTx(tx, peer_id); // have_tx == true -> add_tx == false Assert(!have_tx || !add_tx); } have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); { - LOCK(g_cs_orphans); bool add_tx = orphanage.AddTx(tx, peer_id); // if have_tx is still false, it must be too big Assert(!have_tx == (GetTransactionWeight(*tx) > MAX_STANDARD_TX_WEIGHT)); @@ -120,25 +121,22 @@ FUZZ_TARGET_INIT(txorphan, initialize_orphanage) bool have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); // EraseTx should return 0 if m_orphans doesn't have the tx { - LOCK(g_cs_orphans); Assert(have_tx == orphanage.EraseTx(tx->GetHash())); } have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); // have_tx should be false and EraseTx should fail { - LOCK(g_cs_orphans); Assert(!have_tx && !orphanage.EraseTx(tx->GetHash())); } }, [&] { - LOCK(g_cs_orphans); orphanage.EraseForPeer(peer_id); }, [&] { // test mocktime and expiry SetMockTime(ConsumeTime(fuzzed_data_provider)); auto limit = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); - WITH_LOCK(g_cs_orphans, orphanage.LimitOrphans(limit)); + orphanage.LimitOrphans(limit); Assert(orphanage.Size() <= limit); }); } diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index 0080b1e56e..8babfadf4f 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -3,11 +3,10 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/amount.h> -#include <net_processing.h> -#include <netmessagemaker.h> #include <pubkey.h> #include <test/fuzz/util.h> #include <test/util/script.h> +#include <util/check.h> #include <util/overflow.h> #include <util/rbf.h> #include <util/time.h> @@ -15,290 +14,6 @@ #include <memory> -FuzzedSock::FuzzedSock(FuzzedDataProvider& fuzzed_data_provider) - : m_fuzzed_data_provider{fuzzed_data_provider} -{ - m_socket = fuzzed_data_provider.ConsumeIntegralInRange<SOCKET>(INVALID_SOCKET - 1, INVALID_SOCKET); -} - -FuzzedSock::~FuzzedSock() -{ - // Sock::~Sock() will be called after FuzzedSock::~FuzzedSock() and it will call - // close(m_socket) if m_socket is not INVALID_SOCKET. - // Avoid closing an arbitrary file descriptor (m_socket is just a random very high number which - // theoretically may concide with a real opened file descriptor). - m_socket = INVALID_SOCKET; -} - -FuzzedSock& FuzzedSock::operator=(Sock&& other) -{ - assert(false && "Move of Sock into FuzzedSock not allowed."); - return *this; -} - -ssize_t FuzzedSock::Send(const void* data, size_t len, int flags) const -{ - constexpr std::array send_errnos{ - EACCES, - EAGAIN, - EALREADY, - EBADF, - ECONNRESET, - EDESTADDRREQ, - EFAULT, - EINTR, - EINVAL, - EISCONN, - EMSGSIZE, - ENOBUFS, - ENOMEM, - ENOTCONN, - ENOTSOCK, - EOPNOTSUPP, - EPIPE, - EWOULDBLOCK, - }; - if (m_fuzzed_data_provider.ConsumeBool()) { - return len; - } - const ssize_t r = m_fuzzed_data_provider.ConsumeIntegralInRange<ssize_t>(-1, len); - if (r == -1) { - SetFuzzedErrNo(m_fuzzed_data_provider, send_errnos); - } - return r; -} - -ssize_t FuzzedSock::Recv(void* buf, size_t len, int flags) const -{ - // Have a permanent error at recv_errnos[0] because when the fuzzed data is exhausted - // SetFuzzedErrNo() will always return the first element and we want to avoid Recv() - // returning -1 and setting errno to EAGAIN repeatedly. - constexpr std::array recv_errnos{ - ECONNREFUSED, - EAGAIN, - EBADF, - EFAULT, - EINTR, - EINVAL, - ENOMEM, - ENOTCONN, - ENOTSOCK, - EWOULDBLOCK, - }; - assert(buf != nullptr || len == 0); - if (len == 0 || m_fuzzed_data_provider.ConsumeBool()) { - const ssize_t r = m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; - if (r == -1) { - SetFuzzedErrNo(m_fuzzed_data_provider, recv_errnos); - } - return r; - } - std::vector<uint8_t> random_bytes; - bool pad_to_len_bytes{m_fuzzed_data_provider.ConsumeBool()}; - if (m_peek_data.has_value()) { - // `MSG_PEEK` was used in the preceding `Recv()` call, return `m_peek_data`. - random_bytes.assign({m_peek_data.value()}); - if ((flags & MSG_PEEK) == 0) { - m_peek_data.reset(); - } - pad_to_len_bytes = false; - } else if ((flags & MSG_PEEK) != 0) { - // New call with `MSG_PEEK`. - random_bytes = m_fuzzed_data_provider.ConsumeBytes<uint8_t>(1); - if (!random_bytes.empty()) { - m_peek_data = random_bytes[0]; - pad_to_len_bytes = false; - } - } else { - random_bytes = m_fuzzed_data_provider.ConsumeBytes<uint8_t>( - m_fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, len)); - } - if (random_bytes.empty()) { - const ssize_t r = m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; - if (r == -1) { - SetFuzzedErrNo(m_fuzzed_data_provider, recv_errnos); - } - return r; - } - std::memcpy(buf, random_bytes.data(), random_bytes.size()); - if (pad_to_len_bytes) { - if (len > random_bytes.size()) { - std::memset((char*)buf + random_bytes.size(), 0, len - random_bytes.size()); - } - return len; - } - if (m_fuzzed_data_provider.ConsumeBool() && std::getenv("FUZZED_SOCKET_FAKE_LATENCY") != nullptr) { - std::this_thread::sleep_for(std::chrono::milliseconds{2}); - } - return random_bytes.size(); -} - -int FuzzedSock::Connect(const sockaddr*, socklen_t) const -{ - // Have a permanent error at connect_errnos[0] because when the fuzzed data is exhausted - // SetFuzzedErrNo() will always return the first element and we want to avoid Connect() - // returning -1 and setting errno to EAGAIN repeatedly. - constexpr std::array connect_errnos{ - ECONNREFUSED, - EAGAIN, - ECONNRESET, - EHOSTUNREACH, - EINPROGRESS, - EINTR, - ENETUNREACH, - ETIMEDOUT, - }; - if (m_fuzzed_data_provider.ConsumeBool()) { - SetFuzzedErrNo(m_fuzzed_data_provider, connect_errnos); - return -1; - } - return 0; -} - -int FuzzedSock::Bind(const sockaddr*, socklen_t) const -{ - // Have a permanent error at bind_errnos[0] because when the fuzzed data is exhausted - // SetFuzzedErrNo() will always set the global errno to bind_errnos[0]. We want to - // avoid this method returning -1 and setting errno to a temporary error (like EAGAIN) - // repeatedly because proper code should retry on temporary errors, leading to an - // infinite loop. - constexpr std::array bind_errnos{ - EACCES, - EADDRINUSE, - EADDRNOTAVAIL, - EAGAIN, - }; - if (m_fuzzed_data_provider.ConsumeBool()) { - SetFuzzedErrNo(m_fuzzed_data_provider, bind_errnos); - return -1; - } - return 0; -} - -int FuzzedSock::Listen(int) const -{ - // Have a permanent error at listen_errnos[0] because when the fuzzed data is exhausted - // SetFuzzedErrNo() will always set the global errno to listen_errnos[0]. We want to - // avoid this method returning -1 and setting errno to a temporary error (like EAGAIN) - // repeatedly because proper code should retry on temporary errors, leading to an - // infinite loop. - constexpr std::array listen_errnos{ - EADDRINUSE, - EINVAL, - EOPNOTSUPP, - }; - if (m_fuzzed_data_provider.ConsumeBool()) { - SetFuzzedErrNo(m_fuzzed_data_provider, listen_errnos); - return -1; - } - return 0; -} - -std::unique_ptr<Sock> FuzzedSock::Accept(sockaddr* addr, socklen_t* addr_len) const -{ - constexpr std::array accept_errnos{ - ECONNABORTED, - EINTR, - ENOMEM, - }; - if (m_fuzzed_data_provider.ConsumeBool()) { - SetFuzzedErrNo(m_fuzzed_data_provider, accept_errnos); - return std::unique_ptr<FuzzedSock>(); - } - return std::make_unique<FuzzedSock>(m_fuzzed_data_provider); -} - -int FuzzedSock::GetSockOpt(int level, int opt_name, void* opt_val, socklen_t* opt_len) const -{ - constexpr std::array getsockopt_errnos{ - ENOMEM, - ENOBUFS, - }; - if (m_fuzzed_data_provider.ConsumeBool()) { - SetFuzzedErrNo(m_fuzzed_data_provider, getsockopt_errnos); - return -1; - } - if (opt_val == nullptr) { - return 0; - } - std::memcpy(opt_val, - ConsumeFixedLengthByteVector(m_fuzzed_data_provider, *opt_len).data(), - *opt_len); - return 0; -} - -int FuzzedSock::SetSockOpt(int, int, const void*, socklen_t) const -{ - constexpr std::array setsockopt_errnos{ - ENOMEM, - ENOBUFS, - }; - if (m_fuzzed_data_provider.ConsumeBool()) { - SetFuzzedErrNo(m_fuzzed_data_provider, setsockopt_errnos); - return -1; - } - return 0; -} - -int FuzzedSock::GetSockName(sockaddr* name, socklen_t* name_len) const -{ - constexpr std::array getsockname_errnos{ - ECONNRESET, - ENOBUFS, - }; - if (m_fuzzed_data_provider.ConsumeBool()) { - SetFuzzedErrNo(m_fuzzed_data_provider, getsockname_errnos); - return -1; - } - *name_len = m_fuzzed_data_provider.ConsumeData(name, *name_len); - return 0; -} - -bool FuzzedSock::Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred) const -{ - constexpr std::array wait_errnos{ - EBADF, - EINTR, - EINVAL, - }; - if (m_fuzzed_data_provider.ConsumeBool()) { - SetFuzzedErrNo(m_fuzzed_data_provider, wait_errnos); - return false; - } - if (occurred != nullptr) { - *occurred = m_fuzzed_data_provider.ConsumeBool() ? requested : 0; - } - return true; -} - -bool FuzzedSock::WaitMany(std::chrono::milliseconds timeout, EventsPerSock& events_per_sock) const -{ - for (auto& [sock, events] : events_per_sock) { - (void)sock; - events.occurred = m_fuzzed_data_provider.ConsumeBool() ? events.requested : 0; - } - return true; -} - -bool FuzzedSock::IsConnected(std::string& errmsg) const -{ - if (m_fuzzed_data_provider.ConsumeBool()) { - return true; - } - errmsg = "disconnected at random by the fuzzer"; - return false; -} - -void FillNode(FuzzedDataProvider& fuzzed_data_provider, ConnmanTestMsg& connman, CNode& node) noexcept -{ - connman.Handshake(node, - /*successfully_connected=*/fuzzed_data_provider.ConsumeBool(), - /*remote_services=*/ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS), - /*local_services=*/ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS), - /*version=*/fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(MIN_PEER_PROTO_VERSION, std::numeric_limits<int32_t>::max()), - /*relay_txs=*/fuzzed_data_provider.ConsumeBool()); -} - CAmount ConsumeMoney(FuzzedDataProvider& fuzzed_data_provider, const std::optional<CAmount>& max) noexcept { return fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(0, max.value_or(MAX_MONEY)); @@ -489,33 +204,6 @@ bool ContainsSpentInput(const CTransaction& tx, const CCoinsViewCache& inputs) n return false; } -CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcept -{ - const Network network = fuzzed_data_provider.PickValueInArray({Network::NET_IPV4, Network::NET_IPV6, Network::NET_INTERNAL, Network::NET_ONION}); - CNetAddr net_addr; - if (network == Network::NET_IPV4) { - in_addr v4_addr = {}; - v4_addr.s_addr = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); - net_addr = CNetAddr{v4_addr}; - } else if (network == Network::NET_IPV6) { - if (fuzzed_data_provider.remaining_bytes() >= 16) { - in6_addr v6_addr = {}; - memcpy(v6_addr.s6_addr, fuzzed_data_provider.ConsumeBytes<uint8_t>(16).data(), 16); - net_addr = CNetAddr{v6_addr, fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; - } - } else if (network == Network::NET_INTERNAL) { - net_addr.SetInternal(fuzzed_data_provider.ConsumeBytesAsString(32)); - } else if (network == Network::NET_ONION) { - net_addr.SetSpecial(fuzzed_data_provider.ConsumeBytesAsString(32)); - } - return net_addr; -} - -CAddress ConsumeAddress(FuzzedDataProvider& fuzzed_data_provider) noexcept -{ - return {ConsumeService(fuzzed_data_provider), ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS), NodeSeconds{std::chrono::seconds{fuzzed_data_provider.ConsumeIntegral<uint32_t>()}}}; -} - FILE* FuzzedFileProvider::open() { SetFuzzedErrNo(m_fuzzed_data_provider); diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index a354df2bf7..09c57c7be3 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -12,9 +12,6 @@ #include <consensus/amount.h> #include <consensus/consensus.h> #include <merkleblock.h> -#include <net.h> -#include <netaddress.h> -#include <netbase.h> #include <primitives/transaction.h> #include <script/script.h> #include <script/standard.h> @@ -22,7 +19,6 @@ #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> -#include <test/util/net.h> #include <uint256.h> #include <version.h> @@ -36,54 +32,6 @@ class PeerManager; -class FuzzedSock : public Sock -{ - FuzzedDataProvider& m_fuzzed_data_provider; - - /** - * Data to return when `MSG_PEEK` is used as a `Recv()` flag. - * If `MSG_PEEK` is used, then our `Recv()` returns some random data as usual, but on the next - * `Recv()` call we must return the same data, thus we remember it here. - */ - mutable std::optional<uint8_t> m_peek_data; - -public: - explicit FuzzedSock(FuzzedDataProvider& fuzzed_data_provider); - - ~FuzzedSock() override; - - FuzzedSock& operator=(Sock&& other) override; - - ssize_t Send(const void* data, size_t len, int flags) const override; - - ssize_t Recv(void* buf, size_t len, int flags) const override; - - int Connect(const sockaddr*, socklen_t) const override; - - int Bind(const sockaddr*, socklen_t) const override; - - int Listen(int backlog) const override; - - std::unique_ptr<Sock> Accept(sockaddr* addr, socklen_t* addr_len) const override; - - int GetSockOpt(int level, int opt_name, void* opt_val, socklen_t* opt_len) const override; - - int SetSockOpt(int level, int opt_name, const void* opt_val, socklen_t opt_len) const override; - - int GetSockName(sockaddr* name, socklen_t* name_len) const override; - - bool Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred = nullptr) const override; - - bool WaitMany(std::chrono::milliseconds timeout, EventsPerSock& events_per_sock) const override; - - bool IsConnected(std::string& errmsg) const override; -}; - -[[nodiscard]] inline FuzzedSock ConsumeSock(FuzzedDataProvider& fuzzed_data_provider) -{ - return FuzzedSock{fuzzed_data_provider}; -} - template <typename... Callables> size_t CallOneOf(FuzzedDataProvider& fuzzed_data_provider, Callables... callables) { @@ -272,61 +220,6 @@ inline void SetFuzzedErrNo(FuzzedDataProvider& fuzzed_data_provider) noexcept return result; } -CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcept; - -inline CSubNet ConsumeSubNet(FuzzedDataProvider& fuzzed_data_provider) noexcept -{ - return {ConsumeNetAddr(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<uint8_t>()}; -} - -inline CService ConsumeService(FuzzedDataProvider& fuzzed_data_provider) noexcept -{ - return {ConsumeNetAddr(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<uint16_t>()}; -} - -CAddress ConsumeAddress(FuzzedDataProvider& fuzzed_data_provider) noexcept; - -template <bool ReturnUniquePtr = false> -auto ConsumeNode(FuzzedDataProvider& fuzzed_data_provider, const std::optional<NodeId>& node_id_in = std::nullopt) noexcept -{ - const NodeId node_id = node_id_in.value_or(fuzzed_data_provider.ConsumeIntegralInRange<NodeId>(0, std::numeric_limits<NodeId>::max())); - const auto sock = std::make_shared<FuzzedSock>(fuzzed_data_provider); - const CAddress address = ConsumeAddress(fuzzed_data_provider); - const uint64_t keyed_net_group = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); - const uint64_t local_host_nonce = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); - const CAddress addr_bind = ConsumeAddress(fuzzed_data_provider); - const std::string addr_name = fuzzed_data_provider.ConsumeRandomLengthString(64); - const ConnectionType conn_type = fuzzed_data_provider.PickValueInArray(ALL_CONNECTION_TYPES); - const bool inbound_onion{conn_type == ConnectionType::INBOUND ? fuzzed_data_provider.ConsumeBool() : false}; - NetPermissionFlags permission_flags = ConsumeWeakEnum(fuzzed_data_provider, ALL_NET_PERMISSION_FLAGS); - if constexpr (ReturnUniquePtr) { - return std::make_unique<CNode>(node_id, - sock, - address, - keyed_net_group, - local_host_nonce, - addr_bind, - addr_name, - conn_type, - inbound_onion, - CNodeOptions{ .permission_flags = permission_flags }); - } else { - return CNode{node_id, - sock, - address, - keyed_net_group, - local_host_nonce, - addr_bind, - addr_name, - conn_type, - inbound_onion, - CNodeOptions{ .permission_flags = permission_flags }}; - } -} -inline std::unique_ptr<CNode> ConsumeNodeAsUniquePtr(FuzzedDataProvider& fdp, const std::optional<NodeId>& node_id_in = std::nullopt) { return ConsumeNode<true>(fdp, node_id_in); } - -void FillNode(FuzzedDataProvider& fuzzed_data_provider, ConnmanTestMsg& connman, CNode& node) noexcept EXCLUSIVE_LOCKS_REQUIRED(NetEventsInterface::g_msgproc_mutex); - class FuzzedFileProvider { FuzzedDataProvider& m_fuzzed_data_provider; diff --git a/src/test/fuzz/util/mempool.cpp b/src/test/fuzz/util/mempool.cpp index d0053f77d2..4baca5ec77 100644 --- a/src/test/fuzz/util/mempool.cpp +++ b/src/test/fuzz/util/mempool.cpp @@ -3,12 +3,15 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/amount.h> +#include <consensus/consensus.h> +#include <kernel/mempool_entry.h> #include <primitives/transaction.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/util.h> #include <test/fuzz/util/mempool.h> -#include <txmempool.h> +#include <cassert> +#include <cstdint> #include <limits> CTxMemPoolEntry ConsumeTxMemPoolEntry(FuzzedDataProvider& fuzzed_data_provider, const CTransaction& tx) noexcept diff --git a/src/test/fuzz/util/mempool.h b/src/test/fuzz/util/mempool.h index 4304e5294e..31b578dc4b 100644 --- a/src/test/fuzz/util/mempool.h +++ b/src/test/fuzz/util/mempool.h @@ -5,11 +5,13 @@ #ifndef BITCOIN_TEST_FUZZ_UTIL_MEMPOOL_H #define BITCOIN_TEST_FUZZ_UTIL_MEMPOOL_H -#include <primitives/transaction.h> -#include <test/fuzz/FuzzedDataProvider.h> -#include <txmempool.h> +#include <kernel/mempool_entry.h> #include <validation.h> +class CTransaction; +class CTxMemPool; +class FuzzedDataProvider; + class DummyChainState final : public Chainstate { public: diff --git a/src/test/fuzz/util/net.cpp b/src/test/fuzz/util/net.cpp new file mode 100644 index 0000000000..c6c6e3ad16 --- /dev/null +++ b/src/test/fuzz/util/net.cpp @@ -0,0 +1,358 @@ +// Copyright (c) 2009-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/util/net.h> + +#include <compat/compat.h> +#include <netaddress.h> +#include <protocol.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/util.h> +#include <test/util/net.h> +#include <util/sock.h> +#include <util/time.h> +#include <version.h> + +#include <array> +#include <cassert> +#include <cerrno> +#include <cstdint> +#include <cstdlib> +#include <cstring> +#include <thread> +#include <vector> + +class CNode; + +CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + const Network network = fuzzed_data_provider.PickValueInArray({Network::NET_IPV4, Network::NET_IPV6, Network::NET_INTERNAL, Network::NET_ONION}); + CNetAddr net_addr; + if (network == Network::NET_IPV4) { + in_addr v4_addr = {}; + v4_addr.s_addr = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); + net_addr = CNetAddr{v4_addr}; + } else if (network == Network::NET_IPV6) { + if (fuzzed_data_provider.remaining_bytes() >= 16) { + in6_addr v6_addr = {}; + memcpy(v6_addr.s6_addr, fuzzed_data_provider.ConsumeBytes<uint8_t>(16).data(), 16); + net_addr = CNetAddr{v6_addr, fuzzed_data_provider.ConsumeIntegral<uint32_t>()}; + } + } else if (network == Network::NET_INTERNAL) { + net_addr.SetInternal(fuzzed_data_provider.ConsumeBytesAsString(32)); + } else if (network == Network::NET_ONION) { + auto pub_key{fuzzed_data_provider.ConsumeBytes<uint8_t>(ADDR_TORV3_SIZE)}; + pub_key.resize(ADDR_TORV3_SIZE); + const bool ok{net_addr.SetSpecial(OnionToString(pub_key))}; + assert(ok); + } + return net_addr; +} + +CAddress ConsumeAddress(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + return {ConsumeService(fuzzed_data_provider), ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS), NodeSeconds{std::chrono::seconds{fuzzed_data_provider.ConsumeIntegral<uint32_t>()}}}; +} + +FuzzedSock::FuzzedSock(FuzzedDataProvider& fuzzed_data_provider) + : m_fuzzed_data_provider{fuzzed_data_provider}, m_selectable{fuzzed_data_provider.ConsumeBool()} +{ + m_socket = fuzzed_data_provider.ConsumeIntegralInRange<SOCKET>(INVALID_SOCKET - 1, INVALID_SOCKET); +} + +FuzzedSock::~FuzzedSock() +{ + // Sock::~Sock() will be called after FuzzedSock::~FuzzedSock() and it will call + // close(m_socket) if m_socket is not INVALID_SOCKET. + // Avoid closing an arbitrary file descriptor (m_socket is just a random very high number which + // theoretically may concide with a real opened file descriptor). + m_socket = INVALID_SOCKET; +} + +FuzzedSock& FuzzedSock::operator=(Sock&& other) +{ + assert(false && "Move of Sock into FuzzedSock not allowed."); + return *this; +} + +ssize_t FuzzedSock::Send(const void* data, size_t len, int flags) const +{ + constexpr std::array send_errnos{ + EACCES, + EAGAIN, + EALREADY, + EBADF, + ECONNRESET, + EDESTADDRREQ, + EFAULT, + EINTR, + EINVAL, + EISCONN, + EMSGSIZE, + ENOBUFS, + ENOMEM, + ENOTCONN, + ENOTSOCK, + EOPNOTSUPP, + EPIPE, + EWOULDBLOCK, + }; + if (m_fuzzed_data_provider.ConsumeBool()) { + return len; + } + const ssize_t r = m_fuzzed_data_provider.ConsumeIntegralInRange<ssize_t>(-1, len); + if (r == -1) { + SetFuzzedErrNo(m_fuzzed_data_provider, send_errnos); + } + return r; +} + +ssize_t FuzzedSock::Recv(void* buf, size_t len, int flags) const +{ + // Have a permanent error at recv_errnos[0] because when the fuzzed data is exhausted + // SetFuzzedErrNo() will always return the first element and we want to avoid Recv() + // returning -1 and setting errno to EAGAIN repeatedly. + constexpr std::array recv_errnos{ + ECONNREFUSED, + EAGAIN, + EBADF, + EFAULT, + EINTR, + EINVAL, + ENOMEM, + ENOTCONN, + ENOTSOCK, + EWOULDBLOCK, + }; + assert(buf != nullptr || len == 0); + if (len == 0 || m_fuzzed_data_provider.ConsumeBool()) { + const ssize_t r = m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; + if (r == -1) { + SetFuzzedErrNo(m_fuzzed_data_provider, recv_errnos); + } + return r; + } + std::vector<uint8_t> random_bytes; + bool pad_to_len_bytes{m_fuzzed_data_provider.ConsumeBool()}; + if (m_peek_data.has_value()) { + // `MSG_PEEK` was used in the preceding `Recv()` call, return `m_peek_data`. + random_bytes.assign({m_peek_data.value()}); + if ((flags & MSG_PEEK) == 0) { + m_peek_data.reset(); + } + pad_to_len_bytes = false; + } else if ((flags & MSG_PEEK) != 0) { + // New call with `MSG_PEEK`. + random_bytes = m_fuzzed_data_provider.ConsumeBytes<uint8_t>(1); + if (!random_bytes.empty()) { + m_peek_data = random_bytes[0]; + pad_to_len_bytes = false; + } + } else { + random_bytes = m_fuzzed_data_provider.ConsumeBytes<uint8_t>( + m_fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, len)); + } + if (random_bytes.empty()) { + const ssize_t r = m_fuzzed_data_provider.ConsumeBool() ? 0 : -1; + if (r == -1) { + SetFuzzedErrNo(m_fuzzed_data_provider, recv_errnos); + } + return r; + } + std::memcpy(buf, random_bytes.data(), random_bytes.size()); + if (pad_to_len_bytes) { + if (len > random_bytes.size()) { + std::memset((char*)buf + random_bytes.size(), 0, len - random_bytes.size()); + } + return len; + } + if (m_fuzzed_data_provider.ConsumeBool() && std::getenv("FUZZED_SOCKET_FAKE_LATENCY") != nullptr) { + std::this_thread::sleep_for(std::chrono::milliseconds{2}); + } + return random_bytes.size(); +} + +int FuzzedSock::Connect(const sockaddr*, socklen_t) const +{ + // Have a permanent error at connect_errnos[0] because when the fuzzed data is exhausted + // SetFuzzedErrNo() will always return the first element and we want to avoid Connect() + // returning -1 and setting errno to EAGAIN repeatedly. + constexpr std::array connect_errnos{ + ECONNREFUSED, + EAGAIN, + ECONNRESET, + EHOSTUNREACH, + EINPROGRESS, + EINTR, + ENETUNREACH, + ETIMEDOUT, + }; + if (m_fuzzed_data_provider.ConsumeBool()) { + SetFuzzedErrNo(m_fuzzed_data_provider, connect_errnos); + return -1; + } + return 0; +} + +int FuzzedSock::Bind(const sockaddr*, socklen_t) const +{ + // Have a permanent error at bind_errnos[0] because when the fuzzed data is exhausted + // SetFuzzedErrNo() will always set the global errno to bind_errnos[0]. We want to + // avoid this method returning -1 and setting errno to a temporary error (like EAGAIN) + // repeatedly because proper code should retry on temporary errors, leading to an + // infinite loop. + constexpr std::array bind_errnos{ + EACCES, + EADDRINUSE, + EADDRNOTAVAIL, + EAGAIN, + }; + if (m_fuzzed_data_provider.ConsumeBool()) { + SetFuzzedErrNo(m_fuzzed_data_provider, bind_errnos); + return -1; + } + return 0; +} + +int FuzzedSock::Listen(int) const +{ + // Have a permanent error at listen_errnos[0] because when the fuzzed data is exhausted + // SetFuzzedErrNo() will always set the global errno to listen_errnos[0]. We want to + // avoid this method returning -1 and setting errno to a temporary error (like EAGAIN) + // repeatedly because proper code should retry on temporary errors, leading to an + // infinite loop. + constexpr std::array listen_errnos{ + EADDRINUSE, + EINVAL, + EOPNOTSUPP, + }; + if (m_fuzzed_data_provider.ConsumeBool()) { + SetFuzzedErrNo(m_fuzzed_data_provider, listen_errnos); + return -1; + } + return 0; +} + +std::unique_ptr<Sock> FuzzedSock::Accept(sockaddr* addr, socklen_t* addr_len) const +{ + constexpr std::array accept_errnos{ + ECONNABORTED, + EINTR, + ENOMEM, + }; + if (m_fuzzed_data_provider.ConsumeBool()) { + SetFuzzedErrNo(m_fuzzed_data_provider, accept_errnos); + return std::unique_ptr<FuzzedSock>(); + } + return std::make_unique<FuzzedSock>(m_fuzzed_data_provider); +} + +int FuzzedSock::GetSockOpt(int level, int opt_name, void* opt_val, socklen_t* opt_len) const +{ + constexpr std::array getsockopt_errnos{ + ENOMEM, + ENOBUFS, + }; + if (m_fuzzed_data_provider.ConsumeBool()) { + SetFuzzedErrNo(m_fuzzed_data_provider, getsockopt_errnos); + return -1; + } + if (opt_val == nullptr) { + return 0; + } + std::memcpy(opt_val, + ConsumeFixedLengthByteVector(m_fuzzed_data_provider, *opt_len).data(), + *opt_len); + return 0; +} + +int FuzzedSock::SetSockOpt(int, int, const void*, socklen_t) const +{ + constexpr std::array setsockopt_errnos{ + ENOMEM, + ENOBUFS, + }; + if (m_fuzzed_data_provider.ConsumeBool()) { + SetFuzzedErrNo(m_fuzzed_data_provider, setsockopt_errnos); + return -1; + } + return 0; +} + +int FuzzedSock::GetSockName(sockaddr* name, socklen_t* name_len) const +{ + constexpr std::array getsockname_errnos{ + ECONNRESET, + ENOBUFS, + }; + if (m_fuzzed_data_provider.ConsumeBool()) { + SetFuzzedErrNo(m_fuzzed_data_provider, getsockname_errnos); + return -1; + } + *name_len = m_fuzzed_data_provider.ConsumeData(name, *name_len); + return 0; +} + +bool FuzzedSock::SetNonBlocking() const +{ + constexpr std::array setnonblocking_errnos{ + EBADF, + EPERM, + }; + if (m_fuzzed_data_provider.ConsumeBool()) { + SetFuzzedErrNo(m_fuzzed_data_provider, setnonblocking_errnos); + return false; + } + return true; +} + +bool FuzzedSock::IsSelectable() const +{ + return m_selectable; +} + +bool FuzzedSock::Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred) const +{ + constexpr std::array wait_errnos{ + EBADF, + EINTR, + EINVAL, + }; + if (m_fuzzed_data_provider.ConsumeBool()) { + SetFuzzedErrNo(m_fuzzed_data_provider, wait_errnos); + return false; + } + if (occurred != nullptr) { + *occurred = m_fuzzed_data_provider.ConsumeBool() ? requested : 0; + } + return true; +} + +bool FuzzedSock::WaitMany(std::chrono::milliseconds timeout, EventsPerSock& events_per_sock) const +{ + for (auto& [sock, events] : events_per_sock) { + (void)sock; + events.occurred = m_fuzzed_data_provider.ConsumeBool() ? events.requested : 0; + } + return true; +} + +bool FuzzedSock::IsConnected(std::string& errmsg) const +{ + if (m_fuzzed_data_provider.ConsumeBool()) { + return true; + } + errmsg = "disconnected at random by the fuzzer"; + return false; +} + +void FillNode(FuzzedDataProvider& fuzzed_data_provider, ConnmanTestMsg& connman, CNode& node) noexcept +{ + connman.Handshake(node, + /*successfully_connected=*/fuzzed_data_provider.ConsumeBool(), + /*remote_services=*/ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS), + /*local_services=*/ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS), + /*version=*/fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(MIN_PEER_PROTO_VERSION, std::numeric_limits<int32_t>::max()), + /*relay_txs=*/fuzzed_data_provider.ConsumeBool()); +} diff --git a/src/test/fuzz/util/net.h b/src/test/fuzz/util/net.h new file mode 100644 index 0000000000..74afbe1cd9 --- /dev/null +++ b/src/test/fuzz/util/net.h @@ -0,0 +1,141 @@ +// Copyright (c) 2009-2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_FUZZ_UTIL_NET_H +#define BITCOIN_TEST_FUZZ_UTIL_NET_H + +#include <net.h> +#include <net_permissions.h> +#include <netaddress.h> +#include <node/connection_types.h> +#include <node/eviction.h> +#include <protocol.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/util.h> +#include <test/util/net.h> +#include <threadsafety.h> +#include <util/sock.h> + +#include <chrono> +#include <cstdint> +#include <limits> +#include <memory> +#include <optional> +#include <string> + +CNetAddr ConsumeNetAddr(FuzzedDataProvider& fuzzed_data_provider) noexcept; + +class FuzzedSock : public Sock +{ + FuzzedDataProvider& m_fuzzed_data_provider; + + /** + * Data to return when `MSG_PEEK` is used as a `Recv()` flag. + * If `MSG_PEEK` is used, then our `Recv()` returns some random data as usual, but on the next + * `Recv()` call we must return the same data, thus we remember it here. + */ + mutable std::optional<uint8_t> m_peek_data; + + /** + * Whether to pretend that the socket is select(2)-able. This is randomly set in the + * constructor. It should remain constant so that repeated calls to `IsSelectable()` + * return the same value. + */ + const bool m_selectable; + +public: + explicit FuzzedSock(FuzzedDataProvider& fuzzed_data_provider); + + ~FuzzedSock() override; + + FuzzedSock& operator=(Sock&& other) override; + + ssize_t Send(const void* data, size_t len, int flags) const override; + + ssize_t Recv(void* buf, size_t len, int flags) const override; + + int Connect(const sockaddr*, socklen_t) const override; + + int Bind(const sockaddr*, socklen_t) const override; + + int Listen(int backlog) const override; + + std::unique_ptr<Sock> Accept(sockaddr* addr, socklen_t* addr_len) const override; + + int GetSockOpt(int level, int opt_name, void* opt_val, socklen_t* opt_len) const override; + + int SetSockOpt(int level, int opt_name, const void* opt_val, socklen_t opt_len) const override; + + int GetSockName(sockaddr* name, socklen_t* name_len) const override; + + bool SetNonBlocking() const override; + + bool IsSelectable() const override; + + bool Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred = nullptr) const override; + + bool WaitMany(std::chrono::milliseconds timeout, EventsPerSock& events_per_sock) const override; + + bool IsConnected(std::string& errmsg) const override; +}; + +[[nodiscard]] inline FuzzedSock ConsumeSock(FuzzedDataProvider& fuzzed_data_provider) +{ + return FuzzedSock{fuzzed_data_provider}; +} + +inline CSubNet ConsumeSubNet(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + return {ConsumeNetAddr(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<uint8_t>()}; +} + +inline CService ConsumeService(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + return {ConsumeNetAddr(fuzzed_data_provider), fuzzed_data_provider.ConsumeIntegral<uint16_t>()}; +} + +CAddress ConsumeAddress(FuzzedDataProvider& fuzzed_data_provider) noexcept; + +template <bool ReturnUniquePtr = false> +auto ConsumeNode(FuzzedDataProvider& fuzzed_data_provider, const std::optional<NodeId>& node_id_in = std::nullopt) noexcept +{ + const NodeId node_id = node_id_in.value_or(fuzzed_data_provider.ConsumeIntegralInRange<NodeId>(0, std::numeric_limits<NodeId>::max())); + const auto sock = std::make_shared<FuzzedSock>(fuzzed_data_provider); + const CAddress address = ConsumeAddress(fuzzed_data_provider); + const uint64_t keyed_net_group = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); + const uint64_t local_host_nonce = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); + const CAddress addr_bind = ConsumeAddress(fuzzed_data_provider); + const std::string addr_name = fuzzed_data_provider.ConsumeRandomLengthString(64); + const ConnectionType conn_type = fuzzed_data_provider.PickValueInArray(ALL_CONNECTION_TYPES); + const bool inbound_onion{conn_type == ConnectionType::INBOUND ? fuzzed_data_provider.ConsumeBool() : false}; + NetPermissionFlags permission_flags = ConsumeWeakEnum(fuzzed_data_provider, ALL_NET_PERMISSION_FLAGS); + if constexpr (ReturnUniquePtr) { + return std::make_unique<CNode>(node_id, + sock, + address, + keyed_net_group, + local_host_nonce, + addr_bind, + addr_name, + conn_type, + inbound_onion, + CNodeOptions{ .permission_flags = permission_flags }); + } else { + return CNode{node_id, + sock, + address, + keyed_net_group, + local_host_nonce, + addr_bind, + addr_name, + conn_type, + inbound_onion, + CNodeOptions{ .permission_flags = permission_flags }}; + } +} +inline std::unique_ptr<CNode> ConsumeNodeAsUniquePtr(FuzzedDataProvider& fdp, const std::optional<NodeId>& node_id_in = std::nullopt) { return ConsumeNode<true>(fdp, node_id_in); } + +void FillNode(FuzzedDataProvider& fuzzed_data_provider, ConnmanTestMsg& connman, CNode& node) noexcept EXCLUSIVE_LOCKS_REQUIRED(NetEventsInterface::g_msgproc_mutex); + +#endif // BITCOIN_TEST_FUZZ_UTIL_NET_H diff --git a/src/test/fuzz/validation_load_mempool.cpp b/src/test/fuzz/validation_load_mempool.cpp index d96609416b..9a90de8911 100644 --- a/src/test/fuzz/validation_load_mempool.cpp +++ b/src/test/fuzz/validation_load_mempool.cpp @@ -12,6 +12,7 @@ #include <test/fuzz/util.h> #include <test/fuzz/util/mempool.h> #include <test/util/setup_common.h> +#include <test/util/txmempool.h> #include <txmempool.h> #include <util/time.h> #include <validation.h> diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index 70dd137e22..3643b80d5f 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -429,7 +429,7 @@ BOOST_AUTO_TEST_CASE(logargs) const auto okaylog = std::make_pair("-okaylog", ArgsManager::ALLOW_ANY); const auto dontlog = std::make_pair("-dontlog", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE); SetupArgs(local_args, {okaylog_bool, okaylog_negbool, okaylog, dontlog}); - ResetArgs(local_args, "-okaylog-bool -nookaylog-negbool -okaylog=public -dontlog=private"); + ResetArgs(local_args, "-okaylog-bool -nookaylog-negbool -okaylog=public -dontlog=private42"); // Everything logged to debug.log will also append to str std::string str; @@ -447,7 +447,7 @@ BOOST_AUTO_TEST_CASE(logargs) BOOST_CHECK(str.find("Command-line arg: okaylog-negbool=false") != std::string::npos); BOOST_CHECK(str.find("Command-line arg: okaylog=\"public\"") != std::string::npos); BOOST_CHECK(str.find("dontlog=****") != std::string::npos); - BOOST_CHECK(str.find("private") == std::string::npos); + BOOST_CHECK(str.find("private42") == std::string::npos); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/i2p_tests.cpp b/src/test/i2p_tests.cpp index 9da1ee11f9..7b1bf11cfb 100644 --- a/src/test/i2p_tests.cpp +++ b/src/test/i2p_tests.cpp @@ -8,8 +8,8 @@ #include <test/util/logging.h> #include <test/util/net.h> #include <test/util/setup_common.h> -#include <threadinterrupt.h> #include <util/system.h> +#include <util/threadinterrupt.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 19f457b8dd..b12aac6299 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <policy/policy.h> +#include <test/util/txmempool.h> #include <txmempool.h> #include <util/system.h> #include <util/time.h> @@ -164,7 +165,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) tx5.vout.resize(1); tx5.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx5.vout[0].nValue = 11 * COIN; - entry.nTime = 1; + entry.time = NodeSeconds{1s}; pool.addUnchecked(entry.Fee(10000LL).FromTx(tx5)); BOOST_CHECK_EQUAL(pool.size(), 5U); @@ -224,7 +225,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) tx8.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx8.vout[0].nValue = 10 * COIN; setAncestors.insert(pool.mapTx.find(tx7.GetHash())); - pool.addUnchecked(entry.Fee(0LL).Time(2).FromTx(tx8), setAncestors); + pool.addUnchecked(entry.Fee(0LL).Time(NodeSeconds{2s}).FromTx(tx8), setAncestors); // Now tx8 should be sorted low, but tx6/tx both high sortedOrder.insert(sortedOrder.begin(), tx8.GetHash().ToString()); @@ -238,7 +239,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) tx9.vout.resize(1); tx9.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx9.vout[0].nValue = 1 * COIN; - pool.addUnchecked(entry.Fee(0LL).Time(3).FromTx(tx9), setAncestors); + pool.addUnchecked(entry.Fee(0LL).Time(NodeSeconds{3s}).FromTx(tx9), setAncestors); // tx9 should be sorted low BOOST_CHECK_EQUAL(pool.size(), 9U); @@ -261,7 +262,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) tx10.vout[0].nValue = 10 * COIN; setAncestorsCalculated.clear(); - BOOST_CHECK_EQUAL(pool.CalculateMemPoolAncestors(entry.Fee(200'000LL).Time(4).FromTx(tx10), setAncestorsCalculated, CTxMemPool::Limits::NoLimits(), dummy), true); + BOOST_CHECK_EQUAL(pool.CalculateMemPoolAncestors(entry.Fee(200'000LL).Time(NodeSeconds{4s}).FromTx(tx10), setAncestorsCalculated, CTxMemPool::Limits::NoLimits(), dummy), true); BOOST_CHECK(setAncestorsCalculated == setAncestors); pool.addUnchecked(entry.FromTx(tx10), setAncestors); diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 696a799872..75a46f9aaf 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -9,6 +9,7 @@ #include <node/miner.h> #include <policy/policy.h> #include <script/standard.h> +#include <test/util/txmempool.h> #include <timedata.h> #include <txmempool.h> #include <uint256.h> @@ -86,11 +87,11 @@ constexpr static struct { {0, 792342903}, {6, 678455063}, {6, 773251385}, {5, 186617471}, {6, 883189502}, {7, 396077336}, {8, 254702874}, {0, 455592851}}; -static CBlockIndex CreateBlockIndex(int nHeight, CBlockIndex* active_chain_tip) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +static std::unique_ptr<CBlockIndex> CreateBlockIndex(int nHeight, CBlockIndex* active_chain_tip) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { - CBlockIndex index; - index.nHeight = nHeight; - index.pprev = active_chain_tip; + auto index{std::make_unique<CBlockIndex>()}; + index->nHeight = nHeight; + index->pprev = active_chain_tip; return index; } @@ -115,19 +116,19 @@ void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const tx.vout[0].nValue = 5000000000LL - 1000; // This tx has a low fee: 1000 satoshis uint256 hashParentTx = tx.GetHash(); // save this txid for later use - tx_mempool.addUnchecked(entry.Fee(1000).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(1000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); // This tx has a medium fee: 10000 satoshis tx.vin[0].prevout.hash = txFirst[1]->GetHash(); tx.vout[0].nValue = 5000000000LL - 10000; uint256 hashMediumFeeTx = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(10000).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(10000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); // This tx has a high fee, but depends on the first transaction tx.vin[0].prevout.hash = hashParentTx; tx.vout[0].nValue = 5000000000LL - 1000 - 50000; // 50k satoshi fee uint256 hashHighFeeTx = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(50000).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(50000).Time(Now<NodeSeconds>()).SpendsCoinbase(false).FromTx(tx)); std::unique_ptr<CBlockTemplate> pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); BOOST_REQUIRE_EQUAL(pblocktemplate->block.vtx.size(), 4U); @@ -238,7 +239,7 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: hash = tx.GetHash(); bool spendsCoinbase = i == 0; // only first tx spends coinbase // If we don't set the # of sig ops in the CTxMemPoolEntry, template creation fails - tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); tx.vin[0].prevout.hash = hash; } @@ -256,7 +257,7 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: hash = tx.GetHash(); bool spendsCoinbase = i == 0; // only first tx spends coinbase // If we do set the # of sig ops in the CTxMemPoolEntry, template creation passes - tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).SigOpsCost(80).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(spendsCoinbase).SigOpsCost(80).FromTx(tx)); tx.vin[0].prevout.hash = hash; } BOOST_CHECK(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey)); @@ -280,7 +281,7 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: tx.vout[0].nValue -= LOWFEE; hash = tx.GetHash(); bool spendsCoinbase = i == 0; // only first tx spends coinbase - tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(spendsCoinbase).FromTx(tx)); tx.vin[0].prevout.hash = hash; } BOOST_CHECK(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey)); @@ -292,7 +293,7 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: // orphan in tx_mempool, template creation fails hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).FromTx(tx)); BOOST_CHECK_EXCEPTION(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-txns-inputs-missingorspent")); } @@ -305,7 +306,7 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: tx.vin[0].prevout.hash = txFirst[1]->GetHash(); tx.vout[0].nValue = BLOCKSUBSIDY - HIGHFEE; hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(HIGHFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); tx.vin[0].prevout.hash = hash; tx.vin.resize(2); tx.vin[1].scriptSig = CScript() << OP_1; @@ -313,7 +314,7 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: tx.vin[1].prevout.n = 0; tx.vout[0].nValue = tx.vout[0].nValue + BLOCKSUBSIDY - HIGHERFEE; // First txn output + fresh coinbase - new txn fee hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(HIGHERFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(HIGHERFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); BOOST_CHECK(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey)); } @@ -328,7 +329,7 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: tx.vout[0].nValue = 0; hash = tx.GetHash(); // give it a fee so it'll get mined - tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(false).FromTx(tx)); // Should throw bad-cb-multiple BOOST_CHECK_EXCEPTION(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-cb-multiple")); } @@ -343,10 +344,10 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: tx.vout[0].nValue = BLOCKSUBSIDY - HIGHFEE; tx.vout[0].scriptPubKey = CScript() << OP_1; hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(HIGHFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); tx.vout[0].scriptPubKey = CScript() << OP_2; hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(HIGHFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); BOOST_CHECK_EXCEPTION(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("bad-txns-inputs-missingorspent")); } @@ -389,12 +390,12 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: CScript script = CScript() << OP_0; tx.vout[0].scriptPubKey = GetScriptForDestination(ScriptHash(script)); hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); tx.vin[0].prevout.hash = hash; tx.vin[0].scriptSig = CScript() << std::vector<unsigned char>(script.begin(), script.end()); tx.vout[0].nValue -= LOWFEE; hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(LOWFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(false).FromTx(tx)); // Should throw block-validation-failed BOOST_CHECK_EXCEPTION(AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey), std::runtime_error, HasReason("block-validation-failed")); @@ -431,13 +432,13 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: tx.vout[0].scriptPubKey = CScript() << OP_1; tx.nLockTime = 0; hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(HIGHFEE).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(HIGHFEE).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); BOOST_CHECK(CheckFinalTxAtTip(*Assert(m_node.chainman->ActiveChain().Tip()), CTransaction{tx})); // Locktime passes BOOST_CHECK(!TestSequenceLocks(CTransaction{tx}, tx_mempool)); // Sequence locks fail { CBlockIndex* active_chain_tip = m_node.chainman->ActiveChain().Tip(); - BOOST_CHECK(SequenceLocks(CTransaction(tx), flags, prevheights, CreateBlockIndex(active_chain_tip->nHeight + 2, active_chain_tip))); // Sequence locks pass on 2nd block + BOOST_CHECK(SequenceLocks(CTransaction(tx), flags, prevheights, *CreateBlockIndex(active_chain_tip->nHeight + 2, active_chain_tip))); // Sequence locks pass on 2nd block } // relative time locked @@ -445,7 +446,7 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: tx.vin[0].nSequence = CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG | (((m_node.chainman->ActiveChain().Tip()->GetMedianTimePast()+1-m_node.chainman->ActiveChain()[1]->GetMedianTimePast()) >> CTxIn::SEQUENCE_LOCKTIME_GRANULARITY) + 1); // txFirst[1] is the 3rd block prevheights[0] = baseheight + 2; hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Time(GetTime()).FromTx(tx)); + tx_mempool.addUnchecked(entry.Time(Now<NodeSeconds>()).FromTx(tx)); BOOST_CHECK(CheckFinalTxAtTip(*Assert(m_node.chainman->ActiveChain().Tip()), CTransaction{tx})); // Locktime passes BOOST_CHECK(!TestSequenceLocks(CTransaction{tx}, tx_mempool)); // Sequence locks fail @@ -454,7 +455,7 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: m_node.chainman->ActiveChain().Tip()->GetAncestor(m_node.chainman->ActiveChain().Tip()->nHeight - i)->nTime += SEQUENCE_LOCK_TIME; // Trick the MedianTimePast { CBlockIndex* active_chain_tip = m_node.chainman->ActiveChain().Tip(); - BOOST_CHECK(SequenceLocks(CTransaction(tx), flags, prevheights, CreateBlockIndex(active_chain_tip->nHeight + 1, active_chain_tip))); + BOOST_CHECK(SequenceLocks(CTransaction(tx), flags, prevheights, *CreateBlockIndex(active_chain_tip->nHeight + 1, active_chain_tip))); } for (int i = 0; i < CBlockIndex::nMedianTimeSpan; ++i) { @@ -468,7 +469,7 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: prevheights[0] = baseheight + 3; tx.nLockTime = m_node.chainman->ActiveChain().Tip()->nHeight + 1; hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Time(GetTime()).FromTx(tx)); + tx_mempool.addUnchecked(entry.Time(Now<NodeSeconds>()).FromTx(tx)); BOOST_CHECK(!CheckFinalTxAtTip(*Assert(m_node.chainman->ActiveChain().Tip()), CTransaction{tx})); // Locktime fails BOOST_CHECK(TestSequenceLocks(CTransaction{tx}, tx_mempool)); // Sequence locks pass BOOST_CHECK(IsFinalTx(CTransaction(tx), m_node.chainman->ActiveChain().Tip()->nHeight + 2, m_node.chainman->ActiveChain().Tip()->GetMedianTimePast())); // Locktime passes on 2nd block @@ -479,7 +480,7 @@ void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std:: prevheights.resize(1); prevheights[0] = baseheight + 4; hash = tx.GetHash(); - tx_mempool.addUnchecked(entry.Time(GetTime()).FromTx(tx)); + tx_mempool.addUnchecked(entry.Time(Now<NodeSeconds>()).FromTx(tx)); BOOST_CHECK(!CheckFinalTxAtTip(*Assert(m_node.chainman->ActiveChain().Tip()), CTransaction{tx})); // Locktime fails BOOST_CHECK(TestSequenceLocks(CTransaction{tx}, tx_mempool)); // Sequence locks pass BOOST_CHECK(IsFinalTx(CTransaction(tx), m_node.chainman->ActiveChain().Tip()->nHeight + 2, m_node.chainman->ActiveChain().Tip()->GetMedianTimePast() + 1)); // Locktime passes 1 second later @@ -534,7 +535,7 @@ void MinerTestingSetup::TestPrioritisedMining(const CScript& scriptPubKey, const tx.vout.resize(1); tx.vout[0].nValue = 5000000000LL; // 0 fee uint256 hashFreePrioritisedTx = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(0).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(0).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); tx_mempool.PrioritiseTransaction(hashFreePrioritisedTx, 5 * COIN); tx.vin[0].prevout.hash = txFirst[1]->GetHash(); @@ -542,20 +543,20 @@ void MinerTestingSetup::TestPrioritisedMining(const CScript& scriptPubKey, const tx.vout[0].nValue = 5000000000LL - 1000; // This tx has a low fee: 1000 satoshis uint256 hashParentTx = tx.GetHash(); // save this txid for later use - tx_mempool.addUnchecked(entry.Fee(1000).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(1000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); // This tx has a medium fee: 10000 satoshis tx.vin[0].prevout.hash = txFirst[2]->GetHash(); tx.vout[0].nValue = 5000000000LL - 10000; uint256 hashMediumFeeTx = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(10000).Time(GetTime()).SpendsCoinbase(true).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(10000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); tx_mempool.PrioritiseTransaction(hashMediumFeeTx, -5 * COIN); // This tx also has a low fee, but is prioritised tx.vin[0].prevout.hash = hashParentTx; tx.vout[0].nValue = 5000000000LL - 1000 - 1000; // 1000 satoshi fee uint256 hashPrioritsedChild = tx.GetHash(); - tx_mempool.addUnchecked(entry.Fee(1000).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); + tx_mempool.addUnchecked(entry.Fee(1000).Time(Now<NodeSeconds>()).SpendsCoinbase(false).FromTx(tx)); tx_mempool.PrioritiseTransaction(hashPrioritsedChild, 2 * COIN); // Test that transaction selection properly updates ancestor fee calculations as prioritised diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index c2d2fa37b4..0e1e9ae211 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -84,12 +84,12 @@ BOOST_AUTO_TEST_CASE(netbase_properties) } -bool static TestSplitHost(const std::string& test, const std::string& host, uint16_t port) +bool static TestSplitHost(const std::string& test, const std::string& host, uint16_t port, bool validPort=true) { std::string hostOut; uint16_t portOut{0}; - SplitHostPort(test, portOut, hostOut); - return hostOut == host && port == portOut; + bool validPortOut = SplitHostPort(test, portOut, hostOut); + return hostOut == host && portOut == port && validPortOut == validPort; } BOOST_AUTO_TEST_CASE(netbase_splithost) @@ -109,6 +109,23 @@ BOOST_AUTO_TEST_CASE(netbase_splithost) BOOST_CHECK(TestSplitHost(":8333", "", 8333)); BOOST_CHECK(TestSplitHost("[]:8333", "", 8333)); BOOST_CHECK(TestSplitHost("", "", 0)); + BOOST_CHECK(TestSplitHost(":65535", "", 65535)); + BOOST_CHECK(TestSplitHost(":65536", ":65536", 0, false)); + BOOST_CHECK(TestSplitHost(":-1", ":-1", 0, false)); + BOOST_CHECK(TestSplitHost("[]:70001", "[]:70001", 0, false)); + BOOST_CHECK(TestSplitHost("[]:-1", "[]:-1", 0, false)); + BOOST_CHECK(TestSplitHost("[]:-0", "[]:-0", 0, false)); + BOOST_CHECK(TestSplitHost("[]:0", "", 0, false)); + BOOST_CHECK(TestSplitHost("[]:1/2", "[]:1/2", 0, false)); + BOOST_CHECK(TestSplitHost("[]:1E2", "[]:1E2", 0, false)); + BOOST_CHECK(TestSplitHost("127.0.0.1:65536", "127.0.0.1:65536", 0, false)); + BOOST_CHECK(TestSplitHost("127.0.0.1:0", "127.0.0.1", 0, false)); + BOOST_CHECK(TestSplitHost("127.0.0.1:", "127.0.0.1:", 0, false)); + BOOST_CHECK(TestSplitHost("127.0.0.1:1/2", "127.0.0.1:1/2", 0, false)); + BOOST_CHECK(TestSplitHost("127.0.0.1:1E2", "127.0.0.1:1E2", 0, false)); + BOOST_CHECK(TestSplitHost("www.bitcoincore.org:65536", "www.bitcoincore.org:65536", 0, false)); + BOOST_CHECK(TestSplitHost("www.bitcoincore.org:0", "www.bitcoincore.org", 0, false)); + BOOST_CHECK(TestSplitHost("www.bitcoincore.org:", "www.bitcoincore.org:", 0, false)); } bool static TestParse(std::string src, std::string canon) diff --git a/src/test/orphanage_tests.cpp b/src/test/orphanage_tests.cpp index 842daa8bd4..a55b0bbcd0 100644 --- a/src/test/orphanage_tests.cpp +++ b/src/test/orphanage_tests.cpp @@ -20,13 +20,15 @@ BOOST_FIXTURE_TEST_SUITE(orphanage_tests, TestingSetup) class TxOrphanageTest : public TxOrphanage { public: - inline size_t CountOrphans() const EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans) + inline size_t CountOrphans() const EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) { + LOCK(m_mutex); return m_orphans.size(); } - CTransactionRef RandomOrphan() EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans) + CTransactionRef RandomOrphan() EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) { + LOCK(m_mutex); std::map<uint256, OrphanTx>::iterator it; it = m_orphans.lower_bound(InsecureRand256()); if (it == m_orphans.end()) @@ -59,8 +61,6 @@ BOOST_AUTO_TEST_CASE(DoS_mapOrphans) FillableSigningProvider keystore; BOOST_CHECK(keystore.AddKey(key)); - LOCK(g_cs_orphans); - // 50 orphan transactions: for (int i = 0; i < 50; i++) { diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index 3f66a8fc46..76852d66f7 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -4,6 +4,7 @@ #include <policy/fees.h> #include <policy/policy.h> +#include <test/util/txmempool.h> #include <txmempool.h> #include <uint256.h> #include <util/time.h> @@ -58,7 +59,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) for (int k = 0; k < 4; k++) { // add 4 fee txs tx.vin[0].prevout.n = 10000*blocknum+100*j+k; // make transaction unique uint256 hash = tx.GetHash(); - mpool.addUnchecked(entry.Fee(feeV[j]).Time(GetTime()).Height(blocknum).FromTx(tx)); + mpool.addUnchecked(entry.Fee(feeV[j]).Time(Now<NodeSeconds>()).Height(blocknum).FromTx(tx)); txHashes[j].push_back(hash); } } @@ -129,7 +130,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) for (int k = 0; k < 4; k++) { // add 4 fee txs tx.vin[0].prevout.n = 10000*blocknum+100*j+k; uint256 hash = tx.GetHash(); - mpool.addUnchecked(entry.Fee(feeV[j]).Time(GetTime()).Height(blocknum).FromTx(tx)); + mpool.addUnchecked(entry.Fee(feeV[j]).Time(Now<NodeSeconds>()).Height(blocknum).FromTx(tx)); txHashes[j].push_back(hash); } } @@ -164,7 +165,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) for (int k = 0; k < 4; k++) { // add 4 fee txs tx.vin[0].prevout.n = 10000*blocknum+100*j+k; uint256 hash = tx.GetHash(); - mpool.addUnchecked(entry.Fee(feeV[j]).Time(GetTime()).Height(blocknum).FromTx(tx)); + mpool.addUnchecked(entry.Fee(feeV[j]).Time(Now<NodeSeconds>()).Height(blocknum).FromTx(tx)); CTransactionRef ptx = mpool.get(hash); if (ptx) block.push_back(ptx); diff --git a/src/test/rbf_tests.cpp b/src/test/rbf_tests.cpp index c88cd36688..d362c85560 100644 --- a/src/test/rbf_tests.cpp +++ b/src/test/rbf_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <policy/rbf.h> #include <random.h> +#include <test/util/txmempool.h> #include <txmempool.h> #include <util/system.h> #include <util/time.h> diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index a52530e179..f9b8a47330 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -17,12 +17,49 @@ #include <boost/test/unit_test.hpp> +static UniValue JSON(std::string_view json) +{ + UniValue value; + BOOST_CHECK(value.read(json.data(), json.size())); + return value; +} + +class HasJSON +{ +public: + explicit HasJSON(std::string json) : m_json(std::move(json)) {} + bool operator()(const UniValue& value) const + { + std::string json{value.write()}; + BOOST_CHECK_EQUAL(json, m_json); + return json == m_json; + }; + +private: + const std::string m_json; +}; + class RPCTestingSetup : public TestingSetup { public: + UniValue TransformParams(const UniValue& params, std::vector<std::string> arg_names) const; UniValue CallRPC(std::string args); }; +UniValue RPCTestingSetup::TransformParams(const UniValue& params, std::vector<std::string> arg_names) const +{ + UniValue transformed_params; + CRPCTable table; + CRPCCommand command{"category", "method", [&](const JSONRPCRequest& request, UniValue&, bool) -> bool { transformed_params = request.params; return true; }, arg_names, /*unique_id=*/0}; + table.appendCommand("method", &command); + JSONRPCRequest request; + request.strMethod = "method"; + request.params = params; + if (RPCIsInWarmup(nullptr)) SetRPCWarmupFinished(); + table.execute(request); + return transformed_params; +} + UniValue RPCTestingSetup::CallRPC(std::string args) { std::vector<std::string> vArgs{SplitString(args, ' ')}; @@ -45,6 +82,33 @@ UniValue RPCTestingSetup::CallRPC(std::string args) BOOST_FIXTURE_TEST_SUITE(rpc_tests, RPCTestingSetup) +BOOST_AUTO_TEST_CASE(rpc_namedparams) +{ + const std::vector<std::string> arg_names{"arg1", "arg2", "arg3", "arg4", "arg5"}; + + // Make sure named arguments are transformed into positional arguments in correct places separated by nulls + BOOST_CHECK_EQUAL(TransformParams(JSON(R"({"arg2": 2, "arg4": 4})"), arg_names).write(), "[null,2,null,4]"); + + // Make sure named argument specified multiple times raises an exception + BOOST_CHECK_EXCEPTION(TransformParams(JSON(R"({"arg2": 2, "arg2": 4})"), arg_names), UniValue, + HasJSON(R"({"code":-8,"message":"Parameter arg2 specified multiple times"})")); + + // Make sure named and positional arguments can be combined. + BOOST_CHECK_EQUAL(TransformParams(JSON(R"({"arg5": 5, "args": [1, 2], "arg4": 4})"), arg_names).write(), "[1,2,null,4,5]"); + + // Make sure a unknown named argument raises an exception + BOOST_CHECK_EXCEPTION(TransformParams(JSON(R"({"arg2": 2, "unknown": 6})"), arg_names), UniValue, + HasJSON(R"({"code":-8,"message":"Unknown named parameter unknown"})")); + + // Make sure an overlap between a named argument and positional argument raises an exception + BOOST_CHECK_EXCEPTION(TransformParams(JSON(R"({"args": [1,2,3], "arg4": 4, "arg2": 2})"), arg_names), UniValue, + HasJSON(R"({"code":-8,"message":"Parameter arg2 specified twice both as positional and named argument"})")); + + // Make sure extra positional arguments can be passed through to the method implementation, as long as they don't overlap with named arguments. + BOOST_CHECK_EQUAL(TransformParams(JSON(R"({"args": [1,2,3,4,5,6,7,8,9,10]})"), arg_names).write(), "[1,2,3,4,5,6,7,8,9,10]"); + BOOST_CHECK_EQUAL(TransformParams(JSON(R"([1,2,3,4,5,6,7,8,9,10])"), arg_names).write(), "[1,2,3,4,5,6,7,8,9,10]"); +} + BOOST_AUTO_TEST_CASE(rpc_rawparams) { // Test raw transaction API argument handling diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index 55486db6b2..d78f62972f 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <coins.h> #include <consensus/consensus.h> #include <consensus/tx_verify.h> #include <key.h> diff --git a/src/test/sock_tests.cpp b/src/test/sock_tests.cpp index 8376ec1a68..5bea08c254 100644 --- a/src/test/sock_tests.cpp +++ b/src/test/sock_tests.cpp @@ -4,9 +4,9 @@ #include <compat/compat.h> #include <test/util/setup_common.h> -#include <threadinterrupt.h> #include <util/sock.h> #include <util/system.h> +#include <util/threadinterrupt.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index 0925e2e9ee..b1b262eade 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -253,7 +253,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) BOOST_CHECK(false); } catch (const std::exception& e) { BOOST_CHECK(strstr(e.what(), - "Read attempted past buffer limit") != nullptr); + "Attempt to position past buffer limit") != nullptr); } // The default argument removes the limit completely. BOOST_CHECK(bf.SetLimit()); @@ -322,7 +322,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) BOOST_CHECK(!bf.SetPos(0)); // But we should now be positioned at least as far back as allowed // by the rewind window (relative to our farthest read position, 40). - BOOST_CHECK(bf.GetPos() <= 30); + BOOST_CHECK(bf.GetPos() <= 30U); // We can explicitly close the file, or the destructor will do it. bf.fclose(); @@ -330,6 +330,55 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) fs::remove(streams_test_filename); } +BOOST_AUTO_TEST_CASE(streams_buffered_file_skip) +{ + fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp"; + FILE* file = fsbridge::fopen(streams_test_filename, "w+b"); + // The value at each offset is the byte offset (e.g. byte 1 in the file has the value 0x01). + for (uint8_t j = 0; j < 40; ++j) { + fwrite(&j, 1, 1, file); + } + rewind(file); + + // The buffer is 25 bytes, allow rewinding 10 bytes. + CBufferedFile bf(file, 25, 10, 222, 333); + + uint8_t i; + // This is like bf >> (7-byte-variable), in that it will cause data + // to be read from the file into memory, but it's not copied to us. + bf.SkipTo(7); + BOOST_CHECK_EQUAL(bf.GetPos(), 7U); + bf >> i; + BOOST_CHECK_EQUAL(i, 7); + + // The bytes in the buffer up to offset 7 are valid and can be read. + BOOST_CHECK(bf.SetPos(0)); + bf >> i; + BOOST_CHECK_EQUAL(i, 0); + bf >> i; + BOOST_CHECK_EQUAL(i, 1); + + bf.SkipTo(11); + bf >> i; + BOOST_CHECK_EQUAL(i, 11); + + // SkipTo() honors the transfer limit; we can't position beyond the limit. + bf.SetLimit(13); + try { + bf.SkipTo(14); + BOOST_CHECK(false); + } catch (const std::exception& e) { + BOOST_CHECK(strstr(e.what(), "Attempt to position past buffer limit") != nullptr); + } + + // We can position exactly to the transfer limit. + bf.SkipTo(13); + BOOST_CHECK_EQUAL(bf.GetPos(), 13U); + + bf.fclose(); + fs::remove(streams_test_filename); +} + BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) { // Make this test deterministic. @@ -361,7 +410,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) // sizes; the boundaries of the objects can interact arbitrarily // with the CBufferFile's internal buffer. These first three // cases simulate objects of various sizes (1, 2, 5 bytes). - switch (InsecureRandRange(5)) { + switch (InsecureRandRange(6)) { case 0: { uint8_t a[1]; if (currentPos + 1 > fileSize) @@ -399,6 +448,16 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) break; } case 3: { + // SkipTo is similar to the "read" cases above, except + // we don't receive the data. + size_t skip_length{static_cast<size_t>(InsecureRandRange(5))}; + if (currentPos + skip_length > fileSize) continue; + bf.SetLimit(currentPos + skip_length); + bf.SkipTo(currentPos + skip_length); + currentPos += skip_length; + break; + } + case 4: { // Find a byte value (that is at or ahead of the current position). size_t find = currentPos + InsecureRandRange(8); if (find >= fileSize) @@ -415,7 +474,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) currentPos++; break; } - case 4: { + case 5: { size_t requestPos = InsecureRandRange(maxPos + 4); bool okay = bf.SetPos(requestPos); // The new position may differ from the requested position diff --git a/src/test/sync_tests.cpp b/src/test/sync_tests.cpp index 55c2c5108d..e1270a362b 100644 --- a/src/test/sync_tests.cpp +++ b/src/test/sync_tests.cpp @@ -107,12 +107,12 @@ BOOST_AUTO_TEST_CASE(potential_deadlock_detected) #ifdef DEBUG_LOCKORDER BOOST_AUTO_TEST_CASE(double_lock_mutex) { - TestDoubleLock<Mutex>(true /* should throw */); + TestDoubleLock<Mutex>(/*should_throw=*/true); } BOOST_AUTO_TEST_CASE(double_lock_recursive_mutex) { - TestDoubleLock<RecursiveMutex>(false /* should not throw */); + TestDoubleLock<RecursiveMutex>(/*should_throw=*/false); } #endif /* DEBUG_LOCKORDER */ diff --git a/src/test/system_tests.cpp b/src/test/system_tests.cpp index 11f4be7fef..472b58b4d5 100644 --- a/src/test/system_tests.cpp +++ b/src/test/system_tests.cpp @@ -47,19 +47,13 @@ BOOST_AUTO_TEST_CASE(run_command) BOOST_CHECK(result.isObject()); const UniValue& success = find_value(result, "success"); BOOST_CHECK(!success.isNull()); - BOOST_CHECK_EQUAL(success.getBool(), true); + BOOST_CHECK_EQUAL(success.get_bool(), true); } { // An invalid command is handled by Boost -#ifdef WIN32 - const std::string expected{"The system cannot find the file specified."}; -#else - const std::string expected{"No such file or directory"}; -#endif BOOST_CHECK_EXCEPTION(RunCommandParseJSON("invalid_command"), boost::process::process_error, [&](const boost::process::process_error& e) { - const std::string what(e.what()); - BOOST_CHECK(what.find("RunCommandParseJSON error:") == std::string::npos); - BOOST_CHECK(what.find(expected) != std::string::npos); + BOOST_CHECK(std::string(e.what()).find("RunCommandParseJSON error:") == std::string::npos); + BOOST_CHECK_EQUAL(e.code().value(), 2); return true; }); } @@ -101,7 +95,7 @@ BOOST_AUTO_TEST_CASE(run_command) BOOST_CHECK(result.isObject()); const UniValue& success = find_value(result, "success"); BOOST_CHECK(!success.isNull()); - BOOST_CHECK_EQUAL(success.getBool(), true); + BOOST_CHECK_EQUAL(success.get_bool(), true); } #endif } diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index 62c7ddb673..643d9221fe 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -69,11 +69,16 @@ BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) } } + // It is not safe to stop and destroy the index until it finishes handling + // the last BlockConnected notification. The BlockUntilSyncedToCurrentChain() + // call above is sufficient to ensure this, but the + // SyncWithValidationInterfaceQueue() call below is also needed to ensure + // TSAN always sees the test thread waiting for the notification thread, and + // avoid potential false positive reports. + SyncWithValidationInterfaceQueue(); + // shutdown sequence (c.f. Shutdown() in init.cpp) txindex.Stop(); - - // Let scheduler events finish running to avoid accessing any memory related to txindex after it is destructed - SyncWithValidationInterfaceQueue(); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/txreconciliation_tests.cpp b/src/test/txreconciliation_tests.cpp new file mode 100644 index 0000000000..b018629e76 --- /dev/null +++ b/src/test/txreconciliation_tests.cpp @@ -0,0 +1,84 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/txreconciliation.h> + +#include <test/util/setup_common.h> + +#include <boost/test/unit_test.hpp> + +BOOST_FIXTURE_TEST_SUITE(txreconciliation_tests, BasicTestingSetup) + +BOOST_AUTO_TEST_CASE(RegisterPeerTest) +{ + TxReconciliationTracker tracker(TXRECONCILIATION_VERSION); + const uint64_t salt = 0; + + // Prepare a peer for reconciliation. + tracker.PreRegisterPeer(0); + + // Invalid version. + BOOST_CHECK_EQUAL(tracker.RegisterPeer(/*peer_id=*/0, /*is_peer_inbound=*/true, + /*peer_recon_version=*/0, salt), + ReconciliationRegisterResult::PROTOCOL_VIOLATION); + + // Valid registration (inbound and outbound peers). + BOOST_REQUIRE(!tracker.IsPeerRegistered(0)); + BOOST_REQUIRE_EQUAL(tracker.RegisterPeer(0, true, 1, salt), ReconciliationRegisterResult::SUCCESS); + BOOST_CHECK(tracker.IsPeerRegistered(0)); + BOOST_REQUIRE(!tracker.IsPeerRegistered(1)); + tracker.PreRegisterPeer(1); + BOOST_REQUIRE(tracker.RegisterPeer(1, false, 1, salt) == ReconciliationRegisterResult::SUCCESS); + BOOST_CHECK(tracker.IsPeerRegistered(1)); + + // Reconciliation version is higher than ours, should be able to register. + BOOST_REQUIRE(!tracker.IsPeerRegistered(2)); + tracker.PreRegisterPeer(2); + BOOST_REQUIRE(tracker.RegisterPeer(2, true, 2, salt) == ReconciliationRegisterResult::SUCCESS); + BOOST_CHECK(tracker.IsPeerRegistered(2)); + + // Try registering for the second time. + BOOST_REQUIRE(tracker.RegisterPeer(1, false, 1, salt) == ReconciliationRegisterResult::ALREADY_REGISTERED); + + // Do not register if there were no pre-registration for the peer. + BOOST_REQUIRE_EQUAL(tracker.RegisterPeer(100, true, 1, salt), ReconciliationRegisterResult::NOT_FOUND); + BOOST_CHECK(!tracker.IsPeerRegistered(100)); +} + +BOOST_AUTO_TEST_CASE(ForgetPeerTest) +{ + TxReconciliationTracker tracker(TXRECONCILIATION_VERSION); + NodeId peer_id0 = 0; + + // Removing peer after pre-registring works and does not let to register the peer. + tracker.PreRegisterPeer(peer_id0); + tracker.ForgetPeer(peer_id0); + BOOST_CHECK_EQUAL(tracker.RegisterPeer(peer_id0, true, 1, 1), ReconciliationRegisterResult::NOT_FOUND); + + // Removing peer after it is registered works. + tracker.PreRegisterPeer(peer_id0); + BOOST_REQUIRE(!tracker.IsPeerRegistered(peer_id0)); + BOOST_REQUIRE_EQUAL(tracker.RegisterPeer(peer_id0, true, 1, 1), ReconciliationRegisterResult::SUCCESS); + BOOST_CHECK(tracker.IsPeerRegistered(peer_id0)); + tracker.ForgetPeer(peer_id0); + BOOST_CHECK(!tracker.IsPeerRegistered(peer_id0)); +} + +BOOST_AUTO_TEST_CASE(IsPeerRegisteredTest) +{ + TxReconciliationTracker tracker(TXRECONCILIATION_VERSION); + NodeId peer_id0 = 0; + + BOOST_REQUIRE(!tracker.IsPeerRegistered(peer_id0)); + tracker.PreRegisterPeer(peer_id0); + BOOST_REQUIRE(!tracker.IsPeerRegistered(peer_id0)); + + BOOST_REQUIRE_EQUAL(tracker.RegisterPeer(peer_id0, true, 1, 1), ReconciliationRegisterResult::SUCCESS); + BOOST_CHECK(tracker.IsPeerRegistered(peer_id0)); + + tracker.ForgetPeer(peer_id0); + BOOST_CHECK(!tracker.IsPeerRegistered(peer_id0)); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h index 2f0021b114..0ca63810f3 100644 --- a/src/test/util/chainstate.h +++ b/src/test/util/chainstate.h @@ -11,6 +11,7 @@ #include <node/context.h> #include <node/utxo_snapshot.h> #include <rpc/blockchain.h> +#include <test/util/setup_common.h> #include <validation.h> #include <univalue.h> @@ -20,11 +21,24 @@ const auto NoMalleation = [](AutoFile& file, node::SnapshotMetadata& meta){}; /** * Create and activate a UTXO snapshot, optionally providing a function to * malleate the snapshot. + * + * If `reset_chainstate` is true, reset the original chainstate back to the genesis + * block. This allows us to simulate more realistic conditions in which a snapshot is + * loaded into an otherwise mostly-uninitialized datadir. It also allows us to test + * conditions that would otherwise cause shutdowns based on the IBD chainstate going + * past the snapshot it generated. */ template<typename F = decltype(NoMalleation)> static bool -CreateAndActivateUTXOSnapshot(node::NodeContext& node, const fs::path root, F malleation = NoMalleation) +CreateAndActivateUTXOSnapshot( + TestingSetup* fixture, + F malleation = NoMalleation, + bool reset_chainstate = false, + bool in_memory_chainstate = false) { + node::NodeContext& node = fixture->m_node; + fs::path root = fixture->m_path_root; + // Write out a snapshot to the test's tempdir. // int height; @@ -47,7 +61,38 @@ CreateAndActivateUTXOSnapshot(node::NodeContext& node, const fs::path root, F ma malleation(auto_infile, metadata); - return node.chainman->ActivateSnapshot(auto_infile, metadata, /*in_memory=*/true); + if (reset_chainstate) { + { + // What follows is code to selectively reset chainstate data without + // disturbing the existing BlockManager instance, which is needed to + // recognize the headers chain previously generated by the chainstate we're + // removing. Without those headers, we can't activate the snapshot below. + // + // This is a stripped-down version of node::LoadChainstate which + // preserves the block index. + LOCK(::cs_main); + uint256 gen_hash = node.chainman->ActiveChainstate().m_chain[0]->GetBlockHash(); + node.chainman->ResetChainstates(); + node.chainman->InitializeChainstate(node.mempool.get()); + Chainstate& chain = node.chainman->ActiveChainstate(); + Assert(chain.LoadGenesisBlock()); + // These cache values will be corrected shortly in `MaybeRebalanceCaches`. + chain.InitCoinsDB(1 << 20, true, false, ""); + chain.InitCoinsCache(1 << 20); + chain.CoinsTip().SetBestBlock(gen_hash); + chain.setBlockIndexCandidates.insert(node.chainman->m_blockman.LookupBlockIndex(gen_hash)); + chain.LoadChainTip(); + node.chainman->MaybeRebalanceCaches(); + } + BlockValidationState state; + if (!node.chainman->ActiveChainstate().ActivateBestChain(state)) { + throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString())); + } + Assert( + 0 == WITH_LOCK(node.chainman->GetMutex(), return node.chainman->ActiveHeight())); + } + + return node.chainman->ActivateSnapshot(auto_infile, metadata, in_memory_chainstate); } diff --git a/src/test/util/net.h b/src/test/util/net.h index 73543de4ca..9ae7981980 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -166,6 +166,10 @@ public: return 0; } + bool SetNonBlocking() const override { return true; } + + bool IsSelectable() const override { return true; } + bool Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred = nullptr) const override diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 74b055ee45..c97f400137 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -9,6 +9,7 @@ #include <addrman.h> #include <banman.h> #include <chainparams.h> +#include <common/url.h> #include <consensus/consensus.h> #include <consensus/params.h> #include <consensus/validation.h> @@ -16,6 +17,7 @@ #include <init.h> #include <init/common.h> #include <interfaces/chain.h> +#include <kernel/mempool_entry.h> #include <net.h> #include <net_processing.h> #include <node/blockstorage.h> @@ -36,6 +38,7 @@ #include <shutdown.h> #include <streams.h> #include <test/util/net.h> +#include <test/util/txmempool.h> #include <timedata.h> #include <txdb.h> #include <txmempool.h> @@ -45,7 +48,6 @@ #include <util/threadnames.h> #include <util/time.h> #include <util/translation.h> -#include <util/url.h> #include <util/vector.h> #include <validation.h> #include <validationinterface.h> @@ -60,7 +62,6 @@ using node::ApplyArgsManOptions; using node::BlockAssembler; using node::CalculateCacheSizes; using node::LoadChainstate; -using node::NodeContext; using node::RegenerateCommitments; using node::VerifyLoadedChainstate; @@ -146,7 +147,6 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve Assert(InitScriptExecutionCache(validation_cache_sizes.script_execution_cache_bytes)); m_node.chain = interfaces::MakeChain(m_node); - fCheckBlockIndex = true; static bool noui_connected = false; if (!noui_connected) { noui_connect(); @@ -162,19 +162,6 @@ BasicTestingSetup::~BasicTestingSetup() gArgs.ClearArgs(); } -CTxMemPool::Options MemPoolOptionsForTest(const NodeContext& node) -{ - CTxMemPool::Options mempool_opts{ - .estimator = node.fee_estimator.get(), - // Default to always checking mempool regardless of - // chainparams.DefaultConsistencyChecks for tests - .check_ratio = 1, - }; - const auto err{ApplyArgsManOptions(*node.args, ::Params(), mempool_opts)}; - Assert(!err); - return mempool_opts; -} - ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args) : BasicTestingSetup(chainName, extra_args) { @@ -194,14 +181,13 @@ ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::ve const ChainstateManager::Options chainman_opts{ .chainparams = chainparams, .adjusted_time_callback = GetAdjustedTime, + .check_block_index = true, }; m_node.chainman = std::make_unique<ChainstateManager>(chainman_opts); m_node.chainman->m_blockman.m_block_tree_db = std::make_unique<CBlockTreeDB>(m_cache_sizes.block_tree_db, true); - // Start script-checking threads. Set g_parallel_script_checks to true so they are used. constexpr int script_check_threads = 2; StartScriptCheckWorkerThreads(script_check_threads); - g_parallel_script_checks = true; } ChainTestingSetup::~ChainTestingSetup() @@ -220,17 +206,12 @@ ChainTestingSetup::~ChainTestingSetup() m_node.chainman.reset(); } -TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args) - : ChainTestingSetup(chainName, extra_args) +void TestingSetup::LoadVerifyActivateChainstate() { - // Ideally we'd move all the RPC tests to the functional testing framework - // instead of unit tests, but for now we need these here. - RegisterAllCoreRPCCommands(tableRPC); - node::ChainstateLoadOptions options; options.mempool = Assert(m_node.mempool.get()); - options.block_tree_db_in_memory = true; - options.coins_db_in_memory = true; + options.block_tree_db_in_memory = m_block_tree_db_in_memory; + options.coins_db_in_memory = m_coins_db_in_memory; options.reindex = node::fReindex; options.reindex_chainstate = m_args.GetBoolArg("-reindex-chainstate", false); options.prune = node::fPruneMode; @@ -246,6 +227,22 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const if (!m_node.chainman->ActiveChainstate().ActivateBestChain(state)) { throw std::runtime_error(strprintf("ActivateBestChain failed. (%s)", state.ToString())); } +} + +TestingSetup::TestingSetup( + const std::string& chainName, + const std::vector<const char*>& extra_args, + const bool coins_db_in_memory, + const bool block_tree_db_in_memory) + : ChainTestingSetup(chainName, extra_args), + m_coins_db_in_memory(coins_db_in_memory), + m_block_tree_db_in_memory(block_tree_db_in_memory) +{ + // Ideally we'd move all the RPC tests to the functional testing framework + // instead of unit tests, but for now we need these here. + RegisterAllCoreRPCCommands(tableRPC); + + LoadVerifyActivateChainstate(); m_node.netgroupman = std::make_unique<NetGroupManager>(/*asmap=*/std::vector<bool>()); m_node.addrman = std::make_unique<AddrMan>(*m_node.netgroupman, @@ -263,8 +260,12 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const } } -TestChain100Setup::TestChain100Setup(const std::string& chain_name, const std::vector<const char*>& extra_args) - : TestingSetup{chain_name, extra_args} +TestChain100Setup::TestChain100Setup( + const std::string& chain_name, + const std::vector<const char*>& extra_args, + const bool coins_db_in_memory, + const bool block_tree_db_in_memory) + : TestingSetup{CBaseChainParams::REGTEST, extra_args, coins_db_in_memory, block_tree_db_in_memory} { SetMockTime(1598887952); constexpr std::array<unsigned char, 32> vchKey = { @@ -423,17 +424,6 @@ std::vector<CTransactionRef> TestChain100Setup::PopulateMempool(FastRandomContex return mempool_transactions; } -CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction& tx) const -{ - return FromTx(MakeTransactionRef(tx)); -} - -CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CTransactionRef& tx) const -{ - return CTxMemPoolEntry(tx, nFee, nTime, nHeight, - spendsCoinbase, sigOpCost, lp); -} - /** * @returns a real block (0000000000013b8ab2cd513b0261a14096412195a72a0c4827d229dcc7e0f7af) * with 9 txs. diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 136ee1fd62..dfa36039a2 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -8,21 +8,23 @@ #include <chainparamsbase.h> #include <fs.h> #include <key.h> -#include <util/system.h> #include <node/caches.h> #include <node/context.h> +#include <primitives/transaction.h> #include <pubkey.h> #include <random.h> #include <stdexcept> -#include <txmempool.h> #include <util/check.h> #include <util/string.h> +#include <util/system.h> #include <util/vector.h> #include <functional> #include <type_traits> #include <vector> +class Chainstate; + /** This is connected to the logger. Can be used to redirect logs to any other log */ extern const std::function<void(const std::string&)> G_TEST_LOG_FUN; @@ -90,9 +92,6 @@ struct BasicTestingSetup { ArgsManager m_args; }; - -CTxMemPool::Options MemPoolOptionsForTest(const node::NodeContext& node); - /** Testing setup that performs all steps up until right before * ChainstateManager gets initialized. Meant for testing ChainstateManager * initialization behaviour. @@ -107,7 +106,16 @@ struct ChainTestingSetup : public BasicTestingSetup { /** Testing setup that configures a complete environment. */ struct TestingSetup : public ChainTestingSetup { - explicit TestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); + bool m_coins_db_in_memory{true}; + bool m_block_tree_db_in_memory{true}; + + void LoadVerifyActivateChainstate(); + + explicit TestingSetup( + const std::string& chainName = CBaseChainParams::MAIN, + const std::vector<const char*>& extra_args = {}, + const bool coins_db_in_memory = true, + const bool block_tree_db_in_memory = true); }; /** Identical to TestingSetup, but chain set to regtest */ @@ -124,8 +132,11 @@ class CScript; * Testing fixture that pre-creates a 100-block REGTEST-mode block chain */ struct TestChain100Setup : public TestingSetup { - TestChain100Setup(const std::string& chain_name = CBaseChainParams::REGTEST, - const std::vector<const char*>& extra_args = {}); + TestChain100Setup( + const std::string& chain_name = CBaseChainParams::REGTEST, + const std::vector<const char*>& extra_args = {}, + const bool coins_db_in_memory = true, + const bool block_tree_db_in_memory = true); /** * Create a new block with just given transactions, coinbase paying to @@ -201,33 +212,6 @@ std::unique_ptr<T> MakeNoLogFileContext(const std::string& chain_name = CBaseCha return std::make_unique<T>(chain_name, arguments); } -class CTxMemPoolEntry; - -struct TestMemPoolEntryHelper -{ - // Default values - CAmount nFee; - int64_t nTime; - unsigned int nHeight; - bool spendsCoinbase; - unsigned int sigOpCost; - LockPoints lp; - - TestMemPoolEntryHelper() : - nFee(0), nTime(0), nHeight(1), - spendsCoinbase(false), sigOpCost(4) { } - - CTxMemPoolEntry FromTx(const CMutableTransaction& tx) const; - CTxMemPoolEntry FromTx(const CTransactionRef& tx) const; - - // Change the default value - TestMemPoolEntryHelper &Fee(CAmount _fee) { nFee = _fee; return *this; } - TestMemPoolEntryHelper &Time(int64_t _time) { nTime = _time; return *this; } - TestMemPoolEntryHelper &Height(unsigned int _height) { nHeight = _height; return *this; } - TestMemPoolEntryHelper &SpendsCoinbase(bool _flag) { spendsCoinbase = _flag; return *this; } - TestMemPoolEntryHelper &SigOpsCost(unsigned int _sigopsCost) { sigOpCost = _sigopsCost; return *this; } -}; - CBlock getBlock13b8a(); // define an implicit conversion here so that uint256 may be used directly in BOOST_CHECK_* diff --git a/src/test/util/txmempool.cpp b/src/test/util/txmempool.cpp new file mode 100644 index 0000000000..1873cf5ec8 --- /dev/null +++ b/src/test/util/txmempool.cpp @@ -0,0 +1,38 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/util/txmempool.h> + +#include <chainparams.h> +#include <node/context.h> +#include <node/mempool_args.h> +#include <txmempool.h> +#include <util/check.h> +#include <util/time.h> +#include <util/translation.h> + +using node::NodeContext; + +CTxMemPool::Options MemPoolOptionsForTest(const NodeContext& node) +{ + CTxMemPool::Options mempool_opts{ + .estimator = node.fee_estimator.get(), + // Default to always checking mempool regardless of + // chainparams.DefaultConsistencyChecks for tests + .check_ratio = 1, + }; + const auto err{ApplyArgsManOptions(*node.args, ::Params(), mempool_opts)}; + Assert(!err); + return mempool_opts; +} + +CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction& tx) const +{ + return FromTx(MakeTransactionRef(tx)); +} + +CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CTransactionRef& tx) const +{ + return CTxMemPoolEntry{tx, nFee, TicksSinceEpoch<std::chrono::seconds>(time), nHeight, spendsCoinbase, sigOpCost, lp}; +} diff --git a/src/test/util/txmempool.h b/src/test/util/txmempool.h new file mode 100644 index 0000000000..2fe7d69693 --- /dev/null +++ b/src/test/util/txmempool.h @@ -0,0 +1,37 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_UTIL_TXMEMPOOL_H +#define BITCOIN_TEST_UTIL_TXMEMPOOL_H + +#include <txmempool.h> +#include <util/time.h> + +namespace node { +struct NodeContext; +} + +CTxMemPool::Options MemPoolOptionsForTest(const node::NodeContext& node); + +struct TestMemPoolEntryHelper { + // Default values + CAmount nFee{0}; + NodeSeconds time{}; + unsigned int nHeight{1}; + bool spendsCoinbase{false}; + unsigned int sigOpCost{4}; + LockPoints lp; + + CTxMemPoolEntry FromTx(const CMutableTransaction& tx) const; + CTxMemPoolEntry FromTx(const CTransactionRef& tx) const; + + // Change the default value + TestMemPoolEntryHelper& Fee(CAmount _fee) { nFee = _fee; return *this; } + TestMemPoolEntryHelper& Time(NodeSeconds tp) { time = tp; return *this; } + TestMemPoolEntryHelper& Height(unsigned int _height) { nHeight = _height; return *this; } + TestMemPoolEntryHelper& SpendsCoinbase(bool _flag) { spendsCoinbase = _flag; return *this; } + TestMemPoolEntryHelper& SigOpsCost(unsigned int _sigopsCost) { sigOpCost = _sigopsCost; return *this; } +}; + +#endif // BITCOIN_TEST_UTIL_TXMEMPOOL_H diff --git a/src/test/util/wallet.cpp b/src/test/util/wallet.cpp deleted file mode 100644 index b54774cbb9..0000000000 --- a/src/test/util/wallet.cpp +++ /dev/null @@ -1,27 +0,0 @@ -// Copyright (c) 2019-2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <test/util/wallet.h> - -#include <key_io.h> -#include <outputtype.h> -#include <script/standard.h> -#ifdef ENABLE_WALLET -#include <util/check.h> -#include <util/translation.h> -#include <wallet/wallet.h> -#endif - -using wallet::CWallet; - -const std::string ADDRESS_BCRT1_UNSPENDABLE = "bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj"; - -#ifdef ENABLE_WALLET -std::string getnewaddress(CWallet& w) -{ - constexpr auto output_type = OutputType::BECH32; - return EncodeDestination(*Assert(w.GetNewDestination(output_type, ""))); -} - -#endif // ENABLE_WALLET diff --git a/src/test/util/wallet.h b/src/test/util/wallet.h deleted file mode 100644 index 31281bf70e..0000000000 --- a/src/test/util/wallet.h +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright (c) 2019 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_TEST_UTIL_WALLET_H -#define BITCOIN_TEST_UTIL_WALLET_H - -#include <string> - -namespace wallet { -class CWallet; -} // namespace wallet - -// Constants // - -extern const std::string ADDRESS_BCRT1_UNSPENDABLE; - -// RPC-like // - -/** Import the address to the wallet */ -void importaddress(wallet::CWallet& wallet, const std::string& address); -/** Returns a new address from the wallet */ -std::string getnewaddress(wallet::CWallet& w); - - -#endif // BITCOIN_TEST_UTIL_WALLET_H diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 6e59d2e8e6..602c848c2a 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -9,9 +9,7 @@ #include <hash.h> // For Hash() #include <key.h> // For CKey #include <sync.h> -#include <test/util/logging.h> #include <test/util/setup_common.h> -#include <test/util/str.h> #include <uint256.h> #include <util/getuniquepath.h> #include <util/message.h> // For MessageSign(), MessageVerify(), MESSAGE_MAGIC @@ -26,6 +24,7 @@ #include <util/bitdeque.h> #include <array> +#include <cmath> #include <fstream> #include <limits> #include <map> @@ -54,31 +53,6 @@ namespace BCLog { BOOST_FIXTURE_TEST_SUITE(util_tests, BasicTestingSetup) -BOOST_AUTO_TEST_CASE(util_datadir) -{ - // Use local args variable instead of m_args to avoid making assumptions about test setup - ArgsManager args; - args.ForceSetArg("-datadir", fs::PathToString(m_path_root)); - - const fs::path dd_norm = args.GetDataDirBase(); - - args.ForceSetArg("-datadir", fs::PathToString(dd_norm) + "/"); - args.ClearPathCache(); - BOOST_CHECK_EQUAL(dd_norm, args.GetDataDirBase()); - - args.ForceSetArg("-datadir", fs::PathToString(dd_norm) + "/."); - args.ClearPathCache(); - BOOST_CHECK_EQUAL(dd_norm, args.GetDataDirBase()); - - args.ForceSetArg("-datadir", fs::PathToString(dd_norm) + "/./"); - args.ClearPathCache(); - BOOST_CHECK_EQUAL(dd_norm, args.GetDataDirBase()); - - args.ForceSetArg("-datadir", fs::PathToString(dd_norm) + "/.//"); - args.ClearPathCache(); - BOOST_CHECK_EQUAL(dd_norm, args.GetDataDirBase()); -} - namespace { class NoCopyOrMove { @@ -123,6 +97,11 @@ BOOST_AUTO_TEST_CASE(util_check) // Check nested Asserts BOOST_CHECK_EQUAL(Assert((Assert(x).test() ? 3 : 0)), 3); + + // Check -Wdangling-gsl does not trigger when copying the int. (It would + // trigger on "const int&") + const int nine{*Assert(std::optional<int>{9})}; + BOOST_CHECK_EQUAL(9, nine); } BOOST_AUTO_TEST_CASE(util_criticalsection) @@ -293,1000 +272,6 @@ BOOST_AUTO_TEST_CASE(util_FormatISO8601Date) BOOST_CHECK_EQUAL(FormatISO8601Date(1317425777), "2011-09-30"); } -struct TestArgsManager : public ArgsManager -{ - TestArgsManager() { m_network_only_args.clear(); } - void ReadConfigString(const std::string str_config) - { - std::istringstream streamConfig(str_config); - { - LOCK(cs_args); - m_settings.ro_config.clear(); - m_config_sections.clear(); - } - std::string error; - BOOST_REQUIRE(ReadConfigStream(streamConfig, "", error)); - } - void SetNetworkOnlyArg(const std::string arg) - { - LOCK(cs_args); - m_network_only_args.insert(arg); - } - void SetupArgs(const std::vector<std::pair<std::string, unsigned int>>& args) - { - for (const auto& arg : args) { - AddArg(arg.first, "", arg.second, OptionsCategory::OPTIONS); - } - } - using ArgsManager::GetSetting; - using ArgsManager::GetSettingsList; - using ArgsManager::ReadConfigStream; - using ArgsManager::cs_args; - using ArgsManager::m_network; - using ArgsManager::m_settings; -}; - -//! Test GetSetting and GetArg type coercion, negation, and default value handling. -class CheckValueTest : public TestChain100Setup -{ -public: - struct Expect { - util::SettingsValue setting; - bool default_string = false; - bool default_int = false; - bool default_bool = false; - const char* string_value = nullptr; - std::optional<int64_t> int_value; - std::optional<bool> bool_value; - std::optional<std::vector<std::string>> list_value; - const char* error = nullptr; - - explicit Expect(util::SettingsValue s) : setting(std::move(s)) {} - Expect& DefaultString() { default_string = true; return *this; } - Expect& DefaultInt() { default_int = true; return *this; } - Expect& DefaultBool() { default_bool = true; return *this; } - Expect& String(const char* s) { string_value = s; return *this; } - Expect& Int(int64_t i) { int_value = i; return *this; } - Expect& Bool(bool b) { bool_value = b; return *this; } - Expect& List(std::vector<std::string> m) { list_value = std::move(m); return *this; } - Expect& Error(const char* e) { error = e; return *this; } - }; - - void CheckValue(unsigned int flags, const char* arg, const Expect& expect) - { - TestArgsManager test; - test.SetupArgs({{"-value", flags}}); - const char* argv[] = {"ignored", arg}; - std::string error; - bool success = test.ParseParameters(arg ? 2 : 1, (char**)argv, error); - - BOOST_CHECK_EQUAL(test.GetSetting("-value").write(), expect.setting.write()); - auto settings_list = test.GetSettingsList("-value"); - if (expect.setting.isNull() || expect.setting.isFalse()) { - BOOST_CHECK_EQUAL(settings_list.size(), 0U); - } else { - BOOST_CHECK_EQUAL(settings_list.size(), 1U); - BOOST_CHECK_EQUAL(settings_list[0].write(), expect.setting.write()); - } - - if (expect.error) { - BOOST_CHECK(!success); - BOOST_CHECK_NE(error.find(expect.error), std::string::npos); - } else { - BOOST_CHECK(success); - BOOST_CHECK_EQUAL(error, ""); - } - - if (expect.default_string) { - BOOST_CHECK_EQUAL(test.GetArg("-value", "zzzzz"), "zzzzz"); - } else if (expect.string_value) { - BOOST_CHECK_EQUAL(test.GetArg("-value", "zzzzz"), expect.string_value); - } else { - BOOST_CHECK(!success); - } - - if (expect.default_int) { - BOOST_CHECK_EQUAL(test.GetIntArg("-value", 99999), 99999); - } else if (expect.int_value) { - BOOST_CHECK_EQUAL(test.GetIntArg("-value", 99999), *expect.int_value); - } else { - BOOST_CHECK(!success); - } - - if (expect.default_bool) { - BOOST_CHECK_EQUAL(test.GetBoolArg("-value", false), false); - BOOST_CHECK_EQUAL(test.GetBoolArg("-value", true), true); - } else if (expect.bool_value) { - BOOST_CHECK_EQUAL(test.GetBoolArg("-value", false), *expect.bool_value); - BOOST_CHECK_EQUAL(test.GetBoolArg("-value", true), *expect.bool_value); - } else { - BOOST_CHECK(!success); - } - - if (expect.list_value) { - auto l = test.GetArgs("-value"); - BOOST_CHECK_EQUAL_COLLECTIONS(l.begin(), l.end(), expect.list_value->begin(), expect.list_value->end()); - } else { - BOOST_CHECK(!success); - } - } -}; - -BOOST_FIXTURE_TEST_CASE(util_CheckValue, CheckValueTest) -{ - using M = ArgsManager; - - CheckValue(M::ALLOW_ANY, nullptr, Expect{{}}.DefaultString().DefaultInt().DefaultBool().List({})); - CheckValue(M::ALLOW_ANY, "-novalue", Expect{false}.String("0").Int(0).Bool(false).List({})); - CheckValue(M::ALLOW_ANY, "-novalue=", Expect{false}.String("0").Int(0).Bool(false).List({})); - CheckValue(M::ALLOW_ANY, "-novalue=0", Expect{true}.String("1").Int(1).Bool(true).List({"1"})); - CheckValue(M::ALLOW_ANY, "-novalue=1", Expect{false}.String("0").Int(0).Bool(false).List({})); - CheckValue(M::ALLOW_ANY, "-novalue=2", Expect{false}.String("0").Int(0).Bool(false).List({})); - CheckValue(M::ALLOW_ANY, "-novalue=abc", Expect{true}.String("1").Int(1).Bool(true).List({"1"})); - CheckValue(M::ALLOW_ANY, "-value", Expect{""}.String("").Int(0).Bool(true).List({""})); - CheckValue(M::ALLOW_ANY, "-value=", Expect{""}.String("").Int(0).Bool(true).List({""})); - CheckValue(M::ALLOW_ANY, "-value=0", Expect{"0"}.String("0").Int(0).Bool(false).List({"0"})); - CheckValue(M::ALLOW_ANY, "-value=1", Expect{"1"}.String("1").Int(1).Bool(true).List({"1"})); - CheckValue(M::ALLOW_ANY, "-value=2", Expect{"2"}.String("2").Int(2).Bool(true).List({"2"})); - CheckValue(M::ALLOW_ANY, "-value=abc", Expect{"abc"}.String("abc").Int(0).Bool(false).List({"abc"})); -} - -struct NoIncludeConfTest { - std::string Parse(const char* arg) - { - TestArgsManager test; - test.SetupArgs({{"-includeconf", ArgsManager::ALLOW_ANY}}); - std::array argv{"ignored", arg}; - std::string error; - (void)test.ParseParameters(argv.size(), argv.data(), error); - return error; - } -}; - -BOOST_FIXTURE_TEST_CASE(util_NoIncludeConf, NoIncludeConfTest) -{ - BOOST_CHECK_EQUAL(Parse("-noincludeconf"), ""); - BOOST_CHECK_EQUAL(Parse("-includeconf"), "-includeconf cannot be used from commandline; -includeconf=\"\""); - BOOST_CHECK_EQUAL(Parse("-includeconf=file"), "-includeconf cannot be used from commandline; -includeconf=\"file\""); -} - -BOOST_AUTO_TEST_CASE(util_ParseParameters) -{ - TestArgsManager testArgs; - const auto a = std::make_pair("-a", ArgsManager::ALLOW_ANY); - const auto b = std::make_pair("-b", ArgsManager::ALLOW_ANY); - const auto ccc = std::make_pair("-ccc", ArgsManager::ALLOW_ANY); - const auto d = std::make_pair("-d", ArgsManager::ALLOW_ANY); - - const char *argv_test[] = {"-ignored", "-a", "-b", "-ccc=argument", "-ccc=multiple", "f", "-d=e"}; - - std::string error; - LOCK(testArgs.cs_args); - testArgs.SetupArgs({a, b, ccc, d}); - BOOST_CHECK(testArgs.ParseParameters(0, (char**)argv_test, error)); - BOOST_CHECK(testArgs.m_settings.command_line_options.empty() && testArgs.m_settings.ro_config.empty()); - - BOOST_CHECK(testArgs.ParseParameters(1, (char**)argv_test, error)); - BOOST_CHECK(testArgs.m_settings.command_line_options.empty() && testArgs.m_settings.ro_config.empty()); - - BOOST_CHECK(testArgs.ParseParameters(7, (char**)argv_test, error)); - // expectation: -ignored is ignored (program name argument), - // -a, -b and -ccc end up in map, -d ignored because it is after - // a non-option argument (non-GNU option parsing) - BOOST_CHECK(testArgs.m_settings.command_line_options.size() == 3 && testArgs.m_settings.ro_config.empty()); - BOOST_CHECK(testArgs.IsArgSet("-a") && testArgs.IsArgSet("-b") && testArgs.IsArgSet("-ccc") - && !testArgs.IsArgSet("f") && !testArgs.IsArgSet("-d")); - BOOST_CHECK(testArgs.m_settings.command_line_options.count("a") && testArgs.m_settings.command_line_options.count("b") && testArgs.m_settings.command_line_options.count("ccc") - && !testArgs.m_settings.command_line_options.count("f") && !testArgs.m_settings.command_line_options.count("d")); - - BOOST_CHECK(testArgs.m_settings.command_line_options["a"].size() == 1); - BOOST_CHECK(testArgs.m_settings.command_line_options["a"].front().get_str() == ""); - BOOST_CHECK(testArgs.m_settings.command_line_options["ccc"].size() == 2); - BOOST_CHECK(testArgs.m_settings.command_line_options["ccc"].front().get_str() == "argument"); - BOOST_CHECK(testArgs.m_settings.command_line_options["ccc"].back().get_str() == "multiple"); - BOOST_CHECK(testArgs.GetArgs("-ccc").size() == 2); -} - -BOOST_AUTO_TEST_CASE(util_ParseInvalidParameters) -{ - TestArgsManager test; - test.SetupArgs({{"-registered", ArgsManager::ALLOW_ANY}}); - - const char* argv[] = {"ignored", "-registered"}; - std::string error; - BOOST_CHECK(test.ParseParameters(2, (char**)argv, error)); - BOOST_CHECK_EQUAL(error, ""); - - argv[1] = "-unregistered"; - BOOST_CHECK(!test.ParseParameters(2, (char**)argv, error)); - BOOST_CHECK_EQUAL(error, "Invalid parameter -unregistered"); - - // Make sure registered parameters prefixed with a chain name trigger errors. - // (Previously, they were accepted and ignored.) - argv[1] = "-test.registered"; - BOOST_CHECK(!test.ParseParameters(2, (char**)argv, error)); - BOOST_CHECK_EQUAL(error, "Invalid parameter -test.registered"); -} - -static void TestParse(const std::string& str, bool expected_bool, int64_t expected_int) -{ - TestArgsManager test; - test.SetupArgs({{"-value", ArgsManager::ALLOW_ANY}}); - std::string arg = "-value=" + str; - const char* argv[] = {"ignored", arg.c_str()}; - std::string error; - BOOST_CHECK(test.ParseParameters(2, (char**)argv, error)); - BOOST_CHECK_EQUAL(test.GetBoolArg("-value", false), expected_bool); - BOOST_CHECK_EQUAL(test.GetBoolArg("-value", true), expected_bool); - BOOST_CHECK_EQUAL(test.GetIntArg("-value", 99998), expected_int); - BOOST_CHECK_EQUAL(test.GetIntArg("-value", 99999), expected_int); -} - -// Test bool and int parsing. -BOOST_AUTO_TEST_CASE(util_ArgParsing) -{ - // Some of these cases could be ambiguous or surprising to users, and might - // be worth triggering errors or warnings in the future. But for now basic - // test coverage is useful to avoid breaking backwards compatibility - // unintentionally. - TestParse("", true, 0); - TestParse(" ", false, 0); - TestParse("0", false, 0); - TestParse("0 ", false, 0); - TestParse(" 0", false, 0); - TestParse("+0", false, 0); - TestParse("-0", false, 0); - TestParse("5", true, 5); - TestParse("5 ", true, 5); - TestParse(" 5", true, 5); - TestParse("+5", true, 5); - TestParse("-5", true, -5); - TestParse("0 5", false, 0); - TestParse("5 0", true, 5); - TestParse("050", true, 50); - TestParse("0.", false, 0); - TestParse("5.", true, 5); - TestParse("0.0", false, 0); - TestParse("0.5", false, 0); - TestParse("5.0", true, 5); - TestParse("5.5", true, 5); - TestParse("x", false, 0); - TestParse("x0", false, 0); - TestParse("x5", false, 0); - TestParse("0x", false, 0); - TestParse("5x", true, 5); - TestParse("0x5", false, 0); - TestParse("false", false, 0); - TestParse("true", false, 0); - TestParse("yes", false, 0); - TestParse("no", false, 0); -} - -BOOST_AUTO_TEST_CASE(util_GetBoolArg) -{ - TestArgsManager testArgs; - const auto a = std::make_pair("-a", ArgsManager::ALLOW_ANY); - const auto b = std::make_pair("-b", ArgsManager::ALLOW_ANY); - const auto c = std::make_pair("-c", ArgsManager::ALLOW_ANY); - const auto d = std::make_pair("-d", ArgsManager::ALLOW_ANY); - const auto e = std::make_pair("-e", ArgsManager::ALLOW_ANY); - const auto f = std::make_pair("-f", ArgsManager::ALLOW_ANY); - - const char *argv_test[] = { - "ignored", "-a", "-nob", "-c=0", "-d=1", "-e=false", "-f=true"}; - std::string error; - LOCK(testArgs.cs_args); - testArgs.SetupArgs({a, b, c, d, e, f}); - BOOST_CHECK(testArgs.ParseParameters(7, (char**)argv_test, error)); - - // Each letter should be set. - for (const char opt : "abcdef") - BOOST_CHECK(testArgs.IsArgSet({'-', opt}) || !opt); - - // Nothing else should be in the map - BOOST_CHECK(testArgs.m_settings.command_line_options.size() == 6 && - testArgs.m_settings.ro_config.empty()); - - // The -no prefix should get stripped on the way in. - BOOST_CHECK(!testArgs.IsArgSet("-nob")); - - // The -b option is flagged as negated, and nothing else is - BOOST_CHECK(testArgs.IsArgNegated("-b")); - BOOST_CHECK(!testArgs.IsArgNegated("-a")); - - // Check expected values. - BOOST_CHECK(testArgs.GetBoolArg("-a", false) == true); - BOOST_CHECK(testArgs.GetBoolArg("-b", true) == false); - BOOST_CHECK(testArgs.GetBoolArg("-c", true) == false); - BOOST_CHECK(testArgs.GetBoolArg("-d", false) == true); - BOOST_CHECK(testArgs.GetBoolArg("-e", true) == false); - BOOST_CHECK(testArgs.GetBoolArg("-f", true) == false); -} - -BOOST_AUTO_TEST_CASE(util_GetBoolArgEdgeCases) -{ - // Test some awful edge cases that hopefully no user will ever exercise. - TestArgsManager testArgs; - - // Params test - const auto foo = std::make_pair("-foo", ArgsManager::ALLOW_ANY); - const auto bar = std::make_pair("-bar", ArgsManager::ALLOW_ANY); - const char *argv_test[] = {"ignored", "-nofoo", "-foo", "-nobar=0"}; - testArgs.SetupArgs({foo, bar}); - std::string error; - BOOST_CHECK(testArgs.ParseParameters(4, (char**)argv_test, error)); - - // This was passed twice, second one overrides the negative setting. - BOOST_CHECK(!testArgs.IsArgNegated("-foo")); - BOOST_CHECK(testArgs.GetArg("-foo", "xxx") == ""); - - // A double negative is a positive, and not marked as negated. - BOOST_CHECK(!testArgs.IsArgNegated("-bar")); - BOOST_CHECK(testArgs.GetArg("-bar", "xxx") == "1"); - - // Config test - const char *conf_test = "nofoo=1\nfoo=1\nnobar=0\n"; - BOOST_CHECK(testArgs.ParseParameters(1, (char**)argv_test, error)); - testArgs.ReadConfigString(conf_test); - - // This was passed twice, second one overrides the negative setting, - // and the value. - BOOST_CHECK(!testArgs.IsArgNegated("-foo")); - BOOST_CHECK(testArgs.GetArg("-foo", "xxx") == "1"); - - // A double negative is a positive, and does not count as negated. - BOOST_CHECK(!testArgs.IsArgNegated("-bar")); - BOOST_CHECK(testArgs.GetArg("-bar", "xxx") == "1"); - - // Combined test - const char *combo_test_args[] = {"ignored", "-nofoo", "-bar"}; - const char *combo_test_conf = "foo=1\nnobar=1\n"; - BOOST_CHECK(testArgs.ParseParameters(3, (char**)combo_test_args, error)); - testArgs.ReadConfigString(combo_test_conf); - - // Command line overrides, but doesn't erase old setting - BOOST_CHECK(testArgs.IsArgNegated("-foo")); - BOOST_CHECK(testArgs.GetArg("-foo", "xxx") == "0"); - BOOST_CHECK(testArgs.GetArgs("-foo").size() == 0); - - // Command line overrides, but doesn't erase old setting - BOOST_CHECK(!testArgs.IsArgNegated("-bar")); - BOOST_CHECK(testArgs.GetArg("-bar", "xxx") == ""); - BOOST_CHECK(testArgs.GetArgs("-bar").size() == 1 - && testArgs.GetArgs("-bar").front() == ""); -} - -BOOST_AUTO_TEST_CASE(util_ReadConfigStream) -{ - const char *str_config = - "a=\n" - "b=1\n" - "ccc=argument\n" - "ccc=multiple\n" - "d=e\n" - "nofff=1\n" - "noggg=0\n" - "h=1\n" - "noh=1\n" - "noi=1\n" - "i=1\n" - "sec1.ccc=extend1\n" - "\n" - "[sec1]\n" - "ccc=extend2\n" - "d=eee\n" - "h=1\n" - "[sec2]\n" - "ccc=extend3\n" - "iii=2\n"; - - TestArgsManager test_args; - LOCK(test_args.cs_args); - const auto a = std::make_pair("-a", ArgsManager::ALLOW_ANY); - const auto b = std::make_pair("-b", ArgsManager::ALLOW_ANY); - const auto ccc = std::make_pair("-ccc", ArgsManager::ALLOW_ANY); - const auto d = std::make_pair("-d", ArgsManager::ALLOW_ANY); - const auto e = std::make_pair("-e", ArgsManager::ALLOW_ANY); - const auto fff = std::make_pair("-fff", ArgsManager::ALLOW_ANY); - const auto ggg = std::make_pair("-ggg", ArgsManager::ALLOW_ANY); - const auto h = std::make_pair("-h", ArgsManager::ALLOW_ANY); - const auto i = std::make_pair("-i", ArgsManager::ALLOW_ANY); - const auto iii = std::make_pair("-iii", ArgsManager::ALLOW_ANY); - test_args.SetupArgs({a, b, ccc, d, e, fff, ggg, h, i, iii}); - - test_args.ReadConfigString(str_config); - // expectation: a, b, ccc, d, fff, ggg, h, i end up in map - // so do sec1.ccc, sec1.d, sec1.h, sec2.ccc, sec2.iii - - BOOST_CHECK(test_args.m_settings.command_line_options.empty()); - BOOST_CHECK(test_args.m_settings.ro_config.size() == 3); - BOOST_CHECK(test_args.m_settings.ro_config[""].size() == 8); - BOOST_CHECK(test_args.m_settings.ro_config["sec1"].size() == 3); - BOOST_CHECK(test_args.m_settings.ro_config["sec2"].size() == 2); - - BOOST_CHECK(test_args.m_settings.ro_config[""].count("a")); - BOOST_CHECK(test_args.m_settings.ro_config[""].count("b")); - BOOST_CHECK(test_args.m_settings.ro_config[""].count("ccc")); - BOOST_CHECK(test_args.m_settings.ro_config[""].count("d")); - BOOST_CHECK(test_args.m_settings.ro_config[""].count("fff")); - BOOST_CHECK(test_args.m_settings.ro_config[""].count("ggg")); - BOOST_CHECK(test_args.m_settings.ro_config[""].count("h")); - BOOST_CHECK(test_args.m_settings.ro_config[""].count("i")); - BOOST_CHECK(test_args.m_settings.ro_config["sec1"].count("ccc")); - BOOST_CHECK(test_args.m_settings.ro_config["sec1"].count("h")); - BOOST_CHECK(test_args.m_settings.ro_config["sec2"].count("ccc")); - BOOST_CHECK(test_args.m_settings.ro_config["sec2"].count("iii")); - - BOOST_CHECK(test_args.IsArgSet("-a")); - BOOST_CHECK(test_args.IsArgSet("-b")); - BOOST_CHECK(test_args.IsArgSet("-ccc")); - BOOST_CHECK(test_args.IsArgSet("-d")); - BOOST_CHECK(test_args.IsArgSet("-fff")); - BOOST_CHECK(test_args.IsArgSet("-ggg")); - BOOST_CHECK(test_args.IsArgSet("-h")); - BOOST_CHECK(test_args.IsArgSet("-i")); - BOOST_CHECK(!test_args.IsArgSet("-zzz")); - BOOST_CHECK(!test_args.IsArgSet("-iii")); - - BOOST_CHECK_EQUAL(test_args.GetArg("-a", "xxx"), ""); - BOOST_CHECK_EQUAL(test_args.GetArg("-b", "xxx"), "1"); - BOOST_CHECK_EQUAL(test_args.GetArg("-ccc", "xxx"), "argument"); - BOOST_CHECK_EQUAL(test_args.GetArg("-d", "xxx"), "e"); - BOOST_CHECK_EQUAL(test_args.GetArg("-fff", "xxx"), "0"); - BOOST_CHECK_EQUAL(test_args.GetArg("-ggg", "xxx"), "1"); - BOOST_CHECK_EQUAL(test_args.GetArg("-h", "xxx"), "0"); - BOOST_CHECK_EQUAL(test_args.GetArg("-i", "xxx"), "1"); - BOOST_CHECK_EQUAL(test_args.GetArg("-zzz", "xxx"), "xxx"); - BOOST_CHECK_EQUAL(test_args.GetArg("-iii", "xxx"), "xxx"); - - for (const bool def : {false, true}) { - BOOST_CHECK(test_args.GetBoolArg("-a", def)); - BOOST_CHECK(test_args.GetBoolArg("-b", def)); - BOOST_CHECK(!test_args.GetBoolArg("-ccc", def)); - BOOST_CHECK(!test_args.GetBoolArg("-d", def)); - BOOST_CHECK(!test_args.GetBoolArg("-fff", def)); - BOOST_CHECK(test_args.GetBoolArg("-ggg", def)); - BOOST_CHECK(!test_args.GetBoolArg("-h", def)); - BOOST_CHECK(test_args.GetBoolArg("-i", def)); - BOOST_CHECK(test_args.GetBoolArg("-zzz", def) == def); - BOOST_CHECK(test_args.GetBoolArg("-iii", def) == def); - } - - BOOST_CHECK(test_args.GetArgs("-a").size() == 1 - && test_args.GetArgs("-a").front() == ""); - BOOST_CHECK(test_args.GetArgs("-b").size() == 1 - && test_args.GetArgs("-b").front() == "1"); - BOOST_CHECK(test_args.GetArgs("-ccc").size() == 2 - && test_args.GetArgs("-ccc").front() == "argument" - && test_args.GetArgs("-ccc").back() == "multiple"); - BOOST_CHECK(test_args.GetArgs("-fff").size() == 0); - BOOST_CHECK(test_args.GetArgs("-nofff").size() == 0); - BOOST_CHECK(test_args.GetArgs("-ggg").size() == 1 - && test_args.GetArgs("-ggg").front() == "1"); - BOOST_CHECK(test_args.GetArgs("-noggg").size() == 0); - BOOST_CHECK(test_args.GetArgs("-h").size() == 0); - BOOST_CHECK(test_args.GetArgs("-noh").size() == 0); - BOOST_CHECK(test_args.GetArgs("-i").size() == 1 - && test_args.GetArgs("-i").front() == "1"); - BOOST_CHECK(test_args.GetArgs("-noi").size() == 0); - BOOST_CHECK(test_args.GetArgs("-zzz").size() == 0); - - BOOST_CHECK(!test_args.IsArgNegated("-a")); - BOOST_CHECK(!test_args.IsArgNegated("-b")); - BOOST_CHECK(!test_args.IsArgNegated("-ccc")); - BOOST_CHECK(!test_args.IsArgNegated("-d")); - BOOST_CHECK(test_args.IsArgNegated("-fff")); - BOOST_CHECK(!test_args.IsArgNegated("-ggg")); - BOOST_CHECK(test_args.IsArgNegated("-h")); // last setting takes precedence - BOOST_CHECK(!test_args.IsArgNegated("-i")); // last setting takes precedence - BOOST_CHECK(!test_args.IsArgNegated("-zzz")); - - // Test sections work - test_args.SelectConfigNetwork("sec1"); - - // same as original - BOOST_CHECK_EQUAL(test_args.GetArg("-a", "xxx"), ""); - BOOST_CHECK_EQUAL(test_args.GetArg("-b", "xxx"), "1"); - BOOST_CHECK_EQUAL(test_args.GetArg("-fff", "xxx"), "0"); - BOOST_CHECK_EQUAL(test_args.GetArg("-ggg", "xxx"), "1"); - BOOST_CHECK_EQUAL(test_args.GetArg("-zzz", "xxx"), "xxx"); - BOOST_CHECK_EQUAL(test_args.GetArg("-iii", "xxx"), "xxx"); - // d is overridden - BOOST_CHECK(test_args.GetArg("-d", "xxx") == "eee"); - // section-specific setting - BOOST_CHECK(test_args.GetArg("-h", "xxx") == "1"); - // section takes priority for multiple values - BOOST_CHECK(test_args.GetArg("-ccc", "xxx") == "extend1"); - // check multiple values works - const std::vector<std::string> sec1_ccc_expected = {"extend1","extend2","argument","multiple"}; - const auto& sec1_ccc_res = test_args.GetArgs("-ccc"); - BOOST_CHECK_EQUAL_COLLECTIONS(sec1_ccc_res.begin(), sec1_ccc_res.end(), sec1_ccc_expected.begin(), sec1_ccc_expected.end()); - - test_args.SelectConfigNetwork("sec2"); - - // same as original - BOOST_CHECK(test_args.GetArg("-a", "xxx") == ""); - BOOST_CHECK(test_args.GetArg("-b", "xxx") == "1"); - BOOST_CHECK(test_args.GetArg("-d", "xxx") == "e"); - BOOST_CHECK(test_args.GetArg("-fff", "xxx") == "0"); - BOOST_CHECK(test_args.GetArg("-ggg", "xxx") == "1"); - BOOST_CHECK(test_args.GetArg("-zzz", "xxx") == "xxx"); - BOOST_CHECK(test_args.GetArg("-h", "xxx") == "0"); - // section-specific setting - BOOST_CHECK(test_args.GetArg("-iii", "xxx") == "2"); - // section takes priority for multiple values - BOOST_CHECK(test_args.GetArg("-ccc", "xxx") == "extend3"); - // check multiple values works - const std::vector<std::string> sec2_ccc_expected = {"extend3","argument","multiple"}; - const auto& sec2_ccc_res = test_args.GetArgs("-ccc"); - BOOST_CHECK_EQUAL_COLLECTIONS(sec2_ccc_res.begin(), sec2_ccc_res.end(), sec2_ccc_expected.begin(), sec2_ccc_expected.end()); - - // Test section only options - - test_args.SetNetworkOnlyArg("-d"); - test_args.SetNetworkOnlyArg("-ccc"); - test_args.SetNetworkOnlyArg("-h"); - - test_args.SelectConfigNetwork(CBaseChainParams::MAIN); - BOOST_CHECK(test_args.GetArg("-d", "xxx") == "e"); - BOOST_CHECK(test_args.GetArgs("-ccc").size() == 2); - BOOST_CHECK(test_args.GetArg("-h", "xxx") == "0"); - - test_args.SelectConfigNetwork("sec1"); - BOOST_CHECK(test_args.GetArg("-d", "xxx") == "eee"); - BOOST_CHECK(test_args.GetArgs("-d").size() == 1); - BOOST_CHECK(test_args.GetArgs("-ccc").size() == 2); - BOOST_CHECK(test_args.GetArg("-h", "xxx") == "1"); - - test_args.SelectConfigNetwork("sec2"); - BOOST_CHECK(test_args.GetArg("-d", "xxx") == "xxx"); - BOOST_CHECK(test_args.GetArgs("-d").size() == 0); - BOOST_CHECK(test_args.GetArgs("-ccc").size() == 1); - BOOST_CHECK(test_args.GetArg("-h", "xxx") == "0"); -} - -BOOST_AUTO_TEST_CASE(util_GetArg) -{ - TestArgsManager testArgs; - LOCK(testArgs.cs_args); - testArgs.m_settings.command_line_options.clear(); - testArgs.m_settings.command_line_options["strtest1"] = {"string..."}; - // strtest2 undefined on purpose - testArgs.m_settings.command_line_options["inttest1"] = {"12345"}; - testArgs.m_settings.command_line_options["inttest2"] = {"81985529216486895"}; - // inttest3 undefined on purpose - testArgs.m_settings.command_line_options["booltest1"] = {""}; - // booltest2 undefined on purpose - testArgs.m_settings.command_line_options["booltest3"] = {"0"}; - testArgs.m_settings.command_line_options["booltest4"] = {"1"}; - - // priorities - testArgs.m_settings.command_line_options["pritest1"] = {"a", "b"}; - testArgs.m_settings.ro_config[""]["pritest2"] = {"a", "b"}; - testArgs.m_settings.command_line_options["pritest3"] = {"a"}; - testArgs.m_settings.ro_config[""]["pritest3"] = {"b"}; - testArgs.m_settings.command_line_options["pritest4"] = {"a","b"}; - testArgs.m_settings.ro_config[""]["pritest4"] = {"c","d"}; - - BOOST_CHECK_EQUAL(testArgs.GetArg("strtest1", "default"), "string..."); - BOOST_CHECK_EQUAL(testArgs.GetArg("strtest2", "default"), "default"); - BOOST_CHECK_EQUAL(testArgs.GetIntArg("inttest1", -1), 12345); - BOOST_CHECK_EQUAL(testArgs.GetIntArg("inttest2", -1), 81985529216486895LL); - BOOST_CHECK_EQUAL(testArgs.GetIntArg("inttest3", -1), -1); - BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest1", false), true); - BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest2", false), false); - BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest3", false), false); - BOOST_CHECK_EQUAL(testArgs.GetBoolArg("booltest4", false), true); - - BOOST_CHECK_EQUAL(testArgs.GetArg("pritest1", "default"), "b"); - BOOST_CHECK_EQUAL(testArgs.GetArg("pritest2", "default"), "a"); - BOOST_CHECK_EQUAL(testArgs.GetArg("pritest3", "default"), "a"); - BOOST_CHECK_EQUAL(testArgs.GetArg("pritest4", "default"), "b"); -} - -BOOST_AUTO_TEST_CASE(util_GetChainName) -{ - TestArgsManager test_args; - const auto testnet = std::make_pair("-testnet", ArgsManager::ALLOW_ANY); - const auto regtest = std::make_pair("-regtest", ArgsManager::ALLOW_ANY); - test_args.SetupArgs({testnet, regtest}); - - const char* argv_testnet[] = {"cmd", "-testnet"}; - const char* argv_regtest[] = {"cmd", "-regtest"}; - const char* argv_test_no_reg[] = {"cmd", "-testnet", "-noregtest"}; - const char* argv_both[] = {"cmd", "-testnet", "-regtest"}; - - // equivalent to "-testnet" - // regtest in testnet section is ignored - const char* testnetconf = "testnet=1\nregtest=0\n[test]\nregtest=1"; - std::string error; - - BOOST_CHECK(test_args.ParseParameters(0, (char**)argv_testnet, error)); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "main"); - - BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_testnet, error)); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); - - BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_regtest, error)); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "regtest"); - - BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_test_no_reg, error)); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); - - BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_both, error)); - BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); - - BOOST_CHECK(test_args.ParseParameters(0, (char**)argv_testnet, error)); - test_args.ReadConfigString(testnetconf); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); - - BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_testnet, error)); - test_args.ReadConfigString(testnetconf); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); - - BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_regtest, error)); - test_args.ReadConfigString(testnetconf); - BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); - - BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_test_no_reg, error)); - test_args.ReadConfigString(testnetconf); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); - - BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_both, error)); - test_args.ReadConfigString(testnetconf); - BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); - - // check setting the network to test (and thus making - // [test] regtest=1 potentially relevant) doesn't break things - test_args.SelectConfigNetwork("test"); - - BOOST_CHECK(test_args.ParseParameters(0, (char**)argv_testnet, error)); - test_args.ReadConfigString(testnetconf); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); - - BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_testnet, error)); - test_args.ReadConfigString(testnetconf); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); - - BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_regtest, error)); - test_args.ReadConfigString(testnetconf); - BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); - - BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_test_no_reg, error)); - test_args.ReadConfigString(testnetconf); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); - - BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_both, error)); - test_args.ReadConfigString(testnetconf); - BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); -} - -// Test different ways settings can be merged, and verify results. This test can -// be used to confirm that updates to settings code don't change behavior -// unintentionally. -// -// The test covers: -// -// - Combining different setting actions. Possible actions are: configuring a -// setting, negating a setting (adding "-no" prefix), and configuring/negating -// settings in a network section (adding "main." or "test." prefixes). -// -// - Combining settings from command line arguments and a config file. -// -// - Combining SoftSet and ForceSet calls. -// -// - Testing "main" and "test" network values to make sure settings from network -// sections are applied and to check for mainnet-specific behaviors like -// inheriting settings from the default section. -// -// - Testing network-specific settings like "-wallet", that may be ignored -// outside a network section, and non-network specific settings like "-server" -// that aren't sensitive to the network. -// -struct ArgsMergeTestingSetup : public BasicTestingSetup { - //! Max number of actions to sequence together. Can decrease this when - //! debugging to make test results easier to understand. - static constexpr int MAX_ACTIONS = 3; - - enum Action { NONE, SET, NEGATE, SECTION_SET, SECTION_NEGATE }; - using ActionList = Action[MAX_ACTIONS]; - - //! Enumerate all possible test configurations. - template <typename Fn> - void ForEachMergeSetup(Fn&& fn) - { - ActionList arg_actions = {}; - // command_line_options do not have sections. Only iterate over SET and NEGATE - ForEachNoDup(arg_actions, SET, NEGATE, [&] { - ActionList conf_actions = {}; - ForEachNoDup(conf_actions, SET, SECTION_NEGATE, [&] { - for (bool soft_set : {false, true}) { - for (bool force_set : {false, true}) { - for (const std::string& section : {CBaseChainParams::MAIN, CBaseChainParams::TESTNET, CBaseChainParams::SIGNET}) { - for (const std::string& network : {CBaseChainParams::MAIN, CBaseChainParams::TESTNET, CBaseChainParams::SIGNET}) { - for (bool net_specific : {false, true}) { - fn(arg_actions, conf_actions, soft_set, force_set, section, network, net_specific); - } - } - } - } - } - }); - }); - } - - //! Translate actions into a list of <key>=<value> setting strings. - std::vector<std::string> GetValues(const ActionList& actions, - const std::string& section, - const std::string& name, - const std::string& value_prefix) - { - std::vector<std::string> values; - int suffix = 0; - for (Action action : actions) { - if (action == NONE) break; - std::string prefix; - if (action == SECTION_SET || action == SECTION_NEGATE) prefix = section + "."; - if (action == SET || action == SECTION_SET) { - for (int i = 0; i < 2; ++i) { - values.push_back(prefix + name + "=" + value_prefix + ToString(++suffix)); - } - } - if (action == NEGATE || action == SECTION_NEGATE) { - values.push_back(prefix + "no" + name + "=1"); - } - } - return values; - } -}; - -// Regression test covering different ways config settings can be merged. The -// test parses and merges settings, representing the results as strings that get -// compared against an expected hash. To debug, the result strings can be dumped -// to a file (see comments below). -BOOST_FIXTURE_TEST_CASE(util_ArgsMerge, ArgsMergeTestingSetup) -{ - CHash256 out_sha; - FILE* out_file = nullptr; - if (const char* out_path = getenv("ARGS_MERGE_TEST_OUT")) { - out_file = fsbridge::fopen(out_path, "w"); - if (!out_file) throw std::system_error(errno, std::generic_category(), "fopen failed"); - } - - ForEachMergeSetup([&](const ActionList& arg_actions, const ActionList& conf_actions, bool soft_set, bool force_set, - const std::string& section, const std::string& network, bool net_specific) { - TestArgsManager parser; - LOCK(parser.cs_args); - - std::string desc = "net="; - desc += network; - parser.m_network = network; - - const std::string& name = net_specific ? "wallet" : "server"; - const std::string key = "-" + name; - parser.AddArg(key, name, ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - if (net_specific) parser.SetNetworkOnlyArg(key); - - auto args = GetValues(arg_actions, section, name, "a"); - std::vector<const char*> argv = {"ignored"}; - for (auto& arg : args) { - arg.insert(0, "-"); - desc += " "; - desc += arg; - argv.push_back(arg.c_str()); - } - std::string error; - BOOST_CHECK(parser.ParseParameters(argv.size(), argv.data(), error)); - BOOST_CHECK_EQUAL(error, ""); - - std::string conf; - for (auto& conf_val : GetValues(conf_actions, section, name, "c")) { - desc += " "; - desc += conf_val; - conf += conf_val; - conf += "\n"; - } - std::istringstream conf_stream(conf); - BOOST_CHECK(parser.ReadConfigStream(conf_stream, "filepath", error)); - BOOST_CHECK_EQUAL(error, ""); - - if (soft_set) { - desc += " soft"; - parser.SoftSetArg(key, "soft1"); - parser.SoftSetArg(key, "soft2"); - } - - if (force_set) { - desc += " force"; - parser.ForceSetArg(key, "force1"); - parser.ForceSetArg(key, "force2"); - } - - desc += " || "; - - if (!parser.IsArgSet(key)) { - desc += "unset"; - BOOST_CHECK(!parser.IsArgNegated(key)); - BOOST_CHECK_EQUAL(parser.GetArg(key, "default"), "default"); - BOOST_CHECK(parser.GetArgs(key).empty()); - } else if (parser.IsArgNegated(key)) { - desc += "negated"; - BOOST_CHECK_EQUAL(parser.GetArg(key, "default"), "0"); - BOOST_CHECK(parser.GetArgs(key).empty()); - } else { - desc += parser.GetArg(key, "default"); - desc += " |"; - for (const auto& arg : parser.GetArgs(key)) { - desc += " "; - desc += arg; - } - } - - std::set<std::string> ignored = parser.GetUnsuitableSectionOnlyArgs(); - if (!ignored.empty()) { - desc += " | ignored"; - for (const auto& arg : ignored) { - desc += " "; - desc += arg; - } - } - - desc += "\n"; - - out_sha.Write(MakeUCharSpan(desc)); - if (out_file) { - BOOST_REQUIRE(fwrite(desc.data(), 1, desc.size(), out_file) == desc.size()); - } - }); - - if (out_file) { - if (fclose(out_file)) throw std::system_error(errno, std::generic_category(), "fclose failed"); - out_file = nullptr; - } - - unsigned char out_sha_bytes[CSHA256::OUTPUT_SIZE]; - out_sha.Finalize(out_sha_bytes); - std::string out_sha_hex = HexStr(out_sha_bytes); - - // If check below fails, should manually dump the results with: - // - // ARGS_MERGE_TEST_OUT=results.txt ./test_bitcoin --run_test=util_tests/util_ArgsMerge - // - // And verify diff against previous results to make sure the changes are expected. - // - // Results file is formatted like: - // - // <input> || <IsArgSet/IsArgNegated/GetArg output> | <GetArgs output> | <GetUnsuitable output> - BOOST_CHECK_EQUAL(out_sha_hex, "d1e436c1cd510d0ec44d5205d4b4e3bee6387d316e0075c58206cb16603f3d82"); -} - -// Similar test as above, but for ArgsManager::GetChainName function. -struct ChainMergeTestingSetup : public BasicTestingSetup { - static constexpr int MAX_ACTIONS = 2; - - enum Action { NONE, ENABLE_TEST, DISABLE_TEST, NEGATE_TEST, ENABLE_REG, DISABLE_REG, NEGATE_REG }; - using ActionList = Action[MAX_ACTIONS]; - - //! Enumerate all possible test configurations. - template <typename Fn> - void ForEachMergeSetup(Fn&& fn) - { - ActionList arg_actions = {}; - ForEachNoDup(arg_actions, ENABLE_TEST, NEGATE_REG, [&] { - ActionList conf_actions = {}; - ForEachNoDup(conf_actions, ENABLE_TEST, NEGATE_REG, [&] { fn(arg_actions, conf_actions); }); - }); - } -}; - -BOOST_FIXTURE_TEST_CASE(util_ChainMerge, ChainMergeTestingSetup) -{ - CHash256 out_sha; - FILE* out_file = nullptr; - if (const char* out_path = getenv("CHAIN_MERGE_TEST_OUT")) { - out_file = fsbridge::fopen(out_path, "w"); - if (!out_file) throw std::system_error(errno, std::generic_category(), "fopen failed"); - } - - ForEachMergeSetup([&](const ActionList& arg_actions, const ActionList& conf_actions) { - TestArgsManager parser; - LOCK(parser.cs_args); - parser.AddArg("-regtest", "regtest", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - parser.AddArg("-testnet", "testnet", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - - auto arg = [](Action action) { return action == ENABLE_TEST ? "-testnet=1" : - action == DISABLE_TEST ? "-testnet=0" : - action == NEGATE_TEST ? "-notestnet=1" : - action == ENABLE_REG ? "-regtest=1" : - action == DISABLE_REG ? "-regtest=0" : - action == NEGATE_REG ? "-noregtest=1" : nullptr; }; - - std::string desc; - std::vector<const char*> argv = {"ignored"}; - for (Action action : arg_actions) { - const char* argstr = arg(action); - if (!argstr) break; - argv.push_back(argstr); - desc += " "; - desc += argv.back(); - } - std::string error; - BOOST_CHECK(parser.ParseParameters(argv.size(), argv.data(), error)); - BOOST_CHECK_EQUAL(error, ""); - - std::string conf; - for (Action action : conf_actions) { - const char* argstr = arg(action); - if (!argstr) break; - desc += " "; - desc += argstr + 1; - conf += argstr + 1; - conf += "\n"; - } - std::istringstream conf_stream(conf); - BOOST_CHECK(parser.ReadConfigStream(conf_stream, "filepath", error)); - BOOST_CHECK_EQUAL(error, ""); - - desc += " || "; - try { - desc += parser.GetChainName(); - } catch (const std::runtime_error& e) { - desc += "error: "; - desc += e.what(); - } - desc += "\n"; - - out_sha.Write(MakeUCharSpan(desc)); - if (out_file) { - BOOST_REQUIRE(fwrite(desc.data(), 1, desc.size(), out_file) == desc.size()); - } - }); - - if (out_file) { - if (fclose(out_file)) throw std::system_error(errno, std::generic_category(), "fclose failed"); - out_file = nullptr; - } - - unsigned char out_sha_bytes[CSHA256::OUTPUT_SIZE]; - out_sha.Finalize(out_sha_bytes); - std::string out_sha_hex = HexStr(out_sha_bytes); - - // If check below fails, should manually dump the results with: - // - // CHAIN_MERGE_TEST_OUT=results.txt ./test_bitcoin --run_test=util_tests/util_ChainMerge - // - // And verify diff against previous results to make sure the changes are expected. - // - // Results file is formatted like: - // - // <input> || <output> - BOOST_CHECK_EQUAL(out_sha_hex, "f263493e300023b6509963887444c41386f44b63bc30047eb8402e8c1144854c"); -} - -BOOST_AUTO_TEST_CASE(util_ReadWriteSettings) -{ - // Test writing setting. - TestArgsManager args1; - args1.ForceSetArg("-datadir", fs::PathToString(m_path_root)); - args1.LockSettings([&](util::Settings& settings) { settings.rw_settings["name"] = "value"; }); - args1.WriteSettingsFile(); - - // Test reading setting. - TestArgsManager args2; - args2.ForceSetArg("-datadir", fs::PathToString(m_path_root)); - args2.ReadSettingsFile(); - args2.LockSettings([&](util::Settings& settings) { BOOST_CHECK_EQUAL(settings.rw_settings["name"].get_str(), "value"); }); - - // Test error logging, and remove previously written setting. - { - ASSERT_DEBUG_LOG("Failed renaming settings file"); - fs::remove(args1.GetDataDirBase() / "settings.json"); - fs::create_directory(args1.GetDataDirBase() / "settings.json"); - args2.WriteSettingsFile(); - fs::remove(args1.GetDataDirBase() / "settings.json"); - } -} - BOOST_AUTO_TEST_CASE(util_FormatMoney) { BOOST_CHECK_EQUAL(FormatMoney(0), "0.00"); diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 347a967b33..f868c0d4e6 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -89,7 +89,8 @@ BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) // After adding some blocks to the tip, best block should have changed. BOOST_CHECK(::g_best_block != curr_tip); - BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(m_node, m_path_root)); + BOOST_REQUIRE(CreateAndActivateUTXOSnapshot( + this, NoMalleation, /*reset_chainstate=*/ true)); // Ensure our active chain is the snapshot chainstate. BOOST_CHECK(WITH_LOCK(::cs_main, return chainman.IsSnapshotActive())); diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 9fcb7d315a..22b9af1201 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -10,6 +10,7 @@ #include <sync.h> #include <test/util/chainstate.h> #include <test/util/setup_common.h> +#include <timedata.h> #include <uint256.h> #include <validation.h> #include <validationinterface.h> @@ -63,7 +64,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // Create a snapshot-based chainstate. // const uint256 snapshot_blockhash = GetRandHash(); - Chainstate& c2 = WITH_LOCK(::cs_main, return manager.InitializeChainstate( + Chainstate& c2 = WITH_LOCK(::cs_main, return manager.ActivateExistingSnapshot( &mempool, snapshot_blockhash)); chainstates.push_back(&c2); @@ -133,7 +134,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) // Create a snapshot-based chainstate. // - Chainstate& c2 = WITH_LOCK(cs_main, return manager.InitializeChainstate(&mempool, GetRandHash())); + Chainstate& c2 = WITH_LOCK(cs_main, return manager.ActivateExistingSnapshot(&mempool, GetRandHash())); chainstates.push_back(&c2); c2.InitCoinsDB( /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); @@ -154,162 +155,240 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) BOOST_CHECK_CLOSE(c2.m_coinsdb_cache_size_bytes, max_cache * 0.95, 1); } -//! Test basic snapshot activation. -BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100Setup) -{ - ChainstateManager& chainman = *Assert(m_node.chainman); - - size_t initial_size; - size_t initial_total_coins{100}; - - // Make some initial assertions about the contents of the chainstate. +struct SnapshotTestSetup : TestChain100Setup { + // Run with coinsdb on the filesystem to support, e.g., moving invalidated + // chainstate dirs to "*_invalid". + // + // Note that this means the tests run considerably slower than in-memory DB + // tests, but we can't otherwise test this functionality since it relies on + // destructive filesystem operations. + SnapshotTestSetup() : TestChain100Setup{ + {}, + {}, + /*coins_db_in_memory=*/false, + /*block_tree_db_in_memory=*/false, + } { - LOCK(::cs_main); - CCoinsViewCache& ibd_coinscache = chainman.ActiveChainstate().CoinsTip(); - initial_size = ibd_coinscache.GetCacheSize(); - size_t total_coins{0}; - - for (CTransactionRef& txn : m_coinbase_txns) { - COutPoint op{txn->GetHash(), 0}; - BOOST_CHECK(ibd_coinscache.HaveCoin(op)); - total_coins++; - } - - BOOST_CHECK_EQUAL(total_coins, initial_total_coins); - BOOST_CHECK_EQUAL(initial_size, initial_total_coins); } - // Snapshot should refuse to load at this height. - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(m_node, m_path_root)); - BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash); - BOOST_CHECK(!chainman.SnapshotBlockhash()); - - // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can - // be found. - constexpr int snapshot_height = 110; - mineBlocks(10); - initial_size += 10; - initial_total_coins += 10; - - // Should not load malleated snapshots - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { - // A UTXO is missing but count is correct - metadata.m_coins_count -= 1; - - COutPoint outpoint; - Coin coin; - - auto_infile >> outpoint; - auto_infile >> coin; - })); - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { - // Coins count is larger than coins in file - metadata.m_coins_count += 1; - })); - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { - // Coins count is smaller than coins in file - metadata.m_coins_count -= 1; - })); - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { - // Wrong hash - metadata.m_base_blockhash = uint256::ZERO; - })); - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( - m_node, m_path_root, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { - // Wrong hash - metadata.m_base_blockhash = uint256::ONE; - })); - - BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(m_node, m_path_root)); - - // Ensure our active chain is the snapshot chainstate. - BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash->IsNull()); - BOOST_CHECK_EQUAL( - *chainman.ActiveChainstate().m_from_snapshot_blockhash, - *chainman.SnapshotBlockhash()); - - // Ensure that the genesis block was not marked assumed-valid. - BOOST_CHECK(WITH_LOCK(::cs_main, return !chainman.ActiveChain().Genesis()->IsAssumedValid())); - - const AssumeutxoData& au_data = *ExpectedAssumeutxo(snapshot_height, ::Params()); - const CBlockIndex* tip = WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip()); - - BOOST_CHECK_EQUAL(tip->nChainTx, au_data.nChainTx); - - // To be checked against later when we try loading a subsequent snapshot. - uint256 loaded_snapshot_blockhash{*chainman.SnapshotBlockhash()}; - - // Make some assertions about the both chainstates. These checks ensure the - // legacy chainstate hasn't changed and that the newly created chainstate - // reflects the expected content. + std::tuple<Chainstate*, Chainstate*> SetupSnapshot() { - LOCK(::cs_main); - int chains_tested{0}; + ChainstateManager& chainman = *Assert(m_node.chainman); - for (Chainstate* chainstate : chainman.GetAll()) { - BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); - CCoinsViewCache& coinscache = chainstate->CoinsTip(); + BOOST_CHECK(!chainman.IsSnapshotActive()); - // Both caches will be empty initially. - BOOST_CHECK_EQUAL((unsigned int)0, coinscache.GetCacheSize()); + { + LOCK(::cs_main); + BOOST_CHECK(!chainman.IsSnapshotValidated()); + BOOST_CHECK(!node::FindSnapshotChainstateDir()); + } + size_t initial_size; + size_t initial_total_coins{100}; + + // Make some initial assertions about the contents of the chainstate. + { + LOCK(::cs_main); + CCoinsViewCache& ibd_coinscache = chainman.ActiveChainstate().CoinsTip(); + initial_size = ibd_coinscache.GetCacheSize(); size_t total_coins{0}; for (CTransactionRef& txn : m_coinbase_txns) { COutPoint op{txn->GetHash(), 0}; - BOOST_CHECK(coinscache.HaveCoin(op)); + BOOST_CHECK(ibd_coinscache.HaveCoin(op)); total_coins++; } - BOOST_CHECK_EQUAL(initial_size , coinscache.GetCacheSize()); BOOST_CHECK_EQUAL(total_coins, initial_total_coins); - chains_tested++; + BOOST_CHECK_EQUAL(initial_size, initial_total_coins); } - BOOST_CHECK_EQUAL(chains_tested, 2); - } + Chainstate& validation_chainstate = chainman.ActiveChainstate(); + + // Snapshot should refuse to load at this height. + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(this)); + BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash); + BOOST_CHECK(!chainman.SnapshotBlockhash()); + + // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can + // be found. + constexpr int snapshot_height = 110; + mineBlocks(10); + initial_size += 10; + initial_total_coins += 10; + + // Should not load malleated snapshots + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { + // A UTXO is missing but count is correct + metadata.m_coins_count -= 1; + + COutPoint outpoint; + Coin coin; + + auto_infile >> outpoint; + auto_infile >> coin; + })); + + BOOST_CHECK(!node::FindSnapshotChainstateDir()); + + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { + // Coins count is larger than coins in file + metadata.m_coins_count += 1; + })); + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { + // Coins count is smaller than coins in file + metadata.m_coins_count -= 1; + })); + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { + // Wrong hash + metadata.m_base_blockhash = uint256::ZERO; + })); + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { + // Wrong hash + metadata.m_base_blockhash = uint256::ONE; + })); + + BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(this)); + BOOST_CHECK(fs::exists(*node::FindSnapshotChainstateDir())); + + // Ensure our active chain is the snapshot chainstate. + BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash->IsNull()); + BOOST_CHECK_EQUAL( + *chainman.ActiveChainstate().m_from_snapshot_blockhash, + *chainman.SnapshotBlockhash()); + + Chainstate& snapshot_chainstate = chainman.ActiveChainstate(); + + { + LOCK(::cs_main); - // Mine some new blocks on top of the activated snapshot chainstate. - constexpr size_t new_coins{100}; - mineBlocks(new_coins); // Defined in TestChain100Setup. + fs::path found = *node::FindSnapshotChainstateDir(); - { - LOCK(::cs_main); - size_t coins_in_active{0}; - size_t coins_in_background{0}; - size_t coins_missing_from_background{0}; + // Note: WriteSnapshotBaseBlockhash() is implicitly tested above. + BOOST_CHECK_EQUAL( + *node::ReadSnapshotBaseBlockhash(found), + *chainman.SnapshotBlockhash()); - for (Chainstate* chainstate : chainman.GetAll()) { - BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); - CCoinsViewCache& coinscache = chainstate->CoinsTip(); - bool is_background = chainstate != &chainman.ActiveChainstate(); + // Ensure that the genesis block was not marked assumed-valid. + BOOST_CHECK(!chainman.ActiveChain().Genesis()->IsAssumedValid()); + } - for (CTransactionRef& txn : m_coinbase_txns) { - COutPoint op{txn->GetHash(), 0}; - if (coinscache.HaveCoin(op)) { - (is_background ? coins_in_background : coins_in_active)++; - } else if (is_background) { - coins_missing_from_background++; + const AssumeutxoData& au_data = *ExpectedAssumeutxo(snapshot_height, ::Params()); + const CBlockIndex* tip = WITH_LOCK(chainman.GetMutex(), return chainman.ActiveTip()); + + BOOST_CHECK_EQUAL(tip->nChainTx, au_data.nChainTx); + + // To be checked against later when we try loading a subsequent snapshot. + uint256 loaded_snapshot_blockhash{*chainman.SnapshotBlockhash()}; + + // Make some assertions about the both chainstates. These checks ensure the + // legacy chainstate hasn't changed and that the newly created chainstate + // reflects the expected content. + { + LOCK(::cs_main); + int chains_tested{0}; + + for (Chainstate* chainstate : chainman.GetAll()) { + BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); + CCoinsViewCache& coinscache = chainstate->CoinsTip(); + + // Both caches will be empty initially. + BOOST_CHECK_EQUAL((unsigned int)0, coinscache.GetCacheSize()); + + size_t total_coins{0}; + + for (CTransactionRef& txn : m_coinbase_txns) { + COutPoint op{txn->GetHash(), 0}; + BOOST_CHECK(coinscache.HaveCoin(op)); + total_coins++; } + + BOOST_CHECK_EQUAL(initial_size , coinscache.GetCacheSize()); + BOOST_CHECK_EQUAL(total_coins, initial_total_coins); + chains_tested++; } + + BOOST_CHECK_EQUAL(chains_tested, 2); } - BOOST_CHECK_EQUAL(coins_in_active, initial_total_coins + new_coins); - BOOST_CHECK_EQUAL(coins_in_background, initial_total_coins); - BOOST_CHECK_EQUAL(coins_missing_from_background, new_coins); + // Mine some new blocks on top of the activated snapshot chainstate. + constexpr size_t new_coins{100}; + mineBlocks(new_coins); // Defined in TestChain100Setup. + + { + LOCK(::cs_main); + size_t coins_in_active{0}; + size_t coins_in_background{0}; + size_t coins_missing_from_background{0}; + + for (Chainstate* chainstate : chainman.GetAll()) { + BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); + CCoinsViewCache& coinscache = chainstate->CoinsTip(); + bool is_background = chainstate != &chainman.ActiveChainstate(); + + for (CTransactionRef& txn : m_coinbase_txns) { + COutPoint op{txn->GetHash(), 0}; + if (coinscache.HaveCoin(op)) { + (is_background ? coins_in_background : coins_in_active)++; + } else if (is_background) { + coins_missing_from_background++; + } + } + } + + BOOST_CHECK_EQUAL(coins_in_active, initial_total_coins + new_coins); + BOOST_CHECK_EQUAL(coins_in_background, initial_total_coins); + BOOST_CHECK_EQUAL(coins_missing_from_background, new_coins); + } + + // Snapshot should refuse to load after one has already loaded. + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(this)); + + // Snapshot blockhash should be unchanged. + BOOST_CHECK_EQUAL( + *chainman.ActiveChainstate().m_from_snapshot_blockhash, + loaded_snapshot_blockhash); + return std::make_tuple(&validation_chainstate, &snapshot_chainstate); } - // Snapshot should refuse to load after one has already loaded. - BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(m_node, m_path_root)); + // Simulate a restart of the node by flushing all state to disk, clearing the + // existing ChainstateManager, and unloading the block index. + // + // @returns a reference to the "restarted" ChainstateManager + ChainstateManager& SimulateNodeRestart() + { + ChainstateManager& chainman = *Assert(m_node.chainman); + + BOOST_TEST_MESSAGE("Simulating node restart"); + { + LOCK(::cs_main); + for (Chainstate* cs : chainman.GetAll()) { + cs->ForceFlushStateToDisk(); + } + chainman.ResetChainstates(); + BOOST_CHECK_EQUAL(chainman.GetAll().size(), 0); + const ChainstateManager::Options chainman_opts{ + .chainparams = ::Params(), + .adjusted_time_callback = GetAdjustedTime, + }; + // For robustness, ensure the old manager is destroyed before creating a + // new one. + m_node.chainman.reset(); + m_node.chainman.reset(new ChainstateManager(chainman_opts)); + } + return *Assert(m_node.chainman); + } +}; - // Snapshot blockhash should be unchanged. - BOOST_CHECK_EQUAL( - *chainman.ActiveChainstate().m_from_snapshot_blockhash, - loaded_snapshot_blockhash); +//! Test basic snapshot activation. +BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, SnapshotTestSetup) +{ + this->SetupSnapshot(); } //! Test LoadBlockIndex behavior when multiple chainstates are in use. @@ -374,7 +453,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) BOOST_CHECK_EQUAL(expected_assumed_valid, num_assumed_valid); Chainstate& cs2 = WITH_LOCK(::cs_main, - return chainman.InitializeChainstate(&mempool, GetRandHash())); + return chainman.ActivateExistingSnapshot(&mempool, GetRandHash())); reload_all_block_indexes(); @@ -390,4 +469,59 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.size(), num_indexes); } +//! Ensure that snapshot chainstates initialize properly when found on disk. +BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_init, SnapshotTestSetup) +{ + this->SetupSnapshot(); + + ChainstateManager& chainman = *Assert(m_node.chainman); + + fs::path snapshot_chainstate_dir = *node::FindSnapshotChainstateDir(); + BOOST_CHECK(fs::exists(snapshot_chainstate_dir)); + BOOST_CHECK_EQUAL(snapshot_chainstate_dir, gArgs.GetDataDirNet() / "chainstate_snapshot"); + + BOOST_CHECK(chainman.IsSnapshotActive()); + const uint256 snapshot_tip_hash = WITH_LOCK(chainman.GetMutex(), + return chainman.ActiveTip()->GetBlockHash()); + + auto all_chainstates = chainman.GetAll(); + BOOST_CHECK_EQUAL(all_chainstates.size(), 2); + + // Test that simulating a shutdown (resetting ChainstateManager) and then performing + // chainstate reinitializing successfully cleans up the background-validation + // chainstate data, and we end up with a single chainstate that is at tip. + ChainstateManager& chainman_restarted = this->SimulateNodeRestart(); + + BOOST_TEST_MESSAGE("Performing Load/Verify/Activate of chainstate"); + + // This call reinitializes the chainstates. + this->LoadVerifyActivateChainstate(); + + { + LOCK(chainman_restarted.GetMutex()); + BOOST_CHECK_EQUAL(chainman_restarted.GetAll().size(), 2); + BOOST_CHECK(chainman_restarted.IsSnapshotActive()); + BOOST_CHECK(!chainman_restarted.IsSnapshotValidated()); + + BOOST_CHECK_EQUAL(chainman_restarted.ActiveTip()->GetBlockHash(), snapshot_tip_hash); + BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 210); + } + + BOOST_TEST_MESSAGE( + "Ensure we can mine blocks on top of the initialized snapshot chainstate"); + mineBlocks(10); + { + LOCK(chainman_restarted.GetMutex()); + BOOST_CHECK_EQUAL(chainman_restarted.ActiveHeight(), 220); + + // Background chainstate should be unaware of new blocks on the snapshot + // chainstate. + for (Chainstate* cs : chainman_restarted.GetAll()) { + if (cs != &chainman_restarted.ActiveChainstate()) { + BOOST_CHECK_EQUAL(cs->m_chain.Height(), 110); + } + } + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/tinyformat.h b/src/tinyformat.h index bedaa14007..8eded00add 100644 --- a/src/tinyformat.h +++ b/src/tinyformat.h @@ -508,9 +508,9 @@ class FormatArg { public: FormatArg() - : m_value(NULL), - m_formatImpl(NULL), - m_toIntImpl(NULL) + : m_value(nullptr), + m_formatImpl(nullptr), + m_toIntImpl(nullptr) { } template<typename T> @@ -1005,7 +1005,8 @@ class FormatListN : public FormatList // Special 0-arg version - MSVC says zero-sized C array in struct is nonstandard template<> class FormatListN<0> : public FormatList { - public: FormatListN() : FormatList(0, 0) {} +public: + FormatListN() : FormatList(nullptr, 0) {} }; } // namespace detail diff --git a/src/txdb.h b/src/txdb.h index a04596f7bb..8c41e26f6a 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -9,6 +9,7 @@ #include <coins.h> #include <dbwrapper.h> #include <sync.h> +#include <fs.h> #include <memory> #include <optional> @@ -72,6 +73,9 @@ public: //! Dynamically alter the underlying leveldb cache size. void ResizeCache(size_t new_cache_size) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + //! @returns filesystem path to on-disk storage or std::nullopt if in memory. + std::optional<fs::path> StoragePath() { return m_db->StoragePath(); } }; /** Access to the block database (blocks/index/) */ diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 84ed2e9ef5..12e2d5f224 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -41,42 +41,6 @@ bool TestLockPointValidity(CChain& active_chain, const LockPoints& lp) return true; } -CTxMemPoolEntry::CTxMemPoolEntry(const CTransactionRef& tx, CAmount fee, - int64_t time, unsigned int entry_height, - bool spends_coinbase, int64_t sigops_cost, LockPoints lp) - : tx{tx}, - nFee{fee}, - nTxWeight(GetTransactionWeight(*tx)), - nUsageSize{RecursiveDynamicUsage(tx)}, - nTime{time}, - entryHeight{entry_height}, - spendsCoinbase{spends_coinbase}, - sigOpCost{sigops_cost}, - m_modified_fee{nFee}, - lockPoints{lp}, - nSizeWithDescendants{GetTxSize()}, - nModFeesWithDescendants{nFee}, - nSizeWithAncestors{GetTxSize()}, - nModFeesWithAncestors{nFee}, - nSigOpCostWithAncestors{sigOpCost} {} - -void CTxMemPoolEntry::UpdateModifiedFee(CAmount fee_diff) -{ - nModFeesWithDescendants = SaturatingAdd(nModFeesWithDescendants, fee_diff); - nModFeesWithAncestors = SaturatingAdd(nModFeesWithAncestors, fee_diff); - m_modified_fee = SaturatingAdd(m_modified_fee, fee_diff); -} - -void CTxMemPoolEntry::UpdateLockPoints(const LockPoints& lp) -{ - lockPoints = lp; -} - -size_t CTxMemPoolEntry::GetTxSize() const -{ - return GetVirtualTransactionSize(nTxWeight, sigOpCost, ::nBytesPerSigOp); -} - void CTxMemPool::UpdateForDescendants(txiter updateIt, cacheMap& cachedDescendants, const std::set<uint256>& setExclude, std::set<uint256>& descendants_to_remove) { @@ -1182,3 +1146,17 @@ void CTxMemPool::SetLoadTried(bool load_tried) LOCK(cs); m_load_tried = load_tried; } + + +const std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept +{ + switch (r) { + case MemPoolRemovalReason::EXPIRY: return "expiry"; + case MemPoolRemovalReason::SIZELIMIT: return "sizelimit"; + case MemPoolRemovalReason::REORG: return "reorg"; + case MemPoolRemovalReason::BLOCK: return "block"; + case MemPoolRemovalReason::CONFLICT: return "conflict"; + case MemPoolRemovalReason::REPLACED: return "replaced"; + } + assert(false); +} diff --git a/src/txmempool.h b/src/txmempool.h index 4afaac0506..dd28a84c23 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -20,6 +20,7 @@ #include <coins.h> #include <consensus/amount.h> #include <indirectmap.h> +#include <kernel/mempool_entry.h> #include <policy/feerate.h> #include <policy/packages.h> #include <primitives/transaction.h> @@ -41,132 +42,11 @@ extern RecursiveMutex cs_main; /** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */ static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; -struct LockPoints { - // Will be set to the blockchain height and median time past - // values that would be necessary to satisfy all relative locktime - // constraints (BIP68) of this tx given our view of block chain history - int height{0}; - int64_t time{0}; - // As long as the current chain descends from the highest height block - // containing one of the inputs used in the calculation, then the cached - // values are still valid even after a reorg. - CBlockIndex* maxInputBlock{nullptr}; -}; - /** * Test whether the LockPoints height and time are still valid on the current chain */ bool TestLockPointValidity(CChain& active_chain, const LockPoints& lp) EXCLUSIVE_LOCKS_REQUIRED(cs_main); -struct CompareIteratorByHash { - // SFINAE for T where T is either a pointer type (e.g., a txiter) or a reference_wrapper<T> - // (e.g. a wrapped CTxMemPoolEntry&) - template <typename T> - bool operator()(const std::reference_wrapper<T>& a, const std::reference_wrapper<T>& b) const - { - return a.get().GetTx().GetHash() < b.get().GetTx().GetHash(); - } - template <typename T> - bool operator()(const T& a, const T& b) const - { - return a->GetTx().GetHash() < b->GetTx().GetHash(); - } -}; - -/** \class CTxMemPoolEntry - * - * CTxMemPoolEntry stores data about the corresponding transaction, as well - * as data about all in-mempool transactions that depend on the transaction - * ("descendant" transactions). - * - * When a new entry is added to the mempool, we update the descendant state - * (nCountWithDescendants, nSizeWithDescendants, and nModFeesWithDescendants) for - * all ancestors of the newly added transaction. - * - */ - -class CTxMemPoolEntry -{ -public: - typedef std::reference_wrapper<const CTxMemPoolEntry> CTxMemPoolEntryRef; - // two aliases, should the types ever diverge - typedef std::set<CTxMemPoolEntryRef, CompareIteratorByHash> Parents; - typedef std::set<CTxMemPoolEntryRef, CompareIteratorByHash> Children; - -private: - const CTransactionRef tx; - mutable Parents m_parents; - mutable Children m_children; - const CAmount nFee; //!< Cached to avoid expensive parent-transaction lookups - const size_t nTxWeight; //!< ... and avoid recomputing tx weight (also used for GetTxSize()) - const size_t nUsageSize; //!< ... and total memory usage - const int64_t nTime; //!< Local time when entering the mempool - const unsigned int entryHeight; //!< Chain height when entering the mempool - const bool spendsCoinbase; //!< keep track of transactions that spend a coinbase - const int64_t sigOpCost; //!< Total sigop cost - CAmount m_modified_fee; //!< Used for determining the priority of the transaction for mining in a block - LockPoints lockPoints; //!< Track the height and time at which tx was final - - // Information about descendants of this transaction that are in the - // mempool; if we remove this transaction we must remove all of these - // descendants as well. - uint64_t nCountWithDescendants{1}; //!< number of descendant transactions - uint64_t nSizeWithDescendants; //!< ... and size - CAmount nModFeesWithDescendants; //!< ... and total fees (all including us) - - // Analogous statistics for ancestor transactions - uint64_t nCountWithAncestors{1}; - uint64_t nSizeWithAncestors; - CAmount nModFeesWithAncestors; - int64_t nSigOpCostWithAncestors; - -public: - CTxMemPoolEntry(const CTransactionRef& tx, CAmount fee, - int64_t time, unsigned int entry_height, - bool spends_coinbase, - int64_t sigops_cost, LockPoints lp); - - const CTransaction& GetTx() const { return *this->tx; } - CTransactionRef GetSharedTx() const { return this->tx; } - const CAmount& GetFee() const { return nFee; } - size_t GetTxSize() const; - size_t GetTxWeight() const { return nTxWeight; } - std::chrono::seconds GetTime() const { return std::chrono::seconds{nTime}; } - unsigned int GetHeight() const { return entryHeight; } - int64_t GetSigOpCost() const { return sigOpCost; } - CAmount GetModifiedFee() const { return m_modified_fee; } - size_t DynamicMemoryUsage() const { return nUsageSize; } - const LockPoints& GetLockPoints() const { return lockPoints; } - - // Adjusts the descendant state. - void UpdateDescendantState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount); - // Adjusts the ancestor state - void UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps); - // Updates the modified fees with descendants/ancestors. - void UpdateModifiedFee(CAmount fee_diff); - // Update the LockPoints after a reorg - void UpdateLockPoints(const LockPoints& lp); - - uint64_t GetCountWithDescendants() const { return nCountWithDescendants; } - uint64_t GetSizeWithDescendants() const { return nSizeWithDescendants; } - CAmount GetModFeesWithDescendants() const { return nModFeesWithDescendants; } - - bool GetSpendsCoinbase() const { return spendsCoinbase; } - - uint64_t GetCountWithAncestors() const { return nCountWithAncestors; } - uint64_t GetSizeWithAncestors() const { return nSizeWithAncestors; } - CAmount GetModFeesWithAncestors() const { return nModFeesWithAncestors; } - int64_t GetSigOpCostWithAncestors() const { return nSigOpCostWithAncestors; } - - const Parents& GetMemPoolParentsConst() const { return m_parents; } - const Children& GetMemPoolChildrenConst() const { return m_children; } - Parents& GetMemPoolParents() const { return m_parents; } - Children& GetMemPoolChildren() const { return m_children; } - - mutable size_t vTxHashesIdx; //!< Index in mempool's vTxHashes - mutable Epoch::Marker m_epoch_marker; //!< epoch when last touched, useful for graph algorithms -}; - // extracts a transaction hash from CTxMemPoolEntry or CTransactionRef struct mempoolentry_txid { @@ -355,6 +235,8 @@ enum class MemPoolRemovalReason { REPLACED, //!< Removed for replacement }; +const std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept; + /** * CTxMemPool stores valid-according-to-the-current-best-chain transactions * that may be included in the next block. diff --git a/src/txorphanage.cpp b/src/txorphanage.cpp index 69ae8ea582..b0b71e135c 100644 --- a/src/txorphanage.cpp +++ b/src/txorphanage.cpp @@ -15,11 +15,10 @@ static constexpr int64_t ORPHAN_TX_EXPIRE_TIME = 20 * 60; /** Minimum time between orphan transactions expire time checks in seconds */ static constexpr int64_t ORPHAN_TX_EXPIRE_INTERVAL = 5 * 60; -RecursiveMutex g_cs_orphans; bool TxOrphanage::AddTx(const CTransactionRef& tx, NodeId peer) { - AssertLockHeld(g_cs_orphans); + LOCK(m_mutex); const uint256& hash = tx->GetHash(); if (m_orphans.count(hash)) @@ -55,7 +54,13 @@ bool TxOrphanage::AddTx(const CTransactionRef& tx, NodeId peer) int TxOrphanage::EraseTx(const uint256& txid) { - AssertLockHeld(g_cs_orphans); + LOCK(m_mutex); + return _EraseTx(txid); +} + +int TxOrphanage::_EraseTx(const uint256& txid) +{ + AssertLockHeld(m_mutex); std::map<uint256, OrphanTx>::iterator it = m_orphans.find(txid); if (it == m_orphans.end()) return 0; @@ -87,7 +92,9 @@ int TxOrphanage::EraseTx(const uint256& txid) void TxOrphanage::EraseForPeer(NodeId peer) { - AssertLockHeld(g_cs_orphans); + LOCK(m_mutex); + + m_peer_work_set.erase(peer); int nErased = 0; std::map<uint256, OrphanTx>::iterator iter = m_orphans.begin(); @@ -96,7 +103,7 @@ void TxOrphanage::EraseForPeer(NodeId peer) std::map<uint256, OrphanTx>::iterator maybeErase = iter++; // increment to avoid iterator becoming invalid if (maybeErase->second.fromPeer == peer) { - nErased += EraseTx(maybeErase->second.tx->GetHash()); + nErased += _EraseTx(maybeErase->second.tx->GetHash()); } } if (nErased > 0) LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx from peer=%d\n", nErased, peer); @@ -104,7 +111,7 @@ void TxOrphanage::EraseForPeer(NodeId peer) void TxOrphanage::LimitOrphans(unsigned int max_orphans) { - AssertLockHeld(g_cs_orphans); + LOCK(m_mutex); unsigned int nEvicted = 0; static int64_t nNextSweep; @@ -118,7 +125,7 @@ void TxOrphanage::LimitOrphans(unsigned int max_orphans) { std::map<uint256, OrphanTx>::iterator maybeErase = iter++; if (maybeErase->second.nTimeExpire <= nNow) { - nErased += EraseTx(maybeErase->second.tx->GetHash()); + nErased += _EraseTx(maybeErase->second.tx->GetHash()); } else { nMinExpTime = std::min(maybeErase->second.nTimeExpire, nMinExpTime); } @@ -132,15 +139,19 @@ void TxOrphanage::LimitOrphans(unsigned int max_orphans) { // Evict a random orphan: size_t randompos = rng.randrange(m_orphan_list.size()); - EraseTx(m_orphan_list[randompos]->first); + _EraseTx(m_orphan_list[randompos]->first); ++nEvicted; } if (nEvicted > 0) LogPrint(BCLog::MEMPOOL, "orphanage overflow, removed %u tx\n", nEvicted); } -void TxOrphanage::AddChildrenToWorkSet(const CTransaction& tx, std::set<uint256>& orphan_work_set) const +void TxOrphanage::AddChildrenToWorkSet(const CTransaction& tx, NodeId peer) { - AssertLockHeld(g_cs_orphans); + LOCK(m_mutex); + + // Get this peer's work set, emplacing an empty set it didn't exist + std::set<uint256>& orphan_work_set = m_peer_work_set.try_emplace(peer).first->second; + for (unsigned int i = 0; i < tx.vout.size(); i++) { const auto it_by_prev = m_outpoint_to_orphan_it.find(COutPoint(tx.GetHash(), i)); if (it_by_prev != m_outpoint_to_orphan_it.end()) { @@ -153,7 +164,7 @@ void TxOrphanage::AddChildrenToWorkSet(const CTransaction& tx, std::set<uint256> bool TxOrphanage::HaveTx(const GenTxid& gtxid) const { - LOCK(g_cs_orphans); + LOCK(m_mutex); if (gtxid.IsWtxid()) { return m_wtxid_to_orphan_it.count(gtxid.GetHash()); } else { @@ -161,18 +172,32 @@ bool TxOrphanage::HaveTx(const GenTxid& gtxid) const } } -std::pair<CTransactionRef, NodeId> TxOrphanage::GetTx(const uint256& txid) const +CTransactionRef TxOrphanage::GetTxToReconsider(NodeId peer, NodeId& originator, bool& more) { - AssertLockHeld(g_cs_orphans); - - const auto it = m_orphans.find(txid); - if (it == m_orphans.end()) return {nullptr, -1}; - return {it->second.tx, it->second.fromPeer}; + LOCK(m_mutex); + + auto work_set_it = m_peer_work_set.find(peer); + if (work_set_it != m_peer_work_set.end()) { + auto& work_set = work_set_it->second; + while (!work_set.empty()) { + uint256 txid = *work_set.begin(); + work_set.erase(work_set.begin()); + + const auto orphan_it = m_orphans.find(txid); + if (orphan_it != m_orphans.end()) { + more = !work_set.empty(); + originator = orphan_it->second.fromPeer; + return orphan_it->second.tx; + } + } + } + more = false; + return nullptr; } void TxOrphanage::EraseForBlock(const CBlock& block) { - LOCK(g_cs_orphans); + LOCK(m_mutex); std::vector<uint256> vOrphanErase; @@ -195,7 +220,7 @@ void TxOrphanage::EraseForBlock(const CBlock& block) if (vOrphanErase.size()) { int nErased = 0; for (const uint256& orphanHash : vOrphanErase) { - nErased += EraseTx(orphanHash); + nErased += _EraseTx(orphanHash); } LogPrint(BCLog::MEMPOOL, "Erased %d orphan tx included or conflicted by block\n", nErased); } diff --git a/src/txorphanage.h b/src/txorphanage.h index 9363e6f733..551502d325 100644 --- a/src/txorphanage.h +++ b/src/txorphanage.h @@ -10,8 +10,8 @@ #include <primitives/transaction.h> #include <sync.h> -/** Guards orphan transactions and extra txs for compact blocks */ -extern RecursiveMutex g_cs_orphans; +#include <map> +#include <set> /** A class to track orphan transactions (failed on TX_MISSING_INPUTS) * Since we cannot distinguish orphans from bad transactions with @@ -21,40 +21,46 @@ extern RecursiveMutex g_cs_orphans; class TxOrphanage { public: /** Add a new orphan transaction */ - bool AddTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans); + bool AddTx(const CTransactionRef& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); /** Check if we already have an orphan transaction (by txid or wtxid) */ - bool HaveTx(const GenTxid& gtxid) const LOCKS_EXCLUDED(::g_cs_orphans); - - /** Get an orphan transaction and its originating peer - * (Transaction ref will be nullptr if not found) + bool HaveTx(const GenTxid& gtxid) const EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); + + /** Extract a transaction from a peer's work set + * Returns nullptr and sets more to false if there are no transactions + * to work on. Otherwise returns the transaction reference, removes + * the transaction from the work set, and populates its arguments with + * the originating peer, and whether there are more orphans for this peer + * to work on after this tx. */ - std::pair<CTransactionRef, NodeId> GetTx(const uint256& txid) const EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans); + CTransactionRef GetTxToReconsider(NodeId peer, NodeId& originator, bool& more) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); /** Erase an orphan by txid */ - int EraseTx(const uint256& txid) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans); + int EraseTx(const uint256& txid) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); /** Erase all orphans announced by a peer (eg, after that peer disconnects) */ - void EraseForPeer(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans); + void EraseForPeer(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); /** Erase all orphans included in or invalidated by a new block */ - void EraseForBlock(const CBlock& block) LOCKS_EXCLUDED(::g_cs_orphans); + void EraseForBlock(const CBlock& block) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); /** Limit the orphanage to the given maximum */ - void LimitOrphans(unsigned int max_orphans) EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans); + void LimitOrphans(unsigned int max_orphans) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); - /** Add any orphans that list a particular tx as a parent into a peer's work set - * (ie orphans that may have found their final missing parent, and so should be reconsidered for the mempool) */ - void AddChildrenToWorkSet(const CTransaction& tx, std::set<uint256>& orphan_work_set) const EXCLUSIVE_LOCKS_REQUIRED(g_cs_orphans); + /** Add any orphans that list a particular tx as a parent into a peer's work set */ + void AddChildrenToWorkSet(const CTransaction& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); /** Return how many entries exist in the orphange */ - size_t Size() LOCKS_EXCLUDED(::g_cs_orphans) + size_t Size() EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) { - LOCK(::g_cs_orphans); + LOCK(m_mutex); return m_orphans.size(); } protected: + /** Guards orphan transactions */ + mutable Mutex m_mutex; + struct OrphanTx { CTransactionRef tx; NodeId fromPeer; @@ -64,7 +70,10 @@ protected: /** Map from txid to orphan transaction record. Limited by * -maxorphantx/DEFAULT_MAX_ORPHAN_TRANSACTIONS */ - std::map<uint256, OrphanTx> m_orphans GUARDED_BY(g_cs_orphans); + std::map<uint256, OrphanTx> m_orphans GUARDED_BY(m_mutex); + + /** Which peer provided a parent tx of orphans that need to be reconsidered */ + std::map<NodeId, std::set<uint256>> m_peer_work_set GUARDED_BY(m_mutex); using OrphanMap = decltype(m_orphans); @@ -79,14 +88,17 @@ protected: /** Index from the parents' COutPoint into the m_orphans. Used * to remove orphan transactions from the m_orphans */ - std::map<COutPoint, std::set<OrphanMap::iterator, IteratorComparator>> m_outpoint_to_orphan_it GUARDED_BY(g_cs_orphans); + std::map<COutPoint, std::set<OrphanMap::iterator, IteratorComparator>> m_outpoint_to_orphan_it GUARDED_BY(m_mutex); /** Orphan transactions in vector for quick random eviction */ - std::vector<OrphanMap::iterator> m_orphan_list GUARDED_BY(g_cs_orphans); + std::vector<OrphanMap::iterator> m_orphan_list GUARDED_BY(m_mutex); /** Index from wtxid into the m_orphans to lookup orphan * transactions using their witness ids. */ - std::map<uint256, OrphanMap::iterator> m_wtxid_to_orphan_it GUARDED_BY(g_cs_orphans); + std::map<uint256, OrphanMap::iterator> m_wtxid_to_orphan_it GUARDED_BY(m_mutex); + + /** Erase an orphan by txid */ + int _EraseTx(const uint256& txid) EXCLUSIVE_LOCKS_REQUIRED(m_mutex); }; #endif // BITCOIN_TXORPHANAGE_H diff --git a/src/univalue/include/univalue.h b/src/univalue/include/univalue.h index 850d0a1cbc..16853260b8 100644 --- a/src/univalue/include/univalue.h +++ b/src/univalue/include/univalue.h @@ -25,10 +25,7 @@ public: }; UniValue() { typ = VNULL; } - UniValue(UniValue::VType initialType, const std::string& initialStr = "") { - typ = initialType; - val = initialStr; - } + UniValue(UniValue::VType type, std::string str = {}) : typ{type}, val{std::move(str)} {} template <typename Ref, typename T = std::remove_cv_t<std::remove_reference_t<Ref>>, std::enable_if_t<std::is_floating_point_v<T> || // setFloat std::is_same_v<bool, T> || // setBool @@ -54,12 +51,12 @@ public: void setNull(); void setBool(bool val); - void setNumStr(const std::string& val); + void setNumStr(std::string str); void setInt(uint64_t val); void setInt(int64_t val); void setInt(int val_) { return setInt(int64_t{val_}); } void setFloat(double val); - void setStr(const std::string& val); + void setStr(std::string str); void setArray(); void setObject(); @@ -69,7 +66,6 @@ public: size_t size() const { return values.size(); } - bool getBool() const { return isTrue(); } void getObjMap(std::map<std::string,UniValue>& kv) const; bool checkObject(const std::map<std::string,UniValue::VType>& memberTypes) const; const UniValue& operator[](const std::string& key) const; diff --git a/src/univalue/lib/univalue.cpp b/src/univalue/lib/univalue.cpp index 4448981d3e..5aa39edb75 100644 --- a/src/univalue/lib/univalue.cpp +++ b/src/univalue/lib/univalue.cpp @@ -44,15 +44,15 @@ static bool validNumStr(const std::string& s) return (tt == JTOK_NUMBER); } -void UniValue::setNumStr(const std::string& val_) +void UniValue::setNumStr(std::string str) { - if (!validNumStr(val_)) { - throw std::runtime_error{"The string '" + val_ + "' is not a valid JSON number"}; + if (!validNumStr(str)) { + throw std::runtime_error{"The string '" + str + "' is not a valid JSON number"}; } clear(); typ = VNUM; - val = val_; + val = std::move(str); } void UniValue::setInt(uint64_t val_) @@ -82,11 +82,11 @@ void UniValue::setFloat(double val_) return setNumStr(oss.str()); } -void UniValue::setStr(const std::string& val_) +void UniValue::setStr(std::string str) { clear(); typ = VSTR; - val = val_; + val = std::move(str); } void UniValue::setArray() diff --git a/src/univalue/lib/univalue_get.cpp b/src/univalue/lib/univalue_get.cpp index 5c58f388dd..037449ca08 100644 --- a/src/univalue/lib/univalue_get.cpp +++ b/src/univalue/lib/univalue_get.cpp @@ -60,7 +60,7 @@ const std::vector<UniValue>& UniValue::getValues() const bool UniValue::get_bool() const { checkType(VBOOL); - return getBool(); + return isTrue(); } const std::string& UniValue::get_str() const diff --git a/src/univalue/test/object.cpp b/src/univalue/test/object.cpp index 94d7343ff3..eeaadae3e2 100644 --- a/src/univalue/test/object.cpp +++ b/src/univalue/test/object.cpp @@ -11,6 +11,7 @@ #include <memory> #include <stdexcept> #include <string> +#include <string_view> #include <vector> #define BOOST_CHECK(expr) assert(expr) @@ -160,6 +161,14 @@ void univalue_set() BOOST_CHECK(v.isStr()); BOOST_CHECK_EQUAL(v.getValStr(), "zum"); + { + std::string_view sv{"ab\0c", 4}; + UniValue j{sv}; + BOOST_CHECK(j.isStr()); + BOOST_CHECK_EQUAL(j.getValStr(), sv); + BOOST_CHECK_EQUAL(j.write(), "\"ab\\u0000c\""); + } + v.setFloat(-1.01); BOOST_CHECK(v.isNum()); BOOST_CHECK_EQUAL(v.getValStr(), "-1.01"); @@ -184,13 +193,13 @@ void univalue_set() BOOST_CHECK_EQUAL(v.isBool(), true); BOOST_CHECK_EQUAL(v.isTrue(), false); BOOST_CHECK_EQUAL(v.isFalse(), true); - BOOST_CHECK_EQUAL(v.getBool(), false); + BOOST_CHECK_EQUAL(v.get_bool(), false); v.setBool(true); BOOST_CHECK_EQUAL(v.isBool(), true); BOOST_CHECK_EQUAL(v.isTrue(), true); BOOST_CHECK_EQUAL(v.isFalse(), false); - BOOST_CHECK_EQUAL(v.getBool(), true); + BOOST_CHECK_EQUAL(v.get_bool(), true); BOOST_CHECK_THROW(v.setNumStr("zombocom"), std::runtime_error); diff --git a/src/util/check.cpp b/src/util/check.cpp index 2a9f885560..34b9d376a7 100644 --- a/src/util/check.cpp +++ b/src/util/check.cpp @@ -4,8 +4,30 @@ #include <util/check.h> +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <clientversion.h> #include <tinyformat.h> +#include <cstdio> +#include <cstdlib> +#include <string> + +std::string StrFormatInternalBug(const char* msg, const char* file, int line, const char* func) +{ + return strprintf("Internal bug detected: \"%s\"\n%s:%d (%s)\n" + "%s %s\n" + "Please report this issue here: %s\n", + msg, file, line, func, PACKAGE_NAME, FormatFullVersion(), PACKAGE_BUGREPORT); +} + +NonFatalCheckError::NonFatalCheckError(const char* msg, const char* file, int line, const char* func) + : std::runtime_error{StrFormatInternalBug(msg, file, line, func)} +{ +} + void assertion_fail(const char* file, int line, const char* func, const char* assertion) { auto str = strprintf("%s:%s %s: Assertion `%s' failed.\n", file, line, func, assertion); diff --git a/src/util/check.h b/src/util/check.h index aca957925a..b791944502 100644 --- a/src/util/check.h +++ b/src/util/check.h @@ -5,30 +5,27 @@ #ifndef BITCOIN_UTIL_CHECK_H #define BITCOIN_UTIL_CHECK_H -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - -#include <tinyformat.h> +#include <attributes.h> #include <stdexcept> +#include <utility> + +std::string StrFormatInternalBug(const char* msg, const char* file, int line, const char* func); class NonFatalCheckError : public std::runtime_error { - using std::runtime_error::runtime_error; +public: + NonFatalCheckError(const char* msg, const char* file, int line, const char* func); }; -#define format_internal_error(msg, file, line, func, report) \ - strprintf("Internal bug detected: \"%s\"\n%s:%d (%s)\nPlease report this issue here: %s\n", \ - msg, file, line, func, report) +#define STR_INTERNAL_BUG(msg) StrFormatInternalBug((msg), __FILE__, __LINE__, __func__) /** Helper for CHECK_NONFATAL() */ template <typename T> -T&& inline_check_non_fatal(T&& val, const char* file, int line, const char* func, const char* assertion) +T&& inline_check_non_fatal(LIFETIMEBOUND T&& val, const char* file, int line, const char* func, const char* assertion) { - if (!(val)) { - throw NonFatalCheckError( - format_internal_error(assertion, file, line, func, PACKAGE_BUGREPORT)); + if (!val) { + throw NonFatalCheckError{assertion, file, line, func}; } return std::forward<T>(val); } @@ -56,7 +53,7 @@ void assertion_fail(const char* file, int line, const char* func, const char* as /** Helper for Assert()/Assume() */ template <bool IS_ASSERT, typename T> -T&& inline_assertion_check(T&& val, [[maybe_unused]] const char* file, [[maybe_unused]] int line, [[maybe_unused]] const char* func, [[maybe_unused]] const char* assertion) +T&& inline_assertion_check(LIFETIMEBOUND T&& val, [[maybe_unused]] const char* file, [[maybe_unused]] int line, [[maybe_unused]] const char* func, [[maybe_unused]] const char* assertion) { if constexpr (IS_ASSERT #ifdef ABORT_ON_FAILED_ASSUME @@ -87,11 +84,9 @@ T&& inline_assertion_check(T&& val, [[maybe_unused]] const char* file, [[maybe_u /** * NONFATAL_UNREACHABLE() is a macro that is used to mark unreachable code. It throws a NonFatalCheckError. - * This is used to mark code that is not yet implemented or is not yet reachable. */ #define NONFATAL_UNREACHABLE() \ throw NonFatalCheckError( \ - format_internal_error("Unreachable code reached (non-fatal)", \ - __FILE__, __LINE__, __func__, PACKAGE_BUGREPORT)) + "Unreachable code reached (non-fatal)", __FILE__, __LINE__, __func__) #endif // BITCOIN_UTIL_CHECK_H diff --git a/src/util/error.cpp b/src/util/error.cpp index 33a35a6d59..390cb6c11b 100644 --- a/src/util/error.cpp +++ b/src/util/error.cpp @@ -49,6 +49,11 @@ bilingual_str ResolveErrMsg(const std::string& optname, const std::string& strBi return strprintf(_("Cannot resolve -%s address: '%s'"), optname, strBind); } +bilingual_str InvalidPortErrMsg(const std::string& optname, const std::string& invalid_value) +{ + return strprintf(_("Invalid port specified in %s: '%s'"), optname, invalid_value); +} + bilingual_str AmountHighWarn(const std::string& optname) { return strprintf(_("%s is set very high!"), optname); diff --git a/src/util/error.h b/src/util/error.h index 0429de651a..27916501f0 100644 --- a/src/util/error.h +++ b/src/util/error.h @@ -39,6 +39,8 @@ bilingual_str TransactionErrorString(const TransactionError error); bilingual_str ResolveErrMsg(const std::string& optname, const std::string& strBind); +bilingual_str InvalidPortErrMsg(const std::string& optname, const std::string& strPort); + bilingual_str AmountHighWarn(const std::string& optname); bilingual_str AmountErrMsg(const std::string& optname, const std::string& strValue); diff --git a/src/util/sock.cpp b/src/util/sock.cpp index 125dbc7f18..e3d30c24b3 100644 --- a/src/util/sock.cpp +++ b/src/util/sock.cpp @@ -4,11 +4,11 @@ #include <compat/compat.h> #include <logging.h> -#include <threadinterrupt.h> #include <tinyformat.h> #include <util/sock.h> #include <util/syserror.h> #include <util/system.h> +#include <util/threadinterrupt.h> #include <util/time.h> #include <memory> @@ -117,6 +117,34 @@ int Sock::GetSockName(sockaddr* name, socklen_t* name_len) const return getsockname(m_socket, name, name_len); } +bool Sock::SetNonBlocking() const +{ +#ifdef WIN32 + u_long on{1}; + if (ioctlsocket(m_socket, FIONBIO, &on) == SOCKET_ERROR) { + return false; + } +#else + const int flags{fcntl(m_socket, F_GETFL, 0)}; + if (flags == SOCKET_ERROR) { + return false; + } + if (fcntl(m_socket, F_SETFL, flags | O_NONBLOCK) == SOCKET_ERROR) { + return false; + } +#endif + return true; +} + +bool Sock::IsSelectable() const +{ +#if defined(USE_POLL) || defined(WIN32) + return true; +#else + return m_socket < FD_SETSIZE; +#endif +} + bool Sock::Wait(std::chrono::milliseconds timeout, Event requested, Event* occurred) const { // We need a `shared_ptr` owning `this` for `WaitMany()`, but don't want @@ -185,10 +213,10 @@ bool Sock::WaitMany(std::chrono::milliseconds timeout, EventsPerSock& events_per SOCKET socket_max{0}; for (const auto& [sock, events] : events_per_sock) { - const auto& s = sock->m_socket; - if (!IsSelectableSocket(s)) { + if (!sock->IsSelectable()) { return false; } + const auto& s = sock->m_socket; if (events.requested & RECV) { FD_SET(s, &recv); } diff --git a/src/util/sock.h b/src/util/sock.h index 38a7dc80d6..a7347df8ee 100644 --- a/src/util/sock.h +++ b/src/util/sock.h @@ -6,7 +6,7 @@ #define BITCOIN_UTIL_SOCK_H #include <compat/compat.h> -#include <threadinterrupt.h> +#include <util/threadinterrupt.h> #include <util/time.h> #include <chrono> @@ -133,6 +133,18 @@ public: */ [[nodiscard]] virtual int GetSockName(sockaddr* name, socklen_t* name_len) const; + /** + * Set the non-blocking option on the socket. + * @return true if set successfully + */ + [[nodiscard]] virtual bool SetNonBlocking() const; + + /** + * Check if the underlying socket can be used for `select(2)` (or the `Wait()` method). + * @return true if selectable + */ + [[nodiscard]] virtual bool IsSelectable() const; + using Event = uint8_t; /** diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index b5ac151374..e28ca8e73a 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -97,8 +97,9 @@ std::vector<Byte> ParseHex(std::string_view str) template std::vector<std::byte> ParseHex(std::string_view); template std::vector<uint8_t> ParseHex(std::string_view); -void SplitHostPort(std::string_view in, uint16_t& portOut, std::string& hostOut) +bool SplitHostPort(std::string_view in, uint16_t& portOut, std::string& hostOut) { + bool valid = false; size_t colon = in.find_last_of(':'); // if a : is found, and it either follows a [...], or no other : is in the string, treat it as port separator bool fHaveColon = colon != in.npos; @@ -109,13 +110,18 @@ void SplitHostPort(std::string_view in, uint16_t& portOut, std::string& hostOut) if (ParseUInt16(in.substr(colon + 1), &n)) { in = in.substr(0, colon); portOut = n; + valid = (portOut != 0); } + } else { + valid = true; } if (in.size() > 0 && in[0] == '[' && in[in.size() - 1] == ']') { hostOut = in.substr(1, in.size() - 2); } else { hostOut = in; } + + return valid; } std::string EncodeBase64(Span<const unsigned char> input) diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 14867b21b2..94bc6cc2f3 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -88,7 +88,16 @@ std::string EncodeBase32(Span<const unsigned char> input, bool pad = true); */ std::string EncodeBase32(std::string_view str, bool pad = true); -void SplitHostPort(std::string_view in, uint16_t& portOut, std::string& hostOut); +/** + * Splits socket address string into host string and port value. + * Validates port value. + * + * @param[in] in The socket address string to split. + * @param[out] portOut Port-portion of the input, if found and parsable. + * @param[out] hostOut Host-portion of the input, if found. + * @return true if port-portion is absent or within its allowed range, otherwise false + */ +bool SplitHostPort(std::string_view in, uint16_t& portOut, std::string& hostOut); // LocaleIndependentAtoi is provided for backwards compatibility reasons. // diff --git a/src/util/system.cpp b/src/util/system.cpp index ce5d846eb9..d7a0793ea8 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -922,6 +922,20 @@ static bool GetConfigOptions(std::istream& stream, const std::string& filepath, return true; } +bool IsConfSupported(KeyInfo& key, std::string& error) { + if (key.name == "conf") { + error = "conf cannot be set in the configuration file; use includeconf= if you want to include additional config files"; + return false; + } + if (key.name == "reindex") { + // reindex can be set in a config file but it is strongly discouraged as this will cause the node to reindex on + // every restart. Allow the config but throw a warning + LogPrintf("Warning: reindex=1 is set in the configuration file, which will significantly slow down startup. Consider removing or commenting out this option for better performance, unless there is currently a condition which makes rebuilding the indexes necessary\n"); + return true; + } + return true; +} + bool ArgsManager::ReadConfigStream(std::istream& stream, const std::string& filepath, std::string& error, bool ignore_invalid_keys) { LOCK(cs_args); @@ -932,6 +946,7 @@ bool ArgsManager::ReadConfigStream(std::istream& stream, const std::string& file for (const std::pair<std::string, std::string>& option : options) { KeyInfo key = InterpretKey(option.first); std::optional<unsigned int> flags = GetArgFlags('-' + key.name); + if (!IsConfSupported(key, error)) return false; if (flags) { std::optional<util::SettingsValue> value = InterpretValue(key, &option.second, *flags, error); if (!value) { diff --git a/src/threadinterrupt.cpp b/src/util/threadinterrupt.cpp index e28b447c1d..70731d6f31 100644 --- a/src/threadinterrupt.cpp +++ b/src/util/threadinterrupt.cpp @@ -3,7 +3,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <threadinterrupt.h> +#include <util/threadinterrupt.h> #include <sync.h> diff --git a/src/threadinterrupt.h b/src/util/threadinterrupt.h index 979bc2ee3e..d95cbb9aba 100644 --- a/src/threadinterrupt.h +++ b/src/util/threadinterrupt.h @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_THREADINTERRUPT_H -#define BITCOIN_THREADINTERRUPT_H +#ifndef BITCOIN_UTIL_THREADINTERRUPT_H +#define BITCOIN_UTIL_THREADINTERRUPT_H #include <sync.h> #include <threadsafety.h> @@ -33,4 +33,4 @@ private: std::atomic<bool> flag; }; -#endif // BITCOIN_THREADINTERRUPT_H +#endif // BITCOIN_UTIL_THREADINTERRUPT_H diff --git a/src/validation.cpp b/src/validation.cpp index b6c1c57aea..15d72c903a 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -22,6 +22,7 @@ #include <flatfile.h> #include <fs.h> #include <hash.h> +#include <kernel/mempool_entry.h> #include <logging.h> #include <logging/timer.h> #include <node/blockstorage.h> @@ -120,13 +121,6 @@ RecursiveMutex cs_main; GlobalMutex g_best_block_mutex; std::condition_variable g_best_block_cv; uint256 g_best_block; -bool g_parallel_script_checks{false}; -bool fCheckBlockIndex = false; -bool fCheckpointsEnabled = DEFAULT_CHECKPOINTS_ENABLED; -int64_t nMaxTipAge = DEFAULT_MAX_TIP_AGE; - -uint256 hashAssumeValid; -arith_uint256 nMinimumChainWork; const CBlockIndex* Chainstate::FindForkInGlobalIndex(const CBlockLocator& locator) const { @@ -1411,7 +1405,7 @@ MempoolAcceptResult AcceptToMemoryPool(Chainstate& active_chainstate, const CTra EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { AssertLockHeld(::cs_main); - const CChainParams& chainparams{active_chainstate.m_params}; + const CChainParams& chainparams{active_chainstate.m_chainman.GetParams()}; assert(active_chainstate.GetMempool() != nullptr); CTxMemPool& pool{*active_chainstate.GetMempool()}; @@ -1441,7 +1435,7 @@ PackageMempoolAcceptResult ProcessNewPackage(Chainstate& active_chainstate, CTxM assert(std::all_of(package.cbegin(), package.cend(), [](const auto& tx){return tx != nullptr;})); std::vector<COutPoint> coins_to_uncache; - const CChainParams& chainparams = active_chainstate.m_params; + const CChainParams& chainparams = active_chainstate.m_chainman.GetParams(); const auto result = [&]() EXCLUSIVE_LOCKS_REQUIRED(cs_main) { AssertLockHeld(cs_main); if (test_accept) { @@ -1499,7 +1493,6 @@ Chainstate::Chainstate( std::optional<uint256> from_snapshot_blockhash) : m_mempool(mempool), m_blockman(blockman), - m_params(chainman.GetParams()), m_chainman(chainman), m_from_snapshot_blockhash(from_snapshot_blockhash) {} @@ -1510,7 +1503,7 @@ void Chainstate::InitCoinsDB( fs::path leveldb_name) { if (m_from_snapshot_blockhash) { - leveldb_name += "_" + m_from_snapshot_blockhash->ToString(); + leveldb_name += node::SNAPSHOT_CHAINSTATE_SUFFIX; } m_coins_views = std::make_unique<CoinsViews>( @@ -1543,10 +1536,12 @@ bool Chainstate::IsInitialBlockDownload() const return true; if (m_chain.Tip() == nullptr) return true; - if (m_chain.Tip()->nChainWork < nMinimumChainWork) + if (m_chain.Tip()->nChainWork < m_chainman.MinimumChainWork()) { return true; - if (m_chain.Tip()->GetBlockTime() < (GetTime() - nMaxTipAge)) + } + if (m_chain.Tip()->Time() < Now<NodeSeconds>() - m_chainman.m_options.max_tip_age) { return true; + } LogPrintf("Leaving InitialBlockDownload (latching to false)\n"); m_cached_finished_ibd.store(true, std::memory_order_relaxed); return false; @@ -1835,11 +1830,21 @@ DisconnectResult Chainstate::DisconnectBlock(const CBlock& block, const CBlockIn return DISCONNECT_FAILED; } + // Ignore blocks that contain transactions which are 'overwritten' by later transactions, + // unless those are already completely spent. + // See https://github.com/bitcoin/bitcoin/issues/22596 for additional information. + // Note: the blocks specified here are different than the ones used in ConnectBlock because DisconnectBlock + // unwinds the blocks in reverse. As a result, the inconsistency is not discovered until the earlier + // blocks with the duplicate coinbase transactions are disconnected. + bool fEnforceBIP30 = !((pindex->nHeight==91722 && pindex->GetBlockHash() == uint256S("0x00000000000271a2dc26e7667f8419f2e15416dc6955e5a6c6cdf3f2574dd08e")) || + (pindex->nHeight==91812 && pindex->GetBlockHash() == uint256S("0x00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f"))); + // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { const CTransaction &tx = *(block.vtx[i]); uint256 hash = tx.GetHash(); bool is_coinbase = tx.IsCoinBase(); + bool is_bip30_exception = (is_coinbase && !fEnforceBIP30); // Check that all outputs are available and match the outputs in the block itself // exactly. @@ -1849,7 +1854,9 @@ DisconnectResult Chainstate::DisconnectBlock(const CBlock& block, const CBlockIn Coin coin; bool is_spent = view.SpendCoin(out, &coin); if (!is_spent || tx.vout[o] != coin.out || pindex->nHeight != coin.nHeight || is_coinbase != coin.fCoinBase) { - fClean = false; // transaction output mismatch + if (!is_bip30_exception) { + fClean = false; // transaction output mismatch + } } } } @@ -1980,8 +1987,10 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, uint256 block_hash{block.GetHash()}; assert(*pindex->phashBlock == block_hash); + const bool parallel_script_checks{scriptcheckqueue.HasThreads()}; const auto time_start{SteadyClock::now()}; + const CChainParams& params{m_chainman.GetParams()}; // Check it again in case a previous version let a bad block in // NOTE: We don't currently (re-)invoke ContextualCheckBlock() or @@ -1996,7 +2005,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, // is enforced in ContextualCheckBlockHeader(); we wouldn't want to // re-enforce that rule here (at least until we make it impossible for // m_adjusted_time_callback() to go backward). - if (!CheckBlock(block, state, m_params.GetConsensus(), !fJustCheck, !fJustCheck)) { + if (!CheckBlock(block, state, params.GetConsensus(), !fJustCheck, !fJustCheck)) { if (state.GetResult() == BlockValidationResult::BLOCK_MUTATED) { // We don't write down blocks to disk if they may have been // corrupted, so this should be impossible unless we're having hardware @@ -2014,24 +2023,24 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, // Special case for the genesis block, skipping connection of its transactions // (its coinbase is unspendable) - if (block_hash == m_params.GetConsensus().hashGenesisBlock) { + if (block_hash == params.GetConsensus().hashGenesisBlock) { if (!fJustCheck) view.SetBestBlock(pindex->GetBlockHash()); return true; } bool fScriptChecks = true; - if (!hashAssumeValid.IsNull()) { + if (!m_chainman.AssumedValidBlock().IsNull()) { // We've been configured with the hash of a block which has been externally verified to have a valid history. // A suitable default value is included with the software and updated from time to time. Because validity // relative to a piece of software is an objective fact these defaults can be easily reviewed. // This setting doesn't force the selection of any particular chain but makes validating some faster by // effectively caching the result of part of the verification. - BlockMap::const_iterator it = m_blockman.m_block_index.find(hashAssumeValid); + BlockMap::const_iterator it{m_blockman.m_block_index.find(m_chainman.AssumedValidBlock())}; if (it != m_blockman.m_block_index.end()) { if (it->second.GetAncestor(pindex->nHeight) == pindex && m_chainman.m_best_header->GetAncestor(pindex->nHeight) == pindex && - m_chainman.m_best_header->nChainWork >= nMinimumChainWork) { + m_chainman.m_best_header->nChainWork >= m_chainman.MinimumChainWork()) { // This block is a member of the assumed verified chain and an ancestor of the best header. // Script verification is skipped when connecting blocks under the // assumevalid block. Assuming the assumevalid block is valid this @@ -2044,9 +2053,9 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, // it hard to hide the implication of the demand. This also avoids having release candidates // that are hardly doing any signature verification at all in testing without having to // artificially set the default assumed verified block further back. - // The test against nMinimumChainWork prevents the skipping when denied access to any chain at + // The test against the minimum chain work prevents the skipping when denied access to any chain at // least as good as the expected chain. - fScriptChecks = (GetBlockProofEquivalentTime(*m_chainman.m_best_header, *pindex, *m_chainman.m_best_header, m_params.GetConsensus()) <= 60 * 60 * 24 * 7 * 2); + fScriptChecks = (GetBlockProofEquivalentTime(*m_chainman.m_best_header, *pindex, *m_chainman.m_best_header, params.GetConsensus()) <= 60 * 60 * 24 * 7 * 2); } } } @@ -2068,8 +2077,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, // Now that the whole chain is irreversibly beyond that time it is applied to all blocks except the // two in the chain that violate it. This prevents exploiting the issue against nodes during their // initial block download. - bool fEnforceBIP30 = !((pindex->nHeight==91842 && pindex->GetBlockHash() == uint256S("0x00000000000a4d0a398161ffc163c503763b1f4360639393e0e4c8e300e0caec")) || - (pindex->nHeight==91880 && pindex->GetBlockHash() == uint256S("0x00000000000743f190a18c5577a3c2d2a1f610ae9601ac046a38084ccb7cd721"))); + bool fEnforceBIP30 = !IsBIP30Repeat(*pindex); // Once BIP34 activated it was not possible to create new duplicate coinbases and thus other than starting // with the 2 existing duplicate coinbase pairs, not possible to create overwriting txs. But by the @@ -2127,9 +2135,9 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, // post BIP34 before approximately height 486,000,000. After block // 1,983,702 testnet3 starts doing unnecessary BIP30 checking again. assert(pindex->pprev); - CBlockIndex* pindexBIP34height = pindex->pprev->GetAncestor(m_params.GetConsensus().BIP34Height); + CBlockIndex* pindexBIP34height = pindex->pprev->GetAncestor(params.GetConsensus().BIP34Height); //Only continue to enforce if we're below BIP34 activation height or the block hash at that height doesn't correspond. - fEnforceBIP30 = fEnforceBIP30 && (!pindexBIP34height || !(pindexBIP34height->GetBlockHash() == m_params.GetConsensus().BIP34Hash)); + fEnforceBIP30 = fEnforceBIP30 && (!pindexBIP34height || !(pindexBIP34height->GetBlockHash() == params.GetConsensus().BIP34Hash)); // TODO: Remove BIP30 checking from block height 1,983,702 on, once we have a // consensus change that ensures coinbases at those heights cannot @@ -2168,7 +2176,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, // in multiple threads). Preallocate the vector size so a new allocation // doesn't invalidate pointers into the vector, and keep txsdata in scope // for as long as `control`. - CCheckQueueControl<CScriptCheck> control(fScriptChecks && g_parallel_script_checks ? &scriptcheckqueue : nullptr); + CCheckQueueControl<CScriptCheck> control(fScriptChecks && parallel_script_checks ? &scriptcheckqueue : nullptr); std::vector<PrecomputedTransactionData> txsdata(block.vtx.size()); std::vector<int> prevheights; @@ -2227,7 +2235,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, std::vector<CScriptCheck> vChecks; bool fCacheResults = fJustCheck; /* Don't cache results if we're actually connecting blocks (still consult the cache, though) */ TxValidationState tx_state; - if (fScriptChecks && !CheckInputScripts(tx, tx_state, view, flags, fCacheResults, fCacheResults, txsdata[i], g_parallel_script_checks ? &vChecks : nullptr)) { + if (fScriptChecks && !CheckInputScripts(tx, tx_state, view, flags, fCacheResults, fCacheResults, txsdata[i], parallel_script_checks ? &vChecks : nullptr)) { // Any transaction validation failure in ConnectBlock is a block consensus failure state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, tx_state.GetRejectReason(), tx_state.GetDebugMessage()); @@ -2251,7 +2259,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, Ticks<SecondsDouble>(time_connect), Ticks<MillisecondsDouble>(time_connect) / num_blocks_total); - CAmount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, m_params.GetConsensus()); + CAmount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, params.GetConsensus()); if (block.vtx[0]->GetValueOut() > blockReward) { LogPrintf("ERROR: ConnectBlock(): coinbase pays too much (actual=%d vs limit=%d)\n", block.vtx[0]->GetValueOut(), blockReward); return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-cb-amount"); @@ -2272,7 +2280,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, if (fJustCheck) return true; - if (!m_blockman.WriteUndoDataForBlock(blockundo, state, pindex, m_params)) { + if (!m_blockman.WriteUndoDataForBlock(blockundo, state, pindex, params)) { return false; } @@ -2349,8 +2357,6 @@ bool Chainstate::FlushStateToDisk( { LOCK(cs_main); assert(this->CanFlushToDisk()); - static std::chrono::microseconds nLastWrite{0}; - static std::chrono::microseconds nLastFlush{0}; std::set<int> setFilesToPrune; bool full_flush_completed = false; @@ -2391,7 +2397,7 @@ bool Chainstate::FlushStateToDisk( } else { LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune", BCLog::BENCH); - m_blockman.FindFilesToPrune(setFilesToPrune, m_params.PruneAfterHeight(), m_chain.Height(), last_prune, IsInitialBlockDownload()); + m_blockman.FindFilesToPrune(setFilesToPrune, m_chainman.GetParams().PruneAfterHeight(), m_chain.Height(), last_prune, IsInitialBlockDownload()); m_blockman.m_check_for_pruning = false; } if (!setFilesToPrune.empty()) { @@ -2404,20 +2410,20 @@ bool Chainstate::FlushStateToDisk( } const auto nNow = GetTime<std::chrono::microseconds>(); // Avoid writing/flushing immediately after startup. - if (nLastWrite.count() == 0) { - nLastWrite = nNow; + if (m_last_write.count() == 0) { + m_last_write = nNow; } - if (nLastFlush.count() == 0) { - nLastFlush = nNow; + if (m_last_flush.count() == 0) { + m_last_flush = nNow; } // The cache is large and we're within 10% and 10 MiB of the limit, but we have time now (not in the middle of a block processing). bool fCacheLarge = mode == FlushStateMode::PERIODIC && cache_state >= CoinsCacheSizeState::LARGE; // The cache is over the limit, we have to write now. bool fCacheCritical = mode == FlushStateMode::IF_NEEDED && cache_state >= CoinsCacheSizeState::CRITICAL; // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. - bool fPeriodicWrite = mode == FlushStateMode::PERIODIC && nNow > nLastWrite + DATABASE_WRITE_INTERVAL; + bool fPeriodicWrite = mode == FlushStateMode::PERIODIC && nNow > m_last_write + DATABASE_WRITE_INTERVAL; // It's been very long since we flushed the cache. Do this infrequently, to optimize cache usage. - bool fPeriodicFlush = mode == FlushStateMode::PERIODIC && nNow > nLastFlush + DATABASE_FLUSH_INTERVAL; + bool fPeriodicFlush = mode == FlushStateMode::PERIODIC && nNow > m_last_flush + DATABASE_FLUSH_INTERVAL; // Combine all conditions that result in a full cache flush. fDoFullFlush = (mode == FlushStateMode::ALWAYS) || fCacheLarge || fCacheCritical || fPeriodicFlush || fFlushForPrune; // Write blocks and block index to disk. @@ -2447,7 +2453,7 @@ bool Chainstate::FlushStateToDisk( UnlinkPrunedFiles(setFilesToPrune); } - nLastWrite = nNow; + m_last_write = nNow; } // Flush best chain related state. This can only be done if the blocks / block index write was also done. if (fDoFullFlush && !CoinsTip().GetBestBlock().IsNull()) { @@ -2465,7 +2471,7 @@ bool Chainstate::FlushStateToDisk( // Flush the chainstate (which may refer to block index entries). if (!CoinsTip().Flush()) return AbortNode(state, "Failed to write to coin database"); - nLastFlush = nNow; + m_last_flush = nNow; full_flush_completed = true; TRACE5(utxocache, flush, (int64_t)(GetTimeMicros() - nNow.count()), // in microseconds (µs) @@ -2545,13 +2551,15 @@ void Chainstate::UpdateTip(const CBlockIndex* pindexNew) AssertLockHeld(::cs_main); const auto& coins_tip = this->CoinsTip(); + const CChainParams& params{m_chainman.GetParams()}; + // The remainder of the function isn't relevant if we are not acting on // the active chainstate, so return if need be. if (this != &m_chainman.ActiveChainstate()) { // Only log every so often so that we don't bury log messages at the tip. constexpr int BACKGROUND_LOG_INTERVAL = 2000; if (pindexNew->nHeight % BACKGROUND_LOG_INTERVAL == 0) { - UpdateTipLog(coins_tip, pindexNew, m_params, __func__, "[background validation] ", ""); + UpdateTipLog(coins_tip, pindexNew, params, __func__, "[background validation] ", ""); } return; } @@ -2572,7 +2580,7 @@ void Chainstate::UpdateTip(const CBlockIndex* pindexNew) const CBlockIndex* pindex = pindexNew; for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { WarningBitsConditionChecker checker(m_chainman, bit); - ThresholdState state = checker.GetStateFor(pindex, m_params.GetConsensus(), warningcache.at(bit)); + ThresholdState state = checker.GetStateFor(pindex, params.GetConsensus(), warningcache.at(bit)); if (state == ThresholdState::ACTIVE || state == ThresholdState::LOCKED_IN) { const bilingual_str warning = strprintf(_("Unknown new rules activated (versionbit %i)"), bit); if (state == ThresholdState::ACTIVE) { @@ -2583,7 +2591,7 @@ void Chainstate::UpdateTip(const CBlockIndex* pindexNew) } } } - UpdateTipLog(coins_tip, pindexNew, m_params, __func__, "", warning_messages.original); + UpdateTipLog(coins_tip, pindexNew, params, __func__, "", warning_messages.original); } /** Disconnect m_chain's tip. @@ -2607,7 +2615,7 @@ bool Chainstate::DisconnectTip(BlockValidationState& state, DisconnectedBlockTra // Read block from disk. std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); CBlock& block = *pblock; - if (!ReadBlockFromDisk(block, pindexDelete, m_params.GetConsensus())) { + if (!ReadBlockFromDisk(block, pindexDelete, m_chainman.GetConsensus())) { return error("DisconnectTip(): Failed to read block"); } // Apply the block atomically to the chain state. @@ -2724,7 +2732,7 @@ bool Chainstate::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, std::shared_ptr<const CBlock> pthisBlock; if (!pblock) { std::shared_ptr<CBlock> pblockNew = std::make_shared<CBlock>(); - if (!ReadBlockFromDisk(*pblockNew, pindexNew, m_params.GetConsensus())) { + if (!ReadBlockFromDisk(*pblockNew, pindexNew, m_chainman.GetConsensus())) { return AbortNode(state, "Failed to read block"); } pthisBlock = pblockNew; @@ -3515,7 +3523,7 @@ static bool ContextualCheckBlockHeader(const CBlockHeader& block, BlockValidatio return state.Invalid(BlockValidationResult::BLOCK_INVALID_HEADER, "bad-diffbits", "incorrect proof of work"); // Check against checkpoints - if (fCheckpointsEnabled) { + if (chainman.m_options.checkpoints_enabled) { // Don't accept any forks from the main chain prior to last checkpoint. // GetLastCheckpoint finds the last checkpoint in MapCheckpoints that's in our // BlockIndex(). @@ -3663,12 +3671,12 @@ bool ChainstateManager::AcceptBlockHeader(const CBlockHeader& block, BlockValida CBlockIndex* pindexPrev = nullptr; BlockMap::iterator mi{m_blockman.m_block_index.find(block.hashPrevBlock)}; if (mi == m_blockman.m_block_index.end()) { - LogPrint(BCLog::VALIDATION, "%s: %s prev block not found\n", __func__, hash.ToString()); + LogPrint(BCLog::VALIDATION, "header %s has prev block not found: %s\n", hash.ToString(), block.hashPrevBlock.ToString()); return state.Invalid(BlockValidationResult::BLOCK_MISSING_PREV, "prev-blk-not-found"); } pindexPrev = &((*mi).second); if (pindexPrev->nStatus & BLOCK_FAILED_MASK) { - LogPrint(BCLog::VALIDATION, "%s: %s prev block invalid\n", __func__, hash.ToString()); + LogPrint(BCLog::VALIDATION, "header %s has prev block invalid: %s\n", hash.ToString(), block.hashPrevBlock.ToString()); return state.Invalid(BlockValidationResult::BLOCK_INVALID_PREV, "bad-prevblk"); } if (!ContextualCheckBlockHeader(block, state, m_blockman, *this, pindexPrev, m_options.adjusted_time_callback())) { @@ -3709,7 +3717,7 @@ bool ChainstateManager::AcceptBlockHeader(const CBlockHeader& block, BlockValida m_blockman.m_dirty_blockindex.insert(invalid_walk); invalid_walk = invalid_walk->pprev; } - LogPrint(BCLog::VALIDATION, "%s: %s prev block invalid\n", __func__, hash.ToString()); + LogPrint(BCLog::VALIDATION, "header %s has prev block invalid: %s\n", hash.ToString(), block.hashPrevBlock.ToString()); return state.Invalid(BlockValidationResult::BLOCK_INVALID_PREV, "bad-prevblk"); } } @@ -3829,10 +3837,12 @@ bool Chainstate::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockV // If our tip is behind, a peer could try to send us // low-work blocks on a fake chain that we would never // request; don't process these. - if (pindex->nChainWork < nMinimumChainWork) return true; + if (pindex->nChainWork < m_chainman.MinimumChainWork()) return true; } - if (!CheckBlock(block, state, m_params.GetConsensus()) || + const CChainParams& params{m_chainman.GetParams()}; + + if (!CheckBlock(block, state, params.GetConsensus()) || !ContextualCheckBlock(block, state, m_chainman, pindex->pprev)) { if (state.IsInvalid() && state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { pindex->nStatus |= BLOCK_FAILED_VALID; @@ -3849,7 +3859,7 @@ bool Chainstate::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockV // Write block to history file if (fNewBlock) *fNewBlock = true; try { - FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, pindex->nHeight, m_chain, m_params, dbp)}; + FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, pindex->nHeight, m_chain, params, dbp)}; if (blockPos.IsNull()) { state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); return false; @@ -3993,7 +4003,7 @@ bool Chainstate::LoadChainTip() tip->GetBlockHash().ToString(), m_chain.Height(), FormatISO8601DateTime(tip->GetBlockTime()), - GuessVerificationProgress(m_params.TxData(), tip)); + GuessVerificationProgress(m_chainman.GetParams().TxData(), tip)); return true; } @@ -4129,7 +4139,7 @@ bool Chainstate::RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& in AssertLockHeld(cs_main); // TODO: merge with ConnectBlock CBlock block; - if (!ReadBlockFromDisk(block, pindex, m_params.GetConsensus())) { + if (!ReadBlockFromDisk(block, pindex, m_chainman.GetConsensus())) { return error("ReplayBlock(): ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } @@ -4181,7 +4191,7 @@ bool Chainstate::ReplayBlocks() while (pindexOld != pindexFork) { if (pindexOld->nHeight > 0) { // Never disconnect the genesis block. CBlock block; - if (!ReadBlockFromDisk(block, pindexOld, m_params.GetConsensus())) { + if (!ReadBlockFromDisk(block, pindexOld, m_chainman.GetConsensus())) { return error("RollbackBlock(): ReadBlockFromDisk() failed at %d, hash=%s", pindexOld->nHeight, pindexOld->GetBlockHash().ToString()); } LogPrintf("Rolling back %s (%i)\n", pindexOld->GetBlockHash().ToString(), pindexOld->nHeight); @@ -4333,16 +4343,18 @@ bool Chainstate::LoadGenesisBlock() { LOCK(cs_main); + const CChainParams& params{m_chainman.GetParams()}; + // Check whether we're already initialized by checking for genesis in // m_blockman.m_block_index. Note that we can't use m_chain here, since it is // set based on the coins db, not the block index db, which is the only // thing loaded at this point. - if (m_blockman.m_block_index.count(m_params.GenesisBlock().GetHash())) + if (m_blockman.m_block_index.count(params.GenesisBlock().GetHash())) return true; try { - const CBlock& block = m_params.GenesisBlock(); - FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, 0, m_chain, m_params, nullptr)}; + const CBlock& block = params.GenesisBlock(); + FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, 0, m_chain, params, nullptr)}; if (blockPos.IsNull()) { return error("%s: writing genesis block to disk failed", __func__); } @@ -4366,11 +4378,14 @@ void Chainstate::LoadExternalBlockFile( assert(!dbp == !blocks_with_unknown_parent); const auto start{SteadyClock::now()}; + const CChainParams& params{m_chainman.GetParams()}; int nLoaded = 0; try { // This takes over fileIn and calls fclose() on it in the CBufferedFile destructor CBufferedFile blkdat(fileIn, 2*MAX_BLOCK_SERIALIZED_SIZE, MAX_BLOCK_SERIALIZED_SIZE+8, SER_DISK, CLIENT_VERSION); + // nRewind indicates where to resume scanning in case something goes wrong, + // such as a block fails to deserialize. uint64_t nRewind = blkdat.GetPos(); while (!blkdat.eof()) { if (ShutdownRequested()) return; @@ -4382,10 +4397,10 @@ void Chainstate::LoadExternalBlockFile( try { // locate a header unsigned char buf[CMessageHeader::MESSAGE_START_SIZE]; - blkdat.FindByte(m_params.MessageStart()[0]); + blkdat.FindByte(params.MessageStart()[0]); nRewind = blkdat.GetPos() + 1; blkdat >> buf; - if (memcmp(buf, m_params.MessageStart(), CMessageHeader::MESSAGE_START_SIZE)) { + if (memcmp(buf, params.MessageStart(), CMessageHeader::MESSAGE_START_SIZE)) { continue; } // read size @@ -4394,28 +4409,30 @@ void Chainstate::LoadExternalBlockFile( continue; } catch (const std::exception&) { // no valid block header found; don't complain + // (this happens at the end of every blk.dat file) break; } try { - // read block - uint64_t nBlockPos = blkdat.GetPos(); + // read block header + const uint64_t nBlockPos{blkdat.GetPos()}; if (dbp) dbp->nPos = nBlockPos; blkdat.SetLimit(nBlockPos + nSize); - std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); - CBlock& block = *pblock; - blkdat >> block; - nRewind = blkdat.GetPos(); - - uint256 hash = block.GetHash(); + CBlockHeader header; + blkdat >> header; + const uint256 hash{header.GetHash()}; + // Skip the rest of this block (this may read from disk into memory); position to the marker before the + // next block, but it's still possible to rewind to the start of the current block (without a disk read). + nRewind = nBlockPos + nSize; + blkdat.SkipTo(nRewind); { LOCK(cs_main); // detect out of order blocks, and store them for later - if (hash != m_params.GetConsensus().hashGenesisBlock && !m_blockman.LookupBlockIndex(block.hashPrevBlock)) { + if (hash != params.GetConsensus().hashGenesisBlock && !m_blockman.LookupBlockIndex(header.hashPrevBlock)) { LogPrint(BCLog::REINDEX, "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(), - block.hashPrevBlock.ToString()); + header.hashPrevBlock.ToString()); if (dbp && blocks_with_unknown_parent) { - blocks_with_unknown_parent->emplace(block.hashPrevBlock, *dbp); + blocks_with_unknown_parent->emplace(header.hashPrevBlock, *dbp); } continue; } @@ -4423,20 +4440,26 @@ void Chainstate::LoadExternalBlockFile( // process in case the block isn't known yet const CBlockIndex* pindex = m_blockman.LookupBlockIndex(hash); if (!pindex || (pindex->nStatus & BLOCK_HAVE_DATA) == 0) { - BlockValidationState state; - if (AcceptBlock(pblock, state, nullptr, true, dbp, nullptr, true)) { - nLoaded++; - } - if (state.IsError()) { - break; - } - } else if (hash != m_params.GetConsensus().hashGenesisBlock && pindex->nHeight % 1000 == 0) { + // This block can be processed immediately; rewind to its start, read and deserialize it. + blkdat.SetPos(nBlockPos); + std::shared_ptr<CBlock> pblock{std::make_shared<CBlock>()}; + blkdat >> *pblock; + nRewind = blkdat.GetPos(); + + BlockValidationState state; + if (AcceptBlock(pblock, state, nullptr, true, dbp, nullptr, true)) { + nLoaded++; + } + if (state.IsError()) { + break; + } + } else if (hash != params.GetConsensus().hashGenesisBlock && pindex->nHeight % 1000 == 0) { LogPrint(BCLog::REINDEX, "Block Import: already had block %s at height %d\n", hash.ToString(), pindex->nHeight); } } // Activate the genesis block so normal node progress can continue - if (hash == m_params.GetConsensus().hashGenesisBlock) { + if (hash == params.GetConsensus().hashGenesisBlock) { BlockValidationState state; if (!ActivateBestChain(state, nullptr)) { break; @@ -4457,7 +4480,7 @@ void Chainstate::LoadExternalBlockFile( while (range.first != range.second) { std::multimap<uint256, FlatFilePos>::iterator it = range.first; std::shared_ptr<CBlock> pblockrecursive = std::make_shared<CBlock>(); - if (ReadBlockFromDisk(*pblockrecursive, it->second, m_params.GetConsensus())) { + if (ReadBlockFromDisk(*pblockrecursive, it->second, params.GetConsensus())) { LogPrint(BCLog::REINDEX, "%s: Processing out of order child %s of %s\n", __func__, pblockrecursive->GetHash().ToString(), head.ToString()); LOCK(cs_main); @@ -4473,7 +4496,18 @@ void Chainstate::LoadExternalBlockFile( } } } catch (const std::exception& e) { - LogPrintf("%s: Deserialize or I/O error - %s\n", __func__, e.what()); + // historical bugs added extra data to the block files that does not deserialize cleanly. + // commonly this data is between readable blocks, but it does not really matter. such data is not fatal to the import process. + // the code that reads the block files deals with invalid data by simply ignoring it. + // it continues to search for the next {4 byte magic message start bytes + 4 byte length + block} that does deserialize cleanly + // and passes all of the other block validation checks dealing with POW and the merkle root, etc... + // we merely note with this informational log message when unexpected data is encountered. + // we could also be experiencing a storage system read error, or a read of a previous bad write. these are possible, but + // less likely scenarios. we don't have enough information to tell a difference here. + // the reindex process is not the place to attempt to clean and/or compact the block files. if so desired, a studious node operator + // may use knowledge of the fact that the block files are not entirely pristine in order to prepare a set of pristine, and + // perhaps ordered, block files for later reindexing. + LogPrint(BCLog::REINDEX, "%s: unexpected data at file offset 0x%x - %s. continuing\n", __func__, (nRewind - 1), e.what()); } } } catch (const std::runtime_error& e) { @@ -4484,7 +4518,7 @@ void Chainstate::LoadExternalBlockFile( void Chainstate::CheckBlockIndex() { - if (!fCheckBlockIndex) { + if (!m_chainman.ShouldCheckBlockIndex()) { return; } @@ -4557,7 +4591,7 @@ void Chainstate::CheckBlockIndex() // Begin: actual consistency checks. if (pindex->pprev == nullptr) { // Genesis block checks. - assert(pindex->GetBlockHash() == m_params.GetConsensus().hashGenesisBlock); // Genesis block's hash must match. + assert(pindex->GetBlockHash() == m_chainman.GetConsensus().hashGenesisBlock); // Genesis block's hash must match. assert(pindex == m_chain.Genesis()); // The current active chain's genesis block must be this block. } if (!pindex->HaveTxsDownloaded()) assert(pindex->nSequenceId <= 0); // nSequenceId can't be set positive for blocks that aren't linked (negative is used for preciousblock) @@ -4790,28 +4824,15 @@ std::vector<Chainstate*> ChainstateManager::GetAll() return out; } -Chainstate& ChainstateManager::InitializeChainstate( - CTxMemPool* mempool, const std::optional<uint256>& snapshot_blockhash) +Chainstate& ChainstateManager::InitializeChainstate(CTxMemPool* mempool) { AssertLockHeld(::cs_main); - bool is_snapshot = snapshot_blockhash.has_value(); - std::unique_ptr<Chainstate>& to_modify = - is_snapshot ? m_snapshot_chainstate : m_ibd_chainstate; - - if (to_modify) { - throw std::logic_error("should not be overwriting a chainstate"); - } - to_modify.reset(new Chainstate(mempool, m_blockman, *this, snapshot_blockhash)); + assert(!m_ibd_chainstate); + assert(!m_active_chainstate); - // Snapshot chainstates and initial IBD chaintates always become active. - if (is_snapshot || (!is_snapshot && !m_active_chainstate)) { - LogPrintf("Switching active chainstate to %s\n", to_modify->ToString()); - m_active_chainstate = to_modify.get(); - } else { - throw std::logic_error("unexpected chainstate activation"); - } - - return *to_modify; + m_ibd_chainstate = std::make_unique<Chainstate>(mempool, m_blockman, *this); + m_active_chainstate = m_ibd_chainstate.get(); + return *m_active_chainstate; } const AssumeutxoData* ExpectedAssumeutxo( @@ -4826,6 +4847,46 @@ const AssumeutxoData* ExpectedAssumeutxo( return nullptr; } +static bool DeleteCoinsDBFromDisk(const fs::path db_path, bool is_snapshot) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main) +{ + AssertLockHeld(::cs_main); + + if (is_snapshot) { + fs::path base_blockhash_path = db_path / node::SNAPSHOT_BLOCKHASH_FILENAME; + + if (fs::exists(base_blockhash_path)) { + bool removed = fs::remove(base_blockhash_path); + if (!removed) { + LogPrintf("[snapshot] failed to remove file %s\n", + fs::PathToString(base_blockhash_path)); + } + } else { + LogPrintf("[snapshot] snapshot chainstate dir being removed lacks %s file\n", + fs::PathToString(node::SNAPSHOT_BLOCKHASH_FILENAME)); + } + } + + std::string path_str = fs::PathToString(db_path); + LogPrintf("Removing leveldb dir at %s\n", path_str); + + // We have to destruct before this call leveldb::DB in order to release the db + // lock, otherwise `DestroyDB` will fail. See `leveldb::~DBImpl()`. + const bool destroyed = dbwrapper::DestroyDB(path_str, {}).ok(); + + if (!destroyed) { + LogPrintf("error: leveldb DestroyDB call failed on %s\n", path_str); + } + + // Datadir should be removed from filesystem; otherwise initialization may detect + // it on subsequent statups and get confused. + // + // If the base_blockhash_path removal above fails in the case of snapshot + // chainstates, this will return false since leveldb won't remove a non-empty + // directory. + return destroyed && !fs::exists(db_path); +} + bool ChainstateManager::ActivateSnapshot( AutoFile& coins_file, const SnapshotMetadata& metadata, @@ -4883,11 +4944,34 @@ bool ChainstateManager::ActivateSnapshot( static_cast<size_t>(current_coinstip_cache_size * SNAPSHOT_CACHE_PERC)); } - const bool snapshot_ok = this->PopulateAndValidateSnapshot( + bool snapshot_ok = this->PopulateAndValidateSnapshot( *snapshot_chainstate, coins_file, metadata); + // If not in-memory, persist the base blockhash for use during subsequent + // initialization. + if (!in_memory) { + LOCK(::cs_main); + if (!node::WriteSnapshotBaseBlockhash(*snapshot_chainstate)) { + snapshot_ok = false; + } + } if (!snapshot_ok) { - WITH_LOCK(::cs_main, this->MaybeRebalanceCaches()); + LOCK(::cs_main); + this->MaybeRebalanceCaches(); + + // PopulateAndValidateSnapshot can return (in error) before the leveldb datadir + // has been created, so only attempt removal if we got that far. + if (auto snapshot_datadir = node::FindSnapshotChainstateDir()) { + // We have to destruct leveldb::DB in order to release the db lock, otherwise + // DestroyDB() (in DeleteCoinsDBFromDisk()) will fail. See `leveldb::~DBImpl()`. + // Destructing the chainstate (and so resetting the coinsviews object) does this. + snapshot_chainstate.reset(); + bool removed = DeleteCoinsDBFromDisk(*snapshot_datadir, /*is_snapshot=*/true); + if (!removed) { + AbortNode(strprintf("Failed to remove snapshot chainstate dir (%s). " + "Manually remove it before restarting.\n", fs::PathToString(*snapshot_datadir))); + } + } return false; } @@ -5161,6 +5245,29 @@ void ChainstateManager::MaybeRebalanceCaches() } } +void ChainstateManager::ResetChainstates() +{ + m_ibd_chainstate.reset(); + m_snapshot_chainstate.reset(); + m_active_chainstate = nullptr; +} + +/** + * Apply default chain params to nullopt members. + * This helps to avoid coding errors around the accidental use of the compare + * operators that accept nullopt, thus ignoring the intended default value. + */ +static ChainstateManager::Options&& Flatten(ChainstateManager::Options&& opts) +{ + if (!opts.check_block_index.has_value()) opts.check_block_index = opts.chainparams.DefaultConsistencyChecks(); + if (!opts.minimum_chain_work.has_value()) opts.minimum_chain_work = UintToArith256(opts.chainparams.GetConsensus().nMinimumChainWork); + if (!opts.assumed_valid_block.has_value()) opts.assumed_valid_block = opts.chainparams.GetConsensus().defaultAssumeValid; + Assert(opts.adjusted_time_callback); + return std::move(opts); +} + +ChainstateManager::ChainstateManager(Options options) : m_options{Flatten(std::move(options))} {} + ChainstateManager::~ChainstateManager() { LOCK(::cs_main); @@ -5172,3 +5279,43 @@ ChainstateManager::~ChainstateManager() i.clear(); } } + +bool ChainstateManager::DetectSnapshotChainstate(CTxMemPool* mempool) +{ + assert(!m_snapshot_chainstate); + std::optional<fs::path> path = node::FindSnapshotChainstateDir(); + if (!path) { + return false; + } + std::optional<uint256> base_blockhash = node::ReadSnapshotBaseBlockhash(*path); + if (!base_blockhash) { + return false; + } + LogPrintf("[snapshot] detected active snapshot chainstate (%s) - loading\n", + fs::PathToString(*path)); + + this->ActivateExistingSnapshot(mempool, *base_blockhash); + return true; +} + +Chainstate& ChainstateManager::ActivateExistingSnapshot(CTxMemPool* mempool, uint256 base_blockhash) +{ + assert(!m_snapshot_chainstate); + m_snapshot_chainstate = + std::make_unique<Chainstate>(mempool, m_blockman, *this, base_blockhash); + LogPrintf("[snapshot] switching active chainstate to %s\n", m_snapshot_chainstate->ToString()); + m_active_chainstate = m_snapshot_chainstate.get(); + return *m_snapshot_chainstate; +} + +bool IsBIP30Repeat(const CBlockIndex& block_index) +{ + return (block_index.nHeight==91842 && block_index.GetBlockHash() == uint256S("0x00000000000a4d0a398161ffc163c503763b1f4360639393e0e4c8e300e0caec")) || + (block_index.nHeight==91880 && block_index.GetBlockHash() == uint256S("0x00000000000743f190a18c5577a3c2d2a1f610ae9601ac046a38084ccb7cd721")); +} + +bool IsBIP30Unspendable(const CBlockIndex& block_index) +{ + return (block_index.nHeight==91722 && block_index.GetBlockHash() == uint256S("0x00000000000271a2dc26e7667f8419f2e15416dc6955e5a6c6cdf3f2574dd08e")) || + (block_index.nHeight==91812 && block_index.GetBlockHash() == uint256S("0x00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f")); +} diff --git a/src/validation.h b/src/validation.h index c882eac408..00f7265793 100644 --- a/src/validation.h +++ b/src/validation.h @@ -63,8 +63,6 @@ struct Params; static const int MAX_SCRIPTCHECK_THREADS = 15; /** -par default (number of script-checking threads, 0 = auto) */ static const int DEFAULT_SCRIPTCHECK_THREADS = 0; -static const int64_t DEFAULT_MAX_TIP_AGE = 24 * 60 * 60; -static const bool DEFAULT_CHECKPOINTS_ENABLED = true; /** Default for -stopatheight */ static const int DEFAULT_STOPATHEIGHT = 0; /** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ActiveChain().Tip() will not be pruned. */ @@ -93,20 +91,6 @@ extern GlobalMutex g_best_block_mutex; extern std::condition_variable g_best_block_cv; /** Used to notify getblocktemplate RPC of new tips. */ extern uint256 g_best_block; -/** Whether there are dedicated script-checking threads running. - * False indicates all script checking is done on the main threadMessageHandler thread. - */ -extern bool g_parallel_script_checks; -extern bool fCheckBlockIndex; -extern bool fCheckpointsEnabled; -/** If the tip is older than this (in seconds), the node is considered to be in initial block download. */ -extern int64_t nMaxTipAge; - -/** Block hash whose ancestors we will assume to have valid scripts without checking them. */ -extern uint256 hashAssumeValid; - -/** Minimum work we will assume exists on some valid chain. */ -extern arith_uint256 nMinimumChainWork; /** Documentation for argument 'checklevel'. */ extern const std::vector<std::string> CHECKLEVEL_DOC; @@ -472,10 +456,6 @@ public: //! Chainstate instances. node::BlockManager& m_blockman; - /** Chain parameters for this chainstate */ - /* TODO: replace with m_chainman.GetParams() */ - const CChainParams& m_params; - //! The chainstate manager that owns this chainstate. The reference is //! necessary so that this instance can check whether it is the active //! chainstate within deeply nested method calls. @@ -702,7 +682,7 @@ public: /** * Make various assertions about the state of the block index. * - * By default this only executes fully when using the Regtest chain; see: fCheckBlockIndex. + * By default this only executes fully when using the Regtest chain; see: m_options.check_block_index. */ void CheckBlockIndex(); @@ -763,6 +743,9 @@ private: void UpdateTip(const CBlockIndex* pindexNew) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + std::chrono::microseconds m_last_write{0}; + std::chrono::microseconds m_last_flush{0}; + friend ChainstateManager; }; @@ -867,13 +850,13 @@ private: public: using Options = kernel::ChainstateManagerOpts; - explicit ChainstateManager(Options options) : m_options{std::move(options)} - { - Assert(m_options.adjusted_time_callback); - } + explicit ChainstateManager(Options options); const CChainParams& GetParams() const { return m_options.chainparams; } const Consensus::Params& GetConsensus() const { return m_options.chainparams.GetConsensus(); } + bool ShouldCheckBlockIndex() const { return *Assert(m_options.check_block_index); } + const arith_uint256& MinimumChainWork() const { return *Assert(m_options.minimum_chain_work); } + const uint256& AssumedValidBlock() const { return *Assert(m_options.assumed_valid_block); } /** * Alias for ::cs_main. @@ -926,17 +909,11 @@ public: //! coins databases. This will be split somehow across chainstates. int64_t m_total_coinsdb_cache{0}; - //! Instantiate a new chainstate and assign it based upon whether it is - //! from a snapshot. + //! Instantiate a new chainstate. //! //! @param[in] mempool The mempool to pass to the chainstate // constructor - //! @param[in] snapshot_blockhash If given, signify that this chainstate - //! is based on a snapshot. - Chainstate& InitializeChainstate( - CTxMemPool* mempool, - const std::optional<uint256>& snapshot_blockhash = std::nullopt) - LIFETIMEBOUND EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + Chainstate& InitializeChainstate(CTxMemPool* mempool) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); //! Get all chainstates currently being used. std::vector<Chainstate*> GetAll(); @@ -1050,6 +1027,17 @@ public: * information. */ void ReportHeadersPresync(const arith_uint256& work, int64_t height, int64_t timestamp); + //! When starting up, search the datadir for a chainstate based on a UTXO + //! snapshot that is in the process of being validated. + bool DetectSnapshotChainstate(CTxMemPool* mempool) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + void ResetChainstates() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + + //! Switch the active chainstate to one based on a UTXO snapshot that was loaded + //! previously. + Chainstate& ActivateExistingSnapshot(CTxMemPool* mempool, uint256 base_blockhash) + EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + ~ChainstateManager(); }; @@ -1081,4 +1069,10 @@ bool DeploymentEnabled(const ChainstateManager& chainman, DEP dep) */ const AssumeutxoData* ExpectedAssumeutxo(const int height, const CChainParams& params); +/** Identifies blocks that overwrote an existing coinbase output in the UTXO set (see BIP30) */ +bool IsBIP30Repeat(const CBlockIndex& block_index); + +/** Identifies blocks which coinbase output was subsequently overwritten in the UTXO set (see BIP30) */ +bool IsBIP30Unspendable(const CBlockIndex& block_index); + #endif // BITCOIN_VALIDATION_H diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index 613c5b65ef..740c39d99d 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -17,6 +17,8 @@ #include <unordered_map> #include <utility> +const std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept; + /** * MainSignalsImpl manages a list of shared_ptr<CValidationInterface> callbacks. * @@ -215,9 +217,10 @@ void CMainSignals::TransactionRemovedFromMempool(const CTransactionRef& tx, MemP auto event = [tx, reason, mempool_sequence, this] { m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.TransactionRemovedFromMempool(tx, reason, mempool_sequence); }); }; - ENQUEUE_AND_LOG_EVENT(event, "%s: txid=%s wtxid=%s", __func__, + ENQUEUE_AND_LOG_EVENT(event, "%s: txid=%s wtxid=%s reason=%s", __func__, tx->GetHash().ToString(), - tx->GetWitnessHash().ToString()); + tx->GetWitnessHash().ToString(), + RemovalReasonToString(reason)); } void CMainSignals::BlockConnected(const std::shared_ptr<const CBlock> &pblock, const CBlockIndex *pindex) { diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index d08d3664c4..b56a6d3aee 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -37,7 +37,7 @@ public: bool m_include_unsafe_inputs = false; //! If true, the selection process can add extra unselected inputs from the wallet //! while requires all selected inputs be used - bool m_allow_other_inputs = false; + bool m_allow_other_inputs = true; //! Includes watch only addresses which are solvable bool fAllowWatchOnly = false; //! Override automatic min/max checks on fee, m_feerate must be set if true diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index b568e90998..ba1d60fe44 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -5,6 +5,7 @@ #include <wallet/coinselection.h> #include <consensus/amount.h> +#include <consensus/consensus.h> #include <policy/feerate.h> #include <util/check.h> #include <util/system.h> @@ -354,6 +355,10 @@ void OutputGroup::Insert(const COutput& output, size_t ancestors, size_t descend // descendants is the count as seen from the top ancestor, not the descendants as seen from the // coin itself; thus, this value is counted as the max, not the sum m_descendants = std::max(m_descendants, descendants); + + if (output.input_bytes > 0) { + m_weight += output.input_bytes * WITNESS_SCALE_FACTOR; + } } bool OutputGroup::EligibleForSpending(const CoinEligibilityFilter& eligibility_filter) const @@ -436,22 +441,41 @@ void SelectionResult::Clear() { m_selected_inputs.clear(); m_waste.reset(); + m_weight = 0; } void SelectionResult::AddInput(const OutputGroup& group) { util::insert(m_selected_inputs, group.m_outputs); m_use_effective = !group.m_subtract_fee_outputs; + + m_weight += group.m_weight; +} + +void SelectionResult::AddInputs(const std::set<COutput>& inputs, bool subtract_fee_outputs) +{ + util::insert(m_selected_inputs, inputs); + m_use_effective = !subtract_fee_outputs; + + m_weight += std::accumulate(inputs.cbegin(), inputs.cend(), 0, [](int sum, const auto& coin) { + return sum + std::max(coin.input_bytes, 0) * WITNESS_SCALE_FACTOR; + }); } void SelectionResult::Merge(const SelectionResult& other) { + // Obtain the expected selected inputs count after the merge (for now, duplicates are not allowed) + const size_t expected_count = m_selected_inputs.size() + other.m_selected_inputs.size(); + m_target += other.m_target; m_use_effective |= other.m_use_effective; if (m_algo == SelectionAlgorithm::MANUAL) { m_algo = other.m_algo; } util::insert(m_selected_inputs, other.m_selected_inputs); + assert(m_selected_inputs.size() == expected_count); + + m_weight += other.m_weight; } const std::set<COutput>& SelectionResult::GetInputSet() const diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index 761c2be0b3..ecfc5bfd6b 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -6,6 +6,7 @@ #define BITCOIN_WALLET_COINSELECTION_H #include <consensus/amount.h> +#include <consensus/consensus.h> #include <policy/feerate.h> #include <primitives/transaction.h> #include <random.h> @@ -110,6 +111,8 @@ public: assert(effective_value.has_value()); return effective_value.value(); } + + bool HasEffectiveValue() const { return effective_value.has_value(); } }; /** Parameters for one iteration of Coin Selection. */ @@ -221,6 +224,8 @@ struct OutputGroup /** Indicate that we are subtracting the fee from outputs. * When true, the value that is used for coin selection is the UTXO's real value rather than effective value */ bool m_subtract_fee_outputs{false}; + /** Total weight of the UTXOs in this group. */ + int m_weight{0}; OutputGroup() {} OutputGroup(const CoinSelectionParams& params) : @@ -293,6 +298,8 @@ private: bool m_use_effective{false}; /** The computed waste */ std::optional<CAmount> m_waste; + /** Total weight of the selected inputs */ + int m_weight{0}; public: explicit SelectionResult(const CAmount target, SelectionAlgorithm algo) @@ -308,11 +315,18 @@ public: void Clear(); void AddInput(const OutputGroup& group); + void AddInputs(const std::set<COutput>& inputs, bool subtract_fee_outputs); /** Calculates and stores the waste for this selection via GetSelectionWaste */ void ComputeAndSetWaste(const CAmount min_viable_change, const CAmount change_cost, const CAmount change_fee); [[nodiscard]] CAmount GetWaste() const; + /** + * Combines the @param[in] other selection result into 'this' selection result. + * + * Important note: + * There must be no shared 'COutput' among the two selection results being combined. + */ void Merge(const SelectionResult& other); /** Get m_selected_inputs */ @@ -344,6 +358,8 @@ public: CAmount GetTarget() const { return m_target; } SelectionAlgorithm GetAlgo() const { return m_algo; } + + int GetWeight() const { return m_weight; } }; std::optional<SelectionResult> SelectCoinsBnB(std::vector<OutputGroup>& utxo_pool, const CAmount& selection_target, const CAmount& cost_of_change); diff --git a/src/wallet/dump.cpp b/src/wallet/dump.cpp index f7fee443d0..2e46cf5454 100644 --- a/src/wallet/dump.cpp +++ b/src/wallet/dump.cpp @@ -201,7 +201,7 @@ bool CreateFromDump(const ArgsManager& args, const std::string& name, const fs:: // dummy chain interface bool ret = true; - std::shared_ptr<CWallet> wallet(new CWallet(nullptr /* chain */, name, gArgs, std::move(database)), WalletToolReleaseWallet); + std::shared_ptr<CWallet> wallet(new CWallet(/*chain=*/nullptr, name, gArgs, std::move(database)), WalletToolReleaseWallet); { LOCK(wallet->cs_wallet); DBErrors load_wallet_ret = wallet->LoadWallet(); diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 6d7bb299cc..7343e8860b 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -293,7 +293,22 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo bool SignTransaction(CWallet& wallet, CMutableTransaction& mtx) { LOCK(wallet.cs_wallet); - return wallet.SignTransaction(mtx); + + if (wallet.IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER)) { + // Make a blank psbt + PartiallySignedTransaction psbtx(mtx); + + // First fill transaction with our data without signing, + // so external signers are not asked to sign more than once. + bool complete; + wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, false /* sign */, true /* bip32derivs */); + const TransactionError err = wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, true /* sign */, false /* bip32derivs */); + if (err != TransactionError::OK) return false; + complete = FinalizeAndExtractPSBT(psbtx, mtx); + return complete; + } else { + return wallet.SignTransaction(mtx); + } } Result CommitTransaction(CWallet& wallet, const uint256& txid, CMutableTransaction&& mtx, std::vector<bilingual_str>& errors, uint256& bumped_txid) diff --git a/src/wallet/fees.cpp b/src/wallet/fees.cpp index 3514d018b7..3e442a3f5f 100644 --- a/src/wallet/fees.cpp +++ b/src/wallet/fees.cpp @@ -84,7 +84,7 @@ CFeeRate GetMinimumFeeRate(const CWallet& wallet, const CCoinControl& coin_contr CFeeRate GetDiscardRate(const CWallet& wallet) { unsigned int highest_target = wallet.chain().estimateMaxBlocks(); - CFeeRate discard_rate = wallet.chain().estimateSmartFee(highest_target, false /* conservative */); + CFeeRate discard_rate = wallet.chain().estimateSmartFee(highest_target, /*conservative=*/false); // Don't let discard_rate be greater than longest possible fee estimate if we get a valid fee estimate discard_rate = (discard_rate == CFeeRate(0)) ? wallet.m_discard_rate : std::min(discard_rate, wallet.m_discard_rate); // Discard rate must be at least dust relay feerate diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index 9cf2b677e6..21726a1185 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -36,7 +36,7 @@ using interfaces::Chain; using interfaces::FoundBlock; using interfaces::Handler; -using interfaces::MakeHandler; +using interfaces::MakeSignalHandler; using interfaces::Wallet; using interfaces::WalletAddress; using interfaces::WalletBalances; @@ -481,39 +481,39 @@ public: CAmount getDefaultMaxTxFee() override { return m_wallet->m_default_max_tx_fee; } void remove() override { - RemoveWallet(m_context, m_wallet, false /* load_on_start */); + RemoveWallet(m_context, m_wallet, /*load_on_start=*/false); } bool isLegacy() override { return m_wallet->IsLegacy(); } std::unique_ptr<Handler> handleUnload(UnloadFn fn) override { - return MakeHandler(m_wallet->NotifyUnload.connect(fn)); + return MakeSignalHandler(m_wallet->NotifyUnload.connect(fn)); } std::unique_ptr<Handler> handleShowProgress(ShowProgressFn fn) override { - return MakeHandler(m_wallet->ShowProgress.connect(fn)); + return MakeSignalHandler(m_wallet->ShowProgress.connect(fn)); } std::unique_ptr<Handler> handleStatusChanged(StatusChangedFn fn) override { - return MakeHandler(m_wallet->NotifyStatusChanged.connect([fn](CWallet*) { fn(); })); + return MakeSignalHandler(m_wallet->NotifyStatusChanged.connect([fn](CWallet*) { fn(); })); } std::unique_ptr<Handler> handleAddressBookChanged(AddressBookChangedFn fn) override { - return MakeHandler(m_wallet->NotifyAddressBookChanged.connect( + return MakeSignalHandler(m_wallet->NotifyAddressBookChanged.connect( [fn](const CTxDestination& address, const std::string& label, bool is_mine, const std::string& purpose, ChangeType status) { fn(address, label, is_mine, purpose, status); })); } std::unique_ptr<Handler> handleTransactionChanged(TransactionChangedFn fn) override { - return MakeHandler(m_wallet->NotifyTransactionChanged.connect( + return MakeSignalHandler(m_wallet->NotifyTransactionChanged.connect( [fn](const uint256& txid, ChangeType status) { fn(txid, status); })); } std::unique_ptr<Handler> handleWatchOnlyChanged(WatchOnlyChangedFn fn) override { - return MakeHandler(m_wallet->NotifyWatchonlyChanged.connect(fn)); + return MakeSignalHandler(m_wallet->NotifyWatchonlyChanged.connect(fn)); } std::unique_ptr<Handler> handleCanGetAddressesChanged(CanGetAddressesChangedFn fn) override { - return MakeHandler(m_wallet->NotifyCanGetAddressesChanged.connect(fn)); + return MakeSignalHandler(m_wallet->NotifyCanGetAddressesChanged.connect(fn)); } CWallet* wallet() override { return m_wallet.get(); } diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 8babbb4298..a85302c77d 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -93,6 +93,22 @@ static void RescanWallet(CWallet& wallet, const WalletRescanReserver& reserver, } } +static void EnsureBlockDataFromTime(const CWallet& wallet, int64_t timestamp) +{ + auto& chain{wallet.chain()}; + if (!chain.havePruned()) { + return; + } + + int height{0}; + const bool found{chain.findFirstBlockWithTimeAndHeight(timestamp - TIMESTAMP_WINDOW, 0, FoundBlock().height(height))}; + + uint256 tip_hash{WITH_LOCK(wallet.cs_wallet, return wallet.GetLastBlockHash())}; + if (found && !chain.hasBlocks(tip_hash, height)) { + throw JSONRPCError(RPC_WALLET_ERROR, strprintf("Pruned blocks from height %d required to import keys. Use RPC call getblockchaininfo to determine your pruned height.", height)); + } +} + RPCHelpMan importprivkey() { return RPCHelpMan{"importprivkey", @@ -184,7 +200,7 @@ RPCHelpMan importprivkey() // Add the wpkh script for this key if possible if (pubkey.IsCompressed()) { - pwallet->ImportScripts({GetScriptForDestination(WitnessV0KeyHash(vchAddress))}, 0 /* timestamp */); + pwallet->ImportScripts({GetScriptForDestination(WitnessV0KeyHash(vchAddress))}, /*timestamp=*/0); } } } @@ -273,19 +289,19 @@ RPCHelpMan importaddress() pwallet->MarkDirty(); - pwallet->ImportScriptPubKeys(strLabel, {GetScriptForDestination(dest)}, false /* have_solving_data */, true /* apply_label */, 1 /* timestamp */); + pwallet->ImportScriptPubKeys(strLabel, {GetScriptForDestination(dest)}, /*have_solving_data=*/false, /*apply_label=*/true, /*timestamp=*/1); } else if (IsHex(request.params[0].get_str())) { std::vector<unsigned char> data(ParseHex(request.params[0].get_str())); CScript redeem_script(data.begin(), data.end()); std::set<CScript> scripts = {redeem_script}; - pwallet->ImportScripts(scripts, 0 /* timestamp */); + pwallet->ImportScripts(scripts, /*timestamp=*/0); if (fP2SH) { scripts.insert(GetScriptForDestination(ScriptHash(redeem_script))); } - pwallet->ImportScriptPubKeys(strLabel, scripts, false /* have_solving_data */, true /* apply_label */, 1 /* timestamp */); + pwallet->ImportScriptPubKeys(strLabel, scripts, /*have_solving_data=*/false, /*apply_label=*/true, /*timestamp=*/1); } else { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address or script"); } @@ -293,10 +309,7 @@ RPCHelpMan importaddress() if (fRescan) { RescanWallet(*pwallet, reserver); - { - LOCK(pwallet->cs_wallet); - pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); - } + pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); } return UniValue::VNULL; @@ -467,17 +480,14 @@ RPCHelpMan importpubkey() pwallet->MarkDirty(); - pwallet->ImportScriptPubKeys(strLabel, script_pub_keys, true /* have_solving_data */, true /* apply_label */, 1 /* timestamp */); + pwallet->ImportScriptPubKeys(strLabel, script_pub_keys, /*have_solving_data=*/true, /*apply_label=*/true, /*timestamp=*/1); - pwallet->ImportPubKeys({pubKey.GetID()}, {{pubKey.GetID(), pubKey}} , {} /* key_origins */, false /* add_keypool */, false /* internal */, 1 /* timestamp */); + pwallet->ImportPubKeys({pubKey.GetID()}, {{pubKey.GetID(), pubKey}} , /*key_origins=*/{}, /*add_keypool=*/false, /*internal=*/false, /*timestamp=*/1); } if (fRescan) { RescanWallet(*pwallet, reserver); - { - LOCK(pwallet->cs_wallet); - pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); - } + pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); } return UniValue::VNULL; @@ -510,13 +520,6 @@ RPCHelpMan importwallet() EnsureLegacyScriptPubKeyMan(*pwallet, true); - if (pwallet->chain().havePruned()) { - // Exit early and print an error. - // If a block is pruned after this check, we will import the key(s), - // but fail the rescan with a generic error. - throw JSONRPCError(RPC_WALLET_ERROR, "Importing wallets is disabled when blocks are pruned"); - } - WalletRescanReserver reserver(*pwallet); if (!reserver.reserve()) { throw JSONRPCError(RPC_WALLET_ERROR, "Wallet is currently rescanning. Abort existing rescan or wait."); @@ -571,15 +574,18 @@ RPCHelpMan importwallet() fLabel = true; } } + nTimeBegin = std::min(nTimeBegin, nTime); keys.push_back(std::make_tuple(key, nTime, fLabel, strLabel)); } else if(IsHex(vstr[0])) { std::vector<unsigned char> vData(ParseHex(vstr[0])); CScript script = CScript(vData.begin(), vData.end()); int64_t birth_time = ParseISO8601DateTime(vstr[1]); + if (birth_time > 0) nTimeBegin = std::min(nTimeBegin, birth_time); scripts.push_back(std::pair<CScript, int64_t>(script, birth_time)); } } file.close(); + EnsureBlockDataFromTime(*pwallet, nTimeBegin); // We now know whether we are importing private keys, so we can error if private keys are disabled if (keys.size() > 0 && pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)) { pwallet->chain().showProgress("", 100, false); // hide progress dialog in GUI @@ -608,8 +614,6 @@ RPCHelpMan importwallet() if (has_label) pwallet->SetAddressBook(PKHash(keyid), label, "receive"); - - nTimeBegin = std::min(nTimeBegin, time); progress++; } for (const auto& script_pair : scripts) { @@ -622,16 +626,13 @@ RPCHelpMan importwallet() fGood = false; continue; } - if (time > 0) { - nTimeBegin = std::min(nTimeBegin, time); - } progress++; } pwallet->chain().showProgress("", 100, false); // hide progress dialog in GUI } pwallet->chain().showProgress("", 100, false); // hide progress dialog in GUI - RescanWallet(*pwallet, reserver, nTimeBegin, false /* update */); + RescanWallet(*pwallet, reserver, nTimeBegin, /*update=*/false); pwallet->MarkDirty(); if (!fGood) @@ -1405,11 +1406,8 @@ RPCHelpMan importmulti() } } if (fRescan && fRunScan && requests.size()) { - int64_t scannedTime = pwallet->RescanFromTime(nLowestTimestamp, reserver, true /* update */); - { - LOCK(pwallet->cs_wallet); - pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); - } + int64_t scannedTime = pwallet->RescanFromTime(nLowestTimestamp, reserver, /*update=*/true); + pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); if (pwallet->IsAbortingRescan()) { throw JSONRPCError(RPC_MISC_ERROR, "Rescan aborted by user."); @@ -1598,7 +1596,8 @@ RPCHelpMan importdescriptors() return RPCHelpMan{"importdescriptors", "\nImport descriptors. This will trigger a rescan of the blockchain based on the earliest timestamp of all descriptors being imported. Requires a new wallet backup.\n" "\nNote: This call can take over an hour to complete if using an early timestamp; during that time, other rpc calls\n" - "may report that the imported keys, addresses or scripts exist but related transactions are still missing.\n", + "may report that the imported keys, addresses or scripts exist but related transactions are still missing.\n" + "The rescan is significantly faster if block filters are available (using startup option \"-blockfilterindex=1\").\n", { {"requests", RPCArg::Type::ARR, RPCArg::Optional::NO, "Data to be imported", { @@ -1699,11 +1698,8 @@ RPCHelpMan importdescriptors() // Rescan the blockchain using the lowest timestamp if (rescan) { - int64_t scanned_time = pwallet->RescanFromTime(lowest_timestamp, reserver, true /* update */); - { - LOCK(pwallet->cs_wallet); - pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); - } + int64_t scanned_time = pwallet->RescanFromTime(lowest_timestamp, reserver, /*update=*/true); + pwallet->ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); if (pwallet->IsAbortingRescan()) { throw JSONRPCError(RPC_MISC_ERROR, "Rescan aborted by user."); @@ -1899,7 +1895,9 @@ RPCHelpMan restorewallet() { return RPCHelpMan{ "restorewallet", - "\nRestore and loads a wallet from backup.\n", + "\nRestore and loads a wallet from backup.\n" + "\nThe rescan is significantly faster if a descriptor wallet is restored" + "\nand block filters are available (using startup option \"-blockfilterindex=1\").\n", { {"wallet_name", RPCArg::Type::STR, RPCArg::Optional::NO, "The name that will be applied to the restored wallet"}, {"backup_file", RPCArg::Type::STR, RPCArg::Optional::NO, "The backup file that will be used to restore the wallet."}, diff --git a/src/wallet/rpc/coins.cpp b/src/wallet/rpc/coins.cpp index 9c0c953a7a..6021e4bf4f 100644 --- a/src/wallet/rpc/coins.cpp +++ b/src/wallet/rpc/coins.cpp @@ -515,6 +515,7 @@ RPCHelpMan listunspent() {"maximumAmount", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"unlimited"}, "Maximum value of each UTXO in " + CURRENCY_UNIT + ""}, {"maximumCount", RPCArg::Type::NUM, RPCArg::DefaultHint{"unlimited"}, "Maximum number of UTXOs"}, {"minimumSumAmount", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"unlimited"}, "Minimum sum value of all UTXOs in " + CURRENCY_UNIT + ""}, + {"include_immature_coinbase", RPCArg::Type::BOOL, RPCArg::Default{false}, "Include immature coinbase UTXOs"} }, RPCArgOptions{.oneline_description="query_options"}}, }, @@ -590,10 +591,8 @@ RPCHelpMan listunspent() include_unsafe = request.params[3].get_bool(); } - CAmount nMinimumAmount = 0; - CAmount nMaximumAmount = MAX_MONEY; - CAmount nMinimumSumAmount = MAX_MONEY; - uint64_t nMaximumCount = 0; + CoinFilterParams filter_coins; + filter_coins.min_amount = 0; if (!request.params[4].isNull()) { const UniValue& options = request.params[4].get_obj(); @@ -604,20 +603,25 @@ RPCHelpMan listunspent() {"maximumAmount", UniValueType()}, {"minimumSumAmount", UniValueType()}, {"maximumCount", UniValueType(UniValue::VNUM)}, + {"include_immature_coinbase", UniValueType(UniValue::VBOOL)} }, true, true); if (options.exists("minimumAmount")) - nMinimumAmount = AmountFromValue(options["minimumAmount"]); + filter_coins.min_amount = AmountFromValue(options["minimumAmount"]); if (options.exists("maximumAmount")) - nMaximumAmount = AmountFromValue(options["maximumAmount"]); + filter_coins.max_amount = AmountFromValue(options["maximumAmount"]); if (options.exists("minimumSumAmount")) - nMinimumSumAmount = AmountFromValue(options["minimumSumAmount"]); + filter_coins.min_sum_amount = AmountFromValue(options["minimumSumAmount"]); if (options.exists("maximumCount")) - nMaximumCount = options["maximumCount"].getInt<int64_t>(); + filter_coins.max_count = options["maximumCount"].getInt<int64_t>(); + + if (options.exists("include_immature_coinbase")) { + filter_coins.include_immature_coinbase = options["include_immature_coinbase"].get_bool(); + } } // Make sure the results are valid at least up to the most recent block @@ -633,7 +637,7 @@ RPCHelpMan listunspent() cctl.m_max_depth = nMaxDepth; cctl.m_include_unsafe_inputs = include_unsafe; LOCK(pwallet->cs_wallet); - vecOutputs = AvailableCoinsListUnspent(*pwallet, &cctl, nMinimumAmount, nMaximumAmount, nMinimumSumAmount, nMaximumCount).All(); + vecOutputs = AvailableCoinsListUnspent(*pwallet, &cctl, filter_coins).All(); } LOCK(pwallet->cs_wallet); diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index ebf694157b..24bcae40e0 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -82,10 +82,10 @@ static UniValue FinishTransaction(const std::shared_ptr<CWallet> pwallet, const PartiallySignedTransaction psbtx(rawTx); // First fill transaction with our data without signing, - // so external signers are not asked sign more than once. + // so external signers are not asked to sign more than once. bool complete; - pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, false, true); - const TransactionError err{pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, true, false)}; + pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, /*sign=*/false, /*bip32derivs=*/true); + const TransactionError err{pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, /*sign=*/true, /*bip32derivs=*/false)}; if (err != TransactionError::OK) { throw JSONRPCTransactionError(err); } @@ -161,7 +161,7 @@ UniValue SendMoney(CWallet& wallet, const CCoinControl &coin_control, std::vecto throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, util::ErrorString(res).original); } const CTransactionRef& tx = res->tx; - wallet.CommitTransaction(tx, std::move(map_value), {} /* orderForm */); + wallet.CommitTransaction(tx, std::move(map_value), /*orderForm=*/{}); if (verbose) { UniValue entry(UniValue::VOBJ); entry.pushKV("txid", tx->GetHash().GetHex()); @@ -746,7 +746,7 @@ RPCHelpMan fundrawtransaction() "If that happens, you will need to fund the transaction with different inputs and republish it."}, {"changeAddress", RPCArg::Type::STR, RPCArg::DefaultHint{"automatic"}, "The bitcoin address to receive the change"}, {"changePosition", RPCArg::Type::NUM, RPCArg::DefaultHint{"random"}, "The index of the change output"}, - {"change_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The output type to use. Only valid if changeAddress is not specified. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, + {"change_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The output type to use. Only valid if changeAddress is not specified. Options are \"legacy\", \"p2sh-segwit\", \"bech32\", and \"bech32m\"."}, {"includeWatching", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Also select inputs which are watch only.\n" "Only solvable inputs can be used. Watch-only destinations are solvable if the public key and/or output script was imported,\n" "e.g. with 'importpubkey' or 'importmulti' with the 'pubkeys' or 'desc' field."}, @@ -993,7 +993,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) std::shared_ptr<CWallet> const pwallet = GetWalletForJSONRPCRequest(request); if (!pwallet) return UniValue::VNULL; - if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !want_psbt) { + if (pwallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) && !pwallet->IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER) && !want_psbt) { throw JSONRPCError(RPC_WALLET_ERROR, "bumpfee is not available with wallets that have private keys disabled. Use psbtbumpfee instead."); } @@ -1071,6 +1071,9 @@ static RPCHelpMan bumpfee_helper(std::string method_name) // For psbtbumpfee, return the base64-encoded unsigned PSBT of the new transaction. if (!want_psbt) { if (!feebumper::SignTransaction(*pwallet, mtx)) { + if (pwallet->IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER)) { + throw JSONRPCError(RPC_WALLET_ERROR, "Transaction incomplete. Try psbtbumpfee instead."); + } throw JSONRPCError(RPC_WALLET_ERROR, "Can't sign transaction."); } @@ -1083,7 +1086,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) } else { PartiallySignedTransaction psbtx(mtx); bool complete = false; - const TransactionError err = pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, false /* sign */, true /* bip32derivs */); + const TransactionError err = pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, /*sign=*/false, /*bip32derivs=*/true); CHECK_NONFATAL(err == TransactionError::OK); CHECK_NONFATAL(!complete); CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); @@ -1143,7 +1146,7 @@ RPCHelpMan send() {"add_to_wallet", RPCArg::Type::BOOL, RPCArg::Default{true}, "When false, returns a serialized transaction which will not be added to the wallet or broadcast"}, {"change_address", RPCArg::Type::STR, RPCArg::DefaultHint{"automatic"}, "The bitcoin address to receive the change"}, {"change_position", RPCArg::Type::NUM, RPCArg::DefaultHint{"random"}, "The index of the change output"}, - {"change_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The output type to use. Only valid if change_address is not specified. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, + {"change_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The output type to use. Only valid if change_address is not specified. Options are \"legacy\", \"p2sh-segwit\", \"bech32\" and \"bech32m\"."}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, {"include_watching", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Also select inputs which are watch only.\n" "Only solvable inputs can be used. Watch-only destinations are solvable if the public key and/or output script was imported,\n" @@ -1379,13 +1382,15 @@ RPCHelpMan sendall() throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Input not available. UTXO (%s:%d) was already spent.", input.prevout.hash.ToString(), input.prevout.n)); } const CWalletTx* tx{pwallet->GetWalletTx(input.prevout.hash)}; - if (!tx || pwallet->IsMine(tx->tx->vout[input.prevout.n]) != (coin_control.fAllowWatchOnly ? ISMINE_ALL : ISMINE_SPENDABLE)) { + if (!tx || input.prevout.n >= tx->tx->vout.size() || !(pwallet->IsMine(tx->tx->vout[input.prevout.n]) & (coin_control.fAllowWatchOnly ? ISMINE_ALL : ISMINE_SPENDABLE))) { throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Input not found. UTXO (%s:%d) is not part of wallet.", input.prevout.hash.ToString(), input.prevout.n)); } total_input_value += tx->tx->vout[input.prevout.n].nValue; } } else { - for (const COutput& output : AvailableCoins(*pwallet, &coin_control, fee_rate, /*nMinimumAmount=*/0).All()) { + CoinFilterParams coins_params; + coins_params.min_amount = 0; + for (const COutput& output : AvailableCoins(*pwallet, &coin_control, fee_rate, coins_params).All()) { CHECK_NONFATAL(output.input_bytes > 0); if (send_max && fee_rate.GetFee(output.input_bytes) > output.txout.nValue) { continue; @@ -1596,7 +1601,7 @@ RPCHelpMan walletcreatefundedpsbt() "If that happens, you will need to fund the transaction with different inputs and republish it."}, {"changeAddress", RPCArg::Type::STR, RPCArg::DefaultHint{"automatic"}, "The bitcoin address to receive the change"}, {"changePosition", RPCArg::Type::NUM, RPCArg::DefaultHint{"random"}, "The index of the change output"}, - {"change_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The output type to use. Only valid if changeAddress is not specified. Options are \"legacy\", \"p2sh-segwit\", and \"bech32\"."}, + {"change_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The output type to use. Only valid if changeAddress is not specified. Options are \"legacy\", \"p2sh-segwit\", \"bech32\", and \"bech32m\"."}, {"includeWatching", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Also select inputs which are watch only"}, {"lockUnspents", RPCArg::Type::BOOL, RPCArg::Default{false}, "Lock selected unspent outputs"}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, @@ -1649,11 +1654,8 @@ RPCHelpMan walletcreatefundedpsbt() CAmount fee; int change_position; - bool rbf{wallet.m_signal_rbf}; const UniValue &replaceable_arg = options["replaceable"]; - if (!replaceable_arg.isNull()) { - rbf = replaceable_arg.isTrue(); - } + const bool rbf{replaceable_arg.isNull() ? wallet.m_signal_rbf : replaceable_arg.get_bool()}; CMutableTransaction rawTx = ConstructTransaction(request.params[0], request.params[1], request.params[2], rbf); CCoinControl coin_control; // Automatically select coins, unless at least one is manually selected. Can @@ -1668,7 +1670,7 @@ RPCHelpMan walletcreatefundedpsbt() // Fill transaction with out data but don't sign bool bip32derivs = request.params[4].isNull() ? true : request.params[4].get_bool(); bool complete = true; - const TransactionError err{wallet.FillPSBT(psbtx, complete, 1, false, bip32derivs)}; + const TransactionError err{wallet.FillPSBT(psbtx, complete, 1, /*sign=*/false, /*bip32derivs=*/bip32derivs)}; if (err != TransactionError::OK) { throw JSONRPCTransactionError(err); } diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index 0e13e4756b..02a1ac5ea1 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -316,7 +316,7 @@ static void MaybePushAddress(UniValue & entry, const CTxDestination &dest) */ template <class Vec> static void ListTransactions(const CWallet& wallet, const CWalletTx& wtx, int nMinDepth, bool fLong, - Vec& ret, const isminefilter& filter_ismine, const std::string* filter_label, + Vec& ret, const isminefilter& filter_ismine, const std::optional<std::string>& filter_label, bool include_change = false) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { @@ -329,7 +329,7 @@ static void ListTransactions(const CWallet& wallet, const CWalletTx& wtx, int nM bool involvesWatchonly = CachedTxIsFromMe(wallet, wtx, ISMINE_WATCH_ONLY); // Sent - if (!filter_label) + if (!filter_label.has_value()) { for (const COutputEntry& s : listSent) { @@ -362,7 +362,7 @@ static void ListTransactions(const CWallet& wallet, const CWalletTx& wtx, int nM if (address_book_entry) { label = address_book_entry->GetLabel(); } - if (filter_label && label != *filter_label) { + if (filter_label.has_value() && label != filter_label.value()) { continue; } UniValue entry(UniValue::VOBJ); @@ -447,7 +447,7 @@ RPCHelpMan listtransactions() {RPCResult::Type::OBJ, "", "", Cat(Cat<std::vector<RPCResult>>( { {RPCResult::Type::BOOL, "involvesWatchonly", /*optional=*/true, "Only returns true if imported addresses were involved in transaction."}, - {RPCResult::Type::STR, "address", "The bitcoin address of the transaction."}, + {RPCResult::Type::STR, "address", /*optional=*/true, "The bitcoin address of the transaction (not returned if the output does not have an address, e.g. OP_RETURN null data)."}, {RPCResult::Type::STR, "category", "The transaction category.\n" "\"send\" Transactions sent.\n" "\"receive\" Non-coinbase transactions received.\n" @@ -485,10 +485,10 @@ RPCHelpMan listtransactions() // the user could have gotten from another RPC command prior to now pwallet->BlockUntilSyncedToCurrentChain(); - const std::string* filter_label = nullptr; + std::optional<std::string> filter_label; if (!request.params[0].isNull() && request.params[0].get_str() != "*") { - filter_label = &request.params[0].get_str(); - if (filter_label->empty()) { + filter_label = request.params[0].get_str(); + if (filter_label.value().empty()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Label argument must be a valid label name or \"*\"."); } } @@ -552,6 +552,7 @@ RPCHelpMan listsinceblock() {"include_removed", RPCArg::Type::BOOL, RPCArg::Default{true}, "Show transactions that were removed due to a reorg in the \"removed\" array\n" "(not guaranteed to work on pruned nodes)"}, {"include_change", RPCArg::Type::BOOL, RPCArg::Default{false}, "Also add entries for change outputs.\n"}, + {"label", RPCArg::Type::STR, RPCArg::Optional::OMITTED_NAMED_ARG, "Return only incoming transactions paying to addresses with the specified label.\n"}, }, RPCResult{ RPCResult::Type::OBJ, "", "", @@ -561,7 +562,7 @@ RPCHelpMan listsinceblock() {RPCResult::Type::OBJ, "", "", Cat(Cat<std::vector<RPCResult>>( { {RPCResult::Type::BOOL, "involvesWatchonly", /*optional=*/true, "Only returns true if imported addresses were involved in transaction."}, - {RPCResult::Type::STR, "address", "The bitcoin address of the transaction."}, + {RPCResult::Type::STR, "address", /*optional=*/true, "The bitcoin address of the transaction (not returned if the output does not have an address, e.g. OP_RETURN null data)."}, {RPCResult::Type::STR, "category", "The transaction category.\n" "\"send\" Transactions sent.\n" "\"receive\" Non-coinbase transactions received.\n" @@ -614,7 +615,7 @@ RPCHelpMan listsinceblock() blockId = ParseHashV(request.params[0], "blockhash"); height = int{}; altheight = int{}; - if (!wallet.chain().findCommonAncestor(blockId, wallet.GetLastBlockHash(), /* ancestor out */ FoundBlock().height(*height), /* blockId out */ FoundBlock().height(*altheight))) { + if (!wallet.chain().findCommonAncestor(blockId, wallet.GetLastBlockHash(), /*ancestor_out=*/FoundBlock().height(*height), /*block1_out=*/FoundBlock().height(*altheight))) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Block not found"); } } @@ -634,6 +635,11 @@ RPCHelpMan listsinceblock() bool include_removed = (request.params[3].isNull() || request.params[3].get_bool()); bool include_change = (!request.params[4].isNull() && request.params[4].get_bool()); + std::optional<std::string> filter_label; + if (!request.params[5].isNull()) { + filter_label = request.params[5].get_str(); + } + int depth = height ? wallet.GetLastBlockHeight() + 1 - *height : -1; UniValue transactions(UniValue::VARR); @@ -642,7 +648,7 @@ RPCHelpMan listsinceblock() const CWalletTx& tx = pairWtx.second; if (depth == -1 || abs(wallet.GetTxDepthInMainChain(tx)) < depth) { - ListTransactions(wallet, tx, 0, true, transactions, filter, nullptr /* filter_label */, /*include_change=*/include_change); + ListTransactions(wallet, tx, 0, true, transactions, filter, filter_label, include_change); } } @@ -659,7 +665,7 @@ RPCHelpMan listsinceblock() if (it != wallet.mapWallet.end()) { // We want all transactions regardless of confirmation count to appear here, // even negative confirmation ones, hence the big negative. - ListTransactions(wallet, it->second, -100000000, true, removed, filter, nullptr /* filter_label */, /*include_change=*/include_change); + ListTransactions(wallet, it->second, -100000000, true, removed, filter, filter_label, include_change); } } blockId = block.hashPrevBlock; @@ -777,7 +783,7 @@ RPCHelpMan gettransaction() WalletTxToJSON(*pwallet, wtx, entry); UniValue details(UniValue::VARR); - ListTransactions(*pwallet, wtx, 0, false, details, filter, nullptr /* filter_label */); + ListTransactions(*pwallet, wtx, 0, false, details, filter, /*filter_label=*/std::nullopt); entry.pushKV("details", details); std::string strHex = EncodeHexTx(*wtx.tx, pwallet->chain().rpcSerializationFlags()); @@ -839,7 +845,9 @@ RPCHelpMan rescanblockchain() { return RPCHelpMan{"rescanblockchain", "\nRescan the local blockchain for wallet related transactions.\n" - "Note: Use \"getwalletinfo\" to query the scanning progress.\n", + "Note: Use \"getwalletinfo\" to query the scanning progress.\n" + "The rescan is significantly faster when used on a descriptor wallet\n" + "and block filters are available (using startup option \"-blockfilterindex=1\").\n", { {"start_height", RPCArg::Type::NUM, RPCArg::Default{0}, "block height where the rescan should start"}, {"stop_height", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "the last block height that should be scanned. If none is provided it will rescan up to the tip at return time of this call."}, diff --git a/src/wallet/rpc/util.cpp b/src/wallet/rpc/util.cpp index 1aa2a87e99..26270f23ed 100644 --- a/src/wallet/rpc/util.cpp +++ b/src/wallet/rpc/util.cpp @@ -4,9 +4,9 @@ #include <wallet/rpc/util.h> +#include <common/url.h> #include <rpc/util.h> #include <util/translation.h> -#include <util/url.h> #include <wallet/context.h> #include <wallet/wallet.h> diff --git a/src/wallet/rpc/wallet.cpp b/src/wallet/rpc/wallet.cpp index 675c4a759d..971814e9cd 100644 --- a/src/wallet/rpc/wallet.cpp +++ b/src/wallet/rpc/wallet.cpp @@ -730,7 +730,9 @@ static RPCHelpMan migratewallet() std::shared_ptr<CWallet> wallet = GetWalletForJSONRPCRequest(request); if (!wallet) return NullUniValue; - EnsureWalletIsUnlocked(*wallet); + if (wallet->IsCrypted()) { + throw JSONRPCError(RPC_WALLET_WRONG_ENC_STATE, "Error: migratewallet on encrypted wallets is currently unsupported."); + } WalletContext& context = EnsureWalletContext(request.context); diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 6da0741d59..896ade77dd 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -28,6 +28,9 @@ util::Result<CTxDestination> LegacyScriptPubKeyMan::GetNewDestination(const Outp } assert(type != OutputType::BECH32M); + // Fill-up keypool if needed + TopUp(); + LOCK(cs_KeyStore); // Generate a new key that is added to wallet @@ -304,6 +307,9 @@ util::Result<CTxDestination> LegacyScriptPubKeyMan::GetReservedDestination(const return util::Error{_("Error: Keypool ran out, please call keypoolrefill first")}; } + // Fill-up keypool if needed + TopUp(); + if (!ReserveKeyFromKeyPool(index, keypool, internal)) { return util::Error{_("Error: Keypool ran out, please call keypoolrefill first")}; } @@ -1983,7 +1989,7 @@ util::Result<CTxDestination> DescriptorScriptPubKeyMan::GetNewDestination(const std::optional<OutputType> desc_addr_type = m_wallet_descriptor.descriptor->GetOutputType(); assert(desc_addr_type); if (type != *desc_addr_type) { - throw std::runtime_error(std::string(__func__) + ": Types are inconsistent"); + throw std::runtime_error(std::string(__func__) + ": Types are inconsistent. Stored type does not match type of newly generated address"); } TopUp(); @@ -2001,11 +2007,8 @@ util::Result<CTxDestination> DescriptorScriptPubKeyMan::GetNewDestination(const } CTxDestination dest; - std::optional<OutputType> out_script_type = m_wallet_descriptor.descriptor->GetOutputType(); - if (out_script_type && out_script_type == type) { - ExtractDestination(scripts_temp[0], dest); - } else { - throw std::runtime_error(std::string(__func__) + ": Types are inconsistent. Stored type does not match type of newly generated address"); + if (!ExtractDestination(scripts_temp[0], dest)) { + return util::Error{_("Error: Cannot extract destination from the generated scriptpubkey")}; // shouldn't happen } m_wallet_descriptor.next_index++; WalletBatch(m_storage.GetDatabase()).WriteDescriptor(GetID(), m_wallet_descriptor); @@ -2499,14 +2502,23 @@ TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& keys->Merge(std::move(*script_keys)); } else { // Maybe there are pubkeys listed that we can sign for - script_keys = std::make_unique<FlatSigningProvider>(); - for (const auto& pk_pair : input.hd_keypaths) { - const CPubKey& pubkey = pk_pair.first; - std::unique_ptr<FlatSigningProvider> pk_keys = GetSigningProvider(pubkey); - if (pk_keys) { - keys->Merge(std::move(*pk_keys)); - } + std::vector<CPubKey> pubkeys; + + // ECDSA Pubkeys + for (const auto& [pk, _] : input.hd_keypaths) { + pubkeys.push_back(pk); + } + + // Taproot output pubkey + std::vector<std::vector<unsigned char>> sols; + if (Solver(script, sols) == TxoutType::WITNESS_V1_TAPROOT) { + sols[0].insert(sols[0].begin(), 0x02); + pubkeys.emplace_back(sols[0]); + sols[0][0] = 0x03; + pubkeys.emplace_back(sols[0]); } + + // Taproot pubkeys for (const auto& pk_pair : input.m_tap_bip32_paths) { const XOnlyPubKey& pubkey = pk_pair.first; for (unsigned char prefix : {0x02, 0x03}) { @@ -2514,10 +2526,14 @@ TransactionError DescriptorScriptPubKeyMan::FillPSBT(PartiallySignedTransaction& std::copy(pubkey.begin(), pubkey.end(), b + 1); CPubKey fullpubkey; fullpubkey.Set(b, b + 33); - std::unique_ptr<FlatSigningProvider> pk_keys = GetSigningProvider(fullpubkey); - if (pk_keys) { - keys->Merge(std::move(*pk_keys)); - } + pubkeys.push_back(fullpubkey); + } + } + + for (const auto& pubkey : pubkeys) { + std::unique_ptr<FlatSigningProvider> pk_keys = GetSigningProvider(pubkey); + if (pk_keys) { + keys->Merge(std::move(*pk_keys)); } } } @@ -2642,16 +2658,26 @@ const WalletDescriptor DescriptorScriptPubKeyMan::GetWalletDescriptor() const const std::unordered_set<CScript, SaltedSipHasher> DescriptorScriptPubKeyMan::GetScriptPubKeys() const { + return GetScriptPubKeys(0); +} + +const std::unordered_set<CScript, SaltedSipHasher> DescriptorScriptPubKeyMan::GetScriptPubKeys(int32_t minimum_index) const +{ LOCK(cs_desc_man); std::unordered_set<CScript, SaltedSipHasher> script_pub_keys; script_pub_keys.reserve(m_map_script_pub_keys.size()); - for (auto const& script_pub_key: m_map_script_pub_keys) { - script_pub_keys.insert(script_pub_key.first); + for (auto const& [script_pub_key, index] : m_map_script_pub_keys) { + if (index >= minimum_index) script_pub_keys.insert(script_pub_key); } return script_pub_keys; } +int32_t DescriptorScriptPubKeyMan::GetEndRange() const +{ + return m_max_cached_index + 1; +} + bool DescriptorScriptPubKeyMan::GetDescriptorString(std::string& out, const bool priv) const { LOCK(cs_desc_man); diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 3ab489c374..eb77015956 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -643,6 +643,8 @@ public: const WalletDescriptor GetWalletDescriptor() const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); const std::unordered_set<CScript, SaltedSipHasher> GetScriptPubKeys() const override; + const std::unordered_set<CScript, SaltedSipHasher> GetScriptPubKeys(int32_t minimum_index) const; + int32_t GetEndRange() const; bool GetDescriptorString(std::string& out, const bool priv) const; diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 6833f9a095..33289c1d2f 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -2,10 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <algorithm> #include <consensus/amount.h> #include <consensus/validation.h> #include <interfaces/chain.h> +#include <numeric> #include <policy/policy.h> +#include <primitives/transaction.h> #include <script/signingprovider.h> #include <util/check.h> #include <util/fees.h> @@ -102,15 +105,19 @@ void CoinsResult::Clear() { coins.clear(); } -void CoinsResult::Erase(std::set<COutPoint>& preset_coins) +void CoinsResult::Erase(const std::unordered_set<COutPoint, SaltedOutpointHasher>& coins_to_remove) { - for (auto& it : coins) { - auto& vec = it.second; - auto i = std::find_if(vec.begin(), vec.end(), [&](const COutput &c) { return preset_coins.count(c.outpoint);}); - if (i != vec.end()) { - vec.erase(i); - break; - } + for (auto& [type, vec] : coins) { + auto remove_it = std::remove_if(vec.begin(), vec.end(), [&](const COutput& coin) { + // remove it if it's on the set + if (coins_to_remove.count(coin.outpoint) == 0) return false; + + // update cached amounts + total_amount -= coin.txout.nValue; + if (coin.HasEffectiveValue()) total_effective_amount = *total_effective_amount - coin.GetEffectiveValue(); + return true; + }); + vec.erase(remove_it, vec.end()); } } @@ -124,6 +131,11 @@ void CoinsResult::Shuffle(FastRandomContext& rng_fast) void CoinsResult::Add(OutputType type, const COutput& out) { coins[type].emplace_back(out); + total_amount += out.txout.nValue; + if (out.HasEffectiveValue()) { + total_effective_amount = total_effective_amount.has_value() ? + *total_effective_amount + out.GetEffectiveValue() : out.GetEffectiveValue(); + } } static OutputType GetOutputType(TxoutType type, bool is_from_p2sh) @@ -143,14 +155,55 @@ static OutputType GetOutputType(TxoutType type, bool is_from_p2sh) } } +// Fetch and validate the coin control selected inputs. +// Coins could be internal (from the wallet) or external. +util::Result<PreSelectedInputs> FetchSelectedInputs(const CWallet& wallet, const CCoinControl& coin_control, + const CoinSelectionParams& coin_selection_params) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +{ + PreSelectedInputs result; + std::vector<COutPoint> vPresetInputs; + coin_control.ListSelected(vPresetInputs); + for (const COutPoint& outpoint : vPresetInputs) { + int input_bytes = -1; + CTxOut txout; + if (auto ptr_wtx = wallet.GetWalletTx(outpoint.hash)) { + // Clearly invalid input, fail + if (ptr_wtx->tx->vout.size() <= outpoint.n) { + return util::Error{strprintf(_("Invalid pre-selected input %s"), outpoint.ToString())}; + } + txout = ptr_wtx->tx->vout.at(outpoint.n); + input_bytes = CalculateMaximumSignedInputSize(txout, &wallet, &coin_control); + } else { + // The input is external. We did not find the tx in mapWallet. + if (!coin_control.GetExternalOutput(outpoint, txout)) { + return util::Error{strprintf(_("Not found pre-selected input %s"), outpoint.ToString())}; + } + } + + if (input_bytes == -1) { + input_bytes = CalculateMaximumSignedInputSize(txout, outpoint, &coin_control.m_external_provider, &coin_control); + } + + // If available, override calculated size with coin control specified size + if (coin_control.HasInputWeight(outpoint)) { + input_bytes = GetVirtualTransactionSize(coin_control.GetInputWeight(outpoint), 0, 0); + } + + if (input_bytes == -1) { + return util::Error{strprintf(_("Not solvable pre-selected input %s"), outpoint.ToString())}; // Not solvable, can't estimate size for fee + } + + /* Set some defaults for depth, spendable, solvable, safe, time, and from_me as these don't matter for preset inputs since no selection is being done. */ + COutput output(outpoint, txout, /*depth=*/ 0, input_bytes, /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, /*time=*/ 0, /*from_me=*/ false, coin_selection_params.m_effective_feerate); + result.Insert(output, coin_selection_params.m_subtract_fee_outputs); + } + return result; +} + CoinsResult AvailableCoins(const CWallet& wallet, const CCoinControl* coinControl, std::optional<CFeeRate> feerate, - const CAmount& nMinimumAmount, - const CAmount& nMaximumAmount, - const CAmount& nMinimumSumAmount, - const uint64_t nMaximumCount, - bool only_spendable) + const CoinFilterParams& params) { AssertLockHeld(wallet.cs_wallet); @@ -168,7 +221,7 @@ CoinsResult AvailableCoins(const CWallet& wallet, const uint256& wtxid = entry.first; const CWalletTx& wtx = entry.second; - if (wallet.IsTxImmatureCoinBase(wtx)) + if (wallet.IsTxImmatureCoinBase(wtx) && !params.include_immature_coinbase) continue; int nDepth = wallet.GetTxDepthInMainChain(wtx); @@ -227,10 +280,11 @@ CoinsResult AvailableCoins(const CWallet& wallet, const CTxOut& output = wtx.tx->vout[i]; const COutPoint outpoint(wtxid, i); - if (output.nValue < nMinimumAmount || output.nValue > nMaximumAmount) + if (output.nValue < params.min_amount || output.nValue > params.max_amount) continue; - if (coinControl && coinControl->HasSelected() && !coinControl->m_allow_other_inputs && !coinControl->IsSelected(outpoint)) + // Skip manually selected coins (the caller can fetch them directly) + if (coinControl && coinControl->HasSelected() && coinControl->IsSelected(outpoint)) continue; if (wallet.IsLockedCoin(outpoint)) @@ -258,7 +312,7 @@ CoinsResult AvailableCoins(const CWallet& wallet, bool spendable = ((mine & ISMINE_SPENDABLE) != ISMINE_NO) || (((mine & ISMINE_WATCH_ONLY) != ISMINE_NO) && (coinControl && coinControl->fAllowWatchOnly && solvable)); // Filter by spendable outputs only - if (!spendable && only_spendable) continue; + if (!spendable && params.only_spendable) continue; // Obtain script type std::vector<std::vector<uint8_t>> script_solutions; @@ -279,17 +333,15 @@ CoinsResult AvailableCoins(const CWallet& wallet, result.Add(GetOutputType(type, is_from_p2sh), COutput(outpoint, output, nDepth, input_bytes, spendable, solvable, safeTx, wtx.GetTxTime(), tx_from_me, feerate)); - // Cache total amount as we go - result.total_amount += output.nValue; // Checks the sum amount of all UTXO's. - if (nMinimumSumAmount != MAX_MONEY) { - if (result.total_amount >= nMinimumSumAmount) { + if (params.min_sum_amount != MAX_MONEY) { + if (result.GetTotalAmount() >= params.min_sum_amount) { return result; } } // Checks the maximum number of UTXO's. - if (nMaximumCount > 0 && result.Size() >= nMaximumCount) { + if (params.max_count > 0 && result.Size() >= params.max_count) { return result; } } @@ -298,21 +350,16 @@ CoinsResult AvailableCoins(const CWallet& wallet, return result; } -CoinsResult AvailableCoinsListUnspent(const CWallet& wallet, const CCoinControl* coinControl, const CAmount& nMinimumAmount, const CAmount& nMaximumAmount, const CAmount& nMinimumSumAmount, const uint64_t nMaximumCount) +CoinsResult AvailableCoinsListUnspent(const CWallet& wallet, const CCoinControl* coinControl, CoinFilterParams params) { - return AvailableCoins(wallet, coinControl, /*feerate=*/ std::nullopt, nMinimumAmount, nMaximumAmount, nMinimumSumAmount, nMaximumCount, /*only_spendable=*/false); + params.only_spendable = false; + return AvailableCoins(wallet, coinControl, /*feerate=*/ std::nullopt, params); } CAmount GetAvailableBalance(const CWallet& wallet, const CCoinControl* coinControl) { LOCK(wallet.cs_wallet); - return AvailableCoins(wallet, coinControl, - /*feerate=*/ std::nullopt, - /*nMinimumAmount=*/ 1, - /*nMaximumAmount=*/ MAX_MONEY, - /*nMinimumSumAmount=*/ MAX_MONEY, - /*nMaximumCount=*/ 0 - ).total_amount; + return AvailableCoins(wallet, coinControl).GetTotalAmount(); } const CTxOut& FindNonChangeParentOutput(const CWallet& wallet, const CTransaction& tx, int output) @@ -478,8 +525,9 @@ std::optional<SelectionResult> AttemptSelection(const CWallet& wallet, const CAm if (results.size() > 0) return *std::min_element(results.begin(), results.end()); // If we can't fund the transaction from any individual OutputType, run coin selection one last time - // over all available coins, which would allow mixing - if (allow_mixed_output_types) { + // over all available coins, which would allow mixing. + // If TypesCount() <= 1, there is nothing to mix. + if (allow_mixed_output_types && available_coins.TypesCount() > 1) { if (auto result{ChooseSelectionResult(wallet, nTargetValue, eligibility_filter, available_coins.All(), coin_selection_params)}) { return result; } @@ -511,93 +559,71 @@ std::optional<SelectionResult> ChooseSelectionResult(const CWallet& wallet, cons results.push_back(*srd_result); } - if (results.size() == 0) { + if (results.empty()) { // No solution found return std::nullopt; } + std::vector<SelectionResult> eligible_results; + std::copy_if(results.begin(), results.end(), std::back_inserter(eligible_results), [coin_selection_params](const SelectionResult& result) { + const auto initWeight{coin_selection_params.tx_noinputs_size * WITNESS_SCALE_FACTOR}; + return initWeight + result.GetWeight() <= static_cast<int>(MAX_STANDARD_TX_WEIGHT); + }); + + if (eligible_results.empty()) { + return std::nullopt; + } + // Choose the result with the least waste // If the waste is the same, choose the one which spends more inputs. - auto& best_result = *std::min_element(results.begin(), results.end()); + auto& best_result = *std::min_element(eligible_results.begin(), eligible_results.end()); return best_result; } -std::optional<SelectionResult> SelectCoins(const CWallet& wallet, CoinsResult& available_coins, const CAmount& nTargetValue, const CCoinControl& coin_control, const CoinSelectionParams& coin_selection_params) +std::optional<SelectionResult> SelectCoins(const CWallet& wallet, CoinsResult& available_coins, const PreSelectedInputs& pre_set_inputs, + const CAmount& nTargetValue, const CCoinControl& coin_control, + const CoinSelectionParams& coin_selection_params) { - CAmount value_to_select = nTargetValue; + // Deduct preset inputs amount from the search target + CAmount selection_target = nTargetValue - pre_set_inputs.total_amount; - OutputGroup preset_inputs(coin_selection_params); + // Return if automatic coin selection is disabled, and we don't cover the selection target + if (!coin_control.m_allow_other_inputs && selection_target > 0) return std::nullopt; - // calculate value from preset inputs and store them - std::set<COutPoint> preset_coins; - - std::vector<COutPoint> vPresetInputs; - coin_control.ListSelected(vPresetInputs); - for (const COutPoint& outpoint : vPresetInputs) { - int input_bytes = -1; - CTxOut txout; - auto ptr_wtx = wallet.GetWalletTx(outpoint.hash); - if (ptr_wtx) { - // Clearly invalid input, fail - if (ptr_wtx->tx->vout.size() <= outpoint.n) { - return std::nullopt; - } - txout = ptr_wtx->tx->vout.at(outpoint.n); - input_bytes = CalculateMaximumSignedInputSize(txout, &wallet, &coin_control); - } else { - // The input is external. We did not find the tx in mapWallet. - if (!coin_control.GetExternalOutput(outpoint, txout)) { - return std::nullopt; - } - } - - if (input_bytes == -1) { - input_bytes = CalculateMaximumSignedInputSize(txout, outpoint, &coin_control.m_external_provider, &coin_control); - } - - // If available, override calculated size with coin control specified size - if (coin_control.HasInputWeight(outpoint)) { - input_bytes = GetVirtualTransactionSize(coin_control.GetInputWeight(outpoint), 0, 0); - } - - if (input_bytes == -1) { - return std::nullopt; // Not solvable, can't estimate size for fee - } - - /* Set some defaults for depth, spendable, solvable, safe, time, and from_me as these don't matter for preset inputs since no selection is being done. */ - COutput output(outpoint, txout, /*depth=*/ 0, input_bytes, /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, /*time=*/ 0, /*from_me=*/ false, coin_selection_params.m_effective_feerate); - if (coin_selection_params.m_subtract_fee_outputs) { - value_to_select -= output.txout.nValue; - } else { - value_to_select -= output.GetEffectiveValue(); - } - preset_coins.insert(outpoint); - /* Set ancestors and descendants to 0 as they don't matter for preset inputs since no actual selection is being done. - * positive_only is set to false because we want to include all preset inputs, even if they are dust. - */ - preset_inputs.Insert(output, /*ancestors=*/ 0, /*descendants=*/ 0, /*positive_only=*/ false); - } - - // coin control -> return all selected outputs (we want all selected to go into the transaction for sure) - if (coin_control.HasSelected() && !coin_control.m_allow_other_inputs) { + // Return if we can cover the target only with the preset inputs + if (selection_target <= 0) { SelectionResult result(nTargetValue, SelectionAlgorithm::MANUAL); - result.AddInput(preset_inputs); - - if (!coin_selection_params.m_subtract_fee_outputs && result.GetSelectedEffectiveValue() < nTargetValue) { - return std::nullopt; - } else if (result.GetSelectedValue() < nTargetValue) { - return std::nullopt; - } - + result.AddInputs(pre_set_inputs.coins, coin_selection_params.m_subtract_fee_outputs); result.ComputeAndSetWaste(coin_selection_params.min_viable_change, coin_selection_params.m_cost_of_change, coin_selection_params.m_change_fee); return result; } - // remove preset inputs from coins so that Coin Selection doesn't pick them. - if (coin_control.HasSelected()) { - available_coins.Erase(preset_coins); + // Return early if we cannot cover the target with the wallet's UTXO. + // We use the total effective value if we are not subtracting fee from outputs and 'available_coins' contains the data. + CAmount available_coins_total_amount = coin_selection_params.m_subtract_fee_outputs ? available_coins.GetTotalAmount() : + (available_coins.GetEffectiveTotalAmount().has_value() ? *available_coins.GetEffectiveTotalAmount() : 0); + if (selection_target > available_coins_total_amount) { + return std::nullopt; // Insufficient funds } + // Start wallet Coin Selection procedure + auto op_selection_result = AutomaticCoinSelection(wallet, available_coins, selection_target, coin_control, coin_selection_params); + if (!op_selection_result) return op_selection_result; + + // If needed, add preset inputs to the automatic coin selection result + if (!pre_set_inputs.coins.empty()) { + SelectionResult preselected(pre_set_inputs.total_amount, SelectionAlgorithm::MANUAL); + preselected.AddInputs(pre_set_inputs.coins, coin_selection_params.m_subtract_fee_outputs); + op_selection_result->Merge(preselected); + op_selection_result->ComputeAndSetWaste(coin_selection_params.min_viable_change, + coin_selection_params.m_cost_of_change, + coin_selection_params.m_change_fee); + } + return op_selection_result; +} + +std::optional<SelectionResult> AutomaticCoinSelection(const CWallet& wallet, CoinsResult& available_coins, const CAmount& value_to_select, const CCoinControl& coin_control, const CoinSelectionParams& coin_selection_params) +{ unsigned int limit_ancestor_count = 0; unsigned int limit_descendant_count = 0; wallet.chain().getPackageLimits(limit_ancestor_count, limit_descendant_count); @@ -614,16 +640,10 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, CoinsResult& a available_coins.Shuffle(coin_selection_params.rng_fast); } - SelectionResult preselected(preset_inputs.GetSelectionAmount(), SelectionAlgorithm::MANUAL); - preselected.AddInput(preset_inputs); - // Coin Selection attempts to select inputs from a pool of eligible UTXOs to fund the // transaction at a target feerate. If an attempt fails, more attempts may be made using a more // permissive CoinEligibilityFilter. std::optional<SelectionResult> res = [&] { - // Pre-selected inputs already cover the target amount. - if (value_to_select <= 0) return std::make_optional(SelectionResult(value_to_select, SelectionAlgorithm::MANUAL)); - // If possible, fund the transaction with confirmed UTXOs only. Prefer at least six // confirmations on outputs received from other wallets and only spend confirmed change. if (auto r1{AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(1, 6, 0), available_coins, coin_selection_params, /*allow_mixed_output_types=*/false)}) return r1; @@ -645,7 +665,7 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, CoinsResult& a // If partial groups are allowed, relax the requirement of spending OutputGroups (groups // of UTXOs sent to the same address, which are obviously controlled by a single wallet) // in their entirety. - if (auto r6{AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(0, 1, max_ancestors-1, max_descendants-1, true /* include_partial_groups */), + if (auto r6{AttemptSelection(wallet, value_to_select, CoinEligibilityFilter(0, 1, max_ancestors-1, max_descendants-1, /*include_partial=*/true), available_coins, coin_selection_params, /*allow_mixed_output_types=*/true)}) { return r6; } @@ -653,7 +673,7 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, CoinsResult& a // received from other wallets. if (coin_control.m_include_unsafe_inputs) { if (auto r7{AttemptSelection(wallet, value_to_select, - CoinEligibilityFilter(0 /* conf_mine */, 0 /* conf_theirs */, max_ancestors-1, max_descendants-1, true /* include_partial_groups */), + CoinEligibilityFilter(/*conf_mine=*/0, /*conf_theirs=*/0, max_ancestors-1, max_descendants-1, /*include_partial=*/true), available_coins, coin_selection_params, /*allow_mixed_output_types=*/true)}) { return r7; } @@ -663,7 +683,7 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, CoinsResult& a // OutputGroups use heuristics that may overestimate ancestor/descendant counts. if (!fRejectLongChains) { if (auto r8{AttemptSelection(wallet, value_to_select, - CoinEligibilityFilter(0, 1, std::numeric_limits<uint64_t>::max(), std::numeric_limits<uint64_t>::max(), true /* include_partial_groups */), + CoinEligibilityFilter(0, 1, std::numeric_limits<uint64_t>::max(), std::numeric_limits<uint64_t>::max(), /*include_partial=*/true), available_coins, coin_selection_params, /*allow_mixed_output_types=*/true)}) { return r8; } @@ -673,14 +693,6 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, CoinsResult& a return std::optional<SelectionResult>(); }(); - if (!res) return std::nullopt; - - // Add preset inputs to result - res->Merge(preselected); - if (res->GetAlgo() == SelectionAlgorithm::MANUAL) { - res->ComputeAndSetWaste(coin_selection_params.min_viable_change, coin_selection_params.m_cost_of_change, coin_selection_params.m_change_fee); - } - return res; } @@ -768,7 +780,6 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( AssertLockHeld(wallet.cs_wallet); // out variables, to be packed into returned result structure - CAmount nFeeRet; int nChangePosInOut = change_pos; FastRandomContext rng_fast; @@ -869,19 +880,16 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( const auto change_spend_fee = coin_selection_params.m_discard_feerate.GetFee(coin_selection_params.change_spend_size); coin_selection_params.min_viable_change = std::max(change_spend_fee + 1, dust); + // Static vsize overhead + outputs vsize. 4 nVersion, 4 nLocktime, 1 input count, 1 witness overhead (dummy, flag, stack size) + coin_selection_params.tx_noinputs_size = 10 + GetSizeOfCompactSize(vecSend.size()); // bytes for output count + // vouts to the payees - if (!coin_selection_params.m_subtract_fee_outputs) { - coin_selection_params.tx_noinputs_size = 10; // Static vsize overhead + outputs vsize. 4 nVersion, 4 nLocktime, 1 input count, 1 witness overhead (dummy, flag, stack size) - coin_selection_params.tx_noinputs_size += GetSizeOfCompactSize(vecSend.size()); // bytes for output count - } for (const auto& recipient : vecSend) { CTxOut txout(recipient.nAmount, recipient.scriptPubKey); // Include the fee cost for outputs. - if (!coin_selection_params.m_subtract_fee_outputs) { - coin_selection_params.tx_noinputs_size += ::GetSerializeSize(txout, PROTOCOL_VERSION); - } + coin_selection_params.tx_noinputs_size += ::GetSerializeSize(txout, PROTOCOL_VERSION); if (IsDust(txout, wallet.chain().relayDustFee())) { return util::Error{_("Transaction amount too small")}; @@ -890,20 +898,26 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( } // Include the fees for things that aren't inputs, excluding the change output - const CAmount not_input_fees = coin_selection_params.m_effective_feerate.GetFee(coin_selection_params.tx_noinputs_size); + const CAmount not_input_fees = coin_selection_params.m_effective_feerate.GetFee(coin_selection_params.m_subtract_fee_outputs ? 0 : coin_selection_params.tx_noinputs_size); CAmount selection_target = recipients_sum + not_input_fees; - // Get available coins - auto available_coins = AvailableCoins(wallet, - &coin_control, - coin_selection_params.m_effective_feerate, - 1, /*nMinimumAmount*/ - MAX_MONEY, /*nMaximumAmount*/ - MAX_MONEY, /*nMinimumSumAmount*/ - 0); /*nMaximumCount*/ + // Fetch manually selected coins + PreSelectedInputs preset_inputs; + if (coin_control.HasSelected()) { + auto res_fetch_inputs = FetchSelectedInputs(wallet, coin_control, coin_selection_params); + if (!res_fetch_inputs) return util::Error{util::ErrorString(res_fetch_inputs)}; + preset_inputs = *res_fetch_inputs; + } + + // Fetch wallet available coins if "other inputs" are + // allowed (coins automatically selected by the wallet) + CoinsResult available_coins; + if (coin_control.m_allow_other_inputs) { + available_coins = AvailableCoins(wallet, &coin_control, coin_selection_params.m_effective_feerate); + } // Choose coins to use - std::optional<SelectionResult> result = SelectCoins(wallet, available_coins, /*nTargetValue=*/selection_target, coin_control, coin_selection_params); + std::optional<SelectionResult> result = SelectCoins(wallet, available_coins, preset_inputs, /*nTargetValue=*/selection_target, coin_control, coin_selection_params); if (!result) { return util::Error{_("Insufficient funds")}; } @@ -947,22 +961,28 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( return util::Error{_("Missing solving data for estimating transaction size")}; } CAmount fee_needed = coin_selection_params.m_effective_feerate.GetFee(nBytes); - nFeeRet = result->GetSelectedValue() - recipients_sum - change_amount; + const CAmount output_value = CalculateOutputValue(txNew); + Assume(recipients_sum + change_amount == output_value); + CAmount current_fee = result->GetSelectedValue() - output_value; - // The only time that fee_needed should be less than the amount available for fees is when - // we are subtracting the fee from the outputs. If this occurs at any other time, it is a bug. - assert(coin_selection_params.m_subtract_fee_outputs || fee_needed <= nFeeRet); + // Sanity check that the fee cannot be negative as that means we have more output value than input value + if (current_fee < 0) { + return util::Error{Untranslated(STR_INTERNAL_BUG("Fee paid < 0"))}; + } // If there is a change output and we overpay the fees then increase the change to match the fee needed - if (nChangePosInOut != -1 && fee_needed < nFeeRet) { + if (nChangePosInOut != -1 && fee_needed < current_fee) { auto& change = txNew.vout.at(nChangePosInOut); - change.nValue += nFeeRet - fee_needed; - nFeeRet = fee_needed; + change.nValue += current_fee - fee_needed; + current_fee = result->GetSelectedValue() - CalculateOutputValue(txNew); + if (fee_needed != current_fee) { + return util::Error{Untranslated(STR_INTERNAL_BUG("Change adjustment: Fee needed != fee paid"))}; + } } // Reduce output values for subtractFeeFromAmount if (coin_selection_params.m_subtract_fee_outputs) { - CAmount to_reduce = fee_needed - nFeeRet; + CAmount to_reduce = fee_needed - current_fee; int i = 0; bool fFirst = true; for (const auto& recipient : vecSend) @@ -993,7 +1013,16 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( } ++i; } - nFeeRet = fee_needed; + current_fee = result->GetSelectedValue() - CalculateOutputValue(txNew); + if (fee_needed != current_fee) { + return util::Error{Untranslated(STR_INTERNAL_BUG("SFFO: Fee needed != fee paid"))}; + } + } + + // fee_needed should now always be less than or equal to the current fees that we pay. + // If it is not, it is a bug. + if (fee_needed > current_fee) { + return util::Error{Untranslated(STR_INTERNAL_BUG("Fee needed > fee paid"))}; } // Give up if change keypool ran out and change is required @@ -1015,7 +1044,7 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( return util::Error{_("Transaction too large")}; } - if (nFeeRet > wallet.m_default_max_tx_fee) { + if (current_fee > wallet.m_default_max_tx_fee) { return util::Error{TransactionErrorString(TransactionError::MAX_FEE_EXCEEDED)}; } @@ -1031,14 +1060,14 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( reservedest.KeepDestination(); wallet.WalletLogPrintf("Fee Calculation: Fee:%d Bytes:%u Tgt:%d (requested %d) Reason:\"%s\" Decay %.5f: Estimation: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out) Fail: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out)\n", - nFeeRet, nBytes, feeCalc.returnedTarget, feeCalc.desiredTarget, StringForFeeReason(feeCalc.reason), feeCalc.est.decay, + current_fee, nBytes, feeCalc.returnedTarget, feeCalc.desiredTarget, StringForFeeReason(feeCalc.reason), feeCalc.est.decay, feeCalc.est.pass.start, feeCalc.est.pass.end, (feeCalc.est.pass.totalConfirmed + feeCalc.est.pass.inMempool + feeCalc.est.pass.leftMempool) > 0.0 ? 100 * feeCalc.est.pass.withinTarget / (feeCalc.est.pass.totalConfirmed + feeCalc.est.pass.inMempool + feeCalc.est.pass.leftMempool) : 0.0, feeCalc.est.pass.withinTarget, feeCalc.est.pass.totalConfirmed, feeCalc.est.pass.inMempool, feeCalc.est.pass.leftMempool, feeCalc.est.fail.start, feeCalc.est.fail.end, (feeCalc.est.fail.totalConfirmed + feeCalc.est.fail.inMempool + feeCalc.est.fail.leftMempool) > 0.0 ? 100 * feeCalc.est.fail.withinTarget / (feeCalc.est.fail.totalConfirmed + feeCalc.est.fail.inMempool + feeCalc.est.fail.leftMempool) : 0.0, feeCalc.est.fail.withinTarget, feeCalc.est.fail.totalConfirmed, feeCalc.est.fail.inMempool, feeCalc.est.fail.leftMempool); - return CreatedTransactionResult(tx, nFeeRet, nChangePosInOut, feeCalc); + return CreatedTransactionResult(tx, current_fee, nChangePosInOut, feeCalc); } util::Result<CreatedTransactionResult> CreateTransaction( diff --git a/src/wallet/spend.h b/src/wallet/spend.h index c29e5be5c7..46144d4c92 100644 --- a/src/wallet/spend.h +++ b/src/wallet/spend.h @@ -46,13 +46,36 @@ struct CoinsResult { /** The following methods are provided so that CoinsResult can mimic a vector, * i.e., methods can work with individual OutputType vectors or on the entire object */ size_t Size() const; + /** Return how many different output types this struct stores */ + size_t TypesCount() const { return coins.size(); } void Clear(); - void Erase(std::set<COutPoint>& preset_coins); + void Erase(const std::unordered_set<COutPoint, SaltedOutpointHasher>& coins_to_remove); void Shuffle(FastRandomContext& rng_fast); void Add(OutputType type, const COutput& out); - /** Sum of all available coins */ + CAmount GetTotalAmount() { return total_amount; } + std::optional<CAmount> GetEffectiveTotalAmount() {return total_effective_amount; } + +private: + /** Sum of all available coins raw value */ CAmount total_amount{0}; + /** Sum of all available coins effective value (each output value minus fees required to spend it) */ + std::optional<CAmount> total_effective_amount{0}; +}; + +struct CoinFilterParams { + // Outputs below the minimum amount will not get selected + CAmount min_amount{1}; + // Outputs above the maximum amount will not get selected + CAmount max_amount{MAX_MONEY}; + // Return outputs until the minimum sum amount is covered + CAmount min_sum_amount{MAX_MONEY}; + // Maximum number of outputs that can be returned + uint64_t max_count{0}; + // By default, return only spendable outputs + bool only_spendable{true}; + // By default, do not include immature coinbase outputs + bool include_immature_coinbase{false}; }; /** @@ -61,17 +84,13 @@ struct CoinsResult { CoinsResult AvailableCoins(const CWallet& wallet, const CCoinControl* coinControl = nullptr, std::optional<CFeeRate> feerate = std::nullopt, - const CAmount& nMinimumAmount = 1, - const CAmount& nMaximumAmount = MAX_MONEY, - const CAmount& nMinimumSumAmount = MAX_MONEY, - const uint64_t nMaximumCount = 0, - bool only_spendable = true) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); + const CoinFilterParams& params = {}) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); /** - * Wrapper function for AvailableCoins which skips the `feerate` parameter. Use this function + * Wrapper function for AvailableCoins which skips the `feerate` and `CoinFilterParams::only_spendable` parameters. Use this function * to list all available coins (e.g. listunspent RPC) while not intending to fund a transaction. */ -CoinsResult AvailableCoinsListUnspent(const CWallet& wallet, const CCoinControl* coinControl = nullptr, const CAmount& nMinimumAmount = 1, const CAmount& nMaximumAmount = MAX_MONEY, const CAmount& nMinimumSumAmount = MAX_MONEY, const uint64_t nMaximumCount = 0) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); +CoinsResult AvailableCoinsListUnspent(const CWallet& wallet, const CCoinControl* coinControl = nullptr, CoinFilterParams params = {}) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); CAmount GetAvailableBalance(const CWallet& wallet, const CCoinControl* coinControl = nullptr); @@ -121,9 +140,35 @@ std::optional<SelectionResult> AttemptSelection(const CWallet& wallet, const CAm std::optional<SelectionResult> ChooseSelectionResult(const CWallet& wallet, const CAmount& nTargetValue, const CoinEligibilityFilter& eligibility_filter, const std::vector<COutput>& available_coins, const CoinSelectionParams& coin_selection_params); +// User manually selected inputs that must be part of the transaction +struct PreSelectedInputs +{ + std::set<COutput> coins; + // If subtract fee from outputs is disabled, the 'total_amount' + // will be the sum of each output effective value + // instead of the sum of the outputs amount + CAmount total_amount{0}; + + void Insert(const COutput& output, bool subtract_fee_outputs) + { + if (subtract_fee_outputs) { + total_amount += output.txout.nValue; + } else { + total_amount += output.GetEffectiveValue(); + } + coins.insert(output); + } +}; + +/** + * Fetch and validate coin control selected inputs. + * Coins could be internal (from the wallet) or external. +*/ +util::Result<PreSelectedInputs> FetchSelectedInputs(const CWallet& wallet, const CCoinControl& coin_control, + const CoinSelectionParams& coin_selection_params) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); + /** - * Select a set of coins such that nTargetValue is met and at least - * all coins from coin_control are selected; never select unconfirmed coins if they are not ours + * Select a set of coins such that nTargetValue is met; never select unconfirmed coins if they are not ours * param@[in] wallet The wallet which provides data necessary to spend the selected coins * param@[in] available_coins The struct of coins, organized by OutputType, available for selection prior to filtering * param@[in] nTargetValue The target value @@ -132,9 +177,17 @@ std::optional<SelectionResult> ChooseSelectionResult(const CWallet& wallet, cons * returns If successful, a SelectionResult containing the selected coins * If failed, a nullopt. */ -std::optional<SelectionResult> SelectCoins(const CWallet& wallet, CoinsResult& available_coins, const CAmount& nTargetValue, const CCoinControl& coin_control, +std::optional<SelectionResult> AutomaticCoinSelection(const CWallet& wallet, CoinsResult& available_coins, const CAmount& nTargetValue, const CCoinControl& coin_control, const CoinSelectionParams& coin_selection_params) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); +/** + * Select all coins from coin_control, and if coin_control 'm_allow_other_inputs=true', call 'AutomaticCoinSelection' to + * select a set of coins such that nTargetValue - pre_set_inputs.total_amount is met. + */ +std::optional<SelectionResult> SelectCoins(const CWallet& wallet, CoinsResult& available_coins, const PreSelectedInputs& pre_set_inputs, + const CAmount& nTargetValue, const CCoinControl& coin_control, + const CoinSelectionParams& coin_selection_params) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet); + struct CreatedTransactionResult { CTransactionRef tx; diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 23f024247d..ce875d5442 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -4,6 +4,7 @@ #include <consensus/amount.h> #include <node/context.h> +#include <policy/policy.h> #include <primitives/transaction.h> #include <random.h> #include <test/util/setup_common.h> @@ -83,7 +84,7 @@ static void add_coin(CoinsResult& available_coins, CWallet& wallet, const CAmoun assert(ret.second); CWalletTx& wtx = (*ret.first).second; const auto& txout = wtx.tx->vout.at(nInput); - available_coins.coins[OutputType::BECH32].emplace_back(COutPoint(wtx.GetHash(), nInput), txout, nAge, CalculateMaximumSignedInputSize(txout, &wallet, /*coin_control=*/nullptr), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, wtx.GetTxTime(), fIsFromMe, feerate); + available_coins.Add(OutputType::BECH32, {COutPoint(wtx.GetHash(), nInput), txout, nAge, CalculateMaximumSignedInputSize(txout, &wallet, /*coin_control=*/nullptr), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, wtx.GetTxTime(), fIsFromMe, feerate}); } /** Check if SelectionResult a is equivalent to SelectionResult b. @@ -338,9 +339,13 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) add_coin(available_coins, *wallet, 2 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); CCoinControl coin_control; coin_control.m_allow_other_inputs = true; - coin_control.Select(available_coins.All().at(0).outpoint); + COutput select_coin = available_coins.All().at(0); + coin_control.Select(select_coin.outpoint); + PreSelectedInputs selected_input; + selected_input.Insert(select_coin, coin_selection_params_bnb.m_subtract_fee_outputs); + available_coins.Erase({available_coins.coins[OutputType::BECH32].begin()->outpoint}); coin_selection_params_bnb.m_effective_feerate = CFeeRate(0); - const auto result10 = SelectCoins(*wallet, available_coins, 10 * CENT, coin_control, coin_selection_params_bnb); + const auto result10 = SelectCoins(*wallet, available_coins, selected_input, 10 * CENT, coin_control, coin_selection_params_bnb); BOOST_CHECK(result10); } { @@ -363,7 +368,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) expected_result.Clear(); add_coin(10 * CENT, 2, expected_result); CCoinControl coin_control; - const auto result11 = SelectCoins(*wallet, available_coins, 10 * CENT, coin_control, coin_selection_params_bnb); + const auto result11 = SelectCoins(*wallet, available_coins, /*pre_set_inputs=*/{}, 10 * CENT, coin_control, coin_selection_params_bnb); BOOST_CHECK(EquivalentResult(expected_result, *result11)); available_coins.Clear(); @@ -378,7 +383,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) expected_result.Clear(); add_coin(9 * CENT, 2, expected_result); add_coin(1 * CENT, 2, expected_result); - const auto result12 = SelectCoins(*wallet, available_coins, 10 * CENT, coin_control, coin_selection_params_bnb); + const auto result12 = SelectCoins(*wallet, available_coins, /*pre_set_inputs=*/{}, 10 * CENT, coin_control, coin_selection_params_bnb); BOOST_CHECK(EquivalentResult(expected_result, *result12)); available_coins.Clear(); @@ -394,8 +399,12 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) add_coin(9 * CENT, 2, expected_result); add_coin(1 * CENT, 2, expected_result); coin_control.m_allow_other_inputs = true; - coin_control.Select(available_coins.All().at(1).outpoint); // pre select 9 coin - const auto result13 = SelectCoins(*wallet, available_coins, 10 * CENT, coin_control, coin_selection_params_bnb); + COutput select_coin = available_coins.All().at(1); // pre select 9 coin + coin_control.Select(select_coin.outpoint); + PreSelectedInputs selected_input; + selected_input.Insert(select_coin, coin_selection_params_bnb.m_subtract_fee_outputs); + available_coins.Erase({(++available_coins.coins[OutputType::BECH32].begin())->outpoint}); + const auto result13 = SelectCoins(*wallet, available_coins, selected_input, 10 * CENT, coin_control, coin_selection_params_bnb); BOOST_CHECK(EquivalentResult(expected_result, *result13)); } } @@ -783,7 +792,7 @@ BOOST_AUTO_TEST_CASE(SelectCoins_test) cs_params.m_cost_of_change = 1; cs_params.min_viable_change = 1; CCoinControl cc; - const auto result = SelectCoins(*wallet, available_coins, target, cc, cs_params); + const auto result = SelectCoins(*wallet, available_coins, /*pre_set_inputs=*/{}, target, cc, cs_params); BOOST_CHECK(result); BOOST_CHECK_GE(result->GetSelectedValue(), target); } @@ -922,6 +931,124 @@ BOOST_AUTO_TEST_CASE(effective_value_test) BOOST_CHECK_EQUAL(output5.GetEffectiveValue(), nValue); // The effective value should be equal to the absolute value if input_bytes is -1 } +static std::optional<SelectionResult> select_coins(const CAmount& target, const CoinSelectionParams& cs_params, const CCoinControl& cc, std::function<CoinsResult(CWallet&)> coin_setup, interfaces::Chain* chain, const ArgsManager& args) +{ + std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(chain, "", args, CreateMockWalletDatabase()); + wallet->LoadWallet(); + LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet->SetupDescriptorScriptPubKeyMans(); + + auto available_coins = coin_setup(*wallet); + + const auto result = SelectCoins(*wallet, available_coins, /*pre_set_inputs=*/ {}, target, cc, cs_params); + if (result) { + const auto signedTxSize = 10 + 34 + 68 * result->GetInputSet().size(); // static header size + output size + inputs size (P2WPKH) + BOOST_CHECK_LE(signedTxSize * WITNESS_SCALE_FACTOR, MAX_STANDARD_TX_WEIGHT); + + BOOST_CHECK_GE(result->GetSelectedValue(), target); + } + return result; +} + +static bool has_coin(const CoinSet& set, CAmount amount) +{ + return std::any_of(set.begin(), set.end(), [&](const auto& coin) { return coin.GetEffectiveValue() == amount; }); +} + +BOOST_AUTO_TEST_CASE(check_max_weight) +{ + const CAmount target = 49.5L * COIN; + CCoinControl cc; + + FastRandomContext rand; + CoinSelectionParams cs_params{ + rand, + /*change_output_size=*/34, + /*change_spend_size=*/68, + /*min_change_target=*/CENT, + /*effective_feerate=*/CFeeRate(0), + /*long_term_feerate=*/CFeeRate(0), + /*discard_feerate=*/CFeeRate(0), + /*tx_noinputs_size=*/10 + 34, // static header size + output size + /*avoid_partial=*/false, + }; + + auto chain{m_node.chain.get()}; + + { + // Scenario 1: + // The actor starts with 1x 50.0 BTC and 1515x 0.033 BTC (~100.0 BTC total) unspent outputs + // Then tries to spend 49.5 BTC + // The 50.0 BTC output should be selected, because the transaction would otherwise be too large + + // Perform selection + + const auto result = select_coins( + target, cs_params, cc, [&](CWallet& wallet) { + CoinsResult available_coins; + for (int j = 0; j < 1515; ++j) { + add_coin(available_coins, wallet, CAmount(0.033 * COIN), CFeeRate(0), 144, false, 0, true); + } + + add_coin(available_coins, wallet, CAmount(50 * COIN), CFeeRate(0), 144, false, 0, true); + return available_coins; + }, + chain, m_args); + + BOOST_CHECK(result); + BOOST_CHECK(has_coin(result->GetInputSet(), CAmount(50 * COIN))); + } + + { + // Scenario 2: + + // The actor starts with 400x 0.0625 BTC and 2000x 0.025 BTC (75.0 BTC total) unspent outputs + // Then tries to spend 49.5 BTC + // A combination of coins should be selected, such that the created transaction is not too large + + // Perform selection + const auto result = select_coins( + target, cs_params, cc, [&](CWallet& wallet) { + CoinsResult available_coins; + for (int j = 0; j < 400; ++j) { + add_coin(available_coins, wallet, CAmount(0.0625 * COIN), CFeeRate(0), 144, false, 0, true); + } + for (int j = 0; j < 2000; ++j) { + add_coin(available_coins, wallet, CAmount(0.025 * COIN), CFeeRate(0), 144, false, 0, true); + } + return available_coins; + }, + chain, m_args); + + BOOST_CHECK(has_coin(result->GetInputSet(), CAmount(0.0625 * COIN))); + BOOST_CHECK(has_coin(result->GetInputSet(), CAmount(0.025 * COIN))); + } + + { + // Scenario 3: + + // The actor starts with 1515x 0.033 BTC (49.995 BTC total) unspent outputs + // No results should be returned, because the transaction would be too large + + // Perform selection + const auto result = select_coins( + target, cs_params, cc, [&](CWallet& wallet) { + CoinsResult available_coins; + for (int j = 0; j < 1515; ++j) { + add_coin(available_coins, wallet, CAmount(0.033 * COIN), CFeeRate(0), 144, false, 0, true); + } + return available_coins; + }, + chain, m_args); + + // No results + // 1515 inputs * 68 bytes = 103,020 bytes + // 103,020 bytes * 4 = 412,080 weight, which is above the MAX_STANDARD_TX_WEIGHT of 400,000 + BOOST_CHECK(!result); + } +} + BOOST_AUTO_TEST_CASE(SelectCoins_effective_value_test) { // Test that the effective value is used to check whether preset inputs provide sufficient funds when subtract_fee_outputs is not used. @@ -965,9 +1092,52 @@ BOOST_AUTO_TEST_CASE(SelectCoins_effective_value_test) cc.SetInputWeight(output.outpoint, 148); cc.SelectExternal(output.outpoint, output.txout); - const auto result = SelectCoins(*wallet, available_coins, target, cc, cs_params); + const auto preset_inputs = *Assert(FetchSelectedInputs(*wallet, cc, cs_params)); + available_coins.Erase({available_coins.coins[OutputType::BECH32].begin()->outpoint}); + + const auto result = SelectCoins(*wallet, available_coins, preset_inputs, target, cc, cs_params); BOOST_CHECK(!result); } +BOOST_FIXTURE_TEST_CASE(wallet_coinsresult_test, BasicTestingSetup) +{ + // Test case to verify CoinsResult object sanity. + CoinsResult available_coins; + { + std::unique_ptr<CWallet> dummyWallet = std::make_unique<CWallet>(m_node.chain.get(), "dummy", m_args, CreateMockWalletDatabase()); + BOOST_CHECK_EQUAL(dummyWallet->LoadWallet(), DBErrors::LOAD_OK); + LOCK(dummyWallet->cs_wallet); + dummyWallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + dummyWallet->SetupDescriptorScriptPubKeyMans(); + + // Add some coins to 'available_coins' + for (int i=0; i<10; i++) { + add_coin(available_coins, *dummyWallet, 1 * COIN); + } + } + + { + // First test case, check that 'CoinsResult::Erase' function works as expected. + // By trying to erase two elements from the 'available_coins' object. + std::unordered_set<COutPoint, SaltedOutpointHasher> outs_to_remove; + const auto& coins = available_coins.All(); + for (int i = 0; i < 2; i++) { + outs_to_remove.emplace(coins[i].outpoint); + } + available_coins.Erase(outs_to_remove); + + // Check that the elements were actually removed. + const auto& updated_coins = available_coins.All(); + for (const auto& out: outs_to_remove) { + auto it = std::find_if(updated_coins.begin(), updated_coins.end(), [&out](const COutput &coin) { + return coin.outpoint == out; + }); + BOOST_CHECK(it == updated_coins.end()); + } + // And verify that no extra element were removed + BOOST_CHECK_EQUAL(available_coins.Size(), 8); + } +} + BOOST_AUTO_TEST_SUITE_END() } // namespace wallet diff --git a/src/wallet/test/feebumper_tests.cpp b/src/wallet/test/feebumper_tests.cpp index 6add86dc3d..2480a9b4e1 100644 --- a/src/wallet/test/feebumper_tests.cpp +++ b/src/wallet/test/feebumper_tests.cpp @@ -2,6 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or https://www.opensource.org/licenses/mit-license.php. +#include <consensus/validation.h> +#include <policy/policy.h> #include <primitives/transaction.h> #include <script/script.h> #include <util/strencodings.h> diff --git a/src/wallet/test/ismine_tests.cpp b/src/wallet/test/ismine_tests.cpp index 68146eb079..2c83d25c20 100644 --- a/src/wallet/test/ismine_tests.cpp +++ b/src/wallet/test/ismine_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <key.h> +#include <key_io.h> #include <node/context.h> #include <script/script.h> #include <script/standard.h> @@ -16,6 +17,25 @@ namespace wallet { BOOST_FIXTURE_TEST_SUITE(ismine_tests, BasicTestingSetup) +wallet::ScriptPubKeyMan* CreateDescriptor(CWallet& keystore, const std::string& desc_str, const bool success) +{ + keystore.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + + FlatSigningProvider keys; + std::string error; + std::unique_ptr<Descriptor> parsed_desc = Parse(desc_str, keys, error, false); + BOOST_CHECK(success == (parsed_desc != nullptr)); + if (!success) return nullptr; + + const int64_t range_start = 0, range_end = 1, next_index = 0, timestamp = 1; + + WalletDescriptor w_desc(std::move(parsed_desc), timestamp, range_start, range_end, next_index); + + LOCK(keystore.cs_wallet); + + return Assert(keystore.AddWalletDescriptor(w_desc, keys,/*label=*/"", /*internal=*/false)); +}; + BOOST_AUTO_TEST_CASE(ismine_standard) { CKey keys[2]; @@ -33,7 +53,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) CScript scriptPubKey; isminetype result; - // P2PK compressed + // P2PK compressed - Legacy { CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); @@ -52,7 +72,19 @@ BOOST_AUTO_TEST_CASE(ismine_standard) BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 1); } - // P2PK uncompressed + // P2PK compressed - Descriptor + { + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); + std::string desc_str = "pk(" + EncodeSecret(keys[0]) + ")"; + + auto spk_manager = CreateDescriptor(keystore, desc_str, true); + + scriptPubKey = GetScriptForRawPubKey(pubkeys[0]); + result = spk_manager->IsMine(scriptPubKey); + BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + } + + // P2PK uncompressed - Legacy { CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); @@ -71,7 +103,19 @@ BOOST_AUTO_TEST_CASE(ismine_standard) BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 1); } - // P2PKH compressed + // P2PK uncompressed - Descriptor + { + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); + std::string desc_str = "pk(" + EncodeSecret(uncompressedKey) + ")"; + + auto spk_manager = CreateDescriptor(keystore, desc_str, true); + + scriptPubKey = GetScriptForRawPubKey(uncompressedPubkey); + result = spk_manager->IsMine(scriptPubKey); + BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + } + + // P2PKH compressed - Legacy { CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); @@ -90,7 +134,19 @@ BOOST_AUTO_TEST_CASE(ismine_standard) BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 1); } - // P2PKH uncompressed + // P2PKH compressed - Descriptor + { + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); + std::string desc_str = "pkh(" + EncodeSecret(keys[0]) + ")"; + + auto spk_manager = CreateDescriptor(keystore, desc_str, true); + + scriptPubKey = GetScriptForDestination(PKHash(pubkeys[0])); + result = spk_manager->IsMine(scriptPubKey); + BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + } + + // P2PKH uncompressed - Legacy { CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); @@ -109,7 +165,19 @@ BOOST_AUTO_TEST_CASE(ismine_standard) BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 1); } - // P2SH + // P2PKH uncompressed - Descriptor + { + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); + std::string desc_str = "pkh(" + EncodeSecret(uncompressedKey) + ")"; + + auto spk_manager = CreateDescriptor(keystore, desc_str, true); + + scriptPubKey = GetScriptForDestination(PKHash(uncompressedPubkey)); + result = spk_manager->IsMine(scriptPubKey); + BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + } + + // P2SH - Legacy { CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); @@ -136,7 +204,20 @@ BOOST_AUTO_TEST_CASE(ismine_standard) BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 1); } - // (P2PKH inside) P2SH inside P2SH (invalid) + // P2SH - Descriptor + { + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); + std::string desc_str = "sh(pkh(" + EncodeSecret(keys[0]) + "))"; + + auto spk_manager = CreateDescriptor(keystore, desc_str, true); + + CScript redeemScript = GetScriptForDestination(PKHash(pubkeys[0])); + scriptPubKey = GetScriptForDestination(ScriptHash(redeemScript)); + result = spk_manager->IsMine(scriptPubKey); + BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + } + + // (P2PKH inside) P2SH inside P2SH (invalid) - Legacy { CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); @@ -155,7 +236,16 @@ BOOST_AUTO_TEST_CASE(ismine_standard) BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); } - // (P2PKH inside) P2SH inside P2WSH (invalid) + // (P2PKH inside) P2SH inside P2SH (invalid) - Descriptor + { + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); + std::string desc_str = "sh(sh(" + EncodeSecret(keys[0]) + "))"; + + auto spk_manager = CreateDescriptor(keystore, desc_str, false); + BOOST_CHECK_EQUAL(spk_manager, nullptr); + } + + // (P2PKH inside) P2SH inside P2WSH (invalid) - Legacy { CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); @@ -174,7 +264,16 @@ BOOST_AUTO_TEST_CASE(ismine_standard) BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); } - // P2WPKH inside P2WSH (invalid) + // (P2PKH inside) P2SH inside P2WSH (invalid) - Descriptor + { + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); + std::string desc_str = "wsh(sh(" + EncodeSecret(keys[0]) + "))"; + + auto spk_manager = CreateDescriptor(keystore, desc_str, false); + BOOST_CHECK_EQUAL(spk_manager, nullptr); + } + + // P2WPKH inside P2WSH (invalid) - Legacy { CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); @@ -191,7 +290,16 @@ BOOST_AUTO_TEST_CASE(ismine_standard) BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); } - // (P2PKH inside) P2WSH inside P2WSH (invalid) + // P2WPKH inside P2WSH (invalid) - Descriptor + { + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); + std::string desc_str = "wsh(wpkh(" + EncodeSecret(keys[0]) + "))"; + + auto spk_manager = CreateDescriptor(keystore, desc_str, false); + BOOST_CHECK_EQUAL(spk_manager, nullptr); + } + + // (P2PKH inside) P2WSH inside P2WSH (invalid) - Legacy { CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); @@ -210,7 +318,16 @@ BOOST_AUTO_TEST_CASE(ismine_standard) BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); } - // P2WPKH compressed + // (P2PKH inside) P2WSH inside P2WSH (invalid) - Descriptor + { + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); + std::string desc_str = "wsh(wsh(" + EncodeSecret(keys[0]) + "))"; + + auto spk_manager = CreateDescriptor(keystore, desc_str, false); + BOOST_CHECK_EQUAL(spk_manager, nullptr); + } + + // P2WPKH compressed - Legacy { CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); @@ -226,7 +343,19 @@ BOOST_AUTO_TEST_CASE(ismine_standard) BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 1); } - // P2WPKH uncompressed + // P2WPKH compressed - Descriptor + { + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); + std::string desc_str = "wpkh(" + EncodeSecret(keys[0]) + ")"; + + auto spk_manager = CreateDescriptor(keystore, desc_str, true); + + scriptPubKey = GetScriptForDestination(WitnessV0KeyHash(pubkeys[0])); + result = spk_manager->IsMine(scriptPubKey); + BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + } + + // P2WPKH uncompressed - Legacy { CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); @@ -247,7 +376,16 @@ BOOST_AUTO_TEST_CASE(ismine_standard) BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); } - // scriptPubKey multisig + // P2WPKH uncompressed (invalid) - Descriptor + { + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); + std::string desc_str = "wpkh(" + EncodeSecret(uncompressedKey) + ")"; + + auto spk_manager = CreateDescriptor(keystore, desc_str, false); + BOOST_CHECK_EQUAL(spk_manager, nullptr); + } + + // scriptPubKey multisig - Legacy { CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); @@ -282,7 +420,19 @@ BOOST_AUTO_TEST_CASE(ismine_standard) BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); } - // P2SH multisig + // scriptPubKey multisig - Descriptor + { + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); + std::string desc_str = "multi(2, " + EncodeSecret(uncompressedKey) + ", " + EncodeSecret(keys[1]) + ")"; + + auto spk_manager = CreateDescriptor(keystore, desc_str, true); + + scriptPubKey = GetScriptForMultisig(2, {uncompressedPubkey, pubkeys[1]}); + result = spk_manager->IsMine(scriptPubKey); + BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + } + + // P2SH multisig - Legacy { CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); @@ -305,7 +455,21 @@ BOOST_AUTO_TEST_CASE(ismine_standard) BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 1); } - // P2WSH multisig with compressed keys + // P2SH multisig - Descriptor + { + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); + + std::string desc_str = "sh(multi(2, " + EncodeSecret(uncompressedKey) + ", " + EncodeSecret(keys[1]) + "))"; + + auto spk_manager = CreateDescriptor(keystore, desc_str, true); + + CScript redeemScript = GetScriptForMultisig(2, {uncompressedPubkey, pubkeys[1]}); + scriptPubKey = GetScriptForDestination(ScriptHash(redeemScript)); + result = spk_manager->IsMine(scriptPubKey); + BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + } + + // P2WSH multisig with compressed keys - Legacy { CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); @@ -334,7 +498,21 @@ BOOST_AUTO_TEST_CASE(ismine_standard) BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 1); } - // P2WSH multisig with uncompressed key + // P2WSH multisig with compressed keys - Descriptor + { + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); + + std::string desc_str = "wsh(multi(2, " + EncodeSecret(keys[0]) + ", " + EncodeSecret(keys[1]) + "))"; + + auto spk_manager = CreateDescriptor(keystore, desc_str, true); + + CScript redeemScript = GetScriptForMultisig(2, {pubkeys[0], pubkeys[1]}); + scriptPubKey = GetScriptForDestination(WitnessV0ScriptHash(redeemScript)); + result = spk_manager->IsMine(scriptPubKey); + BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + } + + // P2WSH multisig with uncompressed key - Legacy { CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); @@ -363,7 +541,17 @@ BOOST_AUTO_TEST_CASE(ismine_standard) BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 0); } - // P2WSH multisig wrapped in P2SH + // P2WSH multisig with uncompressed key (invalid) - Descriptor + { + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); + + std::string desc_str = "wsh(multi(2, " + EncodeSecret(uncompressedKey) + ", " + EncodeSecret(keys[1]) + "))"; + + auto spk_manager = CreateDescriptor(keystore, desc_str, false); + BOOST_CHECK_EQUAL(spk_manager, nullptr); + } + + // P2WSH multisig wrapped in P2SH - Legacy { CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); @@ -393,6 +581,83 @@ BOOST_AUTO_TEST_CASE(ismine_standard) BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->GetScriptPubKeys().count(scriptPubKey) == 1); } + // P2WSH multisig wrapped in P2SH - Descriptor + { + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); + + std::string desc_str = "sh(wsh(multi(2, " + EncodeSecret(keys[0]) + ", " + EncodeSecret(keys[1]) + ")))"; + + auto spk_manager = CreateDescriptor(keystore, desc_str, true); + + CScript witnessScript = GetScriptForMultisig(2, {pubkeys[0], pubkeys[1]}); + CScript redeemScript = GetScriptForDestination(WitnessV0ScriptHash(witnessScript)); + scriptPubKey = GetScriptForDestination(ScriptHash(redeemScript)); + result = spk_manager->IsMine(scriptPubKey); + BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + } + + // Combo - Descriptor + { + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); + + std::string desc_str = "combo(" + EncodeSecret(keys[0]) + ")"; + + auto spk_manager = CreateDescriptor(keystore, desc_str, true); + + // Test P2PK + result = spk_manager->IsMine(GetScriptForRawPubKey(pubkeys[0])); + BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + + // Test P2PKH + result = spk_manager->IsMine(GetScriptForDestination(PKHash(pubkeys[0]))); + BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + + // Test P2SH (combo descriptor does not describe P2SH) + CScript redeemScript = GetScriptForDestination(PKHash(pubkeys[0])); + scriptPubKey = GetScriptForDestination(ScriptHash(redeemScript)); + result = spk_manager->IsMine(scriptPubKey); + BOOST_CHECK_EQUAL(result, ISMINE_NO); + + // Test P2WPKH + scriptPubKey = GetScriptForDestination(WitnessV0KeyHash(pubkeys[0])); + result = spk_manager->IsMine(scriptPubKey); + BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + + // P2SH-P2WPKH output + redeemScript = GetScriptForDestination(WitnessV0KeyHash(pubkeys[0])); + scriptPubKey = GetScriptForDestination(ScriptHash(redeemScript)); + result = spk_manager->IsMine(scriptPubKey); + BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + + // Test P2TR (combo descriptor does not describe P2TR) + XOnlyPubKey xpk(pubkeys[0]); + Assert(xpk.IsFullyValid()); + TaprootBuilder builder; + builder.Finalize(xpk); + WitnessV1Taproot output = builder.GetOutput(); + scriptPubKey = GetScriptForDestination(output); + result = spk_manager->IsMine(scriptPubKey); + BOOST_CHECK_EQUAL(result, ISMINE_NO); + } + + // Taproot - Descriptor + { + CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); + + std::string desc_str = "tr(" + EncodeSecret(keys[0]) + ")"; + + auto spk_manager = CreateDescriptor(keystore, desc_str, true); + + XOnlyPubKey xpk(pubkeys[0]); + Assert(xpk.IsFullyValid()); + TaprootBuilder builder; + builder.Finalize(xpk); + WitnessV1Taproot output = builder.GetOutput(); + scriptPubKey = GetScriptForDestination(output); + result = spk_manager->IsMine(scriptPubKey); + BOOST_CHECK_EQUAL(result, ISMINE_SPENDABLE); + } + // OP_RETURN { CWallet keystore(chain.get(), "", m_args, CreateDummyWalletDatabase()); diff --git a/src/wallet/test/spend_tests.cpp b/src/wallet/test/spend_tests.cpp index a75b014870..40756fedfb 100644 --- a/src/wallet/test/spend_tests.cpp +++ b/src/wallet/test/spend_tests.cpp @@ -26,7 +26,7 @@ BOOST_FIXTURE_TEST_CASE(SubtractFee, TestChain100Setup) // leftover input amount which would have been change to the recipient // instead of the miner. auto check_tx = [&wallet](CAmount leftover_input_amount) { - CRecipient recipient{GetScriptForRawPubKey({}), 50 * COIN - leftover_input_amount, true /* subtract fee */}; + CRecipient recipient{GetScriptForRawPubKey({}), 50 * COIN - leftover_input_amount, /*subtract_fee=*/true}; constexpr int RANDOM_CHANGE_POSITION = -1; CCoinControl coin_control; coin_control.m_feerate.emplace(10000); @@ -112,5 +112,50 @@ BOOST_FIXTURE_TEST_CASE(FillInputToWeightTest, BasicTestingSetup) // Note: We don't test the next boundary because of memory allocation constraints. } +BOOST_FIXTURE_TEST_CASE(wallet_duplicated_preset_inputs_test, TestChain100Setup) +{ + // Verify that the wallet's Coin Selection process does not include pre-selected inputs twice in a transaction. + + // Add 4 spendable UTXO, 50 BTC each, to the wallet (total balance 200 BTC) + for (int i = 0; i < 4; i++) CreateAndProcessBlock({}, GetScriptForRawPubKey(coinbaseKey.GetPubKey())); + auto wallet = CreateSyncedWallet(*m_node.chain, WITH_LOCK(Assert(m_node.chainman)->GetMutex(), return m_node.chainman->ActiveChain()), m_args, coinbaseKey); + + LOCK(wallet->cs_wallet); + auto available_coins = AvailableCoins(*wallet); + std::vector<COutput> coins = available_coins.All(); + // Preselect the first 3 UTXO (150 BTC total) + std::set<COutPoint> preset_inputs = {coins[0].outpoint, coins[1].outpoint, coins[2].outpoint}; + + // Try to create a tx that spends more than what preset inputs + wallet selected inputs are covering for. + // The wallet can cover up to 200 BTC, and the tx target is 299 BTC. + std::vector<CRecipient> recipients = {{GetScriptForDestination(*Assert(wallet->GetNewDestination(OutputType::BECH32, "dummy"))), + /*nAmount=*/299 * COIN, /*fSubtractFeeFromAmount=*/true}}; + CCoinControl coin_control; + coin_control.m_allow_other_inputs = true; + for (const auto& outpoint : preset_inputs) { + coin_control.Select(outpoint); + } + + // Attempt to send 299 BTC from a wallet that only has 200 BTC. The wallet should exclude + // the preset inputs from the pool of available coins, realize that there is not enough + // money to fund the 299 BTC payment, and fail with "Insufficient funds". + // + // Even with SFFO, the wallet can only afford to send 200 BTC. + // If the wallet does not properly exclude preset inputs from the pool of available coins + // prior to coin selection, it may create a transaction that does not fund the full payment + // amount or, through SFFO, incorrectly reduce the recipient's amount by the difference + // between the original target and the wrongly counted inputs (in this case 99 BTC) + // so that the recipient's amount is no longer equal to the user's selected target of 299 BTC. + + // First case, use 'subtract_fee_from_outputs=true' + util::Result<CreatedTransactionResult> res_tx = CreateTransaction(*wallet, recipients, /*change_pos*/-1, coin_control); + BOOST_CHECK(!res_tx.has_value()); + + // Second case, don't use 'subtract_fee_from_outputs'. + recipients[0].fSubtractFeeFromAmount = false; + res_tx = CreateTransaction(*wallet, recipients, /*change_pos*/-1, coin_control); + BOOST_CHECK(!res_tx.has_value()); +} + BOOST_AUTO_TEST_SUITE_END() } // namespace wallet diff --git a/src/wallet/test/util.cpp b/src/wallet/test/util.cpp index ab72721f9d..f6c7ecb598 100644 --- a/src/wallet/test/util.cpp +++ b/src/wallet/test/util.cpp @@ -11,8 +11,6 @@ #include <wallet/wallet.h> #include <wallet/walletdb.h> -#include <boost/test/unit_test.hpp> - #include <memory> namespace wallet { @@ -39,10 +37,46 @@ std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cc WalletRescanReserver reserver(*wallet); reserver.reserve(); CWallet::ScanResult result = wallet->ScanForWalletTransactions(cchain.Genesis()->GetBlockHash(), /*start_height=*/0, /*max_height=*/{}, reserver, /*fUpdate=*/false, /*save_progress=*/false); - BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); - BOOST_CHECK_EQUAL(result.last_scanned_block, cchain.Tip()->GetBlockHash()); - BOOST_CHECK_EQUAL(*result.last_scanned_height, cchain.Height()); - BOOST_CHECK(result.last_failed_block.IsNull()); + assert(result.status == CWallet::ScanResult::SUCCESS); + assert(result.last_scanned_block == cchain.Tip()->GetBlockHash()); + assert(*result.last_scanned_height == cchain.Height()); + assert(result.last_failed_block.IsNull()); return wallet; } + +std::unique_ptr<WalletDatabase> DuplicateMockDatabase(WalletDatabase& database, DatabaseOptions& options) +{ + auto new_database = CreateMockWalletDatabase(options); + + // Get a cursor to the original database + auto batch = database.MakeBatch(); + batch->StartCursor(); + + // Get a batch for the new database + auto new_batch = new_database->MakeBatch(); + + // Read all records from the original database and write them to the new one + while (true) { + CDataStream key(SER_DISK, CLIENT_VERSION); + CDataStream value(SER_DISK, CLIENT_VERSION); + bool complete; + batch->ReadAtCursor(key, value, complete); + if (complete) break; + new_batch->Write(key, value); + } + + return new_database; +} + +std::string getnewaddress(CWallet& w) +{ + constexpr auto output_type = OutputType::BECH32; + return EncodeDestination(getNewDestination(w, output_type)); +} + +CTxDestination getNewDestination(CWallet& w, OutputType output_type) +{ + return *Assert(w.GetNewDestination(output_type, "")); +} + } // namespace wallet diff --git a/src/wallet/test/util.h b/src/wallet/test/util.h index 712d0251cd..a2e361ddca 100644 --- a/src/wallet/test/util.h +++ b/src/wallet/test/util.h @@ -5,19 +5,32 @@ #ifndef BITCOIN_WALLET_TEST_UTIL_H #define BITCOIN_WALLET_TEST_UTIL_H +#include <script/standard.h> #include <memory> class ArgsManager; class CChain; class CKey; +enum class OutputType; namespace interfaces { class Chain; } // namespace interfaces namespace wallet { class CWallet; +struct DatabaseOptions; +class WalletDatabase; std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cchain, ArgsManager& args, const CKey& key); + +// Creates a copy of the provided database +std::unique_ptr<WalletDatabase> DuplicateMockDatabase(WalletDatabase& database, DatabaseOptions& options); + +/** Returns a new encoded destination from the wallet (hardcoded to BECH32) */ +std::string getnewaddress(CWallet& w); +/** Returns a new destination, of an specific type, from the wallet */ +CTxDestination getNewDestination(CWallet& w, OutputType output_type); + } // namespace wallet #endif // BITCOIN_WALLET_TEST_UTIL_H diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 60fdbde71b..0f703b7d00 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -587,7 +587,7 @@ BOOST_FIXTURE_TEST_CASE(ListCoinsTest, ListCoinsTestingSetup) // returns the coin associated with the change address underneath the // coinbaseKey pubkey, even though the change address has a different // pubkey. - AddTx(CRecipient{GetScriptForRawPubKey({}), 1 * COIN, false /* subtract fee */}); + AddTx(CRecipient{GetScriptForRawPubKey({}), 1 * COIN, /*subtract_fee=*/false}); { LOCK(wallet->cs_wallet); list = ListCoins(*wallet); diff --git a/src/wallet/test/walletload_tests.cpp b/src/wallet/test/walletload_tests.cpp index f45b69a418..24d21c2f22 100644 --- a/src/wallet/test/walletload_tests.cpp +++ b/src/wallet/test/walletload_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or https://www.opensource.org/licenses/mit-license.php. +#include <wallet/test/util.h> #include <wallet/wallet.h> #include <test/util/setup_common.h> @@ -50,5 +51,139 @@ BOOST_FIXTURE_TEST_CASE(wallet_load_unknown_descriptor, TestingSetup) } } +bool HasAnyRecordOfType(WalletDatabase& db, const std::string& key) +{ + std::unique_ptr<DatabaseBatch> batch = db.MakeBatch(false); + BOOST_CHECK(batch->StartCursor()); + while (true) { + CDataStream ssKey(SER_DISK, CLIENT_VERSION); + CDataStream ssValue(SER_DISK, CLIENT_VERSION); + bool complete; + BOOST_CHECK(batch->ReadAtCursor(ssKey, ssValue, complete)); + if (complete) break; + std::string type; + ssKey >> type; + if (type == key) return true; + } + return false; +} + +BOOST_FIXTURE_TEST_CASE(wallet_load_verif_crypted_key_checksum, TestingSetup) +{ + // The test duplicates the db so each case has its own db instance. + int NUMBER_OF_TESTS = 4; + std::vector<std::unique_ptr<WalletDatabase>> dbs; + CKey first_key; + auto get_db = [](std::vector<std::unique_ptr<WalletDatabase>>& dbs) { + std::unique_ptr<WalletDatabase> db = std::move(dbs.back()); + dbs.pop_back(); + return db; + }; + + { // Context setup. + // Create and encrypt legacy wallet + std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", m_args, CreateMockWalletDatabase())); + LOCK(wallet->cs_wallet); + auto legacy_spkm = wallet->GetOrCreateLegacyScriptPubKeyMan(); + BOOST_CHECK(legacy_spkm->SetupGeneration(true)); + + // Get the first key in the wallet + CTxDestination dest = *Assert(legacy_spkm->GetNewDestination(OutputType::LEGACY)); + CKeyID key_id = GetKeyForDestination(*legacy_spkm, dest); + BOOST_CHECK(legacy_spkm->GetKey(key_id, first_key)); + + // Encrypt the wallet and duplicate database + BOOST_CHECK(wallet->EncryptWallet("encrypt")); + wallet->Flush(); + + DatabaseOptions options; + for (int i=0; i < NUMBER_OF_TESTS; i++) { + dbs.emplace_back(DuplicateMockDatabase(wallet->GetDatabase(), options)); + } + } + + { + // First test case: + // Erase all the crypted keys from db and unlock the wallet. + // The wallet will only re-write the crypted keys to db if any checksum is missing at load time. + // So, if any 'ckey' record re-appears on db, then the checksums were not properly calculated, and we are re-writing + // the records every time that 'CWallet::Unlock' gets called, which is not good. + + // Load the wallet and check that is encrypted + std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", m_args, get_db(dbs))); + BOOST_CHECK_EQUAL(wallet->LoadWallet(), DBErrors::LOAD_OK); + BOOST_CHECK(wallet->IsCrypted()); + BOOST_CHECK(HasAnyRecordOfType(wallet->GetDatabase(), DBKeys::CRYPTED_KEY)); + + // Now delete all records and check that the 'Unlock' function doesn't re-write them + BOOST_CHECK(wallet->GetLegacyScriptPubKeyMan()->DeleteRecords()); + BOOST_CHECK(!HasAnyRecordOfType(wallet->GetDatabase(), DBKeys::CRYPTED_KEY)); + BOOST_CHECK(wallet->Unlock("encrypt")); + BOOST_CHECK(!HasAnyRecordOfType(wallet->GetDatabase(), DBKeys::CRYPTED_KEY)); + } + + { + // Second test case: + // Verify that loading up a 'ckey' with no checksum triggers a complete re-write of the crypted keys. + std::unique_ptr<WalletDatabase> db = get_db(dbs); + { + std::unique_ptr<DatabaseBatch> batch = db->MakeBatch(false); + std::pair<std::vector<unsigned char>, uint256> value; + BOOST_CHECK(batch->Read(std::make_pair(DBKeys::CRYPTED_KEY, first_key.GetPubKey()), value)); + + const auto key = std::make_pair(DBKeys::CRYPTED_KEY, first_key.GetPubKey()); + BOOST_CHECK(batch->Write(key, value.first, /*fOverwrite=*/true)); + } + + // Load the wallet and check that is encrypted + std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", m_args, std::move(db))); + BOOST_CHECK_EQUAL(wallet->LoadWallet(), DBErrors::LOAD_OK); + BOOST_CHECK(wallet->IsCrypted()); + BOOST_CHECK(HasAnyRecordOfType(wallet->GetDatabase(), DBKeys::CRYPTED_KEY)); + + // Now delete all ckey records and check that the 'Unlock' function re-writes them + // (this is because the wallet, at load time, found a ckey record with no checksum) + BOOST_CHECK(wallet->GetLegacyScriptPubKeyMan()->DeleteRecords()); + BOOST_CHECK(!HasAnyRecordOfType(wallet->GetDatabase(), DBKeys::CRYPTED_KEY)); + BOOST_CHECK(wallet->Unlock("encrypt")); + BOOST_CHECK(HasAnyRecordOfType(wallet->GetDatabase(), DBKeys::CRYPTED_KEY)); + } + + { + // Third test case: + // Verify that loading up a 'ckey' with an invalid checksum throws an error. + std::unique_ptr<WalletDatabase> db = get_db(dbs); + { + std::unique_ptr<DatabaseBatch> batch = db->MakeBatch(false); + std::vector<unsigned char> crypted_data; + BOOST_CHECK(batch->Read(std::make_pair(DBKeys::CRYPTED_KEY, first_key.GetPubKey()), crypted_data)); + + // Write an invalid checksum + std::pair<std::vector<unsigned char>, uint256> value = std::make_pair(crypted_data, uint256::ONE); + const auto key = std::make_pair(DBKeys::CRYPTED_KEY, first_key.GetPubKey()); + BOOST_CHECK(batch->Write(key, value, /*fOverwrite=*/true)); + } + + std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", m_args, std::move(db))); + BOOST_CHECK_EQUAL(wallet->LoadWallet(), DBErrors::CORRUPT); + } + + { + // Fourth test case: + // Verify that loading up a 'ckey' with an invalid pubkey throws an error + std::unique_ptr<WalletDatabase> db = get_db(dbs); + { + CPubKey invalid_key; + BOOST_ASSERT(!invalid_key.IsValid()); + const auto key = std::make_pair(DBKeys::CRYPTED_KEY, invalid_key); + std::pair<std::vector<unsigned char>, uint256> value; + BOOST_CHECK(db->MakeBatch(false)->Write(key, value, /*fOverwrite=*/true)); + } + + std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", m_args, std::move(db))); + BOOST_CHECK_EQUAL(wallet->LoadWallet(), DBErrors::CORRUPT); + } +} + BOOST_AUTO_TEST_SUITE_END() } // namespace wallet diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 461e64751a..2c0ce89929 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -5,6 +5,7 @@ #include <wallet/wallet.h> +#include <blockfilter.h> #include <chain.h> #include <consensus/amount.h> #include <consensus/consensus.h> @@ -21,6 +22,7 @@ #include <primitives/block.h> #include <primitives/transaction.h> #include <psbt.h> +#include <random.h> #include <script/descriptor.h> #include <script/script.h> #include <script/signingprovider.h> @@ -169,7 +171,7 @@ std::unique_ptr<interfaces::Handler> HandleLoadWallet(WalletContext& context, Lo { LOCK(context.wallets_mutex); auto it = context.wallet_load_fns.emplace(context.wallet_load_fns.end(), std::move(load_wallet)); - return interfaces::MakeHandler([&context, it] { LOCK(context.wallets_mutex); context.wallet_load_fns.erase(it); }); + return interfaces::MakeCleanupHandler([&context, it] { LOCK(context.wallets_mutex); context.wallet_load_fns.erase(it); }); } void NotifyWalletLoaded(WalletContext& context, const std::shared_ptr<CWallet>& wallet) @@ -260,6 +262,64 @@ std::shared_ptr<CWallet> LoadWalletInternal(WalletContext& context, const std::s return nullptr; } } + +class FastWalletRescanFilter +{ +public: + FastWalletRescanFilter(const CWallet& wallet) : m_wallet(wallet) + { + // fast rescanning via block filters is only supported by descriptor wallets right now + assert(!m_wallet.IsLegacy()); + + // create initial filter with scripts from all ScriptPubKeyMans + for (auto spkm : m_wallet.GetAllScriptPubKeyMans()) { + auto desc_spkm{dynamic_cast<DescriptorScriptPubKeyMan*>(spkm)}; + assert(desc_spkm != nullptr); + AddScriptPubKeys(desc_spkm); + // save each range descriptor's end for possible future filter updates + if (desc_spkm->IsHDEnabled()) { + m_last_range_ends.emplace(desc_spkm->GetID(), desc_spkm->GetEndRange()); + } + } + } + + void UpdateIfNeeded() + { + // repopulate filter with new scripts if top-up has happened since last iteration + for (const auto& [desc_spkm_id, last_range_end] : m_last_range_ends) { + auto desc_spkm{dynamic_cast<DescriptorScriptPubKeyMan*>(m_wallet.GetScriptPubKeyMan(desc_spkm_id))}; + assert(desc_spkm != nullptr); + int32_t current_range_end{desc_spkm->GetEndRange()}; + if (current_range_end > last_range_end) { + AddScriptPubKeys(desc_spkm, last_range_end); + m_last_range_ends.at(desc_spkm->GetID()) = current_range_end; + } + } + } + + std::optional<bool> MatchesBlock(const uint256& block_hash) const + { + return m_wallet.chain().blockFilterMatchesAny(BlockFilterType::BASIC, block_hash, m_filter_set); + } + +private: + const CWallet& m_wallet; + /** Map for keeping track of each range descriptor's last seen end range. + * This information is used to detect whether new addresses were derived + * (that is, if the current end range is larger than the saved end range) + * after processing a block and hence a filter set update is needed to + * take possible keypool top-ups into account. + */ + std::map<uint256, int32_t> m_last_range_ends; + GCSFilter::ElementSet m_filter_set; + + void AddScriptPubKeys(const DescriptorScriptPubKeyMan* desc_spkm, int32_t last_range_end = 0) + { + for (const auto& script_pub_key : desc_spkm->GetScriptPubKeys(last_range_end)) { + m_filter_set.emplace(script_pub_key.begin(), script_pub_key.end()); + } + } +}; } // namespace std::shared_ptr<CWallet> LoadWallet(WalletContext& context, const std::string& name, std::optional<bool> load_on_start, const DatabaseOptions& options, DatabaseStatus& status, bilingual_str& error, std::vector<bilingual_str>& warnings) @@ -1351,7 +1411,7 @@ void CWallet::blockConnected(const interfaces::BlockInfo& block) m_last_block_processed = block.hash; for (size_t index = 0; index < block.data->vtx.size(); index++) { SyncTransaction(block.data->vtx[index], TxStateConfirmed{block.hash, block.height, static_cast<int>(index)}); - transactionRemovedFromMempool(block.data->vtx[index], MemPoolRemovalReason::BLOCK, 0 /* mempool_sequence */); + transactionRemovedFromMempool(block.data->vtx[index], MemPoolRemovalReason::BLOCK, /*mempool_sequence=*/0); } } @@ -1754,7 +1814,11 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc uint256 block_hash = start_block; ScanResult result; - WalletLogPrintf("Rescan started from block %s...\n", start_block.ToString()); + std::unique_ptr<FastWalletRescanFilter> fast_rescan_filter; + if (!IsLegacy() && chain().hasBlockFilterIndex(BlockFilterType::BASIC)) fast_rescan_filter = std::make_unique<FastWalletRescanFilter>(*this); + + WalletLogPrintf("Rescan started from block %s... (%s)\n", start_block.ToString(), + fast_rescan_filter ? "fast variant using block filters" : "slow variant inspecting all blocks"); fAbortRescan = false; ShowProgress(strprintf("%s " + _("Rescanning…").translated, GetDisplayName()), 0); // show rescan progress in GUI as dialog or on splashscreen, if rescan required on startup (e.g. due to corruption) @@ -1781,9 +1845,22 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc WalletLogPrintf("Still rescanning. At block %d. Progress=%f\n", block_height, progress_current); } - // Read block data - CBlock block; - chain().findBlock(block_hash, FoundBlock().data(block)); + bool fetch_block{true}; + if (fast_rescan_filter) { + fast_rescan_filter->UpdateIfNeeded(); + auto matches_block{fast_rescan_filter->MatchesBlock(block_hash)}; + if (matches_block.has_value()) { + if (*matches_block) { + LogPrint(BCLog::SCAN, "Fast rescan: inspect block %d [%s] (filter matched)\n", block_height, block_hash.ToString()); + } else { + result.last_scanned_block = block_hash; + result.last_scanned_height = block_height; + fetch_block = false; + } + } else { + LogPrint(BCLog::SCAN, "Fast rescan: inspect block %d [%s] (WARNING: block filter not found!)\n", block_height, block_hash.ToString()); + } + } // Find next block separately from reading data above, because reading // is slow and there might be a reorg while it is read. @@ -1792,35 +1869,41 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc uint256 next_block_hash; chain().findBlock(block_hash, FoundBlock().inActiveChain(block_still_active).nextBlock(FoundBlock().inActiveChain(next_block).hash(next_block_hash))); - if (!block.IsNull()) { - LOCK(cs_wallet); - if (!block_still_active) { - // Abort scan if current block is no longer active, to prevent - // marking transactions as coming from the wrong block. - result.last_failed_block = block_hash; - result.status = ScanResult::FAILURE; - break; - } - for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { - SyncTransaction(block.vtx[posInBlock], TxStateConfirmed{block_hash, block_height, static_cast<int>(posInBlock)}, fUpdate, /*rescanning_old_block=*/true); - } - // scan succeeded, record block as most recent successfully scanned - result.last_scanned_block = block_hash; - result.last_scanned_height = block_height; + if (fetch_block) { + // Read block data + CBlock block; + chain().findBlock(block_hash, FoundBlock().data(block)); + + if (!block.IsNull()) { + LOCK(cs_wallet); + if (!block_still_active) { + // Abort scan if current block is no longer active, to prevent + // marking transactions as coming from the wrong block. + result.last_failed_block = block_hash; + result.status = ScanResult::FAILURE; + break; + } + for (size_t posInBlock = 0; posInBlock < block.vtx.size(); ++posInBlock) { + SyncTransaction(block.vtx[posInBlock], TxStateConfirmed{block_hash, block_height, static_cast<int>(posInBlock)}, fUpdate, /*rescanning_old_block=*/true); + } + // scan succeeded, record block as most recent successfully scanned + result.last_scanned_block = block_hash; + result.last_scanned_height = block_height; - if (save_progress && next_interval) { - CBlockLocator loc = m_chain->getActiveChainLocator(block_hash); + if (save_progress && next_interval) { + CBlockLocator loc = m_chain->getActiveChainLocator(block_hash); - if (!loc.IsNull()) { - WalletLogPrintf("Saving scan progress %d.\n", block_height); - WalletBatch batch(GetDatabase()); - batch.WriteBestBlock(loc); + if (!loc.IsNull()) { + WalletLogPrintf("Saving scan progress %d.\n", block_height); + WalletBatch batch(GetDatabase()); + batch.WriteBestBlock(loc); + } } + } else { + // could not scan block, keep scanning but record this block as the most recent failure + result.last_failed_block = block_hash; + result.status = ScanResult::FAILURE; } - } else { - // could not scan block, keep scanning but record this block as the most recent failure - result.last_failed_block = block_hash; - result.status = ScanResult::FAILURE; } if (max_height && block_height >= *max_height) { break; @@ -1903,11 +1986,29 @@ std::set<uint256> CWallet::GetTxConflicts(const CWalletTx& wtx) const return result; } +bool CWallet::ShouldResend() const +{ + // Don't attempt to resubmit if the wallet is configured to not broadcast + if (!fBroadcastTransactions) return false; + + // During reindex, importing and IBD, old wallet transactions become + // unconfirmed. Don't resend them as that would spam other nodes. + // We only allow forcing mempool submission when not relaying to avoid this spam. + if (!chain().isReadyToBroadcast()) return false; + + // Do this infrequently and randomly to avoid giving away + // that these are our transactions. + if (NodeClock::now() < m_next_resend) return false; + + return true; +} + +NodeClock::time_point CWallet::GetDefaultNextResend() { return FastRandomContext{}.rand_uniform_delay(NodeClock::now() + 12h, 24h); } + // Resubmit transactions from the wallet to the mempool, optionally asking the // mempool to relay them. On startup, we will do this for all unconfirmed // transactions but will not ask the mempool to relay them. We do this on startup -// to ensure that our own mempool is aware of our transactions, and to also -// initialize m_next_resend so that the actual rebroadcast is scheduled. There +// to ensure that our own mempool is aware of our transactions. There // is a privacy side effect here as not broadcasting on startup also means that we won't // inform the world of our wallet's state, particularly if the wallet (or node) is not // yet synced. @@ -1934,17 +2035,6 @@ void CWallet::ResubmitWalletTransactions(bool relay, bool force) // even if forcing. if (!fBroadcastTransactions) return; - // During reindex, importing and IBD, old wallet transactions become - // unconfirmed. Don't resend them as that would spam other nodes. - // We only allow forcing mempool submission when not relaying to avoid this spam. - if (!force && relay && !chain().isReadyToBroadcast()) return; - - // Do this infrequently and randomly to avoid giving away - // that these are our transactions. - if (!force && GetTime() < m_next_resend) return; - // resend 12-36 hours from now, ~1 day on average. - m_next_resend = GetTime() + (12 * 60 * 60) + GetRand(24 * 60 * 60); - int submitted_tx_count = 0; { // cs_wallet scope @@ -1957,7 +2047,7 @@ void CWallet::ResubmitWalletTransactions(bool relay, bool force) // Only rebroadcast unconfirmed txs if (!wtx.isUnconfirmed()) continue; - // attempt to rebroadcast all txes more than 5 minutes older than + // Attempt to rebroadcast all txes more than 5 minutes older than // the last block, or all txs if forcing. if (!force && wtx.nTimeReceived > m_best_block_time - 5 * 60) continue; to_submit.insert(&wtx); @@ -1979,7 +2069,9 @@ void CWallet::ResubmitWalletTransactions(bool relay, bool force) void MaybeResendWalletTxs(WalletContext& context) { for (const std::shared_ptr<CWallet>& pwallet : GetWallets(context)) { + if (!pwallet->ShouldResend()) continue; pwallet->ResubmitWalletTransactions(/*relay=*/true, /*force=*/false); + pwallet->SetNextResend(); } } @@ -2371,12 +2463,11 @@ bool CWallet::TopUpKeyPool(unsigned int kpSize) util::Result<CTxDestination> CWallet::GetNewDestination(const OutputType type, const std::string label) { LOCK(cs_wallet); - auto spk_man = GetScriptPubKeyMan(type, false /* internal */); + auto spk_man = GetScriptPubKeyMan(type, /*internal=*/false); if (!spk_man) { return util::Error{strprintf(_("Error: No %s addresses available."), FormatOutputType(type))}; } - spk_man->TopUp(); auto op_dest = spk_man->GetNewDestination(type); if (op_dest) { SetAddressBook(*op_dest, label, "receive"); @@ -2470,10 +2561,7 @@ util::Result<CTxDestination> ReserveDestination::GetReservedDestination(bool int return util::Error{strprintf(_("Error: No %s addresses available."), FormatOutputType(type))}; } - if (nIndex == -1) - { - m_spk_man->TopUp(); - + if (nIndex == -1) { CKeyPool keypool; auto op_address = m_spk_man->GetReservedDestination(type, internal, nIndex, keypool); if (!op_address) return op_address; @@ -2831,6 +2919,10 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri "The wallet might had been created on a newer version.\n" "Please try running the latest software version.\n"), walletFile); return nullptr; + } else if (nLoadWalletRet == DBErrors::UNEXPECTED_LEGACY_ENTRY) { + error = strprintf(_("Unexpected legacy entry in descriptor wallet found. Loading wallet %s\n\n" + "The wallet might have been tampered with or created with malicious intent.\n"), walletFile); + return nullptr; } else { error = strprintf(_("Error loading %s"), walletFile); return nullptr; @@ -3198,14 +3290,12 @@ bool CWallet::UpgradeWallet(int version, bilingual_str& error) void CWallet::postInitProcess() { - LOCK(cs_wallet); - // Add wallet transactions that aren't already in a block to mempool // Do this here as mempool requires genesis block to be loaded ResubmitWalletTransactions(/*relay=*/false, /*force=*/true); // Update wallet transactions with current mempool transactions. - chain().requestMempoolTransactions(*this); + WITH_LOCK(cs_wallet, chain().requestMempoolTransactions(*this)); } bool CWallet::BackupWallet(const std::string& strDest) const @@ -4096,8 +4186,8 @@ util::Result<MigrationResult> MigrateLegacyToDescriptor(std::shared_ptr<CWallet> // Make list of wallets to cleanup std::vector<std::shared_ptr<CWallet>> created_wallets; - created_wallets.push_back(std::move(res.watchonly_wallet)); - created_wallets.push_back(std::move(res.solvables_wallet)); + if (res.watchonly_wallet) created_wallets.push_back(std::move(res.watchonly_wallet)); + if (res.solvables_wallet) created_wallets.push_back(std::move(res.solvables_wallet)); // Get the directories to remove after unloading for (std::shared_ptr<CWallet>& w : created_wallets) { diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 953d3e5bde..137320acda 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -20,6 +20,7 @@ #include <util/strencodings.h> #include <util/string.h> #include <util/system.h> +#include <util/time.h> #include <util/ui_change_type.h> #include <validationinterface.h> #include <wallet/crypter.h> @@ -250,7 +251,7 @@ private: int nWalletVersion GUARDED_BY(cs_wallet){FEATURE_BASE}; /** The next scheduled rebroadcast of wallet transactions. */ - std::atomic<int64_t> m_next_resend{}; + NodeClock::time_point m_next_resend{GetDefaultNextResend()}; /** Whether this wallet will submit newly created transactions to the node's mempool and * prompt rebroadcasts (see ResendWalletTransactions()). */ bool fBroadcastTransactions = false; @@ -348,6 +349,8 @@ private: */ static bool AttachChain(const std::shared_ptr<CWallet>& wallet, interfaces::Chain& chain, const bool rescan_required, bilingual_str& error, std::vector<bilingual_str>& warnings); + static NodeClock::time_point GetDefaultNextResend(); + public: /** * Main wallet lock. @@ -399,7 +402,6 @@ public: TxItems wtxOrdered; int64_t nOrderPosNext GUARDED_BY(cs_wallet) = 0; - uint64_t nAccountingEntryNumber = 0; std::map<CTxDestination, CAddressBookData> m_address_book GUARDED_BY(cs_wallet); const CAddressBookData* FindAddressBookEntry(const CTxDestination&, bool allow_change = false) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -537,6 +539,10 @@ public: }; ScanResult ScanForWalletTransactions(const uint256& start_block, int start_height, std::optional<int> max_height, const WalletRescanReserver& reserver, bool fUpdate, const bool save_progress); void transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason, uint64_t mempool_sequence) override; + /** Set the next time this wallet should resend transactions to 12-36 hours from now, ~1 day on average. */ + void SetNextResend() { m_next_resend = GetDefaultNextResend(); } + /** Return true if all conditions for periodically resending transactions are met. */ + bool ShouldResend() const; void ResubmitWalletTransactions(bool relay, bool force); OutputType TransactionChangeType(const std::optional<OutputType>& change_type, const std::vector<CRecipient>& vecSend) const; diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 6a8f0d2481..826cecfb6f 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -315,6 +315,7 @@ public: std::map<uint160, CHDChain> m_hd_chains; bool tx_corrupt{false}; bool descriptor_unknown{false}; + bool unexpected_legacy_entry{false}; CWalletScanState() = default; }; @@ -332,6 +333,11 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, if (filter_fn && !filter_fn(strType)) { return true; } + // Legacy entries in descriptor wallets are not allowed, abort immediately + if (pwallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS) && DBKeys::LEGACY_TYPES.count(strType) > 0) { + wss.unexpected_legacy_entry = true; + return false; + } if (strType == DBKeys::NAME) { std::string strAddress; ssKey >> strAddress; @@ -482,7 +488,7 @@ ReadKeyValue(CWallet* pwallet, CDataStream& ssKey, CDataStream& ssValue, if (!ssValue.eof()) { uint256 checksum; ssValue >> checksum; - if ((checksum_valid = Hash(vchPrivKey) != checksum)) { + if (!(checksum_valid = Hash(vchPrivKey) == checksum)) { strErr = "Error reading wallet database: Encrypted key corrupt"; return false; } @@ -833,6 +839,12 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) std::string strType, strErr; if (!ReadKeyValue(pwallet, ssKey, ssValue, wss, strType, strErr)) { + if (wss.unexpected_legacy_entry) { + strErr = strprintf("Error: Unexpected legacy entry found in descriptor wallet %s. ", pwallet->GetName()); + strErr += "The wallet might have been tampered with or created with malicious intent."; + pwallet->WalletLogPrintf("%s\n", strErr); + return DBErrors::UNEXPECTED_LEGACY_ENTRY; + } // losing keys is considered a catastrophic error, anything else // we assume the user can live with: if (IsKeyType(strType) || strType == DBKeys::DEFAULTKEY) { diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index da6efe534b..27b5dbdd96 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -52,7 +52,8 @@ enum class DBErrors LOAD_FAIL, NEED_REWRITE, NEED_RESCAN, - UNKNOWN_DESCRIPTOR + UNKNOWN_DESCRIPTOR, + UNEXPECTED_LEGACY_ENTRY }; namespace DBKeys { diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index e991bc0814..9ed2a7c18b 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -58,7 +58,7 @@ static const std::shared_ptr<CWallet> MakeWallet(const std::string& name, const } // dummy chain interface - std::shared_ptr<CWallet> wallet_instance{new CWallet(nullptr /* chain */, name, args, std::move(database)), WalletToolReleaseWallet}; + std::shared_ptr<CWallet> wallet_instance{new CWallet(/*chain=*/nullptr, name, args, std::move(database)), WalletToolReleaseWallet}; DBErrors load_wallet_ret; try { load_wallet_ret = wallet_instance->LoadWallet(); diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index eaf3455296..c785a929d3 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -248,18 +248,14 @@ bool CZMQPublishRawBlockNotifier::NotifyBlock(const CBlockIndex *pindex) const Consensus::Params& consensusParams = Params().GetConsensus(); CDataStream ss(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); - { - LOCK(cs_main); - CBlock block; - if(!ReadBlockFromDisk(block, pindex, consensusParams)) - { - zmqError("Can't read block from disk"); - return false; - } - - ss << block; + CBlock block; + if (!ReadBlockFromDisk(block, pindex, consensusParams)) { + zmqError("Can't read block from disk"); + return false; } + ss << block; + return SendZmqMessage(MSG_RAWBLOCK, &(*ss.begin()), ss.size()); } |