diff options
Diffstat (limited to 'src')
309 files changed, 5290 insertions, 2479 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index d12edca64e..b4ff556eb6 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -143,6 +143,8 @@ BITCOIN_CORE_H = \ compat/compat.h \ compat/cpuid.h \ compat/endian.h \ + common/settings.h \ + common/system.h \ compressor.h \ consensus/consensus.h \ consensus/tx_check.h \ @@ -186,6 +188,7 @@ BITCOIN_CORE_H = \ kernel/mempool_limits.h \ kernel/mempool_options.h \ kernel/mempool_persist.h \ + kernel/notifications_interface.h \ kernel/validation_cache_sizes.h \ key.h \ key_io.h \ @@ -214,9 +217,11 @@ BITCOIN_CORE_H = \ node/database_args.h \ node/eviction.h \ node/interface_ui.h \ + node/kernel_notifications.h \ node/mempool_args.h \ node/mempool_persist_args.h \ node/miner.h \ + node/mini_miner.h \ node/minisketchwrapper.h \ node/psbt.h \ node/transaction.h \ @@ -276,10 +281,13 @@ BITCOIN_CORE_H = \ txorphanage.h \ txrequest.h \ undo.h \ + util/any.h \ util/asmap.h \ + util/batchpriority.h \ util/bip32.h \ util/bitdeque.h \ util/bytevectorhash.h \ + util/chaintype.h \ util/check.h \ util/epochguard.h \ util/error.h \ @@ -292,6 +300,7 @@ BITCOIN_CORE_H = \ util/golombrice.h \ util/hash_type.h \ util/hasher.h \ + util/insert.h \ util/macros.h \ util/message.h \ util/moneystr.h \ @@ -301,13 +310,11 @@ BITCOIN_CORE_H = \ util/readwritefile.h \ util/result.h \ util/serfloat.h \ - util/settings.h \ util/sock.h \ util/spanparsing.h \ util/string.h \ util/syscall_sandbox.h \ util/syserror.h \ - util/system.h \ util/thread.h \ util/threadinterrupt.h \ util/threadnames.h \ @@ -356,7 +363,7 @@ BITCOIN_CORE_H = \ obj/build.h: FORCE @$(MKDIR_P) $(builddir)/obj - @$(top_srcdir)/share/genbuild.sh "$(abs_top_builddir)/src/obj/build.h" \ + $(AM_V_GEN) $(top_srcdir)/share/genbuild.sh "$(abs_top_builddir)/src/obj/build.h" \ "$(abs_top_srcdir)" libbitcoin_util_a-clientversion.$(OBJEXT): obj/build.h @@ -406,9 +413,11 @@ libbitcoin_node_a_SOURCES = \ node/eviction.cpp \ node/interface_ui.cpp \ node/interfaces.cpp \ + node/kernel_notifications.cpp \ node/mempool_args.cpp \ node/mempool_persist_args.cpp \ node/miner.cpp \ + node/mini_miner.cpp \ node/minisketchwrapper.cpp \ node/psbt.cpp \ node/transaction.cpp \ @@ -654,6 +663,8 @@ libbitcoin_common_a_SOURCES = \ common/init.cpp \ common/interfaces.cpp \ common/run_command.cpp \ + common/settings.cpp \ + common/system.cpp \ compressor.cpp \ core_read.cpp \ core_write.cpp \ @@ -705,8 +716,10 @@ libbitcoin_util_a_SOURCES = \ support/cleanse.cpp \ sync.cpp \ util/asmap.cpp \ + util/batchpriority.cpp \ util/bip32.cpp \ util/bytevectorhash.cpp \ + util/chaintype.cpp \ util/check.cpp \ util/error.cpp \ util/exception.cpp \ @@ -717,12 +730,10 @@ libbitcoin_util_a_SOURCES = \ util/hasher.cpp \ util/sock.cpp \ util/syserror.cpp \ - util/system.cpp \ util/message.cpp \ util/moneystr.cpp \ util/rbf.cpp \ util/readwritefile.cpp \ - util/settings.cpp \ util/thread.cpp \ util/threadinterrupt.cpp \ util/threadnames.cpp \ @@ -901,12 +912,8 @@ libbitcoinkernel_la_SOURCES = \ kernel/bitcoinkernel.cpp \ arith_uint256.cpp \ chain.cpp \ - chainparamsbase.cpp \ - chainparams.cpp \ clientversion.cpp \ coins.cpp \ - common/args.cpp \ - common/config.cpp \ compressor.cpp \ consensus/merkle.cpp \ consensus/tx_check.cpp \ @@ -956,6 +963,8 @@ libbitcoinkernel_la_SOURCES = \ txdb.cpp \ txmempool.cpp \ uint256.cpp \ + util/batchpriority.cpp \ + util/chaintype.cpp \ util/check.cpp \ util/exception.cpp \ util/fs.cpp \ @@ -965,12 +974,10 @@ libbitcoinkernel_la_SOURCES = \ util/moneystr.cpp \ util/rbf.cpp \ util/serfloat.cpp \ - util/settings.cpp \ util/strencodings.cpp \ util/string.cpp \ util/syscall_sandbox.cpp \ util/syserror.cpp \ - util/system.cpp \ util/thread.cpp \ util/threadnames.cpp \ util/time.cpp \ @@ -1046,7 +1053,7 @@ clean-local: -rm -rf test/__pycache__ .rc.o: - @test -f $(WINDRES) + @test -f $(WINDRES) || (echo "windres $(WINDRES) not found, but is required to compile windows resource files"; exit 1) ## FIXME: How to get the appropriate modulename_CPPFLAGS in here? $(AM_V_GEN) $(WINDRES) $(DEFS) $(DEFAULT_INCLUDES) $(INCLUDES) $(CPPFLAGS) -DWINDRES_PREPROC -i $< -o $@ @@ -1101,12 +1108,11 @@ endif %.raw.h: %.raw @$(MKDIR_P) $(@D) - @{ \ + $(AM_V_GEN) { \ echo "static unsigned const char $(*F)_raw[] = {" && \ $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ echo "};"; \ } > "$@.new" && mv -f "$@.new" "$@" - @echo "Generated $@" include Makefile.minisketch.include diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index c230728a1c..c8e510b482 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -47,6 +47,7 @@ bench_bench_bitcoin_SOURCES = \ bench/rollingbloom.cpp \ bench/rpc_blockchain.cpp \ bench/rpc_mempool.cpp \ + bench/streams_findbyte.cpp \ bench/strencodings.cpp \ bench/util_time.cpp \ bench/verify_script.cpp diff --git a/src/Makefile.qt.include b/src/Makefile.qt.include index 602a118259..7852d1a2fa 100644 --- a/src/Makefile.qt.include +++ b/src/Makefile.qt.include @@ -371,13 +371,13 @@ translate: $(srcdir)/qt/bitcoinstrings.cpp $(QT_FORMS_UI) $(QT_FORMS_UI) $(BITCO @rm -f $(srcdir)/qt/locale/bitcoin_en.xlf.old $(QT_QRC_LOCALE_CPP): $(QT_QRC_LOCALE) $(QT_QM) - @test -f $(RCC) + @test -f $(RCC) || (echo "rcc $(RCC) not found, but is required for generating qrc cpp files"; exit 1) @cp -f $< $(@D)/temp_$(<F) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin_locale --format-version 1 $(@D)/temp_$(<F) > $@ @rm $(@D)/temp_$(<F) $(QT_QRC_CPP): $(QT_QRC) $(QT_FORMS_H) $(QT_RES_FONTS) $(QT_RES_ICONS) $(QT_RES_ANIMATION) - @test -f $(RCC) + @test -f $(RCC) || (echo "rcc $(RCC) not found, but is required for generating qrc cpp files"; exit 1) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(RCC) -name bitcoin --format-version 1 $< > $@ CLEAN_QT = $(nodist_qt_libbitcoinqt_a_SOURCES) $(QT_QM) $(QT_FORMS_H) qt/*.gcda qt/*.gcno qt/temp_bitcoin_locale.qrc @@ -404,7 +404,7 @@ bitcoin_qt_apk: FORCE cd qt/android && ./gradlew build ui_%.h: %.ui - @test -f $(UIC) + @test -f $(UIC) || (echo "uic $(UIC) not found, but is required for generating ui headers"; exit 1) @$(MKDIR_P) $(@D) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(UIC) -o $@ $< || (echo "Error creating $@"; false) @@ -415,6 +415,6 @@ moc_%.cpp: %.h $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(MOC) $(DEFAULT_INCLUDES) $(QT_INCLUDES_UNSUPPRESSED) $(MOC_DEFS) $< > $@ %.qm: %.ts - @test -f $(LRELEASE) + @test -f $(LRELEASE) || (echo "lrelease $(LRELEASE) not found, but is required for generating translations"; exit 1) @$(MKDIR_P) $(@D) $(AM_V_GEN) QT_SELECT=$(QT_SELECT) $(LRELEASE) -silent $< -qm $@ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 69965ed1b8..8e45c797bf 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -106,6 +106,7 @@ BITCOIN_TESTS =\ test/merkle_tests.cpp \ test/merkleblock_tests.cpp \ test/miner_tests.cpp \ + test/miniminer_tests.cpp \ test/miniscript_tests.cpp \ test/minisketch_tests.cpp \ test/multisig_tests.cpp \ @@ -287,6 +288,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/message.cpp \ test/fuzz/miniscript.cpp \ test/fuzz/minisketch.cpp \ + test/fuzz/mini_miner.cpp \ test/fuzz/muhash.cpp \ test/fuzz/multiplication_overflow.cpp \ test/fuzz/net.cpp \ @@ -344,6 +346,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/txorphan.cpp \ test/fuzz/txrequest.cpp \ test/fuzz/utxo_snapshot.cpp \ + test/fuzz/utxo_total_supply.cpp \ test/fuzz/validation_load_mempool.cpp \ test/fuzz/versionbits.cpp endif # ENABLE_FUZZ_BINARY @@ -421,10 +424,9 @@ endif %.json.h: %.json @$(MKDIR_P) $(@D) - @{ \ + $(AM_V_GEN) { \ echo "namespace json_tests{" && \ echo "static unsigned const char $(*F)[] = {" && \ $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ echo "};};"; \ } > "$@.new" && mv -f "$@.new" "$@" - @echo "Generated $@" diff --git a/src/addrdb.cpp b/src/addrdb.cpp index b679ad0602..cb1c49050e 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -9,6 +9,7 @@ #include <chainparams.h> #include <clientversion.h> #include <common/args.h> +#include <common/settings.h> #include <cstdint> #include <hash.h> #include <logging.h> @@ -21,7 +22,6 @@ #include <univalue.h> #include <util/fs.h> #include <util/fs_helpers.h> -#include <util/settings.h> #include <util/translation.h> namespace { @@ -132,7 +132,7 @@ CBanDB::CBanDB(fs::path ban_list_path) bool CBanDB::Write(const banmap_t& banSet) { std::vector<std::string> errors; - if (util::WriteSettings(m_banlist_json, {{JSON_KEY, BanMapToJson(banSet)}}, errors)) { + if (common::WriteSettings(m_banlist_json, {{JSON_KEY, BanMapToJson(banSet)}}, errors)) { return true; } @@ -152,10 +152,10 @@ bool CBanDB::Read(banmap_t& banSet) return false; } - std::map<std::string, util::SettingsValue> settings; + std::map<std::string, common::SettingsValue> settings; std::vector<std::string> errors; - if (!util::ReadSettings(m_banlist_json, settings, errors)) { + if (!common::ReadSettings(m_banlist_json, settings, errors)) { for (const auto& err : errors) { LogPrintf("Cannot load banlist %s: %s\n", fs::PathToString(m_banlist_json), err); } @@ -183,10 +183,10 @@ void ReadFromStream(AddrMan& addr, CDataStream& ssPeers) DeserializeDB(ssPeers, addr, false); } -std::optional<bilingual_str> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman) +util::Result<std::unique_ptr<AddrMan>> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args) { auto check_addrman = std::clamp<int32_t>(args.GetIntArg("-checkaddrman", DEFAULT_ADDRMAN_CONSISTENCY_CHECKS), 0, 1000000); - addrman = std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); + auto addrman{std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman)}; const auto start{SteadyClock::now()}; const auto path_addr{args.GetDataDirNet() / "peers.dat"}; @@ -200,19 +200,18 @@ std::optional<bilingual_str> LoadAddrman(const NetGroupManager& netgroupman, con DumpPeerAddresses(args, *addrman); } catch (const InvalidAddrManVersionError&) { if (!RenameOver(path_addr, (fs::path)path_addr + ".bak")) { - addrman = nullptr; - return strprintf(_("Failed to rename invalid peers.dat file. Please move or delete it and try again.")); + return util::Error{strprintf(_("Failed to rename invalid peers.dat file. Please move or delete it and try again."))}; } // Addrman can be in an inconsistent state after failure, reset it addrman = std::make_unique<AddrMan>(netgroupman, /*deterministic=*/false, /*consistency_check_ratio=*/check_addrman); LogPrintf("Creating new peers.dat because the file version was not compatible (%s). Original backed up to peers.dat.bak\n", fs::quoted(fs::PathToString(path_addr))); DumpPeerAddresses(args, *addrman); } catch (const std::exception& e) { - addrman = nullptr; - return strprintf(_("Invalid or corrupt peers.dat (%s). If you believe this is a bug, please report it to %s. As a workaround, you can move the file (%s) out of the way (rename, move, or delete) to have a new one created on the next start."), - e.what(), PACKAGE_BUGREPORT, fs::quoted(fs::PathToString(path_addr))); + return util::Error{strprintf(_("Invalid or corrupt peers.dat (%s). If you believe this is a bug, please report it to %s. As a workaround, you can move the file (%s) out of the way (rename, move, or delete) to have a new one created on the next start."), + e.what(), PACKAGE_BUGREPORT, fs::quoted(fs::PathToString(path_addr)))}; } - return std::nullopt; + return {std::move(addrman)}; // std::move should be unneccessary but is temporarily needed to work around clang bug + // (https://github.com/bitcoin/bitcoin/pull/25977#issuecomment-1561270092) } void DumpAnchors(const fs::path& anchors_db_path, const std::vector<CAddress>& anchors) diff --git a/src/addrdb.h b/src/addrdb.h index 08d86d0f01..0037495d18 100644 --- a/src/addrdb.h +++ b/src/addrdb.h @@ -6,11 +6,11 @@ #ifndef BITCOIN_ADDRDB_H #define BITCOIN_ADDRDB_H -#include <net_types.h> // For banmap_t -#include <univalue.h> +#include <net_types.h> #include <util/fs.h> +#include <util/result.h> -#include <optional> +#include <memory> #include <vector> class ArgsManager; @@ -18,7 +18,6 @@ class AddrMan; class CAddress; class CDataStream; class NetGroupManager; -struct bilingual_str; bool DumpPeerAddresses(const ArgsManager& args, const AddrMan& addr); /** Only used by tests. */ @@ -49,7 +48,7 @@ public: }; /** Returns an error string on failure */ -std::optional<bilingual_str> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args, std::unique_ptr<AddrMan>& addrman); +util::Result<std::unique_ptr<AddrMan>> LoadAddrman(const NetGroupManager& netgroupman, const ArgsManager& args); /** * Dump the anchor IP address database (anchors.dat) diff --git a/src/attributes.h b/src/attributes.h index 9957bcd84b..a4603b0270 100644 --- a/src/attributes.h +++ b/src/attributes.h @@ -16,4 +16,12 @@ # define LIFETIMEBOUND #endif +#if defined(__GNUC__) +# define ALWAYS_INLINE inline __attribute__((always_inline)) +#elif defined(_MSC_VER) +# define ALWAYS_INLINE __forceinline +#else +# error No known always_inline attribute for this platform. +#endif + #endif // BITCOIN_ATTRIBUTES_H diff --git a/src/banman.cpp b/src/banman.cpp index 5b2049d654..a96b7e3c53 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -5,11 +5,11 @@ #include <banman.h> +#include <common/system.h> #include <logging.h> #include <netaddress.h> #include <node/interface_ui.h> #include <sync.h> -#include <util/system.h> #include <util/time.h> #include <util/translation.h> diff --git a/src/bench/addrman.cpp b/src/bench/addrman.cpp index 8a5cab443f..f044feebba 100644 --- a/src/bench/addrman.cpp +++ b/src/bench/addrman.cpp @@ -72,20 +72,6 @@ static void FillAddrMan(AddrMan& addrman) AddAddressesToAddrMan(addrman); } -static CNetAddr ResolveIP(const std::string& ip) -{ - CNetAddr addr; - LookupHost(ip, addr, false); - return addr; -} - -static CService ResolveService(const std::string& ip, uint16_t port = 0) -{ - CService serv; - Lookup(ip, serv, port, false); - return serv; -} - /* Benchmarks */ static void AddrManAdd(benchmark::Bench& bench) @@ -118,8 +104,8 @@ static void AddrManSelectFromAlmostEmpty(benchmark::Bench& bench) AddrMan addrman{EMPTY_NETGROUPMAN, /*deterministic=*/false, ADDRMAN_CONSISTENCY_CHECK_RATIO}; // Add one address to the new table - CService addr = ResolveService("250.3.1.1", 8333); - addrman.Add({CAddress(addr, NODE_NONE)}, ResolveService("250.3.1.1", 8333)); + CService addr = Lookup("250.3.1.1", 8333, false).value(); + addrman.Add({CAddress(addr, NODE_NONE)}, addr); bench.run([&] { (void)addrman.Select(); @@ -135,7 +121,7 @@ static void AddrManSelectByNetwork(benchmark::Bench& bench) i2p_service.SetSpecial("udhdrtrcetjm5sxzskjyr5ztpeszydbh4dpl3pl4utgqqw2v4jna.b32.i2p"); CAddress i2p_address(i2p_service, NODE_NONE); i2p_address.nTime = Now<NodeSeconds>(); - CNetAddr source = ResolveIP("252.2.2.2"); + const CNetAddr source{LookupHost("252.2.2.2", false).value()}; addrman.Add({i2p_address}, source); FillAddrMan(addrman); diff --git a/src/bench/block_assemble.cpp b/src/bench/block_assemble.cpp index 8dd4117a3e..4d032cefc5 100644 --- a/src/bench/block_assemble.cpp +++ b/src/bench/block_assemble.cpp @@ -27,7 +27,7 @@ static void AssembleBlock(benchmark::Bench& bench) std::array<CTransactionRef, NUM_BLOCKS - COINBASE_MATURITY + 1> txs; for (size_t b{0}; b < NUM_BLOCKS; ++b) { CMutableTransaction tx; - tx.vin.push_back(MineBlock(test_setup->m_node, P2WSH_OP_TRUE)); + tx.vin.push_back(CTxIn{MineBlock(test_setup->m_node, P2WSH_OP_TRUE)}); tx.vin.back().scriptWitness = witness; tx.vout.emplace_back(1337, P2WSH_OP_TRUE); if (NUM_BLOCKS - b >= COINBASE_MATURITY) diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 260c8991ce..269ac847a5 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -9,6 +9,7 @@ #include <common/args.h> #include <consensus/validation.h> #include <streams.h> +#include <util/chaintype.h> #include <validation.h> // These are the two major time-sinks which happen after we have fully received @@ -36,7 +37,7 @@ static void DeserializeAndCheckBlockTest(benchmark::Bench& bench) stream.write({&a, 1}); // Prevent compaction ArgsManager bench_args; - const auto chainParams = CreateChainParams(bench_args, CBaseChainParams::MAIN); + const auto chainParams = CreateChainParams(bench_args, ChainType::MAIN); bench.unit("block").run([&] { CBlock block; // Note that CBlock caches its checked state, so we need to recreate it here diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index 8ad6fde6bf..70e0b86eba 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -4,11 +4,11 @@ #include <bench/bench.h> #include <checkqueue.h> +#include <common/system.h> #include <key.h> #include <prevector.h> #include <pubkey.h> #include <random.h> -#include <util/system.h> #include <vector> diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index 11ce14728d..0e110a653a 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -9,6 +9,7 @@ #include <wallet/coinselection.h> #include <wallet/spend.h> #include <wallet/wallet.h> +#include <wallet/test/util.h> #include <set> @@ -20,7 +21,7 @@ using wallet::CWallet; using wallet::CWalletTx; using wallet::CoinEligibilityFilter; using wallet::CoinSelectionParams; -using wallet::CreateDummyWalletDatabase; +using wallet::CreateMockableWalletDatabase; using wallet::OutputGroup; using wallet::SelectCoinsBnB; using wallet::TxStateInactive; @@ -46,7 +47,7 @@ static void CoinSelection(benchmark::Bench& bench) { NodeContext node; auto chain = interfaces::MakeChain(node); - CWallet wallet(chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(chain.get(), "", CreateMockableWalletDatabase()); std::vector<std::unique_ptr<CWalletTx>> wtxs; LOCK(wallet.cs_wallet); diff --git a/src/bench/load_external.cpp b/src/bench/load_external.cpp index 0fd842c7c3..2ff72a3012 100644 --- a/src/bench/load_external.cpp +++ b/src/bench/load_external.cpp @@ -6,6 +6,7 @@ #include <bench/data.h> #include <chainparams.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> #include <validation.h> /** @@ -22,7 +23,7 @@ */ static void LoadExternalBlockFile(benchmark::Bench& bench) { - const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(CBaseChainParams::MAIN)}; + const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN)}; // Create a single block as in the blocks files (magic bytes, block size, // block data) as a stream object. diff --git a/src/bench/logging.cpp b/src/bench/logging.cpp index 9aedb26236..c97c4e151b 100644 --- a/src/bench/logging.cpp +++ b/src/bench/logging.cpp @@ -5,6 +5,7 @@ #include <bench/bench.h> #include <logging.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> // All but 2 of the benchmarks should have roughly similar performance: // @@ -18,7 +19,7 @@ static void Logging(benchmark::Bench& bench, const std::vector<const char*>& ext LogInstance().DisableCategory(BCLog::LogFlags::ALL); TestingSetup test_setup{ - CBaseChainParams::REGTEST, + ChainType::REGTEST, extra_args, }; diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp index 80c959cdfb..826da73800 100644 --- a/src/bench/mempool_stress.cpp +++ b/src/bench/mempool_stress.cpp @@ -7,6 +7,7 @@ #include <policy/policy.h> #include <test/util/setup_common.h> #include <txmempool.h> +#include <util/chaintype.h> #include <validation.h> #include <vector> @@ -88,7 +89,7 @@ static void ComplexMemPool(benchmark::Bench& bench) childTxs = static_cast<int>(bench.complexityN()); } std::vector<CTransactionRef> ordered_coins = CreateOrderedCoins(det_rand, childTxs, /*min_ancestors=*/1); - const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(CBaseChainParams::MAIN); + const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN); CTxMemPool& pool = *testing_setup.get()->m_node.mempool; LOCK2(cs_main, pool.cs); bench.run([&]() NO_THREAD_SAFETY_ANALYSIS { @@ -103,7 +104,7 @@ static void ComplexMemPool(benchmark::Bench& bench) static void MempoolCheck(benchmark::Bench& bench) { FastRandomContext det_rand{true}; - auto testing_setup = MakeNoLogFileContext<TestChain100Setup>(CBaseChainParams::REGTEST, {"-checkmempool=1"}); + auto testing_setup = MakeNoLogFileContext<TestChain100Setup>(ChainType::REGTEST, {"-checkmempool=1"}); CTxMemPool& pool = *testing_setup.get()->m_node.mempool; LOCK2(cs_main, pool.cs); testing_setup->PopulateMempool(det_rand, 400, true); diff --git a/src/bench/rpc_blockchain.cpp b/src/bench/rpc_blockchain.cpp index f68b6acb5b..a9b197b190 100644 --- a/src/bench/rpc_blockchain.cpp +++ b/src/bench/rpc_blockchain.cpp @@ -8,6 +8,7 @@ #include <rpc/blockchain.h> #include <streams.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> #include <validation.h> #include <univalue.h> @@ -15,7 +16,7 @@ namespace { struct TestBlockAndIndex { - const std::unique_ptr<const TestingSetup> testing_setup{MakeNoLogFileContext<const TestingSetup>(CBaseChainParams::MAIN)}; + const std::unique_ptr<const TestingSetup> testing_setup{MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN)}; CBlock block{}; uint256 blockHash{}; CBlockIndex blockindex{}; diff --git a/src/bench/rpc_mempool.cpp b/src/bench/rpc_mempool.cpp index e3e1a07c83..7e274370e0 100644 --- a/src/bench/rpc_mempool.cpp +++ b/src/bench/rpc_mempool.cpp @@ -3,12 +3,12 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <bench/bench.h> -#include <chainparamsbase.h> #include <kernel/cs_main.h> #include <kernel/mempool_entry.h> #include <rpc/mempool.h> #include <test/util/setup_common.h> #include <txmempool.h> +#include <util/chaintype.h> #include <univalue.h> @@ -21,7 +21,7 @@ static void AddTx(const CTransactionRef& tx, const CAmount& fee, CTxMemPool& poo static void RpcMempool(benchmark::Bench& bench) { - const auto testing_setup = MakeNoLogFileContext<const ChainTestingSetup>(CBaseChainParams::MAIN); + const auto testing_setup = MakeNoLogFileContext<const ChainTestingSetup>(ChainType::MAIN); CTxMemPool& pool = *Assert(testing_setup->m_node.mempool); LOCK2(cs_main, pool.cs); diff --git a/src/bench/streams_findbyte.cpp b/src/bench/streams_findbyte.cpp new file mode 100644 index 0000000000..77f5940926 --- /dev/null +++ b/src/bench/streams_findbyte.cpp @@ -0,0 +1,31 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> + +#include <util/fs.h> +#include <streams.h> + +static void FindByte(benchmark::Bench& bench) +{ + // Setup + FILE* file = fsbridge::fopen("streams_tmp", "w+b"); + const size_t file_size = 200; + uint8_t data[file_size] = {0}; + data[file_size-1] = 1; + fwrite(&data, sizeof(uint8_t), file_size, file); + rewind(file); + CBufferedFile bf(file, /*nBufSize=*/file_size + 1, /*nRewindIn=*/file_size, 0, 0); + + bench.run([&] { + bf.SetPos(0); + bf.FindByte(std::byte(1)); + }); + + // Cleanup + bf.fclose(); + fs::remove("streams_tmp"); +} + +BENCHMARK(FindByte, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/util_time.cpp b/src/bench/util_time.cpp index 8dbbdec28c..4cbc0dfbbd 100644 --- a/src/bench/util_time.cpp +++ b/src/bench/util_time.cpp @@ -32,7 +32,7 @@ static void BenchTimeMillis(benchmark::Bench& bench) static void BenchTimeMillisSys(benchmark::Bench& bench) { bench.run([&] { - (void)GetTimeMillis(); + (void)TicksSinceEpoch<std::chrono::milliseconds>(SystemClock::now()); }); } diff --git a/src/bench/wallet_balance.cpp b/src/bench/wallet_balance.cpp index d5d057e96d..bf2195293e 100644 --- a/src/bench/wallet_balance.cpp +++ b/src/bench/wallet_balance.cpp @@ -4,6 +4,7 @@ #include <bench/bench.h> #include <interfaces/chain.h> +#include <node/chainstate.h> #include <node/context.h> #include <test/util/mining.h> #include <test/util/setup_common.h> @@ -14,26 +15,21 @@ #include <optional> -using wallet::CWallet; -using wallet::CreateMockWalletDatabase; -using wallet::DBErrors; -using wallet::GetBalance; -using wallet::WALLET_FLAG_DESCRIPTORS; - -const std::string ADDRESS_BCRT1_UNSPENDABLE = "bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj"; - +namespace wallet { static void WalletBalance(benchmark::Bench& bench, const bool set_dirty, const bool add_mine) { const auto test_setup = MakeNoLogFileContext<const TestingSetup>(); const auto& ADDRESS_WATCHONLY = ADDRESS_BCRT1_UNSPENDABLE; - CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockWalletDatabase()}; + // Set clock to genesis block, so the descriptors/keys creation time don't interfere with the blocks scanning process. + // The reason is 'generatetoaddress', which creates a chain with deterministic timestamps in the past. + SetMockTime(test_setup->m_node.chainman->GetParams().GenesisBlock().nTime); + CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockableWalletDatabase()}; { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet.SetupDescriptorScriptPubKeyMans(); - if (wallet.LoadWallet() != DBErrors::LOAD_OK) assert(false); } auto handler = test_setup->m_node.chain->handleNotifications({&wallet, [](CWallet*) {}}); @@ -63,3 +59,4 @@ BENCHMARK(WalletBalanceDirty, benchmark::PriorityLevel::HIGH); BENCHMARK(WalletBalanceClean, benchmark::PriorityLevel::HIGH); BENCHMARK(WalletBalanceMine, benchmark::PriorityLevel::HIGH); BENCHMARK(WalletBalanceWatch, benchmark::PriorityLevel::HIGH); +} // namespace wallet diff --git a/src/bench/wallet_create_tx.cpp b/src/bench/wallet_create_tx.cpp index cb31421598..13b0019fd2 100644 --- a/src/bench/wallet_create_tx.cpp +++ b/src/bench/wallet_create_tx.cpp @@ -15,7 +15,7 @@ #include <wallet/wallet.h> using wallet::CWallet; -using wallet::CreateMockWalletDatabase; +using wallet::CreateMockableWalletDatabase; using wallet::DBErrors; using wallet::WALLET_FLAG_DESCRIPTORS; @@ -83,7 +83,7 @@ static void WalletCreateTx(benchmark::Bench& bench, const OutputType output_type { const auto test_setup = MakeNoLogFileContext<const TestingSetup>(); - CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockWalletDatabase()}; + CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockableWalletDatabase()}; { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -136,7 +136,7 @@ static void WalletCreateTx(benchmark::Bench& bench, const OutputType output_type static void AvailableCoins(benchmark::Bench& bench, const std::vector<OutputType>& output_type) { const auto test_setup = MakeNoLogFileContext<const TestingSetup>(); - CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockWalletDatabase()}; + CWallet wallet{test_setup->m_node.chain.get(), "", CreateMockableWalletDatabase()}; { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); diff --git a/src/bench/wallet_loading.cpp b/src/bench/wallet_loading.cpp index 6b09adcc9d..5453238728 100644 --- a/src/bench/wallet_loading.cpp +++ b/src/bench/wallet_loading.cpp @@ -16,33 +16,7 @@ #include <optional> -using wallet::CWallet; -using wallet::DatabaseFormat; -using wallet::DatabaseOptions; -using wallet::TxStateInactive; -using wallet::WALLET_FLAG_DESCRIPTORS; -using wallet::WalletContext; -using wallet::WalletDatabase; - -static std::shared_ptr<CWallet> BenchLoadWallet(std::unique_ptr<WalletDatabase> database, WalletContext& context, DatabaseOptions& options) -{ - bilingual_str error; - std::vector<bilingual_str> warnings; - auto wallet = CWallet::Create(context, "", std::move(database), options.create_flags, error, warnings); - NotifyWalletLoaded(context, wallet); - if (context.chain) { - wallet->postInitProcess(); - } - return wallet; -} - -static void BenchUnloadWallet(std::shared_ptr<CWallet>&& wallet) -{ - SyncWithValidationInterfaceQueue(); - wallet->m_chain_notifications_handler.reset(); - UnloadWallet(std::move(wallet)); -} - +namespace wallet{ static void AddTx(CWallet& wallet) { CMutableTransaction mtx; @@ -55,7 +29,6 @@ static void AddTx(CWallet& wallet) static void WalletLoading(benchmark::Bench& bench, bool legacy_wallet) { const auto test_setup = MakeNoLogFileContext<TestingSetup>(); - test_setup->m_args.ForceSetArg("-unsafesqlitesync", "1"); WalletContext context; context.args = &test_setup->m_args; @@ -63,32 +36,29 @@ static void WalletLoading(benchmark::Bench& bench, bool legacy_wallet) // Setup the wallet // Loading the wallet will also create it - DatabaseOptions options; - if (legacy_wallet) { - options.require_format = DatabaseFormat::BERKELEY; - } else { - options.create_flags = WALLET_FLAG_DESCRIPTORS; - options.require_format = DatabaseFormat::SQLITE; + uint64_t create_flags = 0; + if (!legacy_wallet) { + create_flags = WALLET_FLAG_DESCRIPTORS; } - auto database = CreateMockWalletDatabase(options); - auto wallet = BenchLoadWallet(std::move(database), context, options); + auto database = CreateMockableWalletDatabase(); + auto wallet = TestLoadWallet(std::move(database), context, create_flags); // Generate a bunch of transactions and addresses to put into the wallet for (int i = 0; i < 1000; ++i) { AddTx(*wallet); } - database = DuplicateMockDatabase(wallet->GetDatabase(), options); + database = DuplicateMockDatabase(wallet->GetDatabase()); // reload the wallet for the actual benchmark - BenchUnloadWallet(std::move(wallet)); + TestUnloadWallet(std::move(wallet)); bench.epochs(5).run([&] { - wallet = BenchLoadWallet(std::move(database), context, options); + wallet = TestLoadWallet(std::move(database), context, create_flags); // Cleanup - database = DuplicateMockDatabase(wallet->GetDatabase(), options); - BenchUnloadWallet(std::move(wallet)); + database = DuplicateMockDatabase(wallet->GetDatabase()); + TestUnloadWallet(std::move(wallet)); }); } @@ -101,3 +71,4 @@ BENCHMARK(WalletLoadingLegacy, benchmark::PriorityLevel::HIGH); static void WalletLoadingDescriptors(benchmark::Bench& bench) { WalletLoading(bench, /*legacy_wallet=*/false); } BENCHMARK(WalletLoadingDescriptors, benchmark::PriorityLevel::HIGH); #endif +} // namespace wallet diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index 04f6aae4f7..432bdc8e33 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -12,12 +12,11 @@ // It is part of the libbitcoinkernel project. #include <kernel/chainparams.h> +#include <kernel/chainstatemanager_opts.h> #include <kernel/checks.h> #include <kernel/context.h> #include <kernel/validation_cache_sizes.h> -#include <chainparams.h> -#include <common/args.h> #include <consensus/validation.h> #include <core_io.h> #include <node/blockstorage.h> @@ -25,14 +24,18 @@ #include <node/chainstate.h> #include <scheduler.h> #include <script/sigcache.h> +#include <util/chaintype.h> #include <util/thread.h> #include <validation.h> #include <validationinterface.h> #include <cassert> +#include <cstdint> #include <filesystem> #include <functional> #include <iosfwd> +#include <memory> +#include <string> int main(int argc, char* argv[]) { @@ -48,18 +51,14 @@ int main(int argc, char* argv[]) } std::filesystem::path abs_datadir = std::filesystem::absolute(argv[1]); std::filesystem::create_directories(abs_datadir); - gArgs.ForceSetArg("-datadir", abs_datadir.string()); - // SETUP: Misc Globals - SelectParams(CBaseChainParams::MAIN); - auto chainparams = CChainParams::Main(); - + // SETUP: Context kernel::Context kernel_context{}; // We can't use a goto here, but we can use an assert since none of the // things instantiated so far requires running the epilogue to be torn down // properly - assert(!kernel::SanityChecks(kernel_context).has_value()); + assert(kernel::SanityChecks(kernel_context)); // Necessary for CheckInputScripts (eventually called by ProcessNewBlock), // which will try the script cache first and fall back to actually @@ -79,14 +78,42 @@ int main(int argc, char* argv[]) GetMainSignals().RegisterBackgroundSignalScheduler(scheduler); + class KernelNotifications : public kernel::Notifications + { + public: + void blockTip(SynchronizationState, CBlockIndex&) override + { + std::cout << "Block tip changed" << std::endl; + } + void headerTip(SynchronizationState, int64_t height, int64_t timestamp, bool presync) override + { + std::cout << "Header tip changed: " << height << ", " << timestamp << ", " << presync << std::endl; + } + void progress(const bilingual_str& title, int progress_percent, bool resume_possible) override + { + std::cout << "Progress: " << title.original << ", " << progress_percent << ", " << resume_possible << std::endl; + } + void warning(const bilingual_str& warning) override + { + std::cout << "Warning: " << warning.original << std::endl; + } + }; + auto notifications = std::make_unique<KernelNotifications>(); + // SETUP: Chainstate + auto chainparams = CChainParams::Main(); const ChainstateManager::Options chainman_opts{ .chainparams = *chainparams, - .datadir = gArgs.GetDataDirNet(), + .datadir = abs_datadir, .adjusted_time_callback = NodeClock::now, + .notifications = *notifications, + }; + const node::BlockManager::Options blockman_opts{ + .chainparams = chainman_opts.chainparams, + .blocks_dir = abs_datadir / "blocks", }; - ChainstateManager chainman{chainman_opts, {}}; + ChainstateManager chainman{chainman_opts, blockman_opts}; node::CacheSizes cache_sizes; cache_sizes.block_tree_db = 2 << 20; @@ -117,7 +144,8 @@ int main(int argc, char* argv[]) // Main program logic starts here std::cout << "Hello! I'm going to print out some information about your datadir." << std::endl - << "\t" << "Path: " << gArgs.GetDataDirNet() << std::endl; + << "\t" + << "Path: " << abs_datadir << std::endl; { LOCK(chainman.GetMutex()); std::cout diff --git a/src/bitcoin-cli.cpp b/src/bitcoin-cli.cpp index 63ba6a935d..45db7a9a66 100644 --- a/src/bitcoin-cli.cpp +++ b/src/bitcoin-cli.cpp @@ -10,6 +10,7 @@ #include <chainparamsbase.h> #include <clientversion.h> #include <common/args.h> +#include <common/system.h> #include <common/url.h> #include <compat/compat.h> #include <compat/stdin.h> @@ -20,9 +21,9 @@ #include <rpc/request.h> #include <tinyformat.h> #include <univalue.h> +#include <util/chaintype.h> #include <util/exception.h> #include <util/strencodings.h> -#include <util/system.h> #include <util/time.h> #include <util/translation.h> @@ -73,10 +74,10 @@ static void SetupCliArgs(ArgsManager& argsman) { SetupHelpOptions(argsman); - const auto defaultBaseParams = CreateBaseChainParams(CBaseChainParams::MAIN); - const auto testnetBaseParams = CreateBaseChainParams(CBaseChainParams::TESTNET); - const auto signetBaseParams = CreateBaseChainParams(CBaseChainParams::SIGNET); - const auto regtestBaseParams = CreateBaseChainParams(CBaseChainParams::REGTEST); + const auto defaultBaseParams = CreateBaseChainParams(ChainType::MAIN); + const auto testnetBaseParams = CreateBaseChainParams(ChainType::TESTNET); + const auto signetBaseParams = CreateBaseChainParams(ChainType::SIGNET); + const auto regtestBaseParams = CreateBaseChainParams(ChainType::REGTEST); argsman.AddArg("-version", "Print version and exit", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-conf=<file>", strprintf("Specify configuration file. Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -174,7 +175,7 @@ static int AppInitRPC(int argc, char* argv[]) } // Check for chain settings (BaseParams() calls are only valid after this clause) try { - SelectBaseParams(gArgs.GetChainName()); + SelectBaseParams(gArgs.GetChainType()); } catch (const std::exception& e) { tfm::format(std::cerr, "Error: %s\n", e.what()); return EXIT_FAILURE; @@ -426,10 +427,17 @@ private: std::vector<Peer> m_peers; std::string ChainToString() const { - if (gArgs.GetChainName() == CBaseChainParams::TESTNET) return " testnet"; - if (gArgs.GetChainName() == CBaseChainParams::SIGNET) return " signet"; - if (gArgs.GetChainName() == CBaseChainParams::REGTEST) return " regtest"; - return ""; + switch (gArgs.GetChainType()) { + case ChainType::TESTNET: + return " testnet"; + case ChainType::SIGNET: + return " signet"; + case ChainType::REGTEST: + return " regtest"; + case ChainType::MAIN: + return ""; + } + assert(false); } std::string PingTimeToString(double seconds) const { @@ -863,7 +871,7 @@ static UniValue ConnectAndCallRPC(BaseRequestHandler* rh, const std::string& str try { response = CallRPC(rh, strMethod, args, rpcwallet); if (fWait) { - const UniValue& error = find_value(response, "error"); + const UniValue& error = response.find_value("error"); if (!error.isNull() && error["code"].getInt<int>() == RPC_IN_WARMUP) { throw CConnectionFailed("server in warmup"); } @@ -891,8 +899,8 @@ static void ParseResult(const UniValue& result, std::string& strPrint) static void ParseError(const UniValue& error, std::string& strPrint, int& nRet) { if (error.isObject()) { - const UniValue& err_code = find_value(error, "code"); - const UniValue& err_msg = find_value(error, "message"); + const UniValue& err_code = error.find_value("code"); + const UniValue& err_msg = error.find_value("message"); if (!err_code.isNull()) { strPrint = "error code: " + err_code.getValStr() + "\n"; } @@ -918,15 +926,15 @@ static void GetWalletBalances(UniValue& result) { DefaultRequestHandler rh; const UniValue listwallets = ConnectAndCallRPC(&rh, "listwallets", /* args=*/{}); - if (!find_value(listwallets, "error").isNull()) return; - const UniValue& wallets = find_value(listwallets, "result"); + if (!listwallets.find_value("error").isNull()) return; + const UniValue& wallets = listwallets.find_value("result"); if (wallets.size() <= 1) return; UniValue balances(UniValue::VOBJ); for (const UniValue& wallet : wallets.getValues()) { const std::string& wallet_name = wallet.get_str(); const UniValue getbalances = ConnectAndCallRPC(&rh, "getbalances", /* args=*/{}, wallet_name); - const UniValue& balance = find_value(getbalances, "result")["mine"]["trusted"]; + const UniValue& balance = getbalances.find_value("result")["mine"]["trusted"]; balances.pushKV(wallet_name, balance); } result.pushKV("balances", balances); @@ -962,7 +970,7 @@ static void GetProgressBar(double progress, std::string& progress_bar) */ static void ParseGetInfoResult(UniValue& result) { - if (!find_value(result, "error").isNull()) return; + if (!result.find_value("error").isNull()) return; std::string RESET, GREEN, BLUE, YELLOW, MAGENTA, CYAN; bool should_colorize = false; @@ -1174,9 +1182,9 @@ static int CommandLineRPC(int argc, char *argv[]) rh.reset(new NetinfoRequestHandler()); } else if (gArgs.GetBoolArg("-generate", false)) { const UniValue getnewaddress{GetNewAddress()}; - const UniValue& error{find_value(getnewaddress, "error")}; + const UniValue& error{getnewaddress.find_value("error")}; if (error.isNull()) { - SetGenerateToAddressArgs(find_value(getnewaddress, "result").get_str(), args); + SetGenerateToAddressArgs(getnewaddress.find_value("result").get_str(), args); rh.reset(new GenerateToAddressRequestHandler()); } else { ParseError(error, strPrint, nRet); @@ -1198,8 +1206,8 @@ static int CommandLineRPC(int argc, char *argv[]) const UniValue reply = ConnectAndCallRPC(rh.get(), method, args, wallet_name); // Parse reply - UniValue result = find_value(reply, "result"); - const UniValue& error = find_value(reply, "error"); + UniValue result = reply.find_value("result"); + const UniValue& error = reply.find_value("error"); if (error.isNull()) { if (gArgs.GetBoolArg("-getinfo", false)) { if (!gArgs.IsArgSet("-rpcwallet")) { diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index c35a111313..0c25ddf373 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -6,9 +6,11 @@ #include <config/bitcoin-config.h> #endif +#include <chainparamsbase.h> #include <clientversion.h> #include <coins.h> #include <common/args.h> +#include <common/system.h> #include <compat/compat.h> #include <consensus/amount.h> #include <consensus/consensus.h> @@ -26,7 +28,6 @@ #include <util/rbf.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <util/translation.h> #include <cstdio> @@ -91,7 +92,7 @@ static int AppInitRawTx(int argc, char* argv[]) // Check for chain settings (Params() calls are only valid after this clause) try { - SelectParams(gArgs.GetChainName()); + SelectParams(gArgs.GetChainType()); } catch (const std::exception& e) { tfm::format(std::cerr, "Error: %s\n", e.what()); return EXIT_FAILURE; diff --git a/src/bitcoin-util.cpp b/src/bitcoin-util.cpp index e3f70e1b76..582c18c9c6 100644 --- a/src/bitcoin-util.cpp +++ b/src/bitcoin-util.cpp @@ -12,11 +12,11 @@ #include <chainparamsbase.h> #include <clientversion.h> #include <common/args.h> +#include <common/system.h> #include <compat/compat.h> #include <core_io.h> #include <streams.h> #include <util/exception.h> -#include <util/system.h> #include <util/translation.h> #include <version.h> @@ -75,7 +75,7 @@ static int AppInitUtil(ArgsManager& args, int argc, char* argv[]) // Check for chain settings (Params() calls are only valid after this clause) try { - SelectParams(args.GetChainName()); + SelectParams(args.GetChainType()); } catch (const std::exception& e) { tfm::format(std::cerr, "Error: %s\n", e.what()); return EXIT_FAILURE; diff --git a/src/bitcoin-wallet.cpp b/src/bitcoin-wallet.cpp index 5dbf8a8616..d5dfbbec27 100644 --- a/src/bitcoin-wallet.cpp +++ b/src/bitcoin-wallet.cpp @@ -10,6 +10,7 @@ #include <chainparamsbase.h> #include <clientversion.h> #include <common/args.h> +#include <common/system.h> #include <common/url.h> #include <compat/compat.h> #include <interfaces/init.h> @@ -18,7 +19,6 @@ #include <pubkey.h> #include <tinyformat.h> #include <util/exception.h> -#include <util/system.h> #include <util/translation.h> #include <wallet/wallettool.h> @@ -91,7 +91,7 @@ static std::optional<int> WalletAppInit(ArgsManager& args, int argc, char* argv[ return EXIT_FAILURE; } // Check for chain settings (Params() calls are only valid after this clause) - SelectParams(args.GetChainName()); + SelectParams(args.GetChainType()); return std::nullopt; } diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index e476b06017..c561f9aa14 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -11,6 +11,7 @@ #include <clientversion.h> #include <common/args.h> #include <common/init.h> +#include <common/system.h> #include <common/url.h> #include <compat/compat.h> #include <init.h> @@ -25,7 +26,6 @@ #include <util/strencodings.h> #include <util/syscall_sandbox.h> #include <util/syserror.h> -#include <util/system.h> #include <util/threadnames.h> #include <util/tokenpipe.h> #include <util/translation.h> @@ -112,20 +112,30 @@ int fork_daemon(bool nochdir, bool noclose, TokenPipeEnd& endpoint) #endif -static bool AppInit(NodeContext& node, int argc, char* argv[]) +static bool ParseArgs(ArgsManager& args, int argc, char* argv[]) { - bool fRet = false; - - util::ThreadSetInternalName("init"); - // If Qt is used, parameters/bitcoin.conf are parsed in qt/bitcoin.cpp's main() - ArgsManager& args = *Assert(node.args); SetupServerArgs(args); std::string error; if (!args.ParseParameters(argc, argv, error)) { return InitError(Untranslated(strprintf("Error parsing command line arguments: %s", error))); } + if (auto error = common::InitConfig(args)) { + return InitError(error->message, error->details); + } + + // Error out when loose non-argument tokens are encountered on command line + for (int i = 1; i < argc; i++) { + if (!IsSwitchChar(argv[i][0])) { + return InitError(Untranslated(strprintf("Command line contains unexpected token '%s', see bitcoind -h for a list of options.", argv[i]))); + } + } + return true; +} + +static bool ProcessInitCommands(ArgsManager& args) +{ // Process help and version before taking care about datadir if (HelpRequested(args) || args.IsArgSet("-version")) { std::string strUsage = PACKAGE_NAME " version " + FormatFullVersion() + "\n"; @@ -142,6 +152,14 @@ static bool AppInit(NodeContext& node, int argc, char* argv[]) return true; } + return false; +} + +static bool AppInit(NodeContext& node) +{ + bool fRet = false; + ArgsManager& args = *Assert(node.args); + #if HAVE_DECL_FORK // Communication with parent after daemonizing. This is used for signalling in the following ways: // - a boolean token is sent when the initialization process (all the Init* functions) have finished to indicate @@ -153,23 +171,12 @@ static bool AppInit(NodeContext& node, int argc, char* argv[]) std::any context{&node}; try { - if (auto error = common::InitConfig(args)) { - return InitError(error->message, error->details); - } - - // Error out when loose non-argument tokens are encountered on command line - for (int i = 1; i < argc; i++) { - if (!IsSwitchChar(argv[i][0])) { - return InitError(Untranslated(strprintf("Command line contains unexpected token '%s', see bitcoind -h for a list of options.", argv[i]))); - } - } - // -server defaults to true for bitcoind but not for the GUI so do this here args.SoftSetBoolArg("-server", true); // Set this early so that parameter interactions go to console InitLogging(args); InitParameterInteraction(args); - if (!AppInitBasicSetup(args)) { + if (!AppInitBasicSetup(args, node.exit_status)) { // InitError will have been called with detailed error, which ends up on console return false; } @@ -236,12 +243,6 @@ static bool AppInit(NodeContext& node, int argc, char* argv[]) } #endif SetSyscallSandboxPolicy(SyscallSandboxPolicy::SHUTOFF); - if (fRet) { - WaitForShutdown(); - } - Interrupt(node); - Shutdown(node); - return fRet; } @@ -264,5 +265,22 @@ MAIN_FUNCTION // Connect bitcoind signal handlers noui_connect(); - return (AppInit(node, argc, argv) ? EXIT_SUCCESS : EXIT_FAILURE); + util::ThreadSetInternalName("init"); + + // Interpret command line arguments + ArgsManager& args = *Assert(node.args); + if (!ParseArgs(args, argc, argv)) return EXIT_FAILURE; + // Process early info return commands such as -help or -version + if (ProcessInitCommands(args)) return EXIT_SUCCESS; + + // Start application + if (AppInit(node)) { + WaitForShutdown(); + } else { + node.exit_status = EXIT_FAILURE; + } + Interrupt(node); + Shutdown(node); + + return node.exit_status; } diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index a29e4f794e..9aa0a6ba20 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -3,16 +3,16 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <blockencodings.h> +#include <chainparams.h> +#include <common/system.h> #include <consensus/consensus.h> #include <consensus/validation.h> -#include <chainparams.h> #include <crypto/sha256.h> #include <crypto/siphash.h> #include <random.h> #include <streams.h> #include <txmempool.h> #include <validation.h> -#include <util/system.h> #include <unordered_map> diff --git a/src/chainparams.cpp b/src/chainparams.cpp index d3d358a3f0..539578085b 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -5,16 +5,21 @@ #include <chainparams.h> -#include <chainparamsseeds.h> +#include <chainparamsbase.h> #include <common/args.h> -#include <consensus/merkle.h> +#include <consensus/params.h> #include <deploymentinfo.h> -#include <hash.h> // for signet block challenge hash #include <logging.h> -#include <script/interpreter.h> +#include <tinyformat.h> +#include <util/chaintype.h> +#include <util/strencodings.h> #include <util/string.h> -#include <assert.h> +#include <cassert> +#include <cstdint> +#include <limits> +#include <stdexcept> +#include <vector> void ReadSigNetArgs(const ArgsManager& args, CChainParams::SigNetOptions& options) { @@ -24,9 +29,13 @@ void ReadSigNetArgs(const ArgsManager& args, CChainParams::SigNetOptions& option if (args.IsArgSet("-signetchallenge")) { const auto signet_challenge = args.GetArgs("-signetchallenge"); if (signet_challenge.size() != 1) { - throw std::runtime_error(strprintf("%s: -signetchallenge cannot be multiple values.", __func__)); + throw std::runtime_error("-signetchallenge cannot be multiple values."); } - options.challenge.emplace(ParseHex(signet_challenge[0])); + const auto val{TryParseHex<uint8_t>(signet_challenge[0])}; + if (!val) { + throw std::runtime_error(strprintf("-signetchallenge must be hex, not '%s'.", signet_challenge[0])); + } + options.challenge.emplace(*val); } } @@ -97,26 +106,29 @@ const CChainParams &Params() { return *globalChainParams; } -std::unique_ptr<const CChainParams> CreateChainParams(const ArgsManager& args, const std::string& chain) +std::unique_ptr<const CChainParams> CreateChainParams(const ArgsManager& args, const ChainType chain) { - if (chain == CBaseChainParams::MAIN) { + switch (chain) { + case ChainType::MAIN: return CChainParams::Main(); - } else if (chain == CBaseChainParams::TESTNET) { + case ChainType::TESTNET: return CChainParams::TestNet(); - } else if (chain == CBaseChainParams::SIGNET) { + case ChainType::SIGNET: { auto opts = CChainParams::SigNetOptions{}; ReadSigNetArgs(args, opts); return CChainParams::SigNet(opts); - } else if (chain == CBaseChainParams::REGTEST) { + } + case ChainType::REGTEST: { auto opts = CChainParams::RegTestOptions{}; ReadRegTestArgs(args, opts); return CChainParams::RegTest(opts); } - throw std::runtime_error(strprintf("%s: Unknown chain %s.", __func__, chain)); + } + assert(false); } -void SelectParams(const std::string& network) +void SelectParams(const ChainType chain) { - SelectBaseParams(network); - globalChainParams = CreateChainParams(gArgs, network); + SelectBaseParams(chain); + globalChainParams = CreateChainParams(gArgs, chain); } diff --git a/src/chainparams.h b/src/chainparams.h index cb34d068e1..4743e022db 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -6,27 +6,16 @@ #ifndef BITCOIN_CHAINPARAMS_H #define BITCOIN_CHAINPARAMS_H -#include <kernel/chainparams.h> +#include <kernel/chainparams.h> // IWYU pragma: export -#include <chainparamsbase.h> -#include <consensus/params.h> -#include <netaddress.h> -#include <primitives/block.h> -#include <protocol.h> -#include <util/hash_type.h> - -#include <cstdint> #include <memory> -#include <string> -#include <unordered_map> -#include <vector> + +class ArgsManager; /** * Creates and returns a std::unique_ptr<CChainParams> of the chosen chain. - * @returns a CChainParams* of the chosen chain. - * @throws a std::runtime_error if the chain is not supported. */ -std::unique_ptr<const CChainParams> CreateChainParams(const ArgsManager& args, const std::string& chain); +std::unique_ptr<const CChainParams> CreateChainParams(const ArgsManager& args, const ChainType chain); /** * Return the currently selected parameters. This won't change after app @@ -35,9 +24,8 @@ std::unique_ptr<const CChainParams> CreateChainParams(const ArgsManager& args, c const CChainParams &Params(); /** - * Sets the params returned by Params() to those for the given chain name. - * @throws std::runtime_error when the chain is not supported. + * Sets the params returned by Params() to those for the given chain type. */ -void SelectParams(const std::string& chain); +void SelectParams(const ChainType chain); #endif // BITCOIN_CHAINPARAMS_H diff --git a/src/chainparamsbase.cpp b/src/chainparamsbase.cpp index eb7b31923d..8cbf9e85e0 100644 --- a/src/chainparamsbase.cpp +++ b/src/chainparamsbase.cpp @@ -7,14 +7,10 @@ #include <common/args.h> #include <tinyformat.h> +#include <util/chaintype.h> #include <assert.h> -const std::string CBaseChainParams::MAIN = "main"; -const std::string CBaseChainParams::TESTNET = "test"; -const std::string CBaseChainParams::SIGNET = "signet"; -const std::string CBaseChainParams::REGTEST = "regtest"; - void SetupChainParamsBaseOptions(ArgsManager& argsman) { argsman.AddArg("-chain=<chain>", "Use the chain <chain> (default: main). Allowed values: main, test, signet, regtest", ArgsManager::ALLOW_ANY, OptionsCategory::CHAINPARAMS); @@ -40,22 +36,23 @@ const CBaseChainParams& BaseParams() * Port numbers for incoming Tor connections (8334, 18334, 38334, 18445) have * been chosen arbitrarily to keep ranges of used ports tight. */ -std::unique_ptr<CBaseChainParams> CreateBaseChainParams(const std::string& chain) +std::unique_ptr<CBaseChainParams> CreateBaseChainParams(const ChainType chain) { - if (chain == CBaseChainParams::MAIN) { + switch (chain) { + case ChainType::MAIN: return std::make_unique<CBaseChainParams>("", 8332, 8334); - } else if (chain == CBaseChainParams::TESTNET) { + case ChainType::TESTNET: return std::make_unique<CBaseChainParams>("testnet3", 18332, 18334); - } else if (chain == CBaseChainParams::SIGNET) { + case ChainType::SIGNET: return std::make_unique<CBaseChainParams>("signet", 38332, 38334); - } else if (chain == CBaseChainParams::REGTEST) { + case ChainType::REGTEST: return std::make_unique<CBaseChainParams>("regtest", 18443, 18445); } - throw std::runtime_error(strprintf("%s: Unknown chain %s.", __func__, chain)); + assert(false); } -void SelectBaseParams(const std::string& chain) +void SelectBaseParams(const ChainType chain) { globalChainBaseParams = CreateBaseChainParams(chain); - gArgs.SelectConfigNetwork(chain); + gArgs.SelectConfigNetwork(ChainTypeToString(chain)); } diff --git a/src/chainparamsbase.h b/src/chainparamsbase.h index d593cff722..ea933d1ca8 100644 --- a/src/chainparamsbase.h +++ b/src/chainparamsbase.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_CHAINPARAMSBASE_H #define BITCOIN_CHAINPARAMSBASE_H +#include <util/chaintype.h> + #include <memory> #include <string> @@ -17,14 +19,6 @@ class ArgsManager; class CBaseChainParams { public: - ///@{ - /** Chain name strings */ - static const std::string MAIN; - static const std::string TESTNET; - static const std::string SIGNET; - static const std::string REGTEST; - ///@} - const std::string& DataDir() const { return strDataDir; } uint16_t RPCPort() const { return m_rpc_port; } uint16_t OnionServiceTargetPort() const { return m_onion_service_target_port; } @@ -41,10 +35,8 @@ private: /** * Creates and returns a std::unique_ptr<CBaseChainParams> of the chosen chain. - * @returns a CBaseChainParams* of the chosen chain. - * @throws a std::runtime_error if the chain is not supported. */ -std::unique_ptr<CBaseChainParams> CreateBaseChainParams(const std::string& chain); +std::unique_ptr<CBaseChainParams> CreateBaseChainParams(const ChainType chain); /** *Set the arguments for chainparams @@ -57,7 +49,7 @@ void SetupChainParamsBaseOptions(ArgsManager& argsman); */ const CBaseChainParams& BaseParams(); -/** Sets the params returned by Params() to those for the given network. */ -void SelectBaseParams(const std::string& chain); +/** Sets the params returned by Params() to those for the given chain. */ +void SelectBaseParams(const ChainType chain); #endif // BITCOIN_CHAINPARAMSBASE_H diff --git a/src/common/args.cpp b/src/common/args.cpp index d29b8648bf..643838399f 100644 --- a/src/common/args.cpp +++ b/src/common/args.cpp @@ -6,14 +6,15 @@ #include <common/args.h> #include <chainparamsbase.h> +#include <common/settings.h> #include <logging.h> #include <sync.h> #include <tinyformat.h> #include <univalue.h> +#include <util/chaintype.h> #include <util/check.h> #include <util/fs.h> #include <util/fs_helpers.h> -#include <util/settings.h> #include <util/strencodings.h> #ifdef WIN32 @@ -33,6 +34,7 @@ #include <stdexcept> #include <string> #include <utility> +#include <variant> const char * const BITCOIN_CONF_FILENAME = "bitcoin.conf"; const char * const BITCOIN_SETTINGS_FILENAME = "settings.json"; @@ -102,7 +104,7 @@ KeyInfo InterpretKey(std::string key) * @return parsed settings value if it is valid, otherwise nullopt accompanied * by a descriptive error string */ -std::optional<util::SettingsValue> InterpretValue(const KeyInfo& key, const std::string* value, +std::optional<common::SettingsValue> InterpretValue(const KeyInfo& key, const std::string* value, unsigned int flags, std::string& error) { // Return negated settings as false values. @@ -141,7 +143,7 @@ std::set<std::string> ArgsManager::GetUnsuitableSectionOnlyArgs() const if (m_network.empty()) return std::set<std::string> {}; // if it's okay to use the default section for this network, don't worry - if (m_network == CBaseChainParams::MAIN) return std::set<std::string> {}; + if (m_network == ChainTypeToString(ChainType::MAIN)) return std::set<std::string> {}; for (const auto& arg : m_network_only_args) { if (OnlyHasDefaultSectionSetting(m_settings, m_network, SettingName(arg))) { @@ -155,10 +157,10 @@ std::list<SectionInfo> ArgsManager::GetUnrecognizedSections() const { // Section names to be recognized in the config file. static const std::set<std::string> available_sections{ - CBaseChainParams::REGTEST, - CBaseChainParams::SIGNET, - CBaseChainParams::TESTNET, - CBaseChainParams::MAIN + ChainTypeToString(ChainType::REGTEST), + ChainTypeToString(ChainType::SIGNET), + ChainTypeToString(ChainType::TESTNET), + ChainTypeToString(ChainType::MAIN), }; LOCK(cs_args); @@ -236,15 +238,15 @@ bool ArgsManager::ParseParameters(int argc, const char* const argv[], std::strin return false; } - std::optional<util::SettingsValue> value = InterpretValue(keyinfo, val ? &*val : nullptr, *flags, error); + std::optional<common::SettingsValue> value = InterpretValue(keyinfo, val ? &*val : nullptr, *flags, error); if (!value) return false; m_settings.command_line_options[keyinfo.name].push_back(*value); } // we do not allow -includeconf from command line, only -noincludeconf - if (auto* includes = util::FindKey(m_settings.command_line_options, "includeconf")) { - const util::SettingsSpan values{*includes}; + if (auto* includes = common::FindKey(m_settings.command_line_options, "includeconf")) { + const common::SettingsSpan values{*includes}; // Range may be empty if -noincludeconf was passed if (!values.empty()) { error = "-includeconf cannot be used from commandline; -includeconf=" + values.begin()->write(); @@ -359,7 +361,7 @@ std::optional<const ArgsManager::Command> ArgsManager::GetCommand() const std::vector<std::string> ArgsManager::GetArgs(const std::string& strArg) const { std::vector<std::string> result; - for (const util::SettingsValue& value : GetSettingsList(strArg)) { + for (const common::SettingsValue& value : GetSettingsList(strArg)) { result.push_back(value.isFalse() ? "0" : value.isTrue() ? "1" : value.get_str()); } return result; @@ -406,7 +408,7 @@ bool ArgsManager::ReadSettingsFile(std::vector<std::string>* errors) LOCK(cs_args); m_settings.rw_settings.clear(); std::vector<std::string> read_errors; - if (!util::ReadSettings(path, m_settings.rw_settings, read_errors)) { + if (!common::ReadSettings(path, m_settings.rw_settings, read_errors)) { SaveErrors(read_errors, errors); return false; } @@ -428,7 +430,7 @@ bool ArgsManager::WriteSettingsFile(std::vector<std::string>* errors, bool backu LOCK(cs_args); std::vector<std::string> write_errors; - if (!util::WriteSettings(path_tmp, m_settings.rw_settings, write_errors)) { + if (!common::WriteSettings(path_tmp, m_settings.rw_settings, write_errors)) { SaveErrors(write_errors, errors); return false; } @@ -439,11 +441,11 @@ bool ArgsManager::WriteSettingsFile(std::vector<std::string>* errors, bool backu return true; } -util::SettingsValue ArgsManager::GetPersistentSetting(const std::string& name) const +common::SettingsValue ArgsManager::GetPersistentSetting(const std::string& name) const { LOCK(cs_args); - return util::GetSetting(m_settings, m_network, name, !UseDefaultSection("-" + name), - /*ignore_nonpersistent=*/true, /*get_chain_name=*/false); + return common::GetSetting(m_settings, m_network, name, !UseDefaultSection("-" + name), + /*ignore_nonpersistent=*/true, /*get_chain_type=*/false); } bool ArgsManager::IsArgNegated(const std::string& strArg) const @@ -458,11 +460,11 @@ std::string ArgsManager::GetArg(const std::string& strArg, const std::string& st std::optional<std::string> ArgsManager::GetArg(const std::string& strArg) const { - const util::SettingsValue value = GetSetting(strArg); + const common::SettingsValue value = GetSetting(strArg); return SettingToString(value); } -std::optional<std::string> SettingToString(const util::SettingsValue& value) +std::optional<std::string> SettingToString(const common::SettingsValue& value) { if (value.isNull()) return std::nullopt; if (value.isFalse()) return "0"; @@ -471,7 +473,7 @@ std::optional<std::string> SettingToString(const util::SettingsValue& value) return value.get_str(); } -std::string SettingToString(const util::SettingsValue& value, const std::string& strDefault) +std::string SettingToString(const common::SettingsValue& value, const std::string& strDefault) { return SettingToString(value).value_or(strDefault); } @@ -483,11 +485,11 @@ int64_t ArgsManager::GetIntArg(const std::string& strArg, int64_t nDefault) cons std::optional<int64_t> ArgsManager::GetIntArg(const std::string& strArg) const { - const util::SettingsValue value = GetSetting(strArg); + const common::SettingsValue value = GetSetting(strArg); return SettingToInt(value); } -std::optional<int64_t> SettingToInt(const util::SettingsValue& value) +std::optional<int64_t> SettingToInt(const common::SettingsValue& value) { if (value.isNull()) return std::nullopt; if (value.isFalse()) return 0; @@ -496,7 +498,7 @@ std::optional<int64_t> SettingToInt(const util::SettingsValue& value) return LocaleIndependentAtoi<int64_t>(value.get_str()); } -int64_t SettingToInt(const util::SettingsValue& value, int64_t nDefault) +int64_t SettingToInt(const common::SettingsValue& value, int64_t nDefault) { return SettingToInt(value).value_or(nDefault); } @@ -508,18 +510,18 @@ bool ArgsManager::GetBoolArg(const std::string& strArg, bool fDefault) const std::optional<bool> ArgsManager::GetBoolArg(const std::string& strArg) const { - const util::SettingsValue value = GetSetting(strArg); + const common::SettingsValue value = GetSetting(strArg); return SettingToBool(value); } -std::optional<bool> SettingToBool(const util::SettingsValue& value) +std::optional<bool> SettingToBool(const common::SettingsValue& value) { if (value.isNull()) return std::nullopt; if (value.isBool()) return value.get_bool(); return InterpretBool(value.get_str()); } -bool SettingToBool(const util::SettingsValue& value, bool fDefault) +bool SettingToBool(const common::SettingsValue& value, bool fDefault) { return SettingToBool(value).value_or(fDefault); } @@ -714,62 +716,77 @@ bool CheckDataDirOption(const ArgsManager& args) fs::path ArgsManager::GetConfigFilePath() const { - return GetConfigFile(*this, GetPathArg("-conf", BITCOIN_CONF_FILENAME)); + LOCK(cs_args); + return *Assert(m_config_path); +} + +ChainType ArgsManager::GetChainType() const +{ + std::variant<ChainType, std::string> arg = GetChainArg(); + if (auto* parsed = std::get_if<ChainType>(&arg)) return *parsed; + throw std::runtime_error(strprintf("Unknown chain %s.", std::get<std::string>(arg))); } -std::string ArgsManager::GetChainName() const +std::string ArgsManager::GetChainTypeString() const +{ + auto arg = GetChainArg(); + if (auto* parsed = std::get_if<ChainType>(&arg)) return ChainTypeToString(*parsed); + return std::get<std::string>(arg); +} + +std::variant<ChainType, std::string> ArgsManager::GetChainArg() const { auto get_net = [&](const std::string& arg) { LOCK(cs_args); - util::SettingsValue value = util::GetSetting(m_settings, /* section= */ "", SettingName(arg), + common::SettingsValue value = common::GetSetting(m_settings, /* section= */ "", SettingName(arg), /* ignore_default_section_config= */ false, /*ignore_nonpersistent=*/false, - /* get_chain_name= */ true); + /* get_chain_type= */ true); return value.isNull() ? false : value.isBool() ? value.get_bool() : InterpretBool(value.get_str()); }; const bool fRegTest = get_net("-regtest"); const bool fSigNet = get_net("-signet"); const bool fTestNet = get_net("-testnet"); - const bool is_chain_arg_set = IsArgSet("-chain"); + const auto chain_arg = GetArg("-chain"); - if ((int)is_chain_arg_set + (int)fRegTest + (int)fSigNet + (int)fTestNet > 1) { + if ((int)chain_arg.has_value() + (int)fRegTest + (int)fSigNet + (int)fTestNet > 1) { throw std::runtime_error("Invalid combination of -regtest, -signet, -testnet and -chain. Can use at most one."); } - if (fRegTest) - return CBaseChainParams::REGTEST; - if (fSigNet) { - return CBaseChainParams::SIGNET; + if (chain_arg) { + if (auto parsed = ChainTypeFromString(*chain_arg)) return *parsed; + // Not a known string, so return original string + return *chain_arg; } - if (fTestNet) - return CBaseChainParams::TESTNET; - - return GetArg("-chain", CBaseChainParams::MAIN); + if (fRegTest) return ChainType::REGTEST; + if (fSigNet) return ChainType::SIGNET; + if (fTestNet) return ChainType::TESTNET; + return ChainType::MAIN; } bool ArgsManager::UseDefaultSection(const std::string& arg) const { - return m_network == CBaseChainParams::MAIN || m_network_only_args.count(arg) == 0; + return m_network == ChainTypeToString(ChainType::MAIN) || m_network_only_args.count(arg) == 0; } -util::SettingsValue ArgsManager::GetSetting(const std::string& arg) const +common::SettingsValue ArgsManager::GetSetting(const std::string& arg) const { LOCK(cs_args); - return util::GetSetting( + return common::GetSetting( m_settings, m_network, SettingName(arg), !UseDefaultSection(arg), - /*ignore_nonpersistent=*/false, /*get_chain_name=*/false); + /*ignore_nonpersistent=*/false, /*get_chain_type=*/false); } -std::vector<util::SettingsValue> ArgsManager::GetSettingsList(const std::string& arg) const +std::vector<common::SettingsValue> ArgsManager::GetSettingsList(const std::string& arg) const { LOCK(cs_args); - return util::GetSettingsList(m_settings, m_network, SettingName(arg), !UseDefaultSection(arg)); + return common::GetSettingsList(m_settings, m_network, SettingName(arg), !UseDefaultSection(arg)); } void ArgsManager::logArgsPrefix( const std::string& prefix, const std::string& section, - const std::map<std::string, std::vector<util::SettingsValue>>& args) const + const std::map<std::string, std::vector<common::SettingsValue>>& args) const { std::string section_str = section.empty() ? "" : "[" + section + "] "; for (const auto& arg : args) { diff --git a/src/common/args.h b/src/common/args.h index 430c392e2b..ae3ed02bc7 100644 --- a/src/common/args.h +++ b/src/common/args.h @@ -5,10 +5,11 @@ #ifndef BITCOIN_COMMON_ARGS_H #define BITCOIN_COMMON_ARGS_H +#include <common/settings.h> #include <compat/compat.h> #include <sync.h> +#include <util/chaintype.h> #include <util/fs.h> -#include <util/settings.h> #include <iosfwd> #include <list> @@ -17,6 +18,7 @@ #include <set> #include <stdint.h> #include <string> +#include <variant> #include <vector> class ArgsManager; @@ -26,7 +28,6 @@ extern const char * const BITCOIN_SETTINGS_FILENAME; // Return true if -datadir option points to a valid directory or is not specified. bool CheckDataDirOption(const ArgsManager& args); -fs::path GetConfigFile(const ArgsManager& args, const fs::path& configuration_file_path); /** * Most paths passed as configuration arguments are treated as relative to @@ -74,7 +75,7 @@ struct KeyInfo { KeyInfo InterpretKey(std::string key); -std::optional<util::SettingsValue> InterpretValue(const KeyInfo& key, const std::string* value, +std::optional<common::SettingsValue> InterpretValue(const KeyInfo& key, const std::string* value, unsigned int flags, std::string& error); struct SectionInfo { @@ -83,14 +84,14 @@ struct SectionInfo { int m_line; }; -std::string SettingToString(const util::SettingsValue&, const std::string&); -std::optional<std::string> SettingToString(const util::SettingsValue&); +std::string SettingToString(const common::SettingsValue&, const std::string&); +std::optional<std::string> SettingToString(const common::SettingsValue&); -int64_t SettingToInt(const util::SettingsValue&, int64_t); -std::optional<int64_t> SettingToInt(const util::SettingsValue&); +int64_t SettingToInt(const common::SettingsValue&, int64_t); +std::optional<int64_t> SettingToInt(const common::SettingsValue&); -bool SettingToBool(const util::SettingsValue&, bool); -std::optional<bool> SettingToBool(const util::SettingsValue&); +bool SettingToBool(const common::SettingsValue&, bool); +std::optional<bool> SettingToBool(const common::SettingsValue&); class ArgsManager { @@ -129,13 +130,14 @@ protected: }; mutable RecursiveMutex cs_args; - util::Settings m_settings GUARDED_BY(cs_args); + common::Settings m_settings GUARDED_BY(cs_args); std::vector<std::string> m_command GUARDED_BY(cs_args); std::string m_network GUARDED_BY(cs_args); std::set<std::string> m_network_only_args GUARDED_BY(cs_args); std::map<OptionsCategory, std::map<std::string, Arg>> m_available_args GUARDED_BY(cs_args); bool m_accept_any_command GUARDED_BY(cs_args){true}; std::list<SectionInfo> m_config_sections GUARDED_BY(cs_args); + std::optional<fs::path> m_config_path GUARDED_BY(cs_args); mutable fs::path m_cached_blocks_path GUARDED_BY(cs_args); mutable fs::path m_cached_datadir_path GUARDED_BY(cs_args); mutable fs::path m_cached_network_datadir_path GUARDED_BY(cs_args); @@ -157,12 +159,12 @@ protected: * false if "-nosetting" argument was passed, and a string if a "-setting=value" * argument was passed. */ - util::SettingsValue GetSetting(const std::string& arg) const; + common::SettingsValue GetSetting(const std::string& arg) const; /** * Get list of setting values. */ - std::vector<util::SettingsValue> GetSettingsList(const std::string& arg) const; + std::vector<common::SettingsValue> GetSettingsList(const std::string& arg) const; ArgsManager(); ~ArgsManager(); @@ -323,10 +325,18 @@ protected: void ForceSetArg(const std::string& strArg, const std::string& strValue); /** - * Returns the appropriate chain name from the program arguments. - * @return CBaseChainParams::MAIN by default; raises runtime error if an invalid combination is given. + * Returns the appropriate chain type from the program arguments. + * @return ChainType::MAIN by default; raises runtime error if an invalid + * combination, or unknown chain is given. */ - std::string GetChainName() const; + ChainType GetChainType() const; + + /** + * Returns the appropriate chain type string from the program arguments. + * @return ChainType::MAIN string by default; raises runtime error if an + * invalid combination is given. + */ + std::string GetChainTypeString() const; /** * Add argument @@ -384,7 +394,7 @@ protected: * Get current setting from config file or read/write settings file, * ignoring nonpersistent command line or forced settings values. */ - util::SettingsValue GetPersistentSetting(const std::string& name) const; + common::SettingsValue GetPersistentSetting(const std::string& name) const; /** * Access settings with lock held. @@ -411,11 +421,19 @@ private: */ const fs::path& GetDataDir(bool net_specific) const; + /** + * Return -regtest/-signet/-testnet/-chain= setting as a ChainType enum if a + * recognized chain type was set, or as a string if an unrecognized chain + * name was set. Raise an exception if an invalid combination of flags was + * provided. + */ + std::variant<ChainType, std::string> GetChainArg() const; + // Helper function for LogArgs(). void logArgsPrefix( const std::string& prefix, const std::string& section, - const std::map<std::string, std::vector<util::SettingsValue>>& args) const; + const std::map<std::string, std::vector<common::SettingsValue>>& args) const; }; extern ArgsManager gArgs; diff --git a/src/common/config.cpp b/src/common/config.cpp index 747503ad2a..1c85273f69 100644 --- a/src/common/config.cpp +++ b/src/common/config.cpp @@ -4,12 +4,13 @@ #include <common/args.h> +#include <common/settings.h> #include <logging.h> #include <sync.h> #include <tinyformat.h> #include <univalue.h> +#include <util/chaintype.h> #include <util/fs.h> -#include <util/settings.h> #include <util/string.h> #include <algorithm> @@ -26,11 +27,6 @@ #include <utility> #include <vector> -fs::path GetConfigFile(const ArgsManager& args, const fs::path& configuration_file_path) -{ - return AbsPathForConfigVal(args, configuration_file_path, /*net_specific=*/false); -} - static bool GetConfigOptions(std::istream& stream, const std::string& filepath, std::string& error, std::vector<std::pair<std::string, std::string>>& options, std::list<SectionInfo>& sections) { std::string str, prefix; @@ -102,7 +98,7 @@ bool ArgsManager::ReadConfigStream(std::istream& stream, const std::string& file std::optional<unsigned int> flags = GetArgFlags('-' + key.name); if (!IsConfSupported(key, error)) return false; if (flags) { - std::optional<util::SettingsValue> value = InterpretValue(key, &option.second, *flags, error); + std::optional<common::SettingsValue> value = InterpretValue(key, &option.second, *flags, error); if (!value) { return false; } @@ -125,6 +121,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) LOCK(cs_args); m_settings.ro_config.clear(); m_config_sections.clear(); + m_config_path = AbsPathForConfigVal(*this, GetPathArg("-conf", BITCOIN_CONF_FILENAME), /*net_specific=*/false); } const auto conf_path{GetConfigFilePath()}; @@ -145,22 +142,22 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) bool use_conf_file{true}; { LOCK(cs_args); - if (auto* includes = util::FindKey(m_settings.command_line_options, "includeconf")) { + if (auto* includes = common::FindKey(m_settings.command_line_options, "includeconf")) { // ParseParameters() fails if a non-negated -includeconf is passed on the command-line - assert(util::SettingsSpan(*includes).last_negated()); + assert(common::SettingsSpan(*includes).last_negated()); use_conf_file = false; } } if (use_conf_file) { - std::string chain_id = GetChainName(); + std::string chain_id = GetChainTypeString(); std::vector<std::string> conf_file_names; auto add_includes = [&](const std::string& network, size_t skip = 0) { size_t num_values = 0; LOCK(cs_args); - if (auto* section = util::FindKey(m_settings.ro_config, network)) { - if (auto* values = util::FindKey(*section, "includeconf")) { - for (size_t i = std::max(skip, util::SettingsSpan(*values).negated()); i < values->size(); ++i) { + if (auto* section = common::FindKey(m_settings.ro_config, network)) { + if (auto* values = common::FindKey(*section, "includeconf")) { + for (size_t i = std::max(skip, common::SettingsSpan(*values).negated()); i < values->size(); ++i) { conf_file_names.push_back((*values)[i].get_str()); } num_values = values->size(); @@ -175,7 +172,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) const size_t default_includes = add_includes({}); for (const std::string& conf_file_name : conf_file_names) { - std::ifstream conf_file_stream{GetConfigFile(*this, fs::PathFromString(conf_file_name))}; + std::ifstream conf_file_stream{AbsPathForConfigVal(*this, fs::PathFromString(conf_file_name), /*net_specific=*/false)}; if (conf_file_stream.good()) { if (!ReadConfigStream(conf_file_stream, conf_file_name, error, ignore_invalid_keys)) { return false; @@ -191,7 +188,7 @@ bool ArgsManager::ReadConfigFiles(std::string& error, bool ignore_invalid_keys) conf_file_names.clear(); add_includes(chain_id, /* skip= */ chain_includes); add_includes({}, /* skip= */ default_includes); - std::string chain_id_final = GetChainName(); + std::string chain_id_final = GetChainTypeString(); if (chain_id_final != chain_id) { // Also warn about recursive includeconf for the chain that was specified in one of the includeconfs add_includes(chain_id_final); diff --git a/src/common/init.cpp b/src/common/init.cpp index 6ffa44847a..412d73aec7 100644 --- a/src/common/init.cpp +++ b/src/common/init.cpp @@ -5,6 +5,7 @@ #include <chainparams.h> #include <common/args.h> #include <common/init.h> +#include <logging.h> #include <tinyformat.h> #include <util/fs.h> #include <util/translation.h> @@ -20,13 +21,26 @@ std::optional<ConfigError> InitConfig(ArgsManager& args, SettingsAbortFn setting if (!CheckDataDirOption(args)) { return ConfigError{ConfigStatus::FAILED, strprintf(_("Specified data directory \"%s\" does not exist."), args.GetArg("-datadir", ""))}; } + + // Record original datadir and config paths before parsing the config + // file. It is possible for the config file to contain a datadir= line + // that changes the datadir path after it is parsed. This is useful for + // CLI tools to let them use a different data storage location without + // needing to pass it every time on the command line. (It is not + // possible for the config file to cause another configuration to be + // used, though. Specifying a conf= option in the config file causes a + // parse error, and specifying a datadir= location containing another + // bitcoin.conf file just ignores the other file.) + const fs::path orig_datadir_path{args.GetDataDirBase()}; + const fs::path orig_config_path{AbsPathForConfigVal(args, args.GetPathArg("-conf", BITCOIN_CONF_FILENAME), /*net_specific=*/false)}; + std::string error; if (!args.ReadConfigFiles(error, true)) { return ConfigError{ConfigStatus::FAILED, strprintf(_("Error reading configuration file: %s"), error)}; } // Check for chain settings (Params() calls are only valid after this clause) - SelectParams(args.GetChainName()); + SelectParams(args.GetChainType()); // Create datadir if it does not exist. const auto base_path{args.GetDataDirBase()}; @@ -48,6 +62,32 @@ std::optional<ConfigError> InitConfig(ArgsManager& args, SettingsAbortFn setting fs::create_directories(net_path / "wallets"); } + // Show an error or warning if there is a bitcoin.conf file in the + // datadir that is being ignored. + const fs::path base_config_path = base_path / BITCOIN_CONF_FILENAME; + if (fs::exists(base_config_path) && !fs::equivalent(orig_config_path, base_config_path)) { + const std::string cli_config_path = args.GetArg("-conf", ""); + const std::string config_source = cli_config_path.empty() + ? strprintf("data directory %s", fs::quoted(fs::PathToString(orig_datadir_path))) + : strprintf("command line argument %s", fs::quoted("-conf=" + cli_config_path)); + const std::string error = strprintf( + "Data directory %1$s contains a %2$s file which is ignored, because a different configuration file " + "%3$s from %4$s is being used instead. Possible ways to address this would be to:\n" + "- Delete or rename the %2$s file in data directory %1$s.\n" + "- Change datadir= or conf= options to specify one configuration file, not two, and use " + "includeconf= to include any other configuration files.\n" + "- Set allowignoredconf=1 option to treat this condition as a warning, not an error.", + fs::quoted(fs::PathToString(base_path)), + fs::quoted(BITCOIN_CONF_FILENAME), + fs::quoted(fs::PathToString(orig_config_path)), + config_source); + if (args.GetBoolArg("-allowignoredconf", false)) { + LogPrintf("Warning: %s\n", error); + } else { + return ConfigError{ConfigStatus::FAILED, Untranslated(error)}; + } + } + // Create settings.json if -nosettings was not specified. if (args.GetSettingsPath()) { std::vector<std::string> details; diff --git a/src/util/settings.cpp b/src/common/settings.cpp index bb257c0584..9187f242eb 100644 --- a/src/util/settings.cpp +++ b/src/common/settings.cpp @@ -2,18 +2,21 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <util/fs.h> -#include <util/settings.h> +#include <common/settings.h> #include <tinyformat.h> #include <univalue.h> +#include <util/fs.h> +#include <algorithm> #include <fstream> +#include <iterator> #include <map> #include <string> +#include <utility> #include <vector> -namespace util { +namespace common { namespace { enum class Source { @@ -130,7 +133,7 @@ SettingsValue GetSetting(const Settings& settings, const std::string& name, bool ignore_default_section_config, bool ignore_nonpersistent, - bool get_chain_name) + bool get_chain_type) { SettingsValue result; bool done = false; // Done merging any more settings sources. @@ -145,17 +148,17 @@ SettingsValue GetSetting(const Settings& settings, // assigned value instead of last. In general, later settings take // precedence over early settings, but for backwards compatibility in // the config file the precedence is reversed for all settings except - // chain name settings. + // chain type settings. const bool reverse_precedence = (source == Source::CONFIG_FILE_NETWORK_SECTION || source == Source::CONFIG_FILE_DEFAULT_SECTION) && - !get_chain_name; + !get_chain_type; // Weird behavior preserved for backwards compatibility: Negated // -regtest and -testnet arguments which you would expect to override // values set in the configuration file are currently accepted but // silently ignored. It would be better to apply these just like other // negated values, or at least warn they are ignored. - const bool skip_negated_command_line = get_chain_name; + const bool skip_negated_command_line = get_chain_type; if (done) return; @@ -255,4 +258,4 @@ size_t SettingsSpan::negated() const return 0; } -} // namespace util +} // namespace common diff --git a/src/util/settings.h b/src/common/settings.h index 27e87a4751..0e9d376e23 100644 --- a/src/util/settings.h +++ b/src/common/settings.h @@ -2,18 +2,19 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_UTIL_SETTINGS_H -#define BITCOIN_UTIL_SETTINGS_H +#ifndef BITCOIN_COMMON_SETTINGS_H +#define BITCOIN_COMMON_SETTINGS_H #include <util/fs.h> +#include <cstddef> #include <map> #include <string> #include <vector> class UniValue; -namespace util { +namespace common { //! Settings value type (string/integer/boolean/null variant). //! @@ -60,14 +61,14 @@ bool WriteSettings(const fs::path& path, //! command line). Only return settings in the //! read-only config and read-write settings //! files. -//! @param get_chain_name - enable special backwards compatible behavior -//! for GetChainName +//! @param get_chain_type - enable special backwards compatible behavior +//! for GetChainType SettingsValue GetSetting(const Settings& settings, const std::string& section, const std::string& name, bool ignore_default_section_config, bool ignore_nonpersistent, - bool get_chain_name); + bool get_chain_type); //! Get combined setting value similar to GetSetting(), except if setting was //! specified multiple times, return a list of all the values specified. @@ -109,6 +110,6 @@ auto FindKey(Map&& map, Key&& key) -> decltype(&map.at(key)) return it == map.end() ? nullptr : &it->second; } -} // namespace util +} // namespace common -#endif // BITCOIN_UTIL_SETTINGS_H +#endif // BITCOIN_COMMON_SETTINGS_H diff --git a/src/util/system.cpp b/src/common/system.cpp index 598e6adb88..1d1c5fa56a 100644 --- a/src/util/system.cpp +++ b/src/common/system.cpp @@ -3,20 +3,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <util/system.h> +#include <common/system.h> #include <logging.h> #include <util/string.h> -#include <util/syserror.h> #include <util/time.h> -#if (defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)) -#include <pthread.h> -#include <pthread_np.h> -#endif - #ifndef WIN32 -#include <sched.h> #include <sys/stat.h> #else #include <codecvt> @@ -112,14 +105,3 @@ int64_t GetStartupTime() { return nStartupTime; } - -void ScheduleBatchPriority() -{ -#ifdef SCHED_BATCH - const static sched_param param{}; - const int rc = pthread_setschedparam(pthread_self(), SCHED_BATCH, ¶m); - if (rc != 0) { - LogPrintf("Failed to pthread_setschedparam: %s\n", SysErrorString(rc)); - } -#endif -} diff --git a/src/common/system.h b/src/common/system.h new file mode 100644 index 0000000000..40206aaa01 --- /dev/null +++ b/src/common/system.h @@ -0,0 +1,38 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_COMMON_SYSTEM_H +#define BITCOIN_COMMON_SYSTEM_H + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <compat/assumptions.h> +#include <compat/compat.h> + +#include <set> +#include <stdint.h> +#include <string> + +// Application startup time (used for uptime calculation) +int64_t GetStartupTime(); + +void SetupEnvironment(); +bool SetupNetworking(); +#ifndef WIN32 +std::string ShellEscape(const std::string& arg); +#endif +#if HAVE_SYSTEM +void runCommand(const std::string& strCommand); +#endif + +/** + * Return the number of cores available on the current system. + * @note This does count virtual cores, such as those provided by HyperThreading. + */ +int GetNumCores(); + +#endif // BITCOIN_COMMON_SYSTEM_H diff --git a/src/consensus/validation.h b/src/consensus/validation.h index ad8ee676b2..d5bf08cd61 100644 --- a/src/consensus/validation.h +++ b/src/consensus/validation.h @@ -145,7 +145,7 @@ class BlockValidationState : public ValidationState<BlockValidationResult> {}; // using only serialization with and without witness data. As witness_size // is equal to total_size - stripped_size, this formula is identical to: // weight = (stripped_size * 3) + total_size. -static inline int64_t GetTransactionWeight(const CTransaction& tx) +static inline int32_t GetTransactionWeight(const CTransaction& tx) { return ::GetSerializeSize(tx, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(tx, PROTOCOL_VERSION); } diff --git a/src/core_write.cpp b/src/core_write.cpp index b0e3b0b3c4..54ca306f60 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -4,6 +4,7 @@ #include <core_io.h> +#include <common/system.h> #include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/validation.h> @@ -17,7 +18,6 @@ #include <univalue.h> #include <util/check.h> #include <util/strencodings.h> -#include <util/system.h> #include <map> #include <string> diff --git a/src/crypto/sha256_avx2.cpp b/src/crypto/sha256_avx2.cpp index 624bdb42e4..df8cb7a6c9 100644 --- a/src/crypto/sha256_avx2.cpp +++ b/src/crypto/sha256_avx2.cpp @@ -7,6 +7,7 @@ #include <stdint.h> #include <immintrin.h> +#include <attributes.h> #include <crypto/common.h> namespace sha256d64_avx2 { @@ -36,7 +37,7 @@ __m256i inline sigma0(__m256i x) { return Xor(Or(ShR(x, 7), ShL(x, 25)), Or(ShR( __m256i inline sigma1(__m256i x) { return Xor(Or(ShR(x, 17), ShL(x, 15)), Or(ShR(x, 19), ShL(x, 13)), ShR(x, 10)); } /** One round of SHA-256. */ -void inline __attribute__((always_inline)) Round(__m256i a, __m256i b, __m256i c, __m256i& d, __m256i e, __m256i f, __m256i g, __m256i& h, __m256i k) +void ALWAYS_INLINE Round(__m256i a, __m256i b, __m256i c, __m256i& d, __m256i e, __m256i f, __m256i g, __m256i& h, __m256i k) { __m256i t1 = Add(h, Sigma1(e), Ch(e, f, g), k); __m256i t2 = Add(Sigma0(a), Maj(a, b, c)); diff --git a/src/crypto/sha256_sse41.cpp b/src/crypto/sha256_sse41.cpp index 4eaf7d7b18..d041fdfefc 100644 --- a/src/crypto/sha256_sse41.cpp +++ b/src/crypto/sha256_sse41.cpp @@ -7,6 +7,7 @@ #include <stdint.h> #include <immintrin.h> +#include <attributes.h> #include <crypto/common.h> namespace sha256d64_sse41 { @@ -36,7 +37,7 @@ __m128i inline sigma0(__m128i x) { return Xor(Or(ShR(x, 7), ShL(x, 25)), Or(ShR( __m128i inline sigma1(__m128i x) { return Xor(Or(ShR(x, 17), ShL(x, 15)), Or(ShR(x, 19), ShL(x, 13)), ShR(x, 10)); } /** One round of SHA-256. */ -void inline __attribute__((always_inline)) Round(__m128i a, __m128i b, __m128i c, __m128i& d, __m128i e, __m128i f, __m128i g, __m128i& h, __m128i k) +void ALWAYS_INLINE Round(__m128i a, __m128i b, __m128i c, __m128i& d, __m128i e, __m128i f, __m128i g, __m128i& h, __m128i k) { __m128i t1 = Add(h, Sigma1(e), Ch(e, f, g), k); __m128i t2 = Add(Sigma0(a), Maj(a, b, c)); diff --git a/src/crypto/sha256_x86_shani.cpp b/src/crypto/sha256_x86_shani.cpp index e3143a55c2..79871bfcc1 100644 --- a/src/crypto/sha256_x86_shani.cpp +++ b/src/crypto/sha256_x86_shani.cpp @@ -11,43 +11,45 @@ #include <stdint.h> #include <immintrin.h> +#include <attributes.h> + namespace { alignas(__m128i) const uint8_t MASK[16] = {0x03, 0x02, 0x01, 0x00, 0x07, 0x06, 0x05, 0x04, 0x0b, 0x0a, 0x09, 0x08, 0x0f, 0x0e, 0x0d, 0x0c}; alignas(__m128i) const uint8_t INIT0[16] = {0x8c, 0x68, 0x05, 0x9b, 0x7f, 0x52, 0x0e, 0x51, 0x85, 0xae, 0x67, 0xbb, 0x67, 0xe6, 0x09, 0x6a}; alignas(__m128i) const uint8_t INIT1[16] = {0x19, 0xcd, 0xe0, 0x5b, 0xab, 0xd9, 0x83, 0x1f, 0x3a, 0xf5, 0x4f, 0xa5, 0x72, 0xf3, 0x6e, 0x3c}; -void inline __attribute__((always_inline)) QuadRound(__m128i& state0, __m128i& state1, uint64_t k1, uint64_t k0) +void ALWAYS_INLINE QuadRound(__m128i& state0, __m128i& state1, uint64_t k1, uint64_t k0) { const __m128i msg = _mm_set_epi64x(k1, k0); state1 = _mm_sha256rnds2_epu32(state1, state0, msg); state0 = _mm_sha256rnds2_epu32(state0, state1, _mm_shuffle_epi32(msg, 0x0e)); } -void inline __attribute__((always_inline)) QuadRound(__m128i& state0, __m128i& state1, __m128i m, uint64_t k1, uint64_t k0) +void ALWAYS_INLINE QuadRound(__m128i& state0, __m128i& state1, __m128i m, uint64_t k1, uint64_t k0) { const __m128i msg = _mm_add_epi32(m, _mm_set_epi64x(k1, k0)); state1 = _mm_sha256rnds2_epu32(state1, state0, msg); state0 = _mm_sha256rnds2_epu32(state0, state1, _mm_shuffle_epi32(msg, 0x0e)); } -void inline __attribute__((always_inline)) ShiftMessageA(__m128i& m0, __m128i m1) +void ALWAYS_INLINE ShiftMessageA(__m128i& m0, __m128i m1) { m0 = _mm_sha256msg1_epu32(m0, m1); } -void inline __attribute__((always_inline)) ShiftMessageC(__m128i& m0, __m128i m1, __m128i& m2) +void ALWAYS_INLINE ShiftMessageC(__m128i& m0, __m128i m1, __m128i& m2) { m2 = _mm_sha256msg2_epu32(_mm_add_epi32(m2, _mm_alignr_epi8(m1, m0, 4)), m1); } -void inline __attribute__((always_inline)) ShiftMessageB(__m128i& m0, __m128i m1, __m128i& m2) +void ALWAYS_INLINE ShiftMessageB(__m128i& m0, __m128i m1, __m128i& m2) { ShiftMessageC(m0, m1, m2); ShiftMessageA(m0, m1); } -void inline __attribute__((always_inline)) Shuffle(__m128i& s0, __m128i& s1) +void ALWAYS_INLINE Shuffle(__m128i& s0, __m128i& s1) { const __m128i t1 = _mm_shuffle_epi32(s0, 0xB1); const __m128i t2 = _mm_shuffle_epi32(s1, 0x1B); @@ -55,7 +57,7 @@ void inline __attribute__((always_inline)) Shuffle(__m128i& s0, __m128i& s1) s1 = _mm_blend_epi16(t2, t1, 0xF0); } -void inline __attribute__((always_inline)) Unshuffle(__m128i& s0, __m128i& s1) +void ALWAYS_INLINE Unshuffle(__m128i& s0, __m128i& s1) { const __m128i t1 = _mm_shuffle_epi32(s0, 0x1B); const __m128i t2 = _mm_shuffle_epi32(s1, 0xB1); @@ -63,12 +65,12 @@ void inline __attribute__((always_inline)) Unshuffle(__m128i& s0, __m128i& s1) s1 = _mm_alignr_epi8(t2, t1, 0x08); } -__m128i inline __attribute__((always_inline)) Load(const unsigned char* in) +__m128i ALWAYS_INLINE Load(const unsigned char* in) { return _mm_shuffle_epi8(_mm_loadu_si128((const __m128i*)in), _mm_load_si128((const __m128i*)MASK)); } -void inline __attribute__((always_inline)) Save(unsigned char* out, __m128i s) +void ALWAYS_INLINE Save(unsigned char* out, __m128i s) { _mm_storeu_si128((__m128i*)out, _mm_shuffle_epi8(s, _mm_load_si128((const __m128i*)MASK))); } diff --git a/src/external_signer.cpp b/src/external_signer.cpp index 5524b943f4..6b1e1f0241 100644 --- a/src/external_signer.cpp +++ b/src/external_signer.cpp @@ -30,7 +30,7 @@ bool ExternalSigner::Enumerate(const std::string& command, std::vector<ExternalS } for (const UniValue& signer : result.getValues()) { // Check for error - const UniValue& error = find_value(signer, "error"); + const UniValue& error = signer.find_value("error"); if (!error.isNull()) { if (!error.isStr()) { throw std::runtime_error(strprintf("'%s' error", command)); @@ -38,11 +38,11 @@ bool ExternalSigner::Enumerate(const std::string& command, std::vector<ExternalS throw std::runtime_error(strprintf("'%s' error: %s", command, error.getValStr())); } // Check if fingerprint is present - const UniValue& fingerprint = find_value(signer, "fingerprint"); + const UniValue& fingerprint = signer.find_value("fingerprint"); if (fingerprint.isNull()) { throw std::runtime_error(strprintf("'%s' received invalid response, missing signer fingerprint", command)); } - const std::string fingerprintStr = fingerprint.get_str(); + const std::string& fingerprintStr{fingerprint.get_str()}; // Skip duplicate signer bool duplicate = false; for (const ExternalSigner& signer : signers) { @@ -50,7 +50,7 @@ bool ExternalSigner::Enumerate(const std::string& command, std::vector<ExternalS } if (duplicate) break; std::string name; - const UniValue& model_field = find_value(signer, "model"); + const UniValue& model_field = signer.find_value("model"); if (model_field.isStr() && model_field.getValStr() != "") { name += model_field.getValStr(); } @@ -97,19 +97,19 @@ bool ExternalSigner::SignTransaction(PartiallySignedTransaction& psbtx, std::str const UniValue signer_result = RunCommandParseJSON(command, stdinStr); - if (find_value(signer_result, "error").isStr()) { - error = find_value(signer_result, "error").get_str(); + if (signer_result.find_value("error").isStr()) { + error = signer_result.find_value("error").get_str(); return false; } - if (!find_value(signer_result, "psbt").isStr()) { + if (!signer_result.find_value("psbt").isStr()) { error = "Unexpected result from signer"; return false; } PartiallySignedTransaction signer_psbtx; std::string signer_psbt_error; - if (!DecodeBase64PSBT(signer_psbtx, find_value(signer_result, "psbt").get_str(), signer_psbt_error)) { + if (!DecodeBase64PSBT(signer_psbtx, signer_result.find_value("psbt").get_str(), signer_psbt_error)) { error = strprintf("TX decode failed %s", signer_psbt_error); return false; } diff --git a/src/external_signer.h b/src/external_signer.h index 90f07478e3..81a601811a 100644 --- a/src/external_signer.h +++ b/src/external_signer.h @@ -5,8 +5,8 @@ #ifndef BITCOIN_EXTERNAL_SIGNER_H #define BITCOIN_EXTERNAL_SIGNER_H +#include <common/system.h> #include <univalue.h> -#include <util/system.h> #include <string> #include <vector> diff --git a/src/httprpc.cpp b/src/httprpc.cpp index bf3fa6298d..661406e122 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -76,7 +76,7 @@ static void JSONErrorReply(HTTPRequest* req, const UniValue& objError, const Uni { // Send error reply from json-rpc error object int nStatus = HTTP_INTERNAL_SERVER_ERROR; - int code = find_value(objError, "code").getInt<int>(); + int code = objError.find_value("code").getInt<int>(); if (code == RPC_INVALID_REQUEST) nStatus = HTTP_BAD_REQUEST; @@ -213,7 +213,7 @@ static bool HTTPReq_JSONRPC(const std::any& context, HTTPRequest* req) } else { const UniValue& request = valRequest[reqIdx].get_obj(); // Parse method - std::string strMethod = find_value(request, "method").get_str(); + std::string strMethod = request.find_value("method").get_str(); if (!g_rpc_whitelist[jreq.authUser].count(strMethod)) { LogPrintf("RPC User %s not allowed to call method %s\n", jreq.authUser, strMethod); req->WriteReply(HTTP_FORBIDDEN); diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 4f8a2b4d8d..128c4e3c56 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -170,12 +170,8 @@ static bool ClientAllowed(const CNetAddr& netaddr) static bool InitHTTPAllowList() { rpc_allow_subnets.clear(); - CNetAddr localv4; - CNetAddr localv6; - LookupHost("127.0.0.1", localv4, false); - LookupHost("::1", localv6, false); - rpc_allow_subnets.push_back(CSubNet(localv4, 8)); // always allow IPv4 local subnet - rpc_allow_subnets.push_back(CSubNet(localv6)); // always allow IPv6 localhost + rpc_allow_subnets.push_back(CSubNet{LookupHost("127.0.0.1", false).value(), 8}); // always allow IPv4 local subnet + rpc_allow_subnets.push_back(CSubNet{LookupHost("::1", false).value()}); // always allow IPv6 localhost for (const std::string& strAllow : gArgs.GetArgs("-rpcallowip")) { CSubNet subnet; LookupSubNet(strAllow, subnet); @@ -338,8 +334,8 @@ static bool HTTPBindAddresses(struct evhttp* http) LogPrintf("Binding RPC on address %s port %i\n", i->first, i->second); evhttp_bound_socket *bind_handle = evhttp_bind_socket_with_handle(http, i->first.empty() ? nullptr : i->first.c_str(), i->second); if (bind_handle) { - CNetAddr addr; - if (i->first.empty() || (LookupHost(i->first, addr, false) && addr.IsBindAny())) { + const std::optional<CNetAddr> addr{LookupHost(i->first, false)}; + if (i->first.empty() || (addr.has_value() && addr->IsBindAny())) { LogPrintf("WARNING: the RPC server is not safe to expose to untrusted networks such as the public internet\n"); } boundSockets.push_back(bind_handle); diff --git a/src/i2p.cpp b/src/i2p.cpp index c67b6ed185..f03e375adf 100644 --- a/src/i2p.cpp +++ b/src/i2p.cpp @@ -336,7 +336,7 @@ void Session::GenerateAndSavePrivateKey(const Sock& sock) { DestGenerate(sock); - // umask is set to 0077 in util/system.cpp, which is ok. + // umask is set to 0077 in common/system.cpp, which is ok. if (!WriteBinaryFile(m_private_key_file, std::string(m_private_key.begin(), m_private_key.end()))) { throw std::runtime_error( diff --git a/src/index/base.cpp b/src/index/base.cpp index 237c8e8be0..a713be3480 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -23,7 +23,7 @@ #include <string> #include <utility> -using node::ReadBlockFromDisk; +using node::g_indexes_ready_to_sync; constexpr uint8_t DB_BEST_BLOCK{'B'}; @@ -33,11 +33,7 @@ constexpr auto SYNC_LOCATOR_WRITE_INTERVAL{30s}; template <typename... Args> static void FatalError(const char* fmt, const Args&... args) { - std::string strMessage = tfm::format(fmt, args...); - SetMiscWarning(Untranslated(strMessage)); - LogPrintf("*** %s\n", strMessage); - InitError(_("A fatal internal error occurred, see debug.log for details")); - StartShutdown(); + AbortNode(tfm::format(fmt, args...)); } CBlockLocator GetLocator(interfaces::Chain& chain, const uint256& block_hash) @@ -94,16 +90,21 @@ bool BaseIndex::Init() if (locator.IsNull()) { SetBestBlockIndex(nullptr); } else { - SetBestBlockIndex(m_chainstate->FindForkInGlobalIndex(locator)); + // Setting the best block to the locator's top block. If it is not part of the + // best chain, we will rewind to the fork point during index sync + const CBlockIndex* locator_index{m_chainstate->m_blockman.LookupBlockIndex(locator.vHave.at(0))}; + if (!locator_index) { + return InitError(strprintf(Untranslated("%s: best block of the index not found. Please rebuild the index."), GetName())); + } + SetBestBlockIndex(locator_index); } - // Note: this will latch to true immediately if the user starts up with an empty - // datadir and an index enabled. If this is the case, indexation will happen solely - // via `BlockConnected` signals until, possibly, the next restart. - m_synced = m_best_block_index.load() == active_chain.Tip(); - if (!m_synced) { + // Skip pruning check if indexes are not ready to sync (because reindex-chainstate has wiped the chain). + const CBlockIndex* start_block = m_best_block_index.load(); + bool synced = start_block == active_chain.Tip(); + if (!synced && g_indexes_ready_to_sync) { bool prune_violation = false; - if (!m_best_block_index) { + if (!start_block) { // index is not built yet // make sure we have all block data back to the genesis prune_violation = m_chainstate->m_blockman.GetFirstStoredBlock(*active_chain.Tip()) != active_chain.Genesis(); @@ -111,7 +112,7 @@ bool BaseIndex::Init() // in case the index has a best block set and is not fully synced // check if we have the required blocks to continue building the index else { - const CBlockIndex* block_to_test = m_best_block_index.load(); + const CBlockIndex* block_to_test = start_block; if (!active_chain.Contains(block_to_test)) { // if the bestblock is not part of the mainchain, find the fork // and make sure we have all data down to the fork @@ -135,6 +136,16 @@ bool BaseIndex::Init() return InitError(strprintf(Untranslated("%s best block of the index goes beyond pruned data. Please disable the index or reindex (which will download the whole blockchain again)"), GetName())); } } + + // Child init + if (!CustomInit(start_block ? std::make_optional(interfaces::BlockKey{start_block->GetBlockHash(), start_block->nHeight}) : std::nullopt)) { + return false; + } + + // Note: this will latch to true immediately if the user starts up with an empty + // datadir and an index enabled. If this is the case, indexation will happen solely + // via `BlockConnected` signals until, possibly, the next restart. + m_synced = synced; return true; } @@ -157,10 +168,14 @@ static const CBlockIndex* NextSyncBlock(const CBlockIndex* pindex_prev, CChain& void BaseIndex::ThreadSync() { SetSyscallSandboxPolicy(SyscallSandboxPolicy::TX_INDEX); + // Wait for a possible reindex-chainstate to finish until continuing + // with the index sync + while (!g_indexes_ready_to_sync) { + if (!m_interrupt.sleep_for(std::chrono::milliseconds(500))) return; + } + const CBlockIndex* pindex = m_best_block_index.load(); if (!m_synced) { - auto& consensus_params = Params().GetConsensus(); - std::chrono::steady_clock::time_point last_log_time{0s}; std::chrono::steady_clock::time_point last_locator_write_time{0s}; while (true) { @@ -207,7 +222,7 @@ void BaseIndex::ThreadSync() CBlock block; interfaces::BlockInfo block_info = kernel::MakeBlockInfo(pindex); - if (!ReadBlockFromDisk(block, pindex, consensus_params)) { + if (!m_chainstate->m_blockman.ReadBlockFromDisk(block, *pindex)) { FatalError("%s: Failed to read block %s from disk", __func__, pindex->GetBlockHash().ToString()); return; @@ -396,11 +411,6 @@ bool BaseIndex::Start() RegisterValidationInterface(this); if (!Init()) return false; - const CBlockIndex* index = m_best_block_index.load(); - if (!CustomInit(index ? std::make_optional(interfaces::BlockKey{index->GetBlockHash(), index->nHeight}) : std::nullopt)) { - return false; - } - m_thread_sync = std::thread(&util::TraceThread, GetName(), [this] { ThreadSync(); }); return true; } diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index e6300ce3dd..a860b3a94d 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -12,8 +12,6 @@ #include <util/fs_helpers.h> #include <validation.h> -using node::UndoReadFromDisk; - /* The index database stores three items for each block: the disk location of the encoded filter, * its dSHA256 hash, and the header. Those belonging to blocks on the active chain are indexed by * height, and those belonging to blocks that have been reorganized out of the active chain are @@ -223,7 +221,7 @@ bool BlockFilterIndex::CustomAppend(const interfaces::BlockInfo& block) // pindex variable gives indexing code access to node internals. It // will be removed in upcoming commit const CBlockIndex* pindex = WITH_LOCK(cs_main, return m_chainstate->m_blockman.LookupBlockIndex(block.hash)); - if (!UndoReadFromDisk(block_undo, pindex)) { + if (!m_chainstate->m_blockman.UndoReadFromDisk(block_undo, *pindex)) { return false; } diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index 5bc6ad3d31..d80885f842 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -19,9 +19,6 @@ using kernel::CCoinsStats; using kernel::GetBogoSize; using kernel::TxOutSer; -using node::ReadBlockFromDisk; -using node::UndoReadFromDisk; - static constexpr uint8_t DB_BLOCK_HASH{'s'}; static constexpr uint8_t DB_BLOCK_HEIGHT{'t'}; static constexpr uint8_t DB_MUHASH{'M'}; @@ -125,7 +122,7 @@ bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block) // pindex variable gives indexing code access to node internals. It // will be removed in upcoming commit const CBlockIndex* pindex = WITH_LOCK(cs_main, return m_chainstate->m_blockman.LookupBlockIndex(block.hash)); - if (!UndoReadFromDisk(block_undo, pindex)) { + if (!m_chainstate->m_blockman.UndoReadFromDisk(block_undo, *pindex)) { return false; } @@ -282,12 +279,11 @@ bool CoinStatsIndex::CustomRewind(const interfaces::BlockKey& current_tip, const LOCK(cs_main); const CBlockIndex* iter_tip{m_chainstate->m_blockman.LookupBlockIndex(current_tip.hash)}; const CBlockIndex* new_tip_index{m_chainstate->m_blockman.LookupBlockIndex(new_tip.hash)}; - const auto& consensus_params{Params().GetConsensus()}; do { CBlock block; - if (!ReadBlockFromDisk(block, iter_tip, consensus_params)) { + if (!m_chainstate->m_blockman.ReadBlockFromDisk(block, *iter_tip)) { return error("%s: Failed to read block %s from disk", __func__, iter_tip->GetBlockHash().ToString()); } @@ -409,7 +405,7 @@ bool CoinStatsIndex::ReverseBlock(const CBlock& block, const CBlockIndex* pindex // Ignore genesis block if (pindex->nHeight > 0) { - if (!UndoReadFromDisk(block_undo, pindex)) { + if (!m_chainstate->m_blockman.UndoReadFromDisk(block_undo, *pindex)) { return false; } diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 49bddf2d4d..2e07a35d0d 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -10,8 +10,6 @@ #include <node/blockstorage.h> #include <validation.h> -using node::OpenBlockFile; - constexpr uint8_t DB_TXINDEX{'t'}; std::unique_ptr<TxIndex> g_txindex; @@ -80,7 +78,7 @@ bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRe return false; } - CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); + CAutoFile file(m_chainstate->m_blockman.OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); if (file.IsNull()) { return error("%s: OpenBlockFile failed", __func__); } diff --git a/src/init.cpp b/src/init.cpp index 525648b812..38e1dbb4a2 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -18,7 +18,9 @@ #include <blockfilter.h> #include <chain.h> #include <chainparams.h> +#include <chainparamsbase.h> #include <common/args.h> +#include <common/system.h> #include <consensus/amount.h> #include <deploymentstatus.h> #include <hash.h> @@ -44,6 +46,7 @@ #include <node/chainstatemanager_args.h> #include <node/context.h> #include <node/interface_ui.h> +#include <node/kernel_notifications.h> #include <node/mempool_args.h> #include <node/mempool_persist_args.h> #include <node/miner.h> @@ -69,6 +72,7 @@ #include <txdb.h> #include <txmempool.h> #include <util/asmap.h> +#include <util/chaintype.h> #include <util/check.h> #include <util/fs.h> #include <util/fs_helpers.h> @@ -77,7 +81,6 @@ #include <util/string.h> #include <util/syscall_sandbox.h> #include <util/syserror.h> -#include <util/system.h> #include <util/thread.h> #include <util/threadnames.h> #include <util/translation.h> @@ -110,22 +113,25 @@ #include <zmq/zmqrpc.h> #endif +using kernel::DEFAULT_STOPAFTERBLOCKIMPORT; using kernel::DumpMempool; using kernel::ValidationCacheSizes; using node::ApplyArgsManOptions; +using node::BlockManager; using node::CacheSizes; using node::CalculateCacheSizes; using node::DEFAULT_PERSIST_MEMPOOL; using node::DEFAULT_PRINTPRIORITY; -using node::DEFAULT_STOPAFTERBLOCKIMPORT; +using node::fReindex; +using node::g_indexes_ready_to_sync; +using node::KernelNotifications; using node::LoadChainstate; using node::MempoolPath; -using node::ShouldPersistMempool; using node::NodeContext; +using node::ShouldPersistMempool; using node::ThreadImport; using node::VerifyLoadedChainstate; -using node::fReindex; static constexpr bool DEFAULT_PROXYRANDOMIZE{true}; static constexpr bool DEFAULT_REST_ENABLE{false}; @@ -326,9 +332,8 @@ void Shutdown(NodeContext& node) #if ENABLE_ZMQ if (g_zmq_notification_interface) { - UnregisterValidationInterface(g_zmq_notification_interface); - delete g_zmq_notification_interface; - g_zmq_notification_interface = nullptr; + UnregisterValidationInterface(g_zmq_notification_interface.get()); + g_zmq_notification_interface.reset(); } #endif @@ -408,14 +413,14 @@ void SetupServerArgs(ArgsManager& argsman) init::AddLoggingArgs(argsman); - const auto defaultBaseParams = CreateBaseChainParams(CBaseChainParams::MAIN); - const auto testnetBaseParams = CreateBaseChainParams(CBaseChainParams::TESTNET); - const auto signetBaseParams = CreateBaseChainParams(CBaseChainParams::SIGNET); - const auto regtestBaseParams = CreateBaseChainParams(CBaseChainParams::REGTEST); - const auto defaultChainParams = CreateChainParams(argsman, CBaseChainParams::MAIN); - const auto testnetChainParams = CreateChainParams(argsman, CBaseChainParams::TESTNET); - const auto signetChainParams = CreateChainParams(argsman, CBaseChainParams::SIGNET); - const auto regtestChainParams = CreateChainParams(argsman, CBaseChainParams::REGTEST); + const auto defaultBaseParams = CreateBaseChainParams(ChainType::MAIN); + const auto testnetBaseParams = CreateBaseChainParams(ChainType::TESTNET); + const auto signetBaseParams = CreateBaseChainParams(ChainType::SIGNET); + const auto regtestBaseParams = CreateBaseChainParams(ChainType::REGTEST); + const auto defaultChainParams = CreateChainParams(argsman, ChainType::MAIN); + const auto testnetChainParams = CreateChainParams(argsman, ChainType::TESTNET); + const auto signetChainParams = CreateChainParams(argsman, ChainType::SIGNET); + const auto regtestChainParams = CreateChainParams(argsman, ChainType::REGTEST); // Hidden Options std::vector<std::string> hidden_args = { @@ -441,6 +446,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-dbbatchsize", strprintf("Maximum database write batch size in bytes (default: %u)", nDefaultDbBatchSize), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); argsman.AddArg("-dbcache=<n>", strprintf("Maximum database cache size <n> MiB (%d to %d, default: %d). In addition, unused mempool memory is shared for this cache (see -maxmempool).", nMinDbCache, nMaxDbCache, nDefaultDbCache), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-includeconf=<file>", "Specify additional configuration file, relative to the -datadir path (only useable from configuration file, not command line)", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-allowignoredconf", strprintf("For backwards compatibility, treat an unused %s file in the datadir as a warning, not an error.", BITCOIN_CONF_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-loadblock=<file>", "Imports blocks from external file on startup", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-maxmempool=<n>", strprintf("Keep the transaction memory pool below <n> megabytes (default: %u)", DEFAULT_MAX_MEMPOOL_SIZE_MB), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-maxorphantx=<n>", strprintf("Keep at most <n> unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -454,7 +460,7 @@ void SetupServerArgs(ArgsManager& argsman) "Warning: Reverting this setting requires re-downloading the entire blockchain. " "(default: 0 = disable pruning blocks, 1 = allow manual pruning via RPC, >=%u = automatically prune block files to stay under the specified target size in MiB)", MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-reindex", "Rebuild chain state and block index from the blk*.dat files on disk. This will also rebuild active optional indexes.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); - argsman.AddArg("-reindex-chainstate", "Rebuild chain state from the currently indexed blocks. When in pruning mode or if blocks on disk might be corrupted, use full -reindex instead. Deactivate all optional indexes before running this.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-reindex-chainstate", "Rebuild chain state from the currently indexed blocks. When in pruning mode or if blocks on disk might be corrupted, use full -reindex instead.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-settings=<file>", strprintf("Specify path to dynamic settings data file. Can be disabled with -nosettings. File is written at runtime and not meant to be edited by users (use %s instead for custom settings). Relative paths will be prefixed by datadir location. (default: %s)", BITCOIN_CONF_FILENAME, BITCOIN_SETTINGS_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); #if HAVE_SYSTEM argsman.AddArg("-startupnotify=<cmd>", "Execute command on startup.", ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); @@ -794,7 +800,7 @@ std::set<BlockFilterType> g_enabled_filter_types; std::terminate(); }; -bool AppInitBasicSetup(const ArgsManager& args) +bool AppInitBasicSetup(const ArgsManager& args, std::atomic<int>& exit_status) { // ********************************************************* Step 1: setup #ifdef _MSC_VER @@ -808,7 +814,7 @@ bool AppInitBasicSetup(const ArgsManager& args) // Enable heap terminate-on-corruption HeapSetInformation(nullptr, HeapEnableTerminationOnCorruption, nullptr, 0); #endif - if (!InitShutdownState()) { + if (!InitShutdownState(exit_status)) { return InitError(Untranslated("Initializing wait-for-shutdown state failed.")); } @@ -844,14 +850,14 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb // Error if network-specific options (-addnode, -connect, etc) are // specified in default section of config file, but not overridden - // on the command line or in this network's section of the config file. - std::string network = args.GetChainName(); - if (network == CBaseChainParams::SIGNET) { + // on the command line or in this chain's section of the config file. + ChainType chain = args.GetChainType(); + if (chain == ChainType::SIGNET) { LogPrintf("Signet derived magic (message start): %s\n", HexStr(chainparams.MessageStart())); } bilingual_str errors; for (const auto& arg : args.GetUnsuitableSectionOnlyArgs()) { - errors += strprintf(_("Config setting for %s only applied on %s network when in [%s] section.") + Untranslated("\n"), arg, network, network); + errors += strprintf(_("Config setting for %s only applied on %s network when in [%s] section.") + Untranslated("\n"), arg, ChainTypeToString(chain), ChainTypeToString(chain)); } if (!errors.empty()) { @@ -980,19 +986,6 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb if (args.GetIntArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) > 1) return InitError(Untranslated("Unknown rpcserialversion requested.")); - if (args.GetBoolArg("-reindex-chainstate", false)) { - // indexes that must be deactivated to prevent index corruption, see #24630 - if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) { - return InitError(_("-reindex-chainstate option is not compatible with -coinstatsindex. Please temporarily disable coinstatsindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.")); - } - if (g_enabled_filter_types.count(BlockFilterType::BASIC)) { - return InitError(_("-reindex-chainstate option is not compatible with -blockfilterindex. Please temporarily disable blockfilterindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.")); - } - if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - return InitError(_("-reindex-chainstate option is not compatible with -txindex. Please temporarily disable txindex while using -reindex-chainstate, or replace -reindex-chainstate with -reindex to fully rebuild all indexes.")); - } - } - #if defined(USE_SYSCALL_SANDBOX) if (args.IsArgSet("-sandbox") && !args.IsArgNegated("-sandbox")) { const std::string sandbox_arg{args.GetArg("-sandbox", "")}; @@ -1029,16 +1022,23 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb // Also report errors from parsing before daemonization { + KernelNotifications notifications{}; ChainstateManager::Options chainman_opts_dummy{ .chainparams = chainparams, .datadir = args.GetDataDirNet(), + .notifications = notifications, }; - if (const auto error{ApplyArgsManOptions(args, chainman_opts_dummy)}) { - return InitError(*error); + auto chainman_result{ApplyArgsManOptions(args, chainman_opts_dummy)}; + if (!chainman_result) { + return InitError(util::ErrorString(chainman_result)); } - node::BlockManager::Options blockman_opts_dummy{}; - if (const auto error{ApplyArgsManOptions(args, blockman_opts_dummy)}) { - return InitError(*error); + BlockManager::Options blockman_opts_dummy{ + .chainparams = chainman_opts_dummy.chainparams, + .blocks_dir = args.GetBlocksDirPath(), + }; + auto blockman_result{ApplyArgsManOptions(args, blockman_opts_dummy)}; + if (!blockman_result) { + return InitError(util::ErrorString(blockman_result)); } } @@ -1061,8 +1061,9 @@ static bool LockDataDirectory(bool probeOnly) bool AppInitSanityChecks(const kernel::Context& kernel) { // ********************************************************* Step 4: sanity checks - if (auto error = kernel::SanityChecks(kernel)) { - InitError(*error); + auto result{kernel::SanityChecks(kernel)}; + if (!result) { + InitError(util::ErrorString(result)); return InitError(strprintf(_("Initialization sanity check failed. %s is shutting down."), PACKAGE_NAME)); } @@ -1237,9 +1238,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // Initialize addrman assert(!node.addrman); uiInterface.InitMessage(_("Loading P2P addresses…").translated); - if (const auto error{LoadAddrman(*node.netgroupman, args, node.addrman)}) { - return InitError(*error); - } + auto addrman{LoadAddrman(*node.netgroupman, args)}; + if (!addrman) return InitError(util::ErrorString(addrman)); + node.addrman = std::move(*addrman); } assert(!node.banman); @@ -1358,12 +1359,12 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // -noproxy (or -proxy=0) as well as the empty string can be used to not set a proxy, this is the default std::string proxyArg = args.GetArg("-proxy", ""); if (proxyArg != "" && proxyArg != "0") { - CService proxyAddr; - if (!Lookup(proxyArg, proxyAddr, 9050, fNameLookup)) { + const std::optional<CService> proxyAddr{Lookup(proxyArg, 9050, fNameLookup)}; + if (!proxyAddr.has_value()) { return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'"), proxyArg)); } - Proxy addrProxy = Proxy(proxyAddr, proxyRandomize); + Proxy addrProxy = Proxy(proxyAddr.value(), proxyRandomize); if (!addrProxy.IsValid()) return InitError(strprintf(_("Invalid -proxy address or hostname: '%s'"), proxyArg)); @@ -1389,11 +1390,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) "reaching the Tor network is explicitly forbidden: -onion=0")); } } else { - CService addr; - if (!Lookup(onionArg, addr, 9050, fNameLookup) || !addr.IsValid()) { + const std::optional<CService> addr{Lookup(onionArg, 9050, fNameLookup)}; + if (!addr.has_value() || !addr->IsValid()) { return InitError(strprintf(_("Invalid -onion address or hostname: '%s'"), onionArg)); } - onion_proxy = Proxy{addr, proxyRandomize}; + onion_proxy = Proxy{addr.value(), proxyRandomize}; } } @@ -1413,34 +1414,43 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } for (const std::string& strAddr : args.GetArgs("-externalip")) { - CService addrLocal; - if (Lookup(strAddr, addrLocal, GetListenPort(), fNameLookup) && addrLocal.IsValid()) - AddLocal(addrLocal, LOCAL_MANUAL); + const std::optional<CService> addrLocal{Lookup(strAddr, GetListenPort(), fNameLookup)}; + if (addrLocal.has_value() && addrLocal->IsValid()) + AddLocal(addrLocal.value(), LOCAL_MANUAL); else return InitError(ResolveErrMsg("externalip", strAddr)); } #if ENABLE_ZMQ - g_zmq_notification_interface = CZMQNotificationInterface::Create(); + g_zmq_notification_interface = CZMQNotificationInterface::Create( + [&chainman = node.chainman](CBlock& block, const CBlockIndex& index) { + assert(chainman); + return chainman->m_blockman.ReadBlockFromDisk(block, index); + }); if (g_zmq_notification_interface) { - RegisterValidationInterface(g_zmq_notification_interface); + RegisterValidationInterface(g_zmq_notification_interface.get()); } #endif // ********************************************************* Step 7: load block chain + node.notifications = std::make_unique<KernelNotifications>(); fReindex = args.GetBoolArg("-reindex", false); bool fReindexChainState = args.GetBoolArg("-reindex-chainstate", false); ChainstateManager::Options chainman_opts{ .chainparams = chainparams, .datadir = args.GetDataDirNet(), .adjusted_time_callback = GetAdjustedTime, + .notifications = *node.notifications, }; - Assert(!ApplyArgsManOptions(args, chainman_opts)); // no error can happen, already checked in AppInitParameterInteraction + Assert(ApplyArgsManOptions(args, chainman_opts)); // no error can happen, already checked in AppInitParameterInteraction - node::BlockManager::Options blockman_opts{}; - Assert(!ApplyArgsManOptions(args, blockman_opts)); // no error can happen, already checked in AppInitParameterInteraction + BlockManager::Options blockman_opts{ + .chainparams = chainman_opts.chainparams, + .blocks_dir = args.GetBlocksDirPath(), + }; + Assert(ApplyArgsManOptions(args, blockman_opts)); // no error can happen, already checked in AppInitParameterInteraction // cache size calculations CacheSizes cache_sizes = CalculateCacheSizes(args, g_enabled_filter_types.size()); @@ -1463,8 +1473,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) .estimator = node.fee_estimator.get(), .check_ratio = chainparams.DefaultConsistencyChecks() ? 1 : 0, }; - if (const auto err{ApplyArgsManOptions(args, chainparams, mempool_opts)}) { - return InitError(*err); + auto result{ApplyArgsManOptions(args, chainparams, mempool_opts)}; + if (!result) { + return InitError(util::ErrorString(result)); } mempool_opts.check_ratio = std::clamp<int>(mempool_opts.check_ratio, 0, 1'000'000); @@ -1559,9 +1570,13 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) RegisterValidationInterface(node.peerman.get()); // ********************************************************* Step 8: start indexers + + // If reindex-chainstate was specified, delay syncing indexes until ThreadImport has reindexed the chain + if (!fReindexChainState) g_indexes_ready_to_sync = true; if (args.GetBoolArg("-txindex", DEFAULT_TXINDEX)) { - if (const auto error{WITH_LOCK(cs_main, return CheckLegacyTxindex(*Assert(chainman.m_blockman.m_block_tree_db)))}) { - return InitError(*error); + auto result{WITH_LOCK(cs_main, return CheckLegacyTxindex(*Assert(chainman.m_blockman.m_block_tree_db)))}; + if (!result) { + return InitError(util::ErrorString(result)); } g_txindex = std::make_unique<TxIndex>(interfaces::MakeChain(node), cache_sizes.tx_index, false, fReindex); @@ -1668,7 +1683,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } chainman.m_load_block = std::thread(&util::TraceThread, "loadblk", [=, &chainman, &args] { - ThreadImport(chainman, vImportFiles, args, ShouldPersistMempool(args) ? MempoolPath(args) : fs::path{}); + ThreadImport(chainman, vImportFiles, ShouldPersistMempool(args) ? MempoolPath(args) : fs::path{}); }); // Wait for genesis block to be processed @@ -1739,13 +1754,14 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) }; for (const std::string& bind_arg : args.GetArgs("-bind")) { - CService bind_addr; + std::optional<CService> bind_addr; const size_t index = bind_arg.rfind('='); if (index == std::string::npos) { - if (Lookup(bind_arg, bind_addr, default_bind_port, /*fAllowLookup=*/false)) { - connOptions.vBinds.push_back(bind_addr); - if (IsBadPort(bind_addr.GetPort())) { - InitWarning(BadPortWarning("-bind", bind_addr.GetPort())); + bind_addr = Lookup(bind_arg, default_bind_port, /*fAllowLookup=*/false); + if (bind_addr.has_value()) { + connOptions.vBinds.push_back(bind_addr.value()); + if (IsBadPort(bind_addr.value().GetPort())) { + InitWarning(BadPortWarning("-bind", bind_addr.value().GetPort())); } continue; } @@ -1753,8 +1769,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) const std::string network_type = bind_arg.substr(index + 1); if (network_type == "onion") { const std::string truncated_bind_arg = bind_arg.substr(0, index); - if (Lookup(truncated_bind_arg, bind_addr, BaseParams().OnionServiceTargetPort(), false)) { - connOptions.onion_binds.push_back(bind_addr); + bind_addr = Lookup(truncated_bind_arg, BaseParams().OnionServiceTargetPort(), false); + if (bind_addr.has_value()) { + connOptions.onion_binds.push_back(bind_addr.value()); continue; } } @@ -1832,11 +1849,11 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) const std::string& i2psam_arg = args.GetArg("-i2psam", ""); if (!i2psam_arg.empty()) { - CService addr; - if (!Lookup(i2psam_arg, addr, 7656, fNameLookup) || !addr.IsValid()) { + const std::optional<CService> addr{Lookup(i2psam_arg, 7656, fNameLookup)}; + if (!addr.has_value() || !addr->IsValid()) { return InitError(strprintf(_("Invalid -i2psam address or hostname: '%s'"), i2psam_arg)); } - SetProxy(NET_I2P, Proxy{addr}); + SetProxy(NET_I2P, Proxy{addr.value()}); } else { if (args.IsArgSet("-onlynet") && IsReachable(NET_I2P)) { return InitError( diff --git a/src/init.h b/src/init.h index 8c5b2e77d3..fabb4f66d6 100644 --- a/src/init.h +++ b/src/init.h @@ -38,7 +38,7 @@ void InitParameterInteraction(ArgsManager& args); * @note This can be done before daemonization. Do not call Shutdown() if this function fails. * @pre Parameters should be parsed and config file should be read. */ -bool AppInitBasicSetup(const ArgsManager& args); +bool AppInitBasicSetup(const ArgsManager& args, std::atomic<int>& exit_status); /** * Initialization: parameter interaction. * @note This can be done before daemonization. Do not call Shutdown() if this function fails. diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index a3fa753a98..dd664165d3 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -6,8 +6,8 @@ #define BITCOIN_INTERFACES_CHAIN_H #include <blockfilter.h> +#include <common/settings.h> #include <primitives/transaction.h> // For CTransactionRef -#include <util/settings.h> // For util::SettingsValue #include <functional> #include <memory> @@ -87,6 +87,9 @@ struct BlockInfo { unsigned data_pos = 0; const CBlock* data = nullptr; const CBlockUndo* undo_data = nullptr; + // The maximum time in the chain up to and including this block. + // A timestamp that can only move forward. + unsigned int chain_time_max{0}; BlockInfo(const uint256& hash LIFETIMEBOUND) : hash(hash) {} }; @@ -297,17 +300,17 @@ public: virtual int rpcSerializationFlags() = 0; //! Get settings value. - virtual util::SettingsValue getSetting(const std::string& arg) = 0; + virtual common::SettingsValue getSetting(const std::string& arg) = 0; //! Get list of settings values. - virtual std::vector<util::SettingsValue> getSettingsList(const std::string& arg) = 0; + virtual std::vector<common::SettingsValue> getSettingsList(const std::string& arg) = 0; //! Return <datadir>/settings.json setting value. - virtual util::SettingsValue getRwSetting(const std::string& name) = 0; + virtual common::SettingsValue getRwSetting(const std::string& name) = 0; //! Write a setting to <datadir>/settings.json. Optionally just update the //! setting in memory and do not write the file. - virtual bool updateRwSetting(const std::string& name, const util::SettingsValue& value, bool write=true) = 0; + virtual bool updateRwSetting(const std::string& name, const common::SettingsValue& value, bool write=true) = 0; //! Synchronously send transactionAddedToMempool notifications about all //! current mempool transactions to the specified handler and return after diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 7e87d5a523..3f8df57124 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -5,13 +5,13 @@ #ifndef BITCOIN_INTERFACES_NODE_H #define BITCOIN_INTERFACES_NODE_H +#include <common/settings.h> #include <consensus/amount.h> // For CAmount #include <net.h> // For NodeId #include <net_types.h> // For banmap_t #include <netaddress.h> // For Network #include <netbase.h> // For ConnectionDirection #include <support/allocators/secure.h> // For SecureString -#include <util/settings.h> // For util::SettingsValue #include <util/translation.h> #include <functional> @@ -80,6 +80,9 @@ public: //! Get warnings. virtual bilingual_str getWarnings() = 0; + //! Get exit status. + virtual int getExitStatus() = 0; + // Get log flags. virtual uint32_t getLogCategories() = 0; @@ -103,14 +106,14 @@ public: virtual bool isSettingIgnored(const std::string& name) = 0; //! Return setting value from <datadir>/settings.json or bitcoin.conf. - virtual util::SettingsValue getPersistentSetting(const std::string& name) = 0; + virtual common::SettingsValue getPersistentSetting(const std::string& name) = 0; //! Update a setting in <datadir>/settings.json. - virtual void updateRwSetting(const std::string& name, const util::SettingsValue& value) = 0; + virtual void updateRwSetting(const std::string& name, const common::SettingsValue& value) = 0; //! Force a setting value to be applied, overriding any other configuration //! source, but not being persisted. - virtual void forceSetting(const std::string& name, const util::SettingsValue& value) = 0; + virtual void forceSetting(const std::string& name, const common::SettingsValue& value) = 0; //! Clear all settings in <datadir>/settings.json and store a backup of //! previous settings in <datadir>/settings.json.bak. diff --git a/src/ipc/interfaces.cpp b/src/ipc/interfaces.cpp index d804d9d291..e446cc98db 100644 --- a/src/ipc/interfaces.cpp +++ b/src/ipc/interfaces.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <interfaces/init.h> #include <interfaces/ipc.h> #include <ipc/capnp/protocol.h> @@ -10,7 +11,6 @@ #include <logging.h> #include <tinyformat.h> #include <util/fs.h> -#include <util/system.h> #include <cstdio> #include <cstdlib> diff --git a/src/kernel/blockmanager_opts.h b/src/kernel/blockmanager_opts.h index 9dc93b6dd2..8f26422f72 100644 --- a/src/kernel/blockmanager_opts.h +++ b/src/kernel/blockmanager_opts.h @@ -5,14 +5,26 @@ #ifndef BITCOIN_KERNEL_BLOCKMANAGER_OPTS_H #define BITCOIN_KERNEL_BLOCKMANAGER_OPTS_H +#include <util/fs.h> + +#include <cstdint> + +class CChainParams; + namespace kernel { +static constexpr bool DEFAULT_STOPAFTERBLOCKIMPORT{false}; + /** * An options struct for `BlockManager`, more ergonomically referred to as * `BlockManager::Options` due to the using-declaration in `BlockManager`. */ struct BlockManagerOpts { + const CChainParams& chainparams; uint64_t prune_target{0}; + bool fast_prune{false}; + bool stop_after_block_import{DEFAULT_STOPAFTERBLOCKIMPORT}; + const fs::path blocks_dir; }; } // namespace kernel diff --git a/src/kernel/chain.cpp b/src/kernel/chain.cpp index 82e77125d7..1c877866d0 100644 --- a/src/kernel/chain.cpp +++ b/src/kernel/chain.cpp @@ -16,6 +16,7 @@ interfaces::BlockInfo MakeBlockInfo(const CBlockIndex* index, const CBlock* data if (index) { info.prev_hash = index->pprev ? index->pprev->phashBlock : nullptr; info.height = index->nHeight; + info.chain_time_max = index->GetBlockTimeMax(); LOCK(::cs_main); info.file_number = index->nFile; info.data_pos = index->nDataPos; diff --git a/src/kernel/chainparams.cpp b/src/kernel/chainparams.cpp index 2827737ee1..d9ed1547b3 100644 --- a/src/kernel/chainparams.cpp +++ b/src/kernel/chainparams.cpp @@ -10,13 +10,13 @@ #include <consensus/merkle.h> #include <consensus/params.h> #include <hash.h> -#include <chainparamsbase.h> #include <logging.h> #include <primitives/block.h> #include <primitives/transaction.h> #include <script/interpreter.h> #include <script/script.h> #include <uint256.h> +#include <util/chaintype.h> #include <util/strencodings.h> #include <algorithm> @@ -70,7 +70,7 @@ static CBlock CreateGenesisBlock(uint32_t nTime, uint32_t nNonce, uint32_t nBits class CMainParams : public CChainParams { public: CMainParams() { - strNetworkID = CBaseChainParams::MAIN; + m_chain_type = ChainType::MAIN; consensus.signet_blocks = false; consensus.signet_challenge.clear(); consensus.nSubsidyHalvingInterval = 210000; @@ -192,7 +192,7 @@ public: class CTestNetParams : public CChainParams { public: CTestNetParams() { - strNetworkID = CBaseChainParams::TESTNET; + m_chain_type = ChainType::TESTNET; consensus.signet_blocks = false; consensus.signet_challenge.clear(); consensus.nSubsidyHalvingInterval = 210000; @@ -328,7 +328,7 @@ public: vSeeds = *options.seeds; } - strNetworkID = CBaseChainParams::SIGNET; + m_chain_type = ChainType::SIGNET; consensus.signet_blocks = true; consensus.signet_challenge.assign(bin.begin(), bin.end()); consensus.nSubsidyHalvingInterval = 210000; @@ -397,7 +397,7 @@ class CRegTestParams : public CChainParams public: explicit CRegTestParams(const RegTestOptions& opts) { - strNetworkID = CBaseChainParams::REGTEST; + m_chain_type = ChainType::REGTEST; consensus.signet_blocks = false; consensus.signet_challenge.clear(); consensus.nSubsidyHalvingInterval = 150; diff --git a/src/kernel/chainparams.h b/src/kernel/chainparams.h index 32fe618dbd..ad0b49a885 100644 --- a/src/kernel/chainparams.h +++ b/src/kernel/chainparams.h @@ -11,6 +11,7 @@ #include <primitives/block.h> #include <protocol.h> #include <uint256.h> +#include <util/chaintype.h> #include <util/hash_type.h> #include <cstdint> @@ -114,8 +115,10 @@ public: uint64_t AssumedChainStateSize() const { return m_assumed_chain_state_size; } /** Whether it is possible to mine blocks on demand (no retargeting) */ bool MineBlocksOnDemand() const { return consensus.fPowNoRetargeting; } - /** Return the network string */ - std::string NetworkIDString() const { return strNetworkID; } + /** Return the chain type string */ + std::string GetChainTypeString() const { return ChainTypeToString(m_chain_type); } + /** Return the chain type */ + ChainType GetChainType() const { return m_chain_type; } /** Return the list of hostnames to look up for DNS seeds */ const std::vector<std::string>& DNSSeeds() const { return vSeeds; } const std::vector<unsigned char>& Base58Prefix(Base58Type type) const { return base58Prefixes[type]; } @@ -172,7 +175,7 @@ protected: std::vector<std::string> vSeeds; std::vector<unsigned char> base58Prefixes[MAX_BASE58_TYPES]; std::string bech32_hrp; - std::string strNetworkID; + ChainType m_chain_type; CBlock genesis; std::vector<uint8_t> vFixedSeeds; bool fDefaultConsistencyChecks; diff --git a/src/kernel/chainstatemanager_opts.h b/src/kernel/chainstatemanager_opts.h index 2395f60164..035a913d10 100644 --- a/src/kernel/chainstatemanager_opts.h +++ b/src/kernel/chainstatemanager_opts.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_KERNEL_CHAINSTATEMANAGER_OPTS_H #define BITCOIN_KERNEL_CHAINSTATEMANAGER_OPTS_H +#include <kernel/notifications_interface.h> + #include <arith_uint256.h> #include <dbwrapper.h> #include <txdb.h> @@ -19,6 +21,7 @@ class CChainParams; static constexpr bool DEFAULT_CHECKPOINTS_ENABLED{true}; static constexpr auto DEFAULT_MAX_TIP_AGE{24h}; +static constexpr int DEFAULT_STOPATHEIGHT{0}; namespace kernel { @@ -42,6 +45,8 @@ struct ChainstateManagerOpts { DBOptions block_tree_db{}; DBOptions coins_db{}; CoinsViewOptions coins_view{}; + Notifications& notifications; + int stop_at_height{DEFAULT_STOPATHEIGHT}; }; } // namespace kernel diff --git a/src/kernel/checks.cpp b/src/kernel/checks.cpp index 4c303c172c..bf8a2ec74c 100644 --- a/src/kernel/checks.cpp +++ b/src/kernel/checks.cpp @@ -13,21 +13,21 @@ namespace kernel { -std::optional<bilingual_str> SanityChecks(const Context&) +util::Result<void> SanityChecks(const Context&) { if (!ECC_InitSanityCheck()) { - return Untranslated("Elliptic curve cryptography sanity check failure. Aborting."); + return util::Error{Untranslated("Elliptic curve cryptography sanity check failure. Aborting.")}; } if (!Random_SanityCheck()) { - return Untranslated("OS cryptographic RNG sanity check failure. Aborting."); + return util::Error{Untranslated("OS cryptographic RNG sanity check failure. Aborting.")}; } if (!ChronoSanityCheck()) { - return Untranslated("Clock epoch mismatch. Aborting."); + return util::Error{Untranslated("Clock epoch mismatch. Aborting.")}; } - return std::nullopt; + return {}; } } diff --git a/src/kernel/checks.h b/src/kernel/checks.h index 3eb14824fb..fd8c167015 100644 --- a/src/kernel/checks.h +++ b/src/kernel/checks.h @@ -5,9 +5,7 @@ #ifndef BITCOIN_KERNEL_CHECKS_H #define BITCOIN_KERNEL_CHECKS_H -#include <optional> - -struct bilingual_str; +#include <util/result.h> namespace kernel { @@ -16,8 +14,7 @@ struct Context; /** * Ensure a usable environment with all necessary library support. */ -std::optional<bilingual_str> SanityChecks(const Context&); - -} +[[nodiscard]] util::Result<void> SanityChecks(const Context&); +} // namespace kernel #endif // BITCOIN_KERNEL_CHECKS_H diff --git a/src/kernel/coinstats.cpp b/src/kernel/coinstats.cpp index 4b75c387a6..527433f45e 100644 --- a/src/kernel/coinstats.cpp +++ b/src/kernel/coinstats.cpp @@ -123,7 +123,7 @@ static bool ComputeUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj, c uint256 prevkey; std::map<uint32_t, Coin> outputs; while (pcursor->Valid()) { - interruption_point(); + if (interruption_point) interruption_point(); COutPoint key; Coin coin; if (pcursor->GetKey(key) && pcursor->GetValue(coin)) { diff --git a/src/kernel/mempool_entry.h b/src/kernel/mempool_entry.h index e1ba4296ef..969ddcd1ce 100644 --- a/src/kernel/mempool_entry.h +++ b/src/kernel/mempool_entry.h @@ -75,7 +75,7 @@ private: mutable Parents m_parents; mutable Children m_children; const CAmount nFee; //!< Cached to avoid expensive parent-transaction lookups - const size_t nTxWeight; //!< ... and avoid recomputing tx weight (also used for GetTxSize()) + const int32_t nTxWeight; //!< ... and avoid recomputing tx weight (also used for GetTxSize()) const size_t nUsageSize; //!< ... and total memory usage const int64_t nTime; //!< Local time when entering the mempool const unsigned int entryHeight; //!< Chain height when entering the mempool @@ -88,12 +88,14 @@ private: // mempool; if we remove this transaction we must remove all of these // descendants as well. uint64_t nCountWithDescendants{1}; //!< number of descendant transactions - uint64_t nSizeWithDescendants; //!< ... and size + // Using int64_t instead of int32_t to avoid signed integer overflow issues. + int64_t nSizeWithDescendants; //!< ... and size CAmount nModFeesWithDescendants; //!< ... and total fees (all including us) // Analogous statistics for ancestor transactions uint64_t nCountWithAncestors{1}; - uint64_t nSizeWithAncestors; + // Using int64_t instead of int32_t to avoid signed integer overflow issues. + int64_t nSizeWithAncestors; CAmount nModFeesWithAncestors; int64_t nSigOpCostWithAncestors; @@ -104,7 +106,7 @@ public: int64_t sigops_cost, LockPoints lp) : tx{tx}, nFee{fee}, - nTxWeight(GetTransactionWeight(*tx)), + nTxWeight{GetTransactionWeight(*tx)}, nUsageSize{RecursiveDynamicUsage(tx)}, nTime{time}, entryHeight{entry_height}, @@ -121,11 +123,11 @@ public: const CTransaction& GetTx() const { return *this->tx; } CTransactionRef GetSharedTx() const { return this->tx; } const CAmount& GetFee() const { return nFee; } - size_t GetTxSize() const + int32_t GetTxSize() const { return GetVirtualTransactionSize(nTxWeight, sigOpCost, ::nBytesPerSigOp); } - size_t GetTxWeight() const { return nTxWeight; } + int32_t GetTxWeight() const { return nTxWeight; } std::chrono::seconds GetTime() const { return std::chrono::seconds{nTime}; } unsigned int GetHeight() const { return entryHeight; } int64_t GetSigOpCost() const { return sigOpCost; } @@ -134,9 +136,9 @@ public: const LockPoints& GetLockPoints() const { return lockPoints; } // Adjusts the descendant state. - void UpdateDescendantState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount); + void UpdateDescendantState(int32_t modifySize, CAmount modifyFee, int64_t modifyCount); // Adjusts the ancestor state - void UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps); + void UpdateAncestorState(int32_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps); // Updates the modified fees with descendants/ancestors. void UpdateModifiedFee(CAmount fee_diff) { @@ -152,13 +154,13 @@ public: } uint64_t GetCountWithDescendants() const { return nCountWithDescendants; } - uint64_t GetSizeWithDescendants() const { return nSizeWithDescendants; } + int64_t GetSizeWithDescendants() const { return nSizeWithDescendants; } CAmount GetModFeesWithDescendants() const { return nModFeesWithDescendants; } bool GetSpendsCoinbase() const { return spendsCoinbase; } uint64_t GetCountWithAncestors() const { return nCountWithAncestors; } - uint64_t GetSizeWithAncestors() const { return nSizeWithAncestors; } + int64_t GetSizeWithAncestors() const { return nSizeWithAncestors; } CAmount GetModFeesWithAncestors() const { return nModFeesWithAncestors; } int64_t GetSigOpCostWithAncestors() const { return nSigOpCostWithAncestors; } diff --git a/src/kernel/notifications_interface.h b/src/kernel/notifications_interface.h new file mode 100644 index 0000000000..48248e9aa0 --- /dev/null +++ b/src/kernel/notifications_interface.h @@ -0,0 +1,33 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_KERNEL_NOTIFICATIONS_INTERFACE_H +#define BITCOIN_KERNEL_NOTIFICATIONS_INTERFACE_H + +#include <cstdint> +#include <string> + +class CBlockIndex; +enum class SynchronizationState; +struct bilingual_str; + +namespace kernel { + +/** + * A base class defining functions for notifying about certain kernel + * events. + */ +class Notifications +{ +public: + virtual ~Notifications(){}; + + virtual void blockTip(SynchronizationState state, CBlockIndex& index) {} + virtual void headerTip(SynchronizationState state, int64_t height, int64_t timestamp, bool presync) {} + virtual void progress(const bilingual_str& title, int progress_percent, bool resume_possible) {} + virtual void warning(const bilingual_str& warning) {} +}; +} // namespace kernel + +#endif // BITCOIN_KERNEL_NOTIFICATIONS_INTERFACE_H diff --git a/src/key_io.cpp b/src/key_io.cpp index 4659a59544..454a96df5e 100644 --- a/src/key_io.cpp +++ b/src/key_io.cpp @@ -124,7 +124,11 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par data.clear(); const auto dec = bech32::Decode(str); - if ((dec.encoding == bech32::Encoding::BECH32 || dec.encoding == bech32::Encoding::BECH32M) && dec.data.size() > 0) { + if (dec.encoding == bech32::Encoding::BECH32 || dec.encoding == bech32::Encoding::BECH32M) { + if (dec.data.empty()) { + error_str = "Empty Bech32 data section"; + return CNoDestination(); + } // Bech32 decoding if (dec.hrp != params.Bech32HRP()) { error_str = strprintf("Invalid or unsupported prefix for Segwit (Bech32) address (expected %s, got %s).", params.Bech32HRP(), dec.hrp); @@ -142,6 +146,9 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par // The rest of the symbols are converted witness program bytes. data.reserve(((dec.data.size() - 1) * 5) / 8); if (ConvertBits<5, 8, false>([&](unsigned char c) { data.push_back(c); }, dec.data.begin() + 1, dec.data.end())) { + + std::string_view byte_str{data.size() == 1 ? "byte" : "bytes"}; + if (version == 0) { { WitnessV0KeyHash keyid; @@ -158,7 +165,7 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par } } - error_str = "Invalid Bech32 v0 address data size"; + error_str = strprintf("Invalid Bech32 v0 address program size (%d %s), per BIP141", data.size(), byte_str); return CNoDestination(); } @@ -175,7 +182,7 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par } if (data.size() < 2 || data.size() > BECH32_WITNESS_PROG_MAX_LEN) { - error_str = "Invalid Bech32 address data size"; + error_str = strprintf("Invalid Bech32 address program size (%d %s)", data.size(), byte_str); return CNoDestination(); } @@ -184,6 +191,9 @@ CTxDestination DecodeDestination(const std::string& str, const CChainParams& par std::copy(data.begin(), data.end(), unk.program); unk.length = data.size(); return unk; + } else { + error_str = strprintf("Invalid padding in Bech32 data section"); + return CNoDestination(); } } diff --git a/src/mapport.cpp b/src/mapport.cpp index 994fd12cf5..118827901a 100644 --- a/src/mapport.cpp +++ b/src/mapport.cpp @@ -9,12 +9,12 @@ #include <mapport.h> #include <clientversion.h> +#include <common/system.h> #include <logging.h> #include <net.h> #include <netaddress.h> #include <netbase.h> #include <util/syscall_sandbox.h> -#include <util/system.h> #include <util/thread.h> #include <util/threadinterrupt.h> @@ -175,10 +175,10 @@ static bool ProcessUpnp() LogPrintf("UPnP: GetExternalIPAddress() returned %d\n", r); } else { if (externalIPAddress[0]) { - CNetAddr resolved; - if (LookupHost(externalIPAddress, resolved, false)) { - LogPrintf("UPnP: ExternalIPAddress = %s\n", resolved.ToStringAddr()); - AddLocal(resolved, LOCAL_MAPPED); + std::optional<CNetAddr> resolved{LookupHost(externalIPAddress, false)}; + if (resolved.has_value()) { + LogPrintf("UPnP: ExternalIPAddress = %s\n", resolved->ToStringAddr()); + AddLocal(resolved.value(), LOCAL_MAPPED); } } else { LogPrintf("UPnP: GetExternalIPAddress failed.\n"); diff --git a/src/net.cpp b/src/net.cpp index 337cf60680..282c8fb741 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -130,14 +130,10 @@ uint16_t GetListenPort() { // If -bind= is provided with ":port" part, use that (first one if multiple are provided). for (const std::string& bind_arg : gArgs.GetArgs("-bind")) { - CService bind_addr; constexpr uint16_t dummy_port = 0; - if (Lookup(bind_arg, bind_addr, dummy_port, /*fAllowLookup=*/false)) { - if (bind_addr.GetPort() != dummy_port) { - return bind_addr.GetPort(); - } - } + const std::optional<CService> bind_addr{Lookup(bind_arg, dummy_port, /*fAllowLookup=*/false)}; + if (bind_addr.has_value() && bind_addr->GetPort() != dummy_port) return bind_addr->GetPort(); } // Otherwise, if -whitebind= without NetPermissionFlags::NoBan is provided, use that @@ -461,9 +457,9 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo const uint16_t default_port{pszDest != nullptr ? Params().GetDefaultPort(pszDest) : Params().GetDefaultPort()}; if (pszDest) { - std::vector<CService> resolved; - if (Lookup(pszDest, resolved, default_port, fNameLookup && !HaveNameProxy(), 256) && !resolved.empty()) { - const CService rnd{resolved[GetRand(resolved.size())]}; + const std::vector<CService> resolved{Lookup(pszDest, default_port, fNameLookup && !HaveNameProxy(), 256)}; + if (!resolved.empty()) { + const CService& rnd{resolved[GetRand(resolved.size())]}; addrConnect = CAddress{MaybeFlipIPv6toCJDNS(rnd), NODE_NONE}; if (!addrConnect.IsValid()) { LogPrint(BCLog::NET, "Resolver returned invalid address %s for %s\n", addrConnect.ToStringAddrPort(), pszDest); @@ -1487,7 +1483,6 @@ void CConnman::ThreadDNSAddressSeed() if (HaveNameProxy()) { AddAddrFetch(seed); } else { - std::vector<CNetAddr> vIPs; std::vector<CAddress> vAdd; ServiceFlags requiredServiceBits = GetDesirableServiceFlags(NODE_NONE); std::string host = strprintf("x%x.%s", requiredServiceBits, seed); @@ -1496,8 +1491,9 @@ void CConnman::ThreadDNSAddressSeed() continue; } unsigned int nMaxIPs = 256; // Limits number of IPs learned from a DNS seed - if (LookupHost(host, vIPs, nMaxIPs, true)) { - for (const CNetAddr& ip : vIPs) { + const auto addresses{LookupHost(host, nMaxIPs, true)}; + if (!addresses.empty()) { + for (const CNetAddr& ip : addresses) { CAddress addr = CAddress(CService(ip, Params().GetDefaultPort()), requiredServiceBits); addr.nTime = rng.rand_uniform_delay(Now<NodeSeconds>() - 3 * 24h, -4 * 24h); // use a random age between 3 and 7 days old vAdd.push_back(addr); @@ -1707,7 +1703,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) int nOutboundFullRelay = 0; int nOutboundBlockRelay = 0; int outbound_privacy_network_peers = 0; - std::set<std::vector<unsigned char>> setConnected; // netgroups of our ipv4/ipv6 outbound peers + std::set<std::vector<unsigned char>> outbound_ipv46_peer_netgroups; { LOCK(m_nodes_mutex); @@ -1729,7 +1725,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) case ConnectionType::MANUAL: case ConnectionType::OUTBOUND_FULL_RELAY: case ConnectionType::BLOCK_RELAY: - CAddress address{pnode->addr}; + const CAddress address{pnode->addr}; if (address.IsTor() || address.IsI2P() || address.IsCJDNS()) { // Since our addrman-groups for these networks are // random, without relation to the route we @@ -1740,7 +1736,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // these networks. ++outbound_privacy_network_peers; } else { - setConnected.insert(m_netgroupman.GetGroup(address)); + outbound_ipv46_peer_netgroups.insert(m_netgroupman.GetGroup(address)); } } // no default case, so the compiler can warn about missing cases } @@ -1815,7 +1811,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) m_anchors.pop_back(); if (!addr.IsValid() || IsLocal(addr) || !IsReachable(addr) || !HasAllDesirableServiceFlags(addr.nServices) || - setConnected.count(m_netgroupman.GetGroup(addr))) continue; + outbound_ipv46_peer_netgroups.count(m_netgroupman.GetGroup(addr))) continue; addrConnect = addr; LogPrint(BCLog::NET, "Trying to make an anchor connection to %s\n", addrConnect.ToStringAddrPort()); break; @@ -1855,8 +1851,8 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) std::tie(addr, addr_last_try) = addrman.Select(); } - // Require outbound connections, other than feelers, to be to distinct network groups - if (!fFeeler && setConnected.count(m_netgroupman.GetGroup(addr))) { + // Require outbound IPv4/IPv6 connections, other than feelers, to be to distinct network groups + if (!fFeeler && outbound_ipv46_peer_netgroups.count(m_netgroupman.GetGroup(addr))) { break; } @@ -1902,8 +1898,9 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // Record addrman failure attempts when node has at least 2 persistent outbound connections to peers with // different netgroups in ipv4/ipv6 networks + all peers in Tor/I2P/CJDNS networks. // Don't record addrman failure attempts when node is offline. This can be identified since all local - // network connections(if any) belong in the same netgroup and size of setConnected would only be 1. - OpenNetworkConnection(addrConnect, (int)setConnected.size() + outbound_privacy_network_peers >= std::min(nMaxConnections - 1, 2), &grant, nullptr, conn_type); + // network connections (if any) belong in the same netgroup, and the size of `outbound_ipv46_peer_netgroups` would only be 1. + const bool count_failures{((int)outbound_ipv46_peer_netgroups.size() + outbound_privacy_network_peers) >= std::min(nMaxConnections - 1, 2)}; + OpenNetworkConnection(addrConnect, count_failures, &grant, /*strDest=*/nullptr, conn_type); } } } @@ -2201,14 +2198,11 @@ void Discover() char pszHostName[256] = ""; if (gethostname(pszHostName, sizeof(pszHostName)) != SOCKET_ERROR) { - std::vector<CNetAddr> vaddr; - if (LookupHost(pszHostName, vaddr, 0, true)) + const std::vector<CNetAddr> addresses{LookupHost(pszHostName, 0, true)}; + for (const CNetAddr& addr : addresses) { - for (const CNetAddr &addr : vaddr) - { - if (AddLocal(addr, LOCAL_IF)) - LogPrintf("%s: %s - %s\n", __func__, pszHostName, addr.ToStringAddr()); - } + if (AddLocal(addr, LOCAL_IF)) + LogPrintf("%s: %s - %s\n", __func__, pszHostName, addr.ToStringAddr()); } } #elif (HAVE_DECL_GETIFADDRS && HAVE_DECL_FREEIFADDRS) @@ -200,7 +200,9 @@ public: int nVersion; std::string cleanSubVer; bool fInbound; + // We requested high bandwidth connection to peer bool m_bip152_highbandwidth_to; + // Peer requested high bandwidth connection bool m_bip152_highbandwidth_from; int m_starting_height; uint64_t nSendBytes; diff --git a/src/net_permissions.cpp b/src/net_permissions.cpp index f829e56aa2..23226bbb4f 100644 --- a/src/net_permissions.cpp +++ b/src/net_permissions.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <net_permissions.h> #include <netbase.h> #include <util/error.h> -#include <util/system.h> #include <util/translation.h> const std::vector<std::string> NET_PERMISSIONS_DOC{ @@ -88,18 +88,18 @@ bool NetWhitebindPermissions::TryParse(const std::string& str, NetWhitebindPermi if (!TryParsePermissionFlags(str, flags, offset, error)) return false; const std::string strBind = str.substr(offset); - CService addrBind; - if (!Lookup(strBind, addrBind, 0, false)) { + const std::optional<CService> addrBind{Lookup(strBind, 0, false)}; + if (!addrBind.has_value()) { error = ResolveErrMsg("whitebind", strBind); return false; } - if (addrBind.GetPort() == 0) { + if (addrBind.value().GetPort() == 0) { error = strprintf(_("Need to specify a port with -whitebind: '%s'"), strBind); return false; } output.m_flags = flags; - output.m_service = addrBind; + output.m_service = addrBind.value(); error = Untranslated(""); return true; } diff --git a/src/net_processing.cpp b/src/net_processing.cpp index c50aa2e4f9..6597019797 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -51,12 +51,7 @@ #include <optional> #include <typeinfo> -using node::ReadBlockFromDisk; -using node::ReadRawBlockFromDisk; - -/** How long to cache transactions in mapRelay for normal relay */ -static constexpr auto RELAY_TX_CACHE_TIME = 15min; -/** How long a transaction has to be in the mempool before it can unconditionally be relayed (even when not in mapRelay). */ +/** How long a transaction has to be in the mempool before it can unconditionally be relayed. */ static constexpr auto UNCONDITIONAL_RELAY_DELAY = 2min; /** Headers download timeout. * Timeout = base + per_header * (expected number of headers) */ @@ -292,7 +287,8 @@ struct Peer { * this does not have to be sorted. */ std::set<uint256> m_tx_inventory_to_send GUARDED_BY(m_tx_inventory_mutex); /** Whether the peer has requested us to send our complete mempool. Only - * permitted if the peer has NetPermissionFlags::Mempool. See BIP35. */ + * permitted if the peer has NetPermissionFlags::Mempool or we advertise + * NODE_BLOOM. See BIP35. */ bool m_send_mempool GUARDED_BY(m_tx_inventory_mutex){false}; /** The last time a BIP35 `mempool` request was serviced. */ std::atomic<std::chrono::seconds> m_last_mempool_req{0s}; @@ -433,7 +429,6 @@ struct CNodeState { std::list<QueuedBlock> vBlocksInFlight; //! When the first entry in vBlocksInFlight started downloading. Don't care when vBlocksInFlight is empty. std::chrono::microseconds m_downloading_since{0us}; - int nBlocksInFlight{0}; //! Whether we consider this a preferred download peer. bool fPreferredDownload{false}; /** Whether this peer wants invs or cmpctblocks (when possible) for block announcements. */ @@ -854,6 +849,7 @@ private: std::shared_ptr<const CBlock> m_most_recent_block GUARDED_BY(m_most_recent_block_mutex); std::shared_ptr<const CBlockHeaderAndShortTxIDs> m_most_recent_compact_block GUARDED_BY(m_most_recent_block_mutex); uint256 m_most_recent_block_hash GUARDED_BY(m_most_recent_block_mutex); + std::unique_ptr<const std::map<uint256, CTransactionRef>> m_most_recent_block_txs GUARDED_BY(m_most_recent_block_mutex); // Data about the low-work headers synchronization, aggregated from all peers' HeadersSyncStates. /** Mutex guarding the other m_headers_presync_* variables. */ @@ -879,11 +875,17 @@ private: /** Have we requested this block from a peer */ bool IsBlockRequested(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /** Have we requested this block from an outbound peer */ + bool IsBlockRequestedFromOutbound(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + /** Remove this block from our tracked requested blocks. Called if: * - the block has been received from a peer * - the request for the block has timed out + * If "from_peer" is specified, then only remove the block if it is in + * flight from that peer (to avoid one peer's network traffic from + * affecting another's state). */ - void RemoveBlockRequest(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void RemoveBlockRequest(const uint256& hash, std::optional<NodeId> from_peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /* Mark a block as in flight * Returns false, still setting pit, if the block was already in flight from the same peer @@ -898,14 +900,16 @@ private: */ void FindNextBlocksToDownload(const Peer& peer, unsigned int count, std::vector<const CBlockIndex*>& vBlocks, NodeId& nodeStaller) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> > mapBlocksInFlight GUARDED_BY(cs_main); + /* Multimap used to preserve insertion order */ + typedef std::multimap<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator>> BlockDownloadMap; + BlockDownloadMap mapBlocksInFlight GUARDED_BY(cs_main); /** When our tip was last updated. */ std::atomic<std::chrono::seconds> m_last_tip_update{0s}; /** Determine whether or not a peer can request a transaction, and return it (or nullptr if not found or not allowed). */ CTransactionRef FindTxForGetData(const Peer::TxRelay& tx_relay, const GenTxid& gtxid, const std::chrono::seconds mempool_req, const std::chrono::seconds now) - EXCLUSIVE_LOCKS_REQUIRED(NetEventsInterface::g_msgproc_mutex); + EXCLUSIVE_LOCKS_REQUIRED(!m_most_recent_block_mutex, NetEventsInterface::g_msgproc_mutex); void ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic<bool>& interruptMsgProc) EXCLUSIVE_LOCKS_REQUIRED(!m_most_recent_block_mutex, peer.m_getdata_requests_mutex, NetEventsInterface::g_msgproc_mutex) @@ -914,12 +918,6 @@ private: /** Process a new block. Perform any post-processing housekeeping */ void ProcessBlock(CNode& node, const std::shared_ptr<const CBlock>& block, bool force_processing, bool min_pow_checked); - /** Relay map (txid or wtxid -> CTransactionRef) */ - typedef std::map<uint256, CTransactionRef> MapRelay; - MapRelay mapRelay GUARDED_BY(NetEventsInterface::g_msgproc_mutex); - /** Expiration-time ordered list of (expire time, relay map entry) pairs. */ - std::deque<std::pair<std::chrono::microseconds, MapRelay::iterator>> g_relay_expiration GUARDED_BY(NetEventsInterface::g_msgproc_mutex); - /** * When a peer sends us a valid block, instruct it to announce blocks to us * using CMPCTBLOCK if possible by adding its nodeid to the end of @@ -1116,34 +1114,55 @@ std::chrono::microseconds PeerManagerImpl::NextInvToInbounds(std::chrono::micros bool PeerManagerImpl::IsBlockRequested(const uint256& hash) { - return mapBlocksInFlight.find(hash) != mapBlocksInFlight.end(); + return mapBlocksInFlight.count(hash); } -void PeerManagerImpl::RemoveBlockRequest(const uint256& hash) +bool PeerManagerImpl::IsBlockRequestedFromOutbound(const uint256& hash) { - auto it = mapBlocksInFlight.find(hash); - if (it == mapBlocksInFlight.end()) { - // Block was not requested - return; + for (auto range = mapBlocksInFlight.equal_range(hash); range.first != range.second; range.first++) { + auto [nodeid, block_it] = range.first->second; + CNodeState& nodestate = *Assert(State(nodeid)); + if (!nodestate.m_is_inbound) return true; } - auto [node_id, list_it] = it->second; - CNodeState *state = State(node_id); - assert(state != nullptr); + return false; +} - if (state->vBlocksInFlight.begin() == list_it) { - // First block on the queue was received, update the start download time for the next one - state->m_downloading_since = std::max(state->m_downloading_since, GetTime<std::chrono::microseconds>()); +void PeerManagerImpl::RemoveBlockRequest(const uint256& hash, std::optional<NodeId> from_peer) +{ + auto range = mapBlocksInFlight.equal_range(hash); + if (range.first == range.second) { + // Block was not requested from any peer + return; } - state->vBlocksInFlight.erase(list_it); - state->nBlocksInFlight--; - if (state->nBlocksInFlight == 0) { - // Last validated block on the queue was received. - m_peers_downloading_from--; + // We should not have requested too many of this block + Assume(mapBlocksInFlight.count(hash) <= MAX_CMPCTBLOCKS_INFLIGHT_PER_BLOCK); + + while (range.first != range.second) { + auto [node_id, list_it] = range.first->second; + + if (from_peer && *from_peer != node_id) { + range.first++; + continue; + } + + CNodeState& state = *Assert(State(node_id)); + + if (state.vBlocksInFlight.begin() == list_it) { + // First block on the queue was received, update the start download time for the next one + state.m_downloading_since = std::max(state.m_downloading_since, GetTime<std::chrono::microseconds>()); + } + state.vBlocksInFlight.erase(list_it); + + if (state.vBlocksInFlight.empty()) { + // Last validated block on the queue for this peer was received. + m_peers_downloading_from--; + } + state.m_stalling_since = 0us; + + range.first = mapBlocksInFlight.erase(range.first); } - state->m_stalling_since = 0us; - mapBlocksInFlight.erase(it); } bool PeerManagerImpl::BlockRequested(NodeId nodeid, const CBlockIndex& block, std::list<QueuedBlock>::iterator** pit) @@ -1153,27 +1172,29 @@ bool PeerManagerImpl::BlockRequested(NodeId nodeid, const CBlockIndex& block, st CNodeState *state = State(nodeid); assert(state != nullptr); + Assume(mapBlocksInFlight.count(hash) <= MAX_CMPCTBLOCKS_INFLIGHT_PER_BLOCK); + // Short-circuit most stuff in case it is from the same node - std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); - if (itInFlight != mapBlocksInFlight.end() && itInFlight->second.first == nodeid) { - if (pit) { - *pit = &itInFlight->second.second; + for (auto range = mapBlocksInFlight.equal_range(hash); range.first != range.second; range.first++) { + if (range.first->second.first == nodeid) { + if (pit) { + *pit = &range.first->second.second; + } + return false; } - return false; } - // Make sure it's not listed somewhere already. - RemoveBlockRequest(hash); + // Make sure it's not being fetched already from same peer. + RemoveBlockRequest(hash, nodeid); std::list<QueuedBlock>::iterator it = state->vBlocksInFlight.insert(state->vBlocksInFlight.end(), {&block, std::unique_ptr<PartiallyDownloadedBlock>(pit ? new PartiallyDownloadedBlock(&m_mempool) : nullptr)}); - state->nBlocksInFlight++; - if (state->nBlocksInFlight == 1) { + if (state->vBlocksInFlight.size() == 1) { // We're starting a block download (batch) from this peer. state->m_downloading_since = GetTime<std::chrono::microseconds>(); m_peers_downloading_from++; } - itInFlight = mapBlocksInFlight.insert(std::make_pair(hash, std::make_pair(nodeid, it))).first; + auto itInFlight = mapBlocksInFlight.insert(std::make_pair(hash, std::make_pair(nodeid, it))); if (pit) { *pit = &itInFlight->second.second; } @@ -1376,7 +1397,7 @@ void PeerManagerImpl::FindNextBlocksToDownload(const Peer& peer, unsigned int co } } else if (waitingfor == -1) { // This is the first already-in-flight block. - waitingfor = mapBlocksInFlight[pindex->GetBlockHash()].first; + waitingfor = mapBlocksInFlight.lower_bound(pindex->GetBlockHash())->second.first; } } } @@ -1506,13 +1527,21 @@ void PeerManagerImpl::FinalizeNode(const CNode& node) nSyncStarted--; for (const QueuedBlock& entry : state->vBlocksInFlight) { - mapBlocksInFlight.erase(entry.pindex->GetBlockHash()); + auto range = mapBlocksInFlight.equal_range(entry.pindex->GetBlockHash()); + while (range.first != range.second) { + auto [node_id, list_it] = range.first->second; + if (node_id != nodeid) { + range.first++; + } else { + range.first = mapBlocksInFlight.erase(range.first); + } + } } m_orphanage.EraseForPeer(nodeid); m_txrequest.DisconnectedPeer(nodeid); if (m_txreconciliation) m_txreconciliation->ForgetPeer(nodeid); m_num_preferred_download_peers -= state->fPreferredDownload; - m_peers_downloading_from -= (state->nBlocksInFlight != 0); + m_peers_downloading_from -= (!state->vBlocksInFlight.empty()); assert(m_peers_downloading_from >= 0); m_outbound_peers_with_protect_from_disconnect -= state->m_chain_sync.m_protect; assert(m_outbound_peers_with_protect_from_disconnect >= 0); @@ -1753,11 +1782,10 @@ std::optional<std::string> PeerManagerImpl::FetchBlock(NodeId peer_id, const CBl LOCK(cs_main); - // Mark block as in-flight unless it already is (for this peer). - // If the peer does not send us a block, vBlocksInFlight remains non-empty, - // causing us to timeout and disconnect. - // If a block was already in-flight for a different peer, its BLOCKTXN - // response will be dropped. + // Forget about all prior requests + RemoveBlockRequest(block_index.GetBlockHash(), std::nullopt); + + // Mark block as in-flight if (!BlockRequested(peer_id, block_index)) return "Already requested from this peer"; // Construct message to request the block @@ -1892,10 +1920,17 @@ void PeerManagerImpl::NewPoWValidBlock(const CBlockIndex *pindex, const std::sha std::async(std::launch::deferred, [&] { return msgMaker.Make(NetMsgType::CMPCTBLOCK, *pcmpctblock); })}; { + auto most_recent_block_txs = std::make_unique<std::map<uint256, CTransactionRef>>(); + for (const auto& tx : pblock->vtx) { + most_recent_block_txs->emplace(tx->GetHash(), tx); + most_recent_block_txs->emplace(tx->GetWitnessHash(), tx); + } + LOCK(m_most_recent_block_mutex); m_most_recent_block_hash = hashBlock; m_most_recent_block = pblock; m_most_recent_compact_block = pcmpctblock; + m_most_recent_block_txs = std::move(most_recent_block_txs); } m_connman.ForEachNode([this, pindex, &lazy_ser, &hashBlock](CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { @@ -2179,7 +2214,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& // Fast-path: in this case it is possible to serve the block directly from disk, // as the network format matches the format on disk std::vector<uint8_t> block_data; - if (!ReadRawBlockFromDisk(block_data, pindex->GetBlockPos(), m_chainparams.MessageStart())) { + if (!m_chainman.m_blockman.ReadRawBlockFromDisk(block_data, pindex->GetBlockPos(), m_chainparams.MessageStart())) { assert(!"cannot load block from disk"); } m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, Span{block_data})); @@ -2187,7 +2222,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& } else { // Send block from disk std::shared_ptr<CBlock> pblockRead = std::make_shared<CBlock>(); - if (!ReadBlockFromDisk(*pblockRead, pindex, m_chainparams.GetConsensus())) { + if (!m_chainman.m_blockman.ReadBlockFromDisk(*pblockRead, *pindex)) { assert(!"cannot load block from disk"); } pblock = pblockRead; @@ -2266,13 +2301,17 @@ CTransactionRef PeerManagerImpl::FindTxForGetData(const Peer::TxRelay& tx_relay, } } - // Otherwise, the transaction must have been announced recently. - if (tx_relay.m_recently_announced_invs.contains(gtxid.GetHash())) { - // If it was, it can be relayed from either the mempool... - if (txinfo.tx) return std::move(txinfo.tx); - // ... or the relay pool. - auto mi = mapRelay.find(gtxid.GetHash()); - if (mi != mapRelay.end()) return mi->second; + // Otherwise, the transaction might have been announced recently. + bool recent = tx_relay.m_recently_announced_invs.contains(gtxid.GetHash()); + if (recent && txinfo.tx) return std::move(txinfo.tx); + + // Or it might be from the most recent block + { + LOCK(m_most_recent_block_mutex); + if (m_most_recent_block_txs != nullptr) { + auto it = m_most_recent_block_txs->find(gtxid.GetHash()); + if (it != m_most_recent_block_txs->end()) return it->second; + } } return {}; @@ -2673,7 +2712,7 @@ void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const Peer& peer, c std::vector<CInv> vGetData; // Download as much as possible, from earliest to latest. for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { - if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (nodestate->vBlocksInFlight.size() >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { // Can't download any more from this peer break; } @@ -3154,6 +3193,11 @@ void PeerManagerImpl::ProcessBlock(CNode& node, const std::shared_ptr<const CBlo m_chainman.ProcessNewBlock(block, force_processing, min_pow_checked, &new_block); if (new_block) { node.m_last_block_time = GetTime<std::chrono::seconds>(); + // In case this block came from a different peer than we requested + // from, we can erase the block request now anyway (as we just stored + // this block to disk). + LOCK(cs_main); + RemoveBlockRequest(block->GetHash(), std::nullopt); } else { LOCK(cs_main); mapBlockSource.erase(block->GetHash()); @@ -3378,8 +3422,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // If the peer is old enough to have the old alert system, send it the final alert. if (greatest_common_version <= 70012) { - DataStream finalAlert{ParseHex("60010000000000000000000000ffffff7f00000000ffffff7ffeffff7f01ffffff7f00000000ffffff7f00ffffff7f002f555247454e543a20416c657274206b657920636f6d70726f6d697365642c2075706772616465207265717569726564004630440220653febd6410f470f6bae11cad19c48413becb1ac2c17f908fd0fd53bdc3abd5202206d0e9c96fe88d4a0f01ed9dedae2b6f9e00da94cad0fecaae66ecf689bf71b50")}; - m_connman.PushMessage(&pfrom, CNetMsgMaker(greatest_common_version).Make("alert", finalAlert)); + const auto finalAlert{ParseHex("60010000000000000000000000ffffff7f00000000ffffff7ffeffff7f01ffffff7f00000000ffffff7f00ffffff7f002f555247454e543a20416c657274206b657920636f6d70726f6d697365642c2075706772616465207265717569726564004630440220653febd6410f470f6bae11cad19c48413becb1ac2c17f908fd0fd53bdc3abd5202206d0e9c96fe88d4a0f01ed9dedae2b6f9e00da94cad0fecaae66ecf689bf71b50")}; + m_connman.PushMessage(&pfrom, CNetMsgMaker(greatest_common_version).Make("alert", Span{finalAlert})); } // Feeler connections exist only to verify if address is online. @@ -3874,7 +3918,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (pindex->nHeight >= m_chainman.ActiveChain().Height() - MAX_BLOCKTXN_DEPTH) { CBlock block; - bool ret = ReadBlockFromDisk(block, pindex, m_chainparams.GetConsensus()); + const bool ret{m_chainman.m_blockman.ReadBlockFromDisk(block, *pindex)}; assert(ret); SendBlockTransactions(pfrom, *peer, block, req); @@ -4262,15 +4306,27 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, nodestate->m_last_block_announcement = GetTime(); } - std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> >::iterator blockInFlightIt = mapBlocksInFlight.find(pindex->GetBlockHash()); - bool fAlreadyInFlight = blockInFlightIt != mapBlocksInFlight.end(); - if (pindex->nStatus & BLOCK_HAVE_DATA) // Nothing to do here return; + auto range_flight = mapBlocksInFlight.equal_range(pindex->GetBlockHash()); + size_t already_in_flight = std::distance(range_flight.first, range_flight.second); + bool requested_block_from_this_peer{false}; + + // Multimap ensures ordering of outstanding requests. It's either empty or first in line. + bool first_in_flight = already_in_flight == 0 || (range_flight.first->second.first == pfrom.GetId()); + + while (range_flight.first != range_flight.second) { + if (range_flight.first->second.first == pfrom.GetId()) { + requested_block_from_this_peer = true; + break; + } + range_flight.first++; + } + if (pindex->nChainWork <= m_chainman.ActiveChain().Tip()->nChainWork || // We know something better pindex->nTx != 0) { // We had this block at some point, but pruned it - if (fAlreadyInFlight) { + if (requested_block_from_this_peer) { // We requested this block for some reason, but our mempool will probably be useless // so we just grab the block via normal getdata std::vector<CInv> vInv(1); @@ -4281,15 +4337,15 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } // If we're not close to tip yet, give up and let parallel block fetch work its magic - if (!fAlreadyInFlight && !CanDirectFetch()) { + if (!already_in_flight && !CanDirectFetch()) { return; } // We want to be a bit conservative just to be extra careful about DoS // possibilities in compact block processing... if (pindex->nHeight <= m_chainman.ActiveChain().Height() + 2) { - if ((!fAlreadyInFlight && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) || - (fAlreadyInFlight && blockInFlightIt->second.first == pfrom.GetId())) { + if ((already_in_flight < MAX_CMPCTBLOCKS_INFLIGHT_PER_BLOCK && nodestate->vBlocksInFlight.size() < MAX_BLOCKS_IN_TRANSIT_PER_PEER) || + requested_block_from_this_peer) { std::list<QueuedBlock>::iterator* queuedBlockIt = nullptr; if (!BlockRequested(pfrom.GetId(), *pindex, &queuedBlockIt)) { if (!(*queuedBlockIt)->partialBlock) @@ -4304,14 +4360,19 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, PartiallyDownloadedBlock& partialBlock = *(*queuedBlockIt)->partialBlock; ReadStatus status = partialBlock.InitData(cmpctblock, vExtraTxnForCompact); if (status == READ_STATUS_INVALID) { - RemoveBlockRequest(pindex->GetBlockHash()); // Reset in-flight state in case Misbehaving does not result in a disconnect + RemoveBlockRequest(pindex->GetBlockHash(), pfrom.GetId()); // Reset in-flight state in case Misbehaving does not result in a disconnect Misbehaving(*peer, 100, "invalid compact block"); return; } else if (status == READ_STATUS_FAILED) { - // Duplicate txindexes, the block is now in-flight, so just request it - std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(*peer), blockhash); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + if (first_in_flight) { + // Duplicate txindexes, the block is now in-flight, so just request it + std::vector<CInv> vInv(1); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(*peer), blockhash); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + } else { + // Give up for this peer and wait for other peer(s) + RemoveBlockRequest(pindex->GetBlockHash(), pfrom.GetId()); + } return; } @@ -4326,9 +4387,24 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, txn.blockhash = blockhash; blockTxnMsg << txn; fProcessBLOCKTXN = true; - } else { + } else if (first_in_flight) { + // We will try to round-trip any compact blocks we get on failure, + // as long as it's first... + req.blockhash = pindex->GetBlockHash(); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); + } else if (pfrom.m_bip152_highbandwidth_to && + (!pfrom.IsInboundConn() || + IsBlockRequestedFromOutbound(blockhash) || + already_in_flight < MAX_CMPCTBLOCKS_INFLIGHT_PER_BLOCK - 1)) { + // ... or it's a hb relay peer and: + // - peer is outbound, or + // - we already have an outbound attempt in flight(so we'll take what we can get), or + // - it's not the final parallel download slot (which we may reserve for first outbound) req.blockhash = pindex->GetBlockHash(); m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); + } else { + // Give up for this peer and wait for other peer(s) + RemoveBlockRequest(pindex->GetBlockHash(), pfrom.GetId()); } } else { // This block is either already in flight from a different @@ -4349,7 +4425,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } } } else { - if (fAlreadyInFlight) { + if (requested_block_from_this_peer) { // We requested this block, but its far into the future, so our // mempool will probably be useless - request the block normally std::vector<CInv> vInv(1); @@ -4399,7 +4475,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // process from some other peer. We do this after calling // ProcessNewBlock so that a malleated cmpctblock announcement // can't be used to interfere with block relay. - RemoveBlockRequest(pblock->GetHash()); + RemoveBlockRequest(pblock->GetHash(), std::nullopt); } } return; @@ -4421,24 +4497,44 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, { LOCK(cs_main); - std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); - if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || - it->second.first != pfrom.GetId()) { + auto range_flight = mapBlocksInFlight.equal_range(resp.blockhash); + size_t already_in_flight = std::distance(range_flight.first, range_flight.second); + bool requested_block_from_this_peer{false}; + + // Multimap ensures ordering of outstanding requests. It's either empty or first in line. + bool first_in_flight = already_in_flight == 0 || (range_flight.first->second.first == pfrom.GetId()); + + while (range_flight.first != range_flight.second) { + auto [node_id, block_it] = range_flight.first->second; + if (node_id == pfrom.GetId() && block_it->partialBlock) { + requested_block_from_this_peer = true; + break; + } + range_flight.first++; + } + + if (!requested_block_from_this_peer) { LogPrint(BCLog::NET, "Peer %d sent us block transactions for block we weren't expecting\n", pfrom.GetId()); return; } - PartiallyDownloadedBlock& partialBlock = *it->second.second->partialBlock; + PartiallyDownloadedBlock& partialBlock = *range_flight.first->second.second->partialBlock; ReadStatus status = partialBlock.FillBlock(*pblock, resp.txn); if (status == READ_STATUS_INVALID) { - RemoveBlockRequest(resp.blockhash); // Reset in-flight state in case Misbehaving does not result in a disconnect + RemoveBlockRequest(resp.blockhash, pfrom.GetId()); // Reset in-flight state in case Misbehaving does not result in a disconnect Misbehaving(*peer, 100, "invalid compact block/non-matching block transactions"); return; } else if (status == READ_STATUS_FAILED) { - // Might have collided, fall back to getdata now :( - std::vector<CInv> invs; - invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(*peer), resp.blockhash)); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); + if (first_in_flight) { + // Might have collided, fall back to getdata now :( + std::vector<CInv> invs; + invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(*peer), resp.blockhash)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); + } else { + RemoveBlockRequest(resp.blockhash, pfrom.GetId()); + LogPrint(BCLog::NET, "Peer %d sent us a compact block but it failed to reconstruct, waiting on first download to complete\n", pfrom.GetId()); + return; + } } else { // Block is either okay, or possibly we received // READ_STATUS_CHECKBLOCK_FAILED. @@ -4457,7 +4553,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // though the block was successfully read, and rely on the // handling in ProcessNewBlock to ensure the block index is // updated, etc. - RemoveBlockRequest(resp.blockhash); // it is now an empty pointer + RemoveBlockRequest(resp.blockhash, pfrom.GetId()); // it is now an empty pointer fBlockRead = true; // mapBlockSource is used for potentially punishing peers and // updating which peers send us compact blocks, so the race @@ -4546,7 +4642,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Always process the block if we requested it, since we may // need it even when it's not a candidate for a new best tip. forceProcessing = IsBlockRequested(hash); - RemoveBlockRequest(hash); + RemoveBlockRequest(hash, pfrom.GetId()); // mapBlockSource is only used for punishing peers and setting // which peers send us compact blocks, so the race between here and // cs_main in ProcessNewBlock is fine. @@ -4600,6 +4696,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } if (msg_type == NetMsgType::MEMPOOL) { + // Only process received mempool messages if we advertise NODE_BLOOM + // or if the peer has mempool permissions. if (!(peer->m_our_services & NODE_BLOOM) && !pfrom.HasPermission(NetPermissionFlags::Mempool)) { if (!pfrom.HasPermission(NetPermissionFlags::NoBan)) @@ -5029,14 +5127,14 @@ void PeerManagerImpl::EvictExtraOutboundPeers(std::chrono::seconds now) // valid headers chain with at least as much work as our tip. CNodeState *node_state = State(pnode->GetId()); if (node_state == nullptr || - (now - pnode->m_connected >= MINIMUM_CONNECT_TIME && node_state->nBlocksInFlight == 0)) { + (now - pnode->m_connected >= MINIMUM_CONNECT_TIME && node_state->vBlocksInFlight.empty())) { pnode->fDisconnect = true; LogPrint(BCLog::NET, "disconnecting extra block-relay-only peer=%d (last block received at time %d)\n", pnode->GetId(), count_seconds(pnode->m_last_block_time)); return true; } else { LogPrint(BCLog::NET, "keeping block-relay-only peer=%d chosen for eviction (connect time: %d, blocks_in_flight: %d)\n", - pnode->GetId(), count_seconds(pnode->m_connected), node_state->nBlocksInFlight); + pnode->GetId(), count_seconds(pnode->m_connected), node_state->vBlocksInFlight.size()); } return false; }); @@ -5076,13 +5174,13 @@ void PeerManagerImpl::EvictExtraOutboundPeers(std::chrono::seconds now) // Also don't disconnect any peer we're trying to download a // block from. CNodeState &state = *State(pnode->GetId()); - if (now - pnode->m_connected > MINIMUM_CONNECT_TIME && state.nBlocksInFlight == 0) { + if (now - pnode->m_connected > MINIMUM_CONNECT_TIME && state.vBlocksInFlight.empty()) { LogPrint(BCLog::NET, "disconnecting extra outbound peer=%d (last block announcement received at time %d)\n", pnode->GetId(), oldest_block_announcement); pnode->fDisconnect = true; return true; } else { LogPrint(BCLog::NET, "keeping outbound peer=%d chosen for eviction (connect time: %d, blocks_in_flight: %d)\n", - pnode->GetId(), count_seconds(pnode->m_connected), state.nBlocksInFlight); + pnode->GetId(), count_seconds(pnode->m_connected), state.vBlocksInFlight.size()); return false; } }); @@ -5529,7 +5627,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) m_connman.PushMessage(pto, std::move(cached_cmpctblock_msg.value())); } else { CBlock block; - bool ret = ReadBlockFromDisk(block, pBestIndex, consensusParams); + const bool ret{m_chainman.m_blockman.ReadBlockFromDisk(block, *pBestIndex)}; assert(ret); CBlockHeaderAndShortTxIDs cmpctblock{block}; m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); @@ -5663,7 +5761,9 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // especially since we have many peers and some will draw much shorter delays. unsigned int nRelayedTransactions = 0; LOCK(tx_relay->m_bloom_filter_mutex); - while (!vInvTx.empty() && nRelayedTransactions < INVENTORY_BROADCAST_MAX) { + size_t broadcast_max{INVENTORY_BROADCAST_MAX + (tx_relay->m_tx_inventory_to_send.size()/1000)*5}; + broadcast_max = std::min<size_t>(1000, broadcast_max); + while (!vInvTx.empty() && nRelayedTransactions < broadcast_max) { // Fetch the top element from the heap std::pop_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); std::set<uint256>::iterator it = vInvTx.back(); @@ -5682,7 +5782,6 @@ bool PeerManagerImpl::SendMessages(CNode* pto) continue; } auto txid = txinfo.tx->GetHash(); - auto wtxid = txinfo.tx->GetWitnessHash(); // Peer told you to not send transactions at that feerate? Don't bother sending it. if (txinfo.fee < filterrate.GetFee(txinfo.vsize)) { continue; @@ -5692,24 +5791,6 @@ bool PeerManagerImpl::SendMessages(CNode* pto) tx_relay->m_recently_announced_invs.insert(hash); vInv.push_back(inv); nRelayedTransactions++; - { - // Expire old relay messages - while (!g_relay_expiration.empty() && g_relay_expiration.front().first < current_time) - { - mapRelay.erase(g_relay_expiration.front().second); - g_relay_expiration.pop_front(); - } - - auto ret = mapRelay.emplace(txid, std::move(txinfo.tx)); - if (ret.second) { - g_relay_expiration.emplace_back(current_time + RELAY_TX_CACHE_TIME, ret.first); - } - // Add wtxid-based lookup into mapRelay as well, so that peers can request by wtxid - auto ret2 = mapRelay.emplace(wtxid, ret.first->second); - if (ret2.second) { - g_relay_expiration.emplace_back(current_time + RELAY_TX_CACHE_TIME, ret2.first); - } - } if (vInv.size() == MAX_INV_SZ) { m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); @@ -5735,7 +5816,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // Stalling only triggers when the block download window cannot move. During normal steady state, // the download window should be much larger than the to-be-downloaded set of blocks, so disconnection // should only happen during initial block download. - LogPrintf("Peer=%d is stalling block download, disconnecting\n", pto->GetId()); + LogPrintf("Peer=%d%s is stalling block download, disconnecting\n", pto->GetId(), fLogIPs ? strprintf(" peeraddr=%s", pto->addr.ToStringAddrPort()) : ""); pto->fDisconnect = true; // Increase timeout for the next peer so that we don't disconnect multiple peers if our own // bandwidth is insufficient. @@ -5754,7 +5835,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) QueuedBlock &queuedBlock = state.vBlocksInFlight.front(); int nOtherPeersWithValidatedDownloads = m_peers_downloading_from - 1; if (current_time > state.m_downloading_since + std::chrono::seconds{consensusParams.nPowTargetSpacing} * (BLOCK_DOWNLOAD_TIMEOUT_BASE + BLOCK_DOWNLOAD_TIMEOUT_PER_PEER * nOtherPeersWithValidatedDownloads)) { - LogPrintf("Timeout downloading block %s from peer=%d, disconnecting\n", queuedBlock.pindex->GetBlockHash().ToString(), pto->GetId()); + LogPrintf("Timeout downloading block %s from peer=%d%s, disconnecting\n", queuedBlock.pindex->GetBlockHash().ToString(), pto->GetId(), fLogIPs ? strprintf(" peeraddr=%s", pto->addr.ToStringAddrPort()) : ""); pto->fDisconnect = true; return true; } @@ -5770,11 +5851,11 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // disconnect our sync peer for stalling; we have bigger // problems if we can't get any outbound peers. if (!pto->HasPermission(NetPermissionFlags::NoBan)) { - LogPrintf("Timeout downloading headers from peer=%d, disconnecting\n", pto->GetId()); + LogPrintf("Timeout downloading headers from peer=%d%s, disconnecting\n", pto->GetId(), fLogIPs ? strprintf(" peeraddr=%s", pto->addr.ToStringAddrPort()) : ""); pto->fDisconnect = true; return true; } else { - LogPrintf("Timeout downloading headers from noban peer=%d, not disconnecting\n", pto->GetId()); + LogPrintf("Timeout downloading headers from noban peer=%d%s, not disconnecting\n", pto->GetId(), fLogIPs ? strprintf(" peeraddr=%s", pto->addr.ToStringAddrPort()) : ""); // Reset the headers sync state so that we have a // chance to try downloading from a different peer. // Note: this will also result in at least one more @@ -5800,10 +5881,10 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // Message: getdata (blocks) // std::vector<CInv> vGetData; - if (CanServeBlocks(*peer) && ((sync_blocks_and_headers_from_peer && !IsLimitedPeer(*peer)) || !m_chainman.ActiveChainstate().IsInitialBlockDownload()) && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (CanServeBlocks(*peer) && ((sync_blocks_and_headers_from_peer && !IsLimitedPeer(*peer)) || !m_chainman.ActiveChainstate().IsInitialBlockDownload()) && state.vBlocksInFlight.size() < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { std::vector<const CBlockIndex*> vToDownload; NodeId staller = -1; - FindNextBlocksToDownload(*peer, MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller); + FindNextBlocksToDownload(*peer, MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.vBlocksInFlight.size(), vToDownload, staller); for (const CBlockIndex *pindex : vToDownload) { uint32_t nFetchFlags = GetFetchFlags(*peer); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); @@ -5811,7 +5892,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) LogPrint(BCLog::NET, "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), pindex->nHeight, pto->GetId()); } - if (state.nBlocksInFlight == 0 && staller != -1) { + if (state.vBlocksInFlight.empty() && staller != -1) { if (State(staller)->m_stalling_since == 0us) { State(staller)->m_stalling_since = current_time; LogPrint(BCLog::NET, "Stall started peer=%d\n", staller); diff --git a/src/net_processing.h b/src/net_processing.h index af9a02139b..deebb24c94 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -22,6 +22,8 @@ static const bool DEFAULT_PEERBLOOMFILTERS = false; static const bool DEFAULT_PEERBLOCKFILTERS = false; /** Threshold for marking a node to be discouraged, e.g. disconnected and added to the discouragement filter. */ static const int DISCOURAGEMENT_THRESHOLD{100}; +/** Maximum number of outstanding CMPCTBLOCK requests for the same block. */ +static const unsigned int MAX_CMPCTBLOCKS_INFLIGHT_PER_BLOCK = 3; struct CNodeStateStats { int nSyncHeight = -1; diff --git a/src/netbase.cpp b/src/netbase.cpp index f39a3635f4..8f6f92ea7d 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -36,8 +36,8 @@ static Proxy nameProxy GUARDED_BY(g_proxyinfo_mutex); int nConnectTimeout = DEFAULT_CONNECT_TIMEOUT; bool fNameLookup = DEFAULT_NAME_LOOKUP; -// Need ample time for negotiation for very slow proxies such as Tor (milliseconds) -int g_socks5_recv_timeout = 20 * 1000; +// Need ample time for negotiation for very slow proxies such as Tor +std::chrono::milliseconds g_socks5_recv_timeout = 20s; static std::atomic<bool> interruptSocks5Recv(false); std::vector<CNetAddr> WrappedGetAddrInfo(const std::string& name, bool allow_lookup) @@ -132,14 +132,9 @@ std::vector<std::string> GetNetworkNames(bool append_unroutable) return names; } -static bool LookupIntern(const std::string& name, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function) +static std::vector<CNetAddr> LookupIntern(const std::string& name, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function) { - vIP.clear(); - - if (!ContainsNoNUL(name)) { - return false; - } - + if (!ContainsNoNUL(name)) return {}; { CNetAddr addr; // From our perspective, onion addresses are not hostnames but rather @@ -148,83 +143,65 @@ static bool LookupIntern(const std::string& name, std::vector<CNetAddr>& vIP, un // getaddrinfo to decode them and it wouldn't make sense to resolve // them, we return a network address representing it instead. See // CNetAddr::SetSpecial(const std::string&) for more details. - if (addr.SetSpecial(name)) { - vIP.push_back(addr); - return true; - } + if (addr.SetSpecial(name)) return {addr}; } + std::vector<CNetAddr> addresses; + for (const CNetAddr& resolved : dns_lookup_function(name, fAllowLookup)) { - if (nMaxSolutions > 0 && vIP.size() >= nMaxSolutions) { + if (nMaxSolutions > 0 && addresses.size() >= nMaxSolutions) { break; } /* Never allow resolving to an internal address. Consider any such result invalid */ if (!resolved.IsInternal()) { - vIP.push_back(resolved); + addresses.push_back(resolved); } } - return (vIP.size() > 0); + return addresses; } -bool LookupHost(const std::string& name, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function) +std::vector<CNetAddr> LookupHost(const std::string& name, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function) { - if (!ContainsNoNUL(name)) { - return false; - } + if (!ContainsNoNUL(name)) return {}; std::string strHost = name; - if (strHost.empty()) - return false; + if (strHost.empty()) return {}; if (strHost.front() == '[' && strHost.back() == ']') { strHost = strHost.substr(1, strHost.size() - 2); } - return LookupIntern(strHost, vIP, nMaxSolutions, fAllowLookup, dns_lookup_function); + return LookupIntern(strHost, nMaxSolutions, fAllowLookup, dns_lookup_function); } -bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup, DNSLookupFn dns_lookup_function) +std::optional<CNetAddr> LookupHost(const std::string& name, bool fAllowLookup, DNSLookupFn dns_lookup_function) { - if (!ContainsNoNUL(name)) { - return false; - } - std::vector<CNetAddr> vIP; - LookupHost(name, vIP, 1, fAllowLookup, dns_lookup_function); - if(vIP.empty()) - return false; - addr = vIP.front(); - return true; + const std::vector<CNetAddr> addresses{LookupHost(name, 1, fAllowLookup, dns_lookup_function)}; + return addresses.empty() ? std::nullopt : std::make_optional(addresses.front()); } -bool Lookup(const std::string& name, std::vector<CService>& vAddr, uint16_t portDefault, bool fAllowLookup, unsigned int nMaxSolutions, DNSLookupFn dns_lookup_function) +std::vector<CService> Lookup(const std::string& name, uint16_t portDefault, bool fAllowLookup, unsigned int nMaxSolutions, DNSLookupFn dns_lookup_function) { if (name.empty() || !ContainsNoNUL(name)) { - return false; + return {}; } uint16_t port{portDefault}; std::string hostname; SplitHostPort(name, port, hostname); - std::vector<CNetAddr> vIP; - bool fRet = LookupIntern(hostname, vIP, nMaxSolutions, fAllowLookup, dns_lookup_function); - if (!fRet) - return false; - vAddr.resize(vIP.size()); - for (unsigned int i = 0; i < vIP.size(); i++) - vAddr[i] = CService(vIP[i], port); - return true; + const std::vector<CNetAddr> addresses{LookupIntern(hostname, nMaxSolutions, fAllowLookup, dns_lookup_function)}; + if (addresses.empty()) return {}; + std::vector<CService> services; + services.reserve(addresses.size()); + for (const auto& addr : addresses) + services.emplace_back(addr, port); + return services; } -bool Lookup(const std::string& name, CService& addr, uint16_t portDefault, bool fAllowLookup, DNSLookupFn dns_lookup_function) +std::optional<CService> Lookup(const std::string& name, uint16_t portDefault, bool fAllowLookup, DNSLookupFn dns_lookup_function) { - if (!ContainsNoNUL(name)) { - return false; - } - std::vector<CService> vService; - bool fRet = Lookup(name, vService, portDefault, fAllowLookup, 1, dns_lookup_function); - if (!fRet) - return false; - addr = vService[0]; - return true; + const std::vector<CService> services{Lookup(name, portDefault, fAllowLookup, 1, dns_lookup_function)}; + + return services.empty() ? std::nullopt : std::make_optional(services.front()); } CService LookupNumeric(const std::string& name, uint16_t portDefault, DNSLookupFn dns_lookup_function) @@ -232,12 +209,9 @@ CService LookupNumeric(const std::string& name, uint16_t portDefault, DNSLookupF if (!ContainsNoNUL(name)) { return {}; } - CService addr; // "1.2:345" will fail to resolve the ip, but will still set the port. // If the ip fails to resolve, re-init the result. - if(!Lookup(name, addr, portDefault, false, dns_lookup_function)) - addr = CService(); - return addr; + return Lookup(name, portDefault, /*fAllowLookup=*/false, dns_lookup_function).value_or(CService{}); } /** SOCKS version */ @@ -296,7 +270,7 @@ enum class IntrRecvError { * * @param data The buffer where the read bytes should be stored. * @param len The number of bytes to read into the specified buffer. - * @param timeout The total timeout in milliseconds for this read. + * @param timeout The total timeout for this read. * @param sock The socket (has to be in non-blocking mode) from which to read bytes. * * @returns An IntrRecvError indicating the resulting status of this read. @@ -306,10 +280,10 @@ enum class IntrRecvError { * @see This function can be interrupted by calling InterruptSocks5(bool). * Sockets can be made non-blocking with Sock::SetNonBlocking(). */ -static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, int timeout, const Sock& sock) +static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, std::chrono::milliseconds timeout, const Sock& sock) { - int64_t curTime = GetTimeMillis(); - int64_t endTime = curTime + timeout; + auto curTime{Now<SteadyMilliseconds>()}; + const auto endTime{curTime + timeout}; while (len > 0 && curTime < endTime) { ssize_t ret = sock.Recv(data, len, 0); // Optimistically try the recv first if (ret > 0) { @@ -333,7 +307,7 @@ static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, int timeout, c } if (interruptSocks5Recv) return IntrRecvError::Interrupted; - curTime = GetTimeMillis(); + curTime = Now<SteadyMilliseconds>(); } return len == 0 ? IntrRecvError::OK : IntrRecvError::Timeout; } @@ -689,27 +663,27 @@ bool LookupSubNet(const std::string& subnet_str, CSubNet& subnet_out) const size_t slash_pos{subnet_str.find_last_of('/')}; const std::string str_addr{subnet_str.substr(0, slash_pos)}; - CNetAddr addr; + const std::optional<CNetAddr> addr{LookupHost(str_addr, /*fAllowLookup=*/false)}; - if (LookupHost(str_addr, addr, /*fAllowLookup=*/false)) { + if (addr.has_value()) { if (slash_pos != subnet_str.npos) { const std::string netmask_str{subnet_str.substr(slash_pos + 1)}; uint8_t netmask; if (ParseUInt8(netmask_str, &netmask)) { // Valid number; assume CIDR variable-length subnet masking. - subnet_out = CSubNet{addr, netmask}; + subnet_out = CSubNet{addr.value(), netmask}; return subnet_out.IsValid(); } else { // Invalid number; try full netmask syntax. Never allow lookup for netmask. - CNetAddr full_netmask; - if (LookupHost(netmask_str, full_netmask, /*fAllowLookup=*/false)) { - subnet_out = CSubNet{addr, full_netmask}; + const std::optional<CNetAddr> full_netmask{LookupHost(netmask_str, /*fAllowLookup=*/false)}; + if (full_netmask.has_value()) { + subnet_out = CSubNet{addr.value(), full_netmask.value()}; return subnet_out.IsValid(); } } } else { // Single IP subnet (<ipv4>/32 or <ipv6>/128). - subnet_out = CSubNet{addr}; + subnet_out = CSubNet{addr.value()}; return subnet_out.IsValid(); } } diff --git a/src/netbase.h b/src/netbase.h index a43f22f240..1da4f5c51d 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -105,24 +105,25 @@ extern DNSLookupFn g_dns_lookup; * @param name The string representing a host. Could be a name or a numerical * IP address (IPv6 addresses in their bracketed form are * allowed). - * @param[out] vIP The resulting network addresses to which the specified host - * string resolved. * - * @returns Whether or not the specified host string successfully resolved to - * any resulting network addresses. + * @returns The resulting network addresses to which the specified host + * string resolved. * - * @see Lookup(const std::string&, std::vector<CService>&, uint16_t, bool, unsigned int, DNSLookupFn) + * @see Lookup(const std::string&, uint16_t, bool, unsigned int, DNSLookupFn) * for additional parameter descriptions. */ -bool LookupHost(const std::string& name, std::vector<CNetAddr>& vIP, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function = g_dns_lookup); +std::vector<CNetAddr> LookupHost(const std::string& name, unsigned int nMaxSolutions, bool fAllowLookup, DNSLookupFn dns_lookup_function = g_dns_lookup); /** * Resolve a host string to its first corresponding network address. * - * @see LookupHost(const std::string&, std::vector<CNetAddr>&, uint16_t, bool, DNSLookupFn) + * @returns The resulting network address to which the specified host + * string resolved or std::nullopt if host does not resolve to an address. + * + * @see LookupHost(const std::string&, unsigned int, bool, DNSLookupFn) * for additional parameter descriptions. */ -bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup, DNSLookupFn dns_lookup_function = g_dns_lookup); +std::optional<CNetAddr> LookupHost(const std::string& name, bool fAllowLookup, DNSLookupFn dns_lookup_function = g_dns_lookup); /** * Resolve a service string to its corresponding service. @@ -132,8 +133,6 @@ bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup, DNSL * disambiguated bracketed form), optionally followed by a uint16_t port * number. (e.g. example.com:8333 or * [2001:db8:85a3:8d3:1319:8a2e:370:7348]:420) - * @param[out] vAddr The resulting services to which the specified service string - * resolved. * @param portDefault The default port for resulting services if not specified * by the service string. * @param fAllowLookup Whether or not hostname lookups are permitted. If yes, @@ -141,18 +140,18 @@ bool LookupHost(const std::string& name, CNetAddr& addr, bool fAllowLookup, DNSL * @param nMaxSolutions The maximum number of results we want, specifying 0 * means "as many solutions as we get." * - * @returns Whether or not the service string successfully resolved to any - * resulting services. + * @returns The resulting services to which the specified service string + * resolved. */ -bool Lookup(const std::string& name, std::vector<CService>& vAddr, uint16_t portDefault, bool fAllowLookup, unsigned int nMaxSolutions, DNSLookupFn dns_lookup_function = g_dns_lookup); +std::vector<CService> Lookup(const std::string& name, uint16_t portDefault, bool fAllowLookup, unsigned int nMaxSolutions, DNSLookupFn dns_lookup_function = g_dns_lookup); /** * Resolve a service string to its first corresponding service. * - * @see Lookup(const std::string&, std::vector<CService>&, uint16_t, bool, unsigned int, DNSLookupFn) + * @see Lookup(const std::string&, uint16_t, bool, unsigned int, DNSLookupFn) * for additional parameter descriptions. */ -bool Lookup(const std::string& name, CService& addr, uint16_t portDefault, bool fAllowLookup, DNSLookupFn dns_lookup_function = g_dns_lookup); +std::optional<CService> Lookup(const std::string& name, uint16_t portDefault, bool fAllowLookup, DNSLookupFn dns_lookup_function = g_dns_lookup); /** * Resolve a service string with a numeric IP to its first corresponding @@ -160,7 +159,7 @@ bool Lookup(const std::string& name, CService& addr, uint16_t portDefault, bool * * @returns The resulting CService if the resolution was successful, [::]:0 otherwise. * - * @see Lookup(const std::string&, std::vector<CService>&, uint16_t, bool, unsigned int, DNSLookupFn) + * @see Lookup(const std::string&, uint16_t, bool, unsigned int, DNSLookupFn) * for additional parameter descriptions. */ CService LookupNumeric(const std::string& name, uint16_t portDefault = 0, DNSLookupFn dns_lookup_function = g_dns_lookup); diff --git a/src/node/blockmanager_args.cpp b/src/node/blockmanager_args.cpp index 06a1934947..4b296db1b0 100644 --- a/src/node/blockmanager_args.cpp +++ b/src/node/blockmanager_args.cpp @@ -5,26 +5,35 @@ #include <node/blockmanager_args.h> #include <common/args.h> +#include <node/blockstorage.h> +#include <tinyformat.h> +#include <util/result.h> +#include <util/translation.h> #include <validation.h> +#include <cstdint> + namespace node { -std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, BlockManager::Options& opts) +util::Result<void> ApplyArgsManOptions(const ArgsManager& args, BlockManager::Options& opts) { // block pruning; get the amount of disk space (in MiB) to allot for block & undo files int64_t nPruneArg{args.GetIntArg("-prune", opts.prune_target)}; if (nPruneArg < 0) { - return _("Prune cannot be configured with a negative value."); + return util::Error{_("Prune cannot be configured with a negative value.")}; } uint64_t nPruneTarget{uint64_t(nPruneArg) * 1024 * 1024}; if (nPruneArg == 1) { // manual pruning: -prune=1 nPruneTarget = BlockManager::PRUNE_TARGET_MANUAL; } else if (nPruneTarget) { if (nPruneTarget < MIN_DISK_SPACE_FOR_BLOCK_FILES) { - return strprintf(_("Prune configured below the minimum of %d MiB. Please use a higher number."), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024); + return util::Error{strprintf(_("Prune configured below the minimum of %d MiB. Please use a higher number."), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024)}; } } opts.prune_target = nPruneTarget; - return std::nullopt; + if (auto value{args.GetBoolArg("-fastprune")}) opts.fast_prune = *value; + if (auto value{args.GetBoolArg("-stopafterblockimport")}) opts.stop_after_block_import = *value; + + return {}; } } // namespace node diff --git a/src/node/blockmanager_args.h b/src/node/blockmanager_args.h index e657c6bb45..de9fe83a5c 100644 --- a/src/node/blockmanager_args.h +++ b/src/node/blockmanager_args.h @@ -7,14 +7,12 @@ #define BITCOIN_NODE_BLOCKMANAGER_ARGS_H #include <node/blockstorage.h> - -#include <optional> +#include <util/result.h> class ArgsManager; -struct bilingual_str; namespace node { -std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, BlockManager::Options& opts); +[[nodiscard]] util::Result<void> ApplyArgsManOptions(const ArgsManager& args, BlockManager::Options& opts); } // namespace node #endif // BITCOIN_NODE_BLOCKMANAGER_ARGS_H diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index abd71ca50b..1368ae6f6d 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -6,7 +6,6 @@ #include <chain.h> #include <clientversion.h> -#include <common/args.h> #include <consensus/validation.h> #include <flatfile.h> #include <hash.h> @@ -18,9 +17,9 @@ #include <signet.h> #include <streams.h> #include <undo.h> +#include <util/batchpriority.h> #include <util/fs.h> #include <util/syscall_sandbox.h> -#include <util/system.h> #include <validation.h> #include <map> @@ -28,6 +27,7 @@ namespace node { std::atomic_bool fReindex(false); +std::atomic_bool g_indexes_ready_to_sync{false}; bool CBlockIndexWorkComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const { @@ -53,10 +53,6 @@ bool CBlockIndexHeightOnlyComparator::operator()(const CBlockIndex* pa, const CB return pa->nHeight < pb->nHeight; } -static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false); -static FlatFileSeq BlockFileSeq(); -static FlatFileSeq UndoFileSeq(); - std::vector<CBlockIndex*> BlockManager::GetAllBlockIndices() { AssertLockHeld(cs_main); @@ -253,9 +249,9 @@ CBlockIndex* BlockManager::InsertBlockIndex(const uint256& hash) return pindex; } -bool BlockManager::LoadBlockIndex(const Consensus::Params& consensus_params) +bool BlockManager::LoadBlockIndex() { - if (!m_block_tree_db->LoadBlockIndexGuts(consensus_params, [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); })) { + if (!m_block_tree_db->LoadBlockIndexGuts(GetConsensus(), [this](const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return this->InsertBlockIndex(hash); })) { return false; } @@ -318,9 +314,9 @@ bool BlockManager::WriteBlockIndexDB() return true; } -bool BlockManager::LoadBlockIndexDB(const Consensus::Params& consensus_params) +bool BlockManager::LoadBlockIndexDB() { - if (!LoadBlockIndex(consensus_params)) { + if (!LoadBlockIndex()) { return false; } @@ -423,7 +419,7 @@ const CBlockIndex* BlockManager::GetFirstStoredBlock(const CBlockIndex& start_bl // rev files since they'll be rewritten by the reindex anyway. This ensures that m_blockfile_info // is in sync with what's actually on disk by the time we start downloading, so that pruning // works correctly. -void CleanupBlockRevFiles() +void BlockManager::CleanupBlockRevFiles() const { std::map<std::string, fs::path> mapBlockFiles; @@ -431,8 +427,7 @@ void CleanupBlockRevFiles() // Remove the rev files immediately and insert the blk file paths into an // ordered map keyed by block file index. LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n"); - const fs::path& blocksdir = gArgs.GetBlocksDirPath(); - for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) { + for (fs::directory_iterator it(m_opts.blocks_dir); it != fs::directory_iterator(); it++) { const std::string path = fs::PathToString(it->path().filename()); if (fs::is_regular_file(*it) && path.length() == 12 && @@ -467,7 +462,7 @@ CBlockFileInfo* BlockManager::GetBlockFileInfo(size_t n) return &m_blockfile_info.at(n); } -static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) +bool BlockManager::UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) const { // Open history file to append AutoFile fileout{OpenUndoFile(pos)}; @@ -496,9 +491,9 @@ static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const return true; } -bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) +bool BlockManager::UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& index) const { - const FlatFilePos pos{WITH_LOCK(::cs_main, return pindex->GetUndoPos())}; + const FlatFilePos pos{WITH_LOCK(::cs_main, return index.GetUndoPos())}; if (pos.IsNull()) { return error("%s: no undo data available", __func__); @@ -514,7 +509,7 @@ bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) uint256 hashChecksum; HashVerifier verifier{filein}; // Use HashVerifier as reserializing may lose data, c.f. commit d342424301013ec47dc146a4beb49d5c9319d80a try { - verifier << pindex->pprev->GetBlockHash(); + verifier << index.pprev->GetBlockHash(); verifier >> blockundo; filein >> hashChecksum; } catch (const std::exception& e) { @@ -570,7 +565,7 @@ uint64_t BlockManager::CalculateCurrentUsage() return retval; } -void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) +void BlockManager::UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const { std::error_code ec; for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { @@ -583,28 +578,28 @@ void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) } } -static FlatFileSeq BlockFileSeq() +FlatFileSeq BlockManager::BlockFileSeq() const { - return FlatFileSeq(gArgs.GetBlocksDirPath(), "blk", gArgs.GetBoolArg("-fastprune", false) ? 0x4000 /* 16kb */ : BLOCKFILE_CHUNK_SIZE); + return FlatFileSeq(m_opts.blocks_dir, "blk", m_opts.fast_prune ? 0x4000 /* 16kb */ : BLOCKFILE_CHUNK_SIZE); } -static FlatFileSeq UndoFileSeq() +FlatFileSeq BlockManager::UndoFileSeq() const { - return FlatFileSeq(gArgs.GetBlocksDirPath(), "rev", UNDOFILE_CHUNK_SIZE); + return FlatFileSeq(m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE); } -FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) +FILE* BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const { return BlockFileSeq().Open(pos, fReadOnly); } /** Open an undo file (rev?????.dat) */ -static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) +FILE* BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const { return UndoFileSeq().Open(pos, fReadOnly); } -fs::path GetBlockPosFilename(const FlatFilePos& pos) +fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const { return BlockFileSeq().FileName(pos); } @@ -623,7 +618,7 @@ bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigne unsigned int max_blockfile_size{MAX_BLOCKFILE_SIZE}; // Use smaller blockfiles in test-only -fastprune mode - but avoid // the possibility of having a block not fit into the block file. - if (gArgs.GetBoolArg("-fastprune", false)) { + if (m_opts.fast_prune) { max_blockfile_size = 0x10000; // 64kiB if (nAddSize >= max_blockfile_size) { // dynamically adjust the blockfile size to be larger than the added size @@ -697,7 +692,7 @@ bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFileP return true; } -static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) +bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) const { // Open history file to append CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION); @@ -720,16 +715,16 @@ static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessa return true; } -bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) +bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block) { AssertLockHeld(::cs_main); // Write undo information to disk - if (pindex->GetUndoPos().IsNull()) { + if (block.GetUndoPos().IsNull()) { FlatFilePos _pos; - if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) { + if (!FindUndoPos(state, block.nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) { return error("ConnectBlock(): FindUndoPos failed"); } - if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) { + if (!UndoWriteToDisk(blockundo, _pos, block.pprev->GetBlockHash(), GetParams().MessageStart())) { return AbortNode(state, "Failed to write undo data"); } // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order) @@ -737,20 +732,20 @@ bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValid // in the block file info as below; note that this does not catch the case where the undo writes are keeping up // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in // the FindBlockPos function - if (_pos.nFile < m_last_blockfile && static_cast<uint32_t>(pindex->nHeight) == m_blockfile_info[_pos.nFile].nHeightLast) { + if (_pos.nFile < m_last_blockfile && static_cast<uint32_t>(block.nHeight) == m_blockfile_info[_pos.nFile].nHeightLast) { FlushUndoFile(_pos.nFile, true); } // update nUndoPos in block index - pindex->nUndoPos = _pos.nPos; - pindex->nStatus |= BLOCK_HAVE_UNDO; - m_dirty_blockindex.insert(pindex); + block.nUndoPos = _pos.nPos; + block.nStatus |= BLOCK_HAVE_UNDO; + m_dirty_blockindex.insert(&block); } return true; } -bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams) +bool BlockManager::ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) const { block.SetNull(); @@ -768,33 +763,33 @@ bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::P } // Check the header - if (!CheckProofOfWork(block.GetHash(), block.nBits, consensusParams)) { + if (!CheckProofOfWork(block.GetHash(), block.nBits, GetConsensus())) { return error("ReadBlockFromDisk: Errors in block header at %s", pos.ToString()); } // Signet only: check block solution - if (consensusParams.signet_blocks && !CheckSignetBlockSolution(block, consensusParams)) { + if (GetConsensus().signet_blocks && !CheckSignetBlockSolution(block, GetConsensus())) { return error("ReadBlockFromDisk: Errors in block solution at %s", pos.ToString()); } return true; } -bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams) +bool BlockManager::ReadBlockFromDisk(CBlock& block, const CBlockIndex& index) const { - const FlatFilePos block_pos{WITH_LOCK(cs_main, return pindex->GetBlockPos())}; + const FlatFilePos block_pos{WITH_LOCK(cs_main, return index.GetBlockPos())}; - if (!ReadBlockFromDisk(block, block_pos, consensusParams)) { + if (!ReadBlockFromDisk(block, block_pos)) { return false; } - if (block.GetHash() != pindex->GetBlockHash()) { + if (block.GetHash() != index.GetBlockHash()) { return error("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() doesn't match index for %s at %s", - pindex->ToString(), block_pos.ToString()); + index.ToString(), block_pos.ToString()); } return true; } -bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start) +bool BlockManager::ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start) const { FlatFilePos hpos = pos; hpos.nPos -= 8; // Seek back 8 bytes for meta header @@ -829,7 +824,7 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, c return true; } -FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp) +FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const FlatFilePos* dbp) { unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); FlatFilePos blockPos; @@ -847,7 +842,7 @@ FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, CCha return FlatFilePos(); } if (!position_known) { - if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) { + if (!WriteBlockToDisk(block, blockPos, GetParams().MessageStart())) { AbortNode("Failed to write block"); return FlatFilePos(); } @@ -872,7 +867,7 @@ public: } }; -void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args, const fs::path& mempool_path) +void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const fs::path& mempool_path) { SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION_LOAD_BLOCKS); ScheduleBatchPriority(); @@ -888,10 +883,10 @@ void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFile std::multimap<uint256, FlatFilePos> blocks_with_unknown_parent; while (true) { FlatFilePos pos(nFile, 0); - if (!fs::exists(GetBlockPosFilename(pos))) { + if (!fs::exists(chainman.m_blockman.GetBlockPosFilename(pos))) { break; // No block files left to reindex } - FILE* file = OpenBlockFile(pos, true); + FILE* file = chainman.m_blockman.OpenBlockFile(pos, true); if (!file) { break; // This error is logged in OpenBlockFile } @@ -933,18 +928,18 @@ void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFile for (Chainstate* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) { BlockValidationState state; if (!chainstate->ActivateBestChain(state, nullptr)) { - LogPrintf("Failed to connect best block (%s)\n", state.ToString()); - StartShutdown(); + AbortNode(strprintf("Failed to connect best block (%s)", state.ToString())); return; } } - if (args.GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) { + if (chainman.m_blockman.StopAfterBlockImport()) { LogPrintf("Stopping after block import\n"); StartShutdown(); return; } } // End scope of ImportingNow chainman.ActiveChainstate().LoadMempool(mempool_path); + g_indexes_ready_to_sync = true; } } // namespace node diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 3eb27cc72d..a1ebb0df0a 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -8,6 +8,7 @@ #include <attributes.h> #include <chain.h> #include <kernel/blockmanager_opts.h> +#include <kernel/chainparams.h> #include <kernel/cs_main.h> #include <protocol.h> #include <sync.h> @@ -19,7 +20,6 @@ #include <unordered_map> #include <vector> -class ArgsManager; class BlockValidationState; class CBlock; class CBlockFileInfo; @@ -35,7 +35,6 @@ struct Params; } namespace node { -static constexpr bool DEFAULT_STOPAFTERBLOCKIMPORT{false}; /** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB @@ -48,6 +47,7 @@ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = CMessageHeader::MESSAGE_START_SIZE + sizeof(unsigned int); extern std::atomic_bool fReindex; +extern std::atomic_bool g_indexes_ready_to_sync; // Because validation code takes pointers to the map's CBlockIndex objects, if // we ever switch to another associative container, we need to either use a @@ -81,18 +81,28 @@ class BlockManager friend ChainstateManager; private: + const CChainParams& GetParams() const { return m_opts.chainparams; } + const Consensus::Params& GetConsensus() const { return m_opts.chainparams.GetConsensus(); } /** * Load the blocktree off disk and into memory. Populate certain metadata * per index entry (nStatus, nChainWork, nTimeMax, etc.) as well as peripheral * collections like m_dirty_blockindex. */ - bool LoadBlockIndex(const Consensus::Params& consensus_params) + bool LoadBlockIndex() EXCLUSIVE_LOCKS_REQUIRED(cs_main); void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false); void FlushUndoFile(int block_file, bool finalize = false); bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown); bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize); + FlatFileSeq BlockFileSeq() const; + FlatFileSeq UndoFileSeq() const; + + FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const; + + bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) const; + bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) const; + /* Calculate the block/rev files to delete based on height specified by user with RPC command pruneblockchain */ void FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight, int chain_tip_height); @@ -162,7 +172,7 @@ public: std::unique_ptr<CBlockTreeDB> m_block_tree_db GUARDED_BY(::cs_main); bool WriteBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - bool LoadBlockIndexDB(const Consensus::Params& consensus_params) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + bool LoadBlockIndexDB() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** * Remove any pruned block & undo files that are still on disk. @@ -184,11 +194,11 @@ public: /** Get block file info entry for one block file */ CBlockFileInfo* GetBlockFileInfo(size_t n); - bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) + bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** Store block on disk. If dbp is not nullptr, then it provides the known position of the block within a block file on disk. */ - FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp); + FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const FlatFilePos* dbp); /** Whether running in -prune mode. */ [[nodiscard]] bool IsPruneMode() const { return m_prune_mode; } @@ -199,6 +209,8 @@ public: [[nodiscard]] bool LoadingBlocks() const { return m_importing || fReindex; } + [[nodiscard]] bool StopAfterBlockImport() const { return m_opts.stop_after_block_import; } + /** Calculate the amount of disk space the block & undo files currently use */ uint64_t CalculateCurrentUsage(); @@ -216,28 +228,29 @@ public: //! Create or update a prune lock identified by its name void UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); -}; -void CleanupBlockRevFiles(); + /** Open a block file (blk?????.dat) */ + FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false) const; -/** Open a block file (blk?????.dat) */ -FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false); -/** Translation to a filesystem path */ -fs::path GetBlockPosFilename(const FlatFilePos& pos); + /** Translation to a filesystem path */ + fs::path GetBlockPosFilename(const FlatFilePos& pos) const; -/** - * Actually unlink the specified files - */ -void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune); + /** + * Actually unlink the specified files + */ + void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) const; -/** Functions for disk access for blocks */ -bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams); -bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams); -bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start); + /** Functions for disk access for blocks */ + bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) const; + bool ReadBlockFromDisk(CBlock& block, const CBlockIndex& index) const; + bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start) const; -bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); + bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& index) const; + + void CleanupBlockRevFiles() const; +}; -void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args, const fs::path& mempool_path); +void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const fs::path& mempool_path); } // namespace node #endif // BITCOIN_NODE_BLOCKSTORAGE_H diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index f60ff83a0d..8f997b0594 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -51,7 +51,7 @@ static ChainstateLoadResult CompleteChainstateInitialization( pblocktree->WriteReindexing(true); //If we're reindexing in prune mode, wipe away unusable block files and all undo data files if (options.prune) { - CleanupBlockRevFiles(); + chainman.m_blockman.CleanupBlockRevFiles(); } } @@ -253,7 +253,7 @@ ChainstateLoadResult VerifyLoadedChainstate(ChainstateManager& chainman, const C "Only rebuild the block database if you are sure that your computer's date and time are correct")}; } - VerifyDBResult result = CVerifyDB().VerifyDB( + VerifyDBResult result = CVerifyDB(chainman.GetNotifications()).VerifyDB( *chainstate, chainman.GetConsensus(), chainstate->CoinsDB(), options.check_level, options.check_blocks); diff --git a/src/node/chainstatemanager_args.cpp b/src/node/chainstatemanager_args.cpp index b97344c9aa..a7f7303348 100644 --- a/src/node/chainstatemanager_args.cpp +++ b/src/node/chainstatemanager_args.cpp @@ -11,16 +11,16 @@ #include <node/database_args.h> #include <tinyformat.h> #include <uint256.h> +#include <util/result.h> #include <util/strencodings.h> #include <util/translation.h> #include <validation.h> #include <chrono> -#include <optional> #include <string> namespace node { -std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts) +util::Result<void> ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts) { if (auto value{args.GetBoolArg("-checkblockindex")}) opts.check_block_index = *value; @@ -28,7 +28,7 @@ std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, Chains if (auto value{args.GetArg("-minimumchainwork")}) { if (!IsHexNumber(*value)) { - return strprintf(Untranslated("Invalid non-hex (%s) minimum chain work value specified"), *value); + return util::Error{strprintf(Untranslated("Invalid non-hex (%s) minimum chain work value specified"), *value)}; } opts.minimum_chain_work = UintToArith256(uint256S(*value)); } @@ -37,10 +37,12 @@ std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, Chains if (auto value{args.GetIntArg("-maxtipage")}) opts.max_tip_age = std::chrono::seconds{*value}; + if (auto value{args.GetIntArg("-stopatheight")}) opts.stop_at_height = *value; + ReadDatabaseArgs(args, opts.block_tree_db); ReadDatabaseArgs(args, opts.coins_db); ReadCoinsViewArgs(args, opts.coins_view); - return std::nullopt; + return {}; } } // namespace node diff --git a/src/node/chainstatemanager_args.h b/src/node/chainstatemanager_args.h index 6c46b998f2..701515953e 100644 --- a/src/node/chainstatemanager_args.h +++ b/src/node/chainstatemanager_args.h @@ -5,15 +5,13 @@ #ifndef BITCOIN_NODE_CHAINSTATEMANAGER_ARGS_H #define BITCOIN_NODE_CHAINSTATEMANAGER_ARGS_H +#include <util/result.h> #include <validation.h> -#include <optional> - class ArgsManager; -struct bilingual_str; namespace node { -std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts); +[[nodiscard]] util::Result<void> ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts); } // namespace node #endif // BITCOIN_NODE_CHAINSTATEMANAGER_ARGS_H diff --git a/src/node/context.cpp b/src/node/context.cpp index af59ab932b..ca56fa0b86 100644 --- a/src/node/context.cpp +++ b/src/node/context.cpp @@ -11,6 +11,7 @@ #include <net.h> #include <net_processing.h> #include <netgroup.h> +#include <node/kernel_notifications.h> #include <policy/fees.h> #include <scheduler.h> #include <txmempool.h> diff --git a/src/node/context.h b/src/node/context.h index 84f4053c84..91b68fa5bb 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -7,7 +7,9 @@ #include <kernel/context.h> +#include <atomic> #include <cassert> +#include <cstdlib> #include <functional> #include <memory> #include <vector> @@ -30,6 +32,8 @@ class WalletLoader; } // namespace interfaces namespace node { +class KernelNotifications; + //! NodeContext struct containing references to chain state and connection //! state. //! @@ -62,6 +66,8 @@ struct NodeContext { interfaces::WalletLoader* wallet_loader{nullptr}; std::unique_ptr<CScheduler> scheduler; std::function<void()> rpc_interruption_point = [] {}; + std::unique_ptr<KernelNotifications> notifications; + std::atomic<int> exit_status{EXIT_SUCCESS}; //! Declare default constructor and destructor that are not inline, so code //! instantiating the NodeContext struct doesn't need to #include class diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 5fcace833f..94b607b1de 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -89,10 +89,11 @@ public: void initLogging() override { InitLogging(args()); } void initParameterInteraction() override { InitParameterInteraction(args()); } bilingual_str getWarnings() override { return GetWarnings(true); } + int getExitStatus() override { return Assert(m_context)->exit_status.load(); } uint32_t getLogCategories() override { return LogInstance().GetCategoryMask(); } bool baseInitialize() override { - if (!AppInitBasicSetup(args())) return false; + if (!AppInitBasicSetup(args(), Assert(context())->exit_status)) return false; if (!AppInitParameterInteraction(args(), /*use_syscall_sandbox=*/false)) return false; m_context->kernel = std::make_unique<kernel::Context>(); @@ -105,7 +106,10 @@ public: } bool appInitMain(interfaces::BlockAndHeaderTipInfo* tip_info) override { - return AppInitMain(*m_context, tip_info); + if (AppInitMain(*m_context, tip_info)) return true; + // Error during initialization, set exit status before continue + m_context->exit_status.store(EXIT_FAILURE); + return false; } void appShutdown() override { @@ -125,17 +129,17 @@ public: bool isSettingIgnored(const std::string& name) override { bool ignored = false; - args().LockSettings([&](util::Settings& settings) { - if (auto* options = util::FindKey(settings.command_line_options, name)) { + args().LockSettings([&](common::Settings& settings) { + if (auto* options = common::FindKey(settings.command_line_options, name)) { ignored = !options->empty(); } }); return ignored; } - util::SettingsValue getPersistentSetting(const std::string& name) override { return args().GetPersistentSetting(name); } - void updateRwSetting(const std::string& name, const util::SettingsValue& value) override + common::SettingsValue getPersistentSetting(const std::string& name) override { return args().GetPersistentSetting(name); } + void updateRwSetting(const std::string& name, const common::SettingsValue& value) override { - args().LockSettings([&](util::Settings& settings) { + args().LockSettings([&](common::Settings& settings) { if (value.isNull()) { settings.rw_settings.erase(name); } else { @@ -144,9 +148,9 @@ public: }); args().WriteSettingsFile(); } - void forceSetting(const std::string& name, const util::SettingsValue& value) override + void forceSetting(const std::string& name, const common::SettingsValue& value) override { - args().LockSettings([&](util::Settings& settings) { + args().LockSettings([&](common::Settings& settings) { if (value.isNull()) { settings.forced_settings.erase(name); } else { @@ -157,7 +161,7 @@ public: void resetSettings() override { args().WriteSettingsFile(/*errors=*/nullptr, /*backup=*/true); - args().LockSettings([&](util::Settings& settings) { + args().LockSettings([&](common::Settings& settings) { settings.rw_settings.clear(); }); args().WriteSettingsFile(); @@ -239,7 +243,7 @@ public: std::vector<ExternalSigner> signers = {}; const std::string command = args().GetArg("-signer", ""); if (command == "") return {}; - ExternalSigner::Enumerate(command, signers, Params().NetworkIDString()); + ExternalSigner::Enumerate(command, signers, Params().GetChainTypeString()); std::vector<std::unique_ptr<interfaces::ExternalSigner>> result; result.reserve(signers.size()); for (auto& signer : signers) { @@ -394,7 +398,7 @@ public: NodeContext* m_context{nullptr}; }; -bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<RecursiveMutex>& lock, const CChain& active) +bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<RecursiveMutex>& lock, const CChain& active, const BlockManager& blockman) { if (!index) return false; if (block.m_hash) *block.m_hash = index->GetBlockHash(); @@ -404,10 +408,10 @@ bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<Rec if (block.m_mtp_time) *block.m_mtp_time = index->GetMedianTimePast(); if (block.m_in_active_chain) *block.m_in_active_chain = active[index->nHeight] == index; if (block.m_locator) { *block.m_locator = GetLocator(index); } - if (block.m_next_block) FillBlock(active[index->nHeight] == index ? active[index->nHeight + 1] : nullptr, *block.m_next_block, lock, active); + if (block.m_next_block) FillBlock(active[index->nHeight] == index ? active[index->nHeight + 1] : nullptr, *block.m_next_block, lock, active, blockman); if (block.m_data) { REVERSE_LOCK(lock); - if (!ReadBlockFromDisk(*block.m_data, index, Params().GetConsensus())) block.m_data->SetNull(); + if (!blockman.ReadBlockFromDisk(*block.m_data, *index)) block.m_data->SetNull(); } block.found = true; return true; @@ -557,13 +561,13 @@ public: bool findBlock(const uint256& hash, const FoundBlock& block) override { WAIT_LOCK(cs_main, lock); - return FillBlock(chainman().m_blockman.LookupBlockIndex(hash), block, lock, chainman().ActiveChain()); + return FillBlock(chainman().m_blockman.LookupBlockIndex(hash), block, lock, chainman().ActiveChain(), chainman().m_blockman); } bool findFirstBlockWithTimeAndHeight(int64_t min_time, int min_height, const FoundBlock& block) override { WAIT_LOCK(cs_main, lock); const CChain& active = chainman().ActiveChain(); - return FillBlock(active.FindEarliestAtLeast(min_time, min_height), block, lock, active); + return FillBlock(active.FindEarliestAtLeast(min_time, min_height), block, lock, active, chainman().m_blockman); } bool findAncestorByHeight(const uint256& block_hash, int ancestor_height, const FoundBlock& ancestor_out) override { @@ -571,10 +575,10 @@ public: const CChain& active = chainman().ActiveChain(); if (const CBlockIndex* block = chainman().m_blockman.LookupBlockIndex(block_hash)) { if (const CBlockIndex* ancestor = block->GetAncestor(ancestor_height)) { - return FillBlock(ancestor, ancestor_out, lock, active); + return FillBlock(ancestor, ancestor_out, lock, active, chainman().m_blockman); } } - return FillBlock(nullptr, ancestor_out, lock, active); + return FillBlock(nullptr, ancestor_out, lock, active, chainman().m_blockman); } bool findAncestorByHash(const uint256& block_hash, const uint256& ancestor_hash, const FoundBlock& ancestor_out) override { @@ -582,7 +586,7 @@ public: const CBlockIndex* block = chainman().m_blockman.LookupBlockIndex(block_hash); const CBlockIndex* ancestor = chainman().m_blockman.LookupBlockIndex(ancestor_hash); if (block && ancestor && block->GetAncestor(ancestor->nHeight) != ancestor) ancestor = nullptr; - return FillBlock(ancestor, ancestor_out, lock, chainman().ActiveChain()); + return FillBlock(ancestor, ancestor_out, lock, chainman().ActiveChain(), chainman().m_blockman); } bool findCommonAncestor(const uint256& block_hash1, const uint256& block_hash2, const FoundBlock& ancestor_out, const FoundBlock& block1_out, const FoundBlock& block2_out) override { @@ -594,9 +598,9 @@ public: // Using & instead of && below to avoid short circuiting and leaving // output uninitialized. Cast bool to int to avoid -Wbitwise-instead-of-logical // compiler warnings. - return int{FillBlock(ancestor, ancestor_out, lock, active)} & - int{FillBlock(block1, block1_out, lock, active)} & - int{FillBlock(block2, block2_out, lock, active)}; + return int{FillBlock(ancestor, ancestor_out, lock, active, chainman().m_blockman)} & + int{FillBlock(block1, block1_out, lock, active, chainman().m_blockman)} & + int{FillBlock(block2, block2_out, lock, active, chainman().m_blockman)}; } void findCoins(std::map<COutPoint, Coin>& coins) override { return FindCoins(m_node, coins); } double guessVerificationProgress(const uint256& block_hash) override @@ -744,27 +748,27 @@ public: RPCRunLater(name, std::move(fn), seconds); } int rpcSerializationFlags() override { return RPCSerializationFlags(); } - util::SettingsValue getSetting(const std::string& name) override + common::SettingsValue getSetting(const std::string& name) override { return args().GetSetting(name); } - std::vector<util::SettingsValue> getSettingsList(const std::string& name) override + std::vector<common::SettingsValue> getSettingsList(const std::string& name) override { return args().GetSettingsList(name); } - util::SettingsValue getRwSetting(const std::string& name) override + common::SettingsValue getRwSetting(const std::string& name) override { - util::SettingsValue result; - args().LockSettings([&](const util::Settings& settings) { - if (const util::SettingsValue* value = util::FindKey(settings.rw_settings, name)) { + common::SettingsValue result; + args().LockSettings([&](const common::Settings& settings) { + if (const common::SettingsValue* value = common::FindKey(settings.rw_settings, name)) { result = *value; } }); return result; } - bool updateRwSetting(const std::string& name, const util::SettingsValue& value, bool write) override + bool updateRwSetting(const std::string& name, const common::SettingsValue& value, bool write) override { - args().LockSettings([&](util::Settings& settings) { + args().LockSettings([&](common::Settings& settings) { if (value.isNull()) { settings.rw_settings.erase(name); } else { diff --git a/src/node/kernel_notifications.cpp b/src/node/kernel_notifications.cpp new file mode 100644 index 0000000000..926b157f3b --- /dev/null +++ b/src/node/kernel_notifications.cpp @@ -0,0 +1,75 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/kernel_notifications.h> + +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif + +#include <common/args.h> +#include <common/system.h> +#include <node/interface_ui.h> +#include <util/strencodings.h> +#include <util/string.h> +#include <util/translation.h> +#include <warnings.h> + +#include <cstdint> +#include <string> +#include <thread> + +static void AlertNotify(const std::string& strMessage) +{ + uiInterface.NotifyAlertChanged(); +#if HAVE_SYSTEM + std::string strCmd = gArgs.GetArg("-alertnotify", ""); + if (strCmd.empty()) return; + + // Alert text should be plain ascii coming from a trusted source, but to + // be safe we first strip anything not in safeChars, then add single quotes around + // the whole string before passing it to the shell: + std::string singleQuote("'"); + std::string safeStatus = SanitizeString(strMessage); + safeStatus = singleQuote+safeStatus+singleQuote; + ReplaceAll(strCmd, "%s", safeStatus); + + std::thread t(runCommand, strCmd); + t.detach(); // thread runs free +#endif +} + +static void DoWarning(const bilingual_str& warning) +{ + static bool fWarned = false; + SetMiscWarning(warning); + if (!fWarned) { + AlertNotify(warning.original); + fWarned = true; + } +} + +namespace node { + +void KernelNotifications::blockTip(SynchronizationState state, CBlockIndex& index) +{ + uiInterface.NotifyBlockTip(state, &index); +} + +void KernelNotifications::headerTip(SynchronizationState state, int64_t height, int64_t timestamp, bool presync) +{ + uiInterface.NotifyHeaderTip(state, height, timestamp, presync); +} + +void KernelNotifications::progress(const bilingual_str& title, int progress_percent, bool resume_possible) +{ + uiInterface.ShowProgress(title.translated, progress_percent, resume_possible); +} + +void KernelNotifications::warning(const bilingual_str& warning) +{ + DoWarning(warning); +} + +} // namespace node diff --git a/src/node/kernel_notifications.h b/src/node/kernel_notifications.h new file mode 100644 index 0000000000..3e665bbf14 --- /dev/null +++ b/src/node/kernel_notifications.h @@ -0,0 +1,31 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_KERNEL_NOTIFICATIONS_H +#define BITCOIN_NODE_KERNEL_NOTIFICATIONS_H + +#include <kernel/notifications_interface.h> + +#include <cstdint> +#include <string> + +class CBlockIndex; +enum class SynchronizationState; +struct bilingual_str; + +namespace node { +class KernelNotifications : public kernel::Notifications +{ +public: + void blockTip(SynchronizationState state, CBlockIndex& index) override; + + void headerTip(SynchronizationState state, int64_t height, int64_t timestamp, bool presync) override; + + void progress(const bilingual_str& title, int progress_percent, bool resume_possible) override; + + void warning(const bilingual_str& warning) override; +}; +} // namespace node + +#endif // BITCOIN_NODE_KERNEL_NOTIFICATIONS_H diff --git a/src/node/mempool_args.cpp b/src/node/mempool_args.cpp index f193469506..5381902263 100644 --- a/src/node/mempool_args.cpp +++ b/src/node/mempool_args.cpp @@ -38,7 +38,7 @@ void ApplyArgsManOptions(const ArgsManager& argsman, MemPoolLimits& mempool_limi } } -std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& argsman, const CChainParams& chainparams, MemPoolOptions& mempool_opts) +util::Result<void> ApplyArgsManOptions(const ArgsManager& argsman, const CChainParams& chainparams, MemPoolOptions& mempool_opts) { mempool_opts.check_ratio = argsman.GetIntArg("-checkmempool", mempool_opts.check_ratio); @@ -52,7 +52,7 @@ std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& argsman, con if (std::optional<CAmount> inc_relay_fee = ParseMoney(argsman.GetArg("-incrementalrelayfee", ""))) { mempool_opts.incremental_relay_feerate = CFeeRate{inc_relay_fee.value()}; } else { - return AmountErrMsg("incrementalrelayfee", argsman.GetArg("-incrementalrelayfee", "")); + return util::Error{AmountErrMsg("incrementalrelayfee", argsman.GetArg("-incrementalrelayfee", ""))}; } } @@ -61,7 +61,7 @@ std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& argsman, con // High fee check is done afterward in CWallet::Create() mempool_opts.min_relay_feerate = CFeeRate{min_relay_feerate.value()}; } else { - return AmountErrMsg("minrelaytxfee", argsman.GetArg("-minrelaytxfee", "")); + return util::Error{AmountErrMsg("minrelaytxfee", argsman.GetArg("-minrelaytxfee", ""))}; } } else if (mempool_opts.incremental_relay_feerate > mempool_opts.min_relay_feerate) { // Allow only setting incremental fee to control both @@ -75,7 +75,7 @@ std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& argsman, con if (std::optional<CAmount> parsed = ParseMoney(argsman.GetArg("-dustrelayfee", ""))) { mempool_opts.dust_relay_feerate = CFeeRate{parsed.value()}; } else { - return AmountErrMsg("dustrelayfee", argsman.GetArg("-dustrelayfee", "")); + return util::Error{AmountErrMsg("dustrelayfee", argsman.GetArg("-dustrelayfee", ""))}; } } @@ -89,12 +89,12 @@ std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& argsman, con mempool_opts.require_standard = !argsman.GetBoolArg("-acceptnonstdtxn", !chainparams.RequireStandard()); if (!chainparams.IsTestChain() && !mempool_opts.require_standard) { - return strprintf(Untranslated("acceptnonstdtxn is not currently supported for %s chain"), chainparams.NetworkIDString()); + return util::Error{strprintf(Untranslated("acceptnonstdtxn is not currently supported for %s chain"), chainparams.GetChainTypeString())}; } mempool_opts.full_rbf = argsman.GetBoolArg("-mempoolfullrbf", mempool_opts.full_rbf); ApplyArgsManOptions(argsman, mempool_opts.limits); - return std::nullopt; + return {}; } diff --git a/src/node/mempool_args.h b/src/node/mempool_args.h index 52d8b4f265..630fee6421 100644 --- a/src/node/mempool_args.h +++ b/src/node/mempool_args.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_NODE_MEMPOOL_ARGS_H #define BITCOIN_NODE_MEMPOOL_ARGS_H -#include <optional> +#include <util/result.h> class ArgsManager; class CChainParams; @@ -21,7 +21,7 @@ struct MemPoolOptions; * @param[in] argsman The ArgsManager in which to check set options. * @param[in,out] mempool_opts The MemPoolOptions to modify according to \p argsman. */ -[[nodiscard]] std::optional<bilingual_str> ApplyArgsManOptions(const ArgsManager& argsman, const CChainParams& chainparams, kernel::MemPoolOptions& mempool_opts); +[[nodiscard]] util::Result<void> ApplyArgsManOptions(const ArgsManager& argsman, const CChainParams& chainparams, kernel::MemPoolOptions& mempool_opts); #endif // BITCOIN_NODE_MEMPOOL_ARGS_H diff --git a/src/node/miner.h b/src/node/miner.h index f1ccffff55..70de9e1db0 100644 --- a/src/node/miner.h +++ b/src/node/miner.h @@ -14,7 +14,10 @@ #include <optional> #include <stdint.h> +#include <boost/multi_index/identity.hpp> +#include <boost/multi_index/indexed_by.hpp> #include <boost/multi_index/ordered_index.hpp> +#include <boost/multi_index/tag.hpp> #include <boost/multi_index_container.hpp> class ArgsManager; diff --git a/src/node/mini_miner.cpp b/src/node/mini_miner.cpp new file mode 100644 index 0000000000..6f253eddfa --- /dev/null +++ b/src/node/mini_miner.cpp @@ -0,0 +1,371 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/mini_miner.h> + +#include <consensus/amount.h> +#include <policy/feerate.h> +#include <primitives/transaction.h> +#include <timedata.h> +#include <util/check.h> +#include <util/moneystr.h> + +#include <algorithm> +#include <numeric> +#include <utility> + +namespace node { + +MiniMiner::MiniMiner(const CTxMemPool& mempool, const std::vector<COutPoint>& outpoints) +{ + LOCK(mempool.cs); + // Find which outpoints to calculate bump fees for. + // Anything that's spent by the mempool is to-be-replaced + // Anything otherwise unavailable just has a bump fee of 0 + for (const auto& outpoint : outpoints) { + if (!mempool.exists(GenTxid::Txid(outpoint.hash))) { + // This UTXO is either confirmed or not yet submitted to mempool. + // If it's confirmed, no bump fee is required. + // If it's not yet submitted, we have no information, so return 0. + m_bump_fees.emplace(outpoint, 0); + continue; + } + + // UXTO is created by transaction in mempool, add to map. + // Note: This will either create a missing entry or add the outpoint to an existing entry + m_requested_outpoints_by_txid[outpoint.hash].push_back(outpoint); + + if (const auto ptx{mempool.GetConflictTx(outpoint)}) { + // This outpoint is already being spent by another transaction in the mempool. We + // assume that the caller wants to replace this transaction and its descendants. It + // would be unusual for the transaction to have descendants as the wallet won’t normally + // attempt to replace transactions with descendants. If the outpoint is from a mempool + // transaction, we still need to calculate its ancestors bump fees (added to + // m_requested_outpoints_by_txid below), but after removing the to-be-replaced entries. + // + // Note that the descendants of a transaction include the transaction itself. Also note, + // that this is only calculating bump fees. RBF fee rules should be handled separately. + CTxMemPool::setEntries descendants; + mempool.CalculateDescendants(mempool.GetIter(ptx->GetHash()).value(), descendants); + for (const auto& desc_txiter : descendants) { + m_to_be_replaced.insert(desc_txiter->GetTx().GetHash()); + } + } + } + + // No unconfirmed UTXOs, so nothing mempool-related needs to be calculated. + if (m_requested_outpoints_by_txid.empty()) return; + + // Calculate the cluster and construct the entry map. + std::vector<uint256> txids_needed; + txids_needed.reserve(m_requested_outpoints_by_txid.size()); + for (const auto& [txid, _]: m_requested_outpoints_by_txid) { + txids_needed.push_back(txid); + } + const auto cluster = mempool.GatherClusters(txids_needed); + if (cluster.empty()) { + // An empty cluster means that at least one of the transactions is missing from the mempool + // (should not be possible given processing above) or DoS limit was hit. + m_ready_to_calculate = false; + return; + } + + // Add every entry to m_entries_by_txid and m_entries, except the ones that will be replaced. + for (const auto& txiter : cluster) { + if (!m_to_be_replaced.count(txiter->GetTx().GetHash())) { + auto [mapiter, success] = m_entries_by_txid.emplace(txiter->GetTx().GetHash(), MiniMinerMempoolEntry(txiter)); + m_entries.push_back(mapiter); + } else { + auto outpoints_it = m_requested_outpoints_by_txid.find(txiter->GetTx().GetHash()); + if (outpoints_it != m_requested_outpoints_by_txid.end()) { + // This UTXO is the output of a to-be-replaced transaction. Bump fee is 0; spending + // this UTXO is impossible as it will no longer exist after the replacement. + for (const auto& outpoint : outpoints_it->second) { + m_bump_fees.emplace(outpoint, 0); + } + m_requested_outpoints_by_txid.erase(outpoints_it); + } + } + } + + // Build the m_descendant_set_by_txid cache. + for (const auto& txiter : cluster) { + const auto& txid = txiter->GetTx().GetHash(); + // Cache descendants for future use. Unlike the real mempool, a descendant MiniMinerMempoolEntry + // will not exist without its ancestor MiniMinerMempoolEntry, so these sets won't be invalidated. + std::vector<MockEntryMap::iterator> cached_descendants; + const bool remove{m_to_be_replaced.count(txid) > 0}; + CTxMemPool::setEntries descendants; + mempool.CalculateDescendants(txiter, descendants); + Assume(descendants.count(txiter) > 0); + for (const auto& desc_txiter : descendants) { + const auto txid_desc = desc_txiter->GetTx().GetHash(); + const bool remove_desc{m_to_be_replaced.count(txid_desc) > 0}; + auto desc_it{m_entries_by_txid.find(txid_desc)}; + Assume((desc_it == m_entries_by_txid.end()) == remove_desc); + if (remove) Assume(remove_desc); + // It's possible that remove=false but remove_desc=true. + if (!remove && !remove_desc) { + cached_descendants.push_back(desc_it); + } + } + if (remove) { + Assume(cached_descendants.empty()); + } else { + m_descendant_set_by_txid.emplace(txid, cached_descendants); + } + } + + // Release the mempool lock; we now have all the information we need for a subset of the entries + // we care about. We will solely operate on the MiniMinerMempoolEntry map from now on. + Assume(m_in_block.empty()); + Assume(m_requested_outpoints_by_txid.size() <= outpoints.size()); + SanityCheck(); +} + +// Compare by min(ancestor feerate, individual feerate), then iterator +// +// Under the ancestor-based mining approach, high-feerate children can pay for parents, but high-feerate +// parents do not incentive inclusion of their children. Therefore the mining algorithm only considers +// transactions for inclusion on basis of the minimum of their own feerate or their ancestor feerate. +struct AncestorFeerateComparator +{ + template<typename I> + bool operator()(const I& a, const I& b) const { + auto min_feerate = [](const MiniMinerMempoolEntry& e) -> CFeeRate { + const CAmount ancestor_fee{e.GetModFeesWithAncestors()}; + const int64_t ancestor_size{e.GetSizeWithAncestors()}; + const CAmount tx_fee{e.GetModifiedFee()}; + const int64_t tx_size{e.GetTxSize()}; + // Comparing ancestor feerate with individual feerate: + // ancestor_fee / ancestor_size <= tx_fee / tx_size + // Avoid division and possible loss of precision by + // multiplying both sides by the sizes: + return ancestor_fee * tx_size < tx_fee * ancestor_size ? + CFeeRate(ancestor_fee, ancestor_size) : + CFeeRate(tx_fee, tx_size); + }; + CFeeRate a_feerate{min_feerate(a->second)}; + CFeeRate b_feerate{min_feerate(b->second)}; + if (a_feerate != b_feerate) { + return a_feerate > b_feerate; + } + // Use txid as tiebreaker for stable sorting + return a->first < b->first; + } +}; + +void MiniMiner::DeleteAncestorPackage(const std::set<MockEntryMap::iterator, IteratorComparator>& ancestors) +{ + Assume(ancestors.size() >= 1); + // "Mine" all transactions in this ancestor set. + for (auto& anc : ancestors) { + Assume(m_in_block.count(anc->first) == 0); + m_in_block.insert(anc->first); + m_total_fees += anc->second.GetModifiedFee(); + m_total_vsize += anc->second.GetTxSize(); + auto it = m_descendant_set_by_txid.find(anc->first); + // Each entry’s descendant set includes itself + Assume(it != m_descendant_set_by_txid.end()); + for (auto& descendant : it->second) { + // If these fail, we must be double-deducting. + Assume(descendant->second.GetModFeesWithAncestors() >= anc->second.GetModifiedFee()); + Assume(descendant->second.vsize_with_ancestors >= anc->second.GetTxSize()); + descendant->second.fee_with_ancestors -= anc->second.GetModifiedFee(); + descendant->second.vsize_with_ancestors -= anc->second.GetTxSize(); + } + } + // Delete these entries. + for (const auto& anc : ancestors) { + m_descendant_set_by_txid.erase(anc->first); + // The above loop should have deducted each ancestor's size and fees from each of their + // respective descendants exactly once. + Assume(anc->second.GetModFeesWithAncestors() == 0); + Assume(anc->second.GetSizeWithAncestors() == 0); + auto vec_it = std::find(m_entries.begin(), m_entries.end(), anc); + Assume(vec_it != m_entries.end()); + m_entries.erase(vec_it); + m_entries_by_txid.erase(anc); + } +} + +void MiniMiner::SanityCheck() const +{ + // m_entries, m_entries_by_txid, and m_descendant_set_by_txid all same size + Assume(m_entries.size() == m_entries_by_txid.size()); + Assume(m_entries.size() == m_descendant_set_by_txid.size()); + // Cached ancestor values should be at least as large as the transaction's own fee and size + Assume(std::all_of(m_entries.begin(), m_entries.end(), [](const auto& entry) { + return entry->second.GetSizeWithAncestors() >= entry->second.GetTxSize() && + entry->second.GetModFeesWithAncestors() >= entry->second.GetModifiedFee();})); + // None of the entries should be to-be-replaced transactions + Assume(std::all_of(m_to_be_replaced.begin(), m_to_be_replaced.end(), + [&](const auto& txid){return m_entries_by_txid.find(txid) == m_entries_by_txid.end();})); +} + +void MiniMiner::BuildMockTemplate(const CFeeRate& target_feerate) +{ + while (!m_entries_by_txid.empty()) { + // Sort again, since transaction removal may change some m_entries' ancestor feerates. + std::sort(m_entries.begin(), m_entries.end(), AncestorFeerateComparator()); + + // Pick highest ancestor feerate entry. + auto best_iter = m_entries.begin(); + Assume(best_iter != m_entries.end()); + const auto ancestor_package_size = (*best_iter)->second.GetSizeWithAncestors(); + const auto ancestor_package_fee = (*best_iter)->second.GetModFeesWithAncestors(); + // Stop here. Everything that didn't "make it into the block" has bumpfee. + if (ancestor_package_fee < target_feerate.GetFee(ancestor_package_size)) { + break; + } + + // Calculate ancestors on the fly. This lookup should be fairly cheap, and ancestor sets + // change at every iteration, so this is more efficient than maintaining a cache. + std::set<MockEntryMap::iterator, IteratorComparator> ancestors; + { + std::set<MockEntryMap::iterator, IteratorComparator> to_process; + to_process.insert(*best_iter); + while (!to_process.empty()) { + auto iter = to_process.begin(); + Assume(iter != to_process.end()); + ancestors.insert(*iter); + for (const auto& input : (*iter)->second.GetTx().vin) { + if (auto parent_it{m_entries_by_txid.find(input.prevout.hash)}; parent_it != m_entries_by_txid.end()) { + if (ancestors.count(parent_it) == 0) { + to_process.insert(parent_it); + } + } + } + to_process.erase(iter); + } + } + DeleteAncestorPackage(ancestors); + SanityCheck(); + } + Assume(m_in_block.empty() || m_total_fees >= target_feerate.GetFee(m_total_vsize)); + // Do not try to continue building the block template with a different feerate. + m_ready_to_calculate = false; +} + +std::map<COutPoint, CAmount> MiniMiner::CalculateBumpFees(const CFeeRate& target_feerate) +{ + if (!m_ready_to_calculate) return {}; + // Build a block template until the target feerate is hit. + BuildMockTemplate(target_feerate); + + // Each transaction that "made it into the block" has a bumpfee of 0, i.e. they are part of an + // ancestor package with at least the target feerate and don't need to be bumped. + for (const auto& txid : m_in_block) { + // Not all of the block transactions were necessarily requested. + auto it = m_requested_outpoints_by_txid.find(txid); + if (it != m_requested_outpoints_by_txid.end()) { + for (const auto& outpoint : it->second) { + m_bump_fees.emplace(outpoint, 0); + } + m_requested_outpoints_by_txid.erase(it); + } + } + + // A transactions and its ancestors will only be picked into a block when + // both the ancestor set feerate and the individual feerate meet the target + // feerate. + // + // We had to convince ourselves that after running the mini miner and + // picking all eligible transactions into our MockBlockTemplate, there + // could still be transactions remaining that have a lower individual + // feerate than their ancestor feerate. So here is an example: + // + // ┌─────────────────┐ + // │ │ + // │ Grandparent │ + // │ 1700 vB │ + // │ 1700 sats │ Target feerate: 10 s/vB + // │ 1 s/vB │ GP Ancestor Set Feerate (ASFR): 1 s/vB + // │ │ P1_ASFR: 9.84 s/vB + // └──────▲───▲──────┘ P2_ASFR: 2.47 s/vB + // │ │ C_ASFR: 10.27 s/vB + // ┌───────────────┐ │ │ ┌──────────────┐ + // │ ├────┘ └────┤ │ ⇒ C_FR < TFR < C_ASFR + // │ Parent 1 │ │ Parent 2 │ + // │ 200 vB │ │ 200 vB │ + // │ 17000 sats │ │ 3000 sats │ + // │ 85 s/vB │ │ 15 s/vB │ + // │ │ │ │ + // └───────────▲───┘ └───▲──────────┘ + // │ │ + // │ ┌───────────┐ │ + // └────┤ ├────┘ + // │ Child │ + // │ 100 vB │ + // │ 900 sats │ + // │ 9 s/vB │ + // │ │ + // └───────────┘ + // + // We therefore calculate both the bump fee that is necessary to elevate + // the individual transaction to the target feerate: + // target_feerate × tx_size - tx_fees + // and the bump fee that is necessary to bump the entire ancestor set to + // the target feerate: + // target_feerate × ancestor_set_size - ancestor_set_fees + // By picking the maximum from the two, we ensure that a transaction meets + // both criteria. + for (const auto& [txid, outpoints] : m_requested_outpoints_by_txid) { + auto it = m_entries_by_txid.find(txid); + Assume(it != m_entries_by_txid.end()); + if (it != m_entries_by_txid.end()) { + Assume(target_feerate.GetFee(it->second.GetSizeWithAncestors()) > std::min(it->second.GetModifiedFee(), it->second.GetModFeesWithAncestors())); + CAmount bump_fee_with_ancestors = target_feerate.GetFee(it->second.GetSizeWithAncestors()) - it->second.GetModFeesWithAncestors(); + CAmount bump_fee_individual = target_feerate.GetFee(it->second.GetTxSize()) - it->second.GetModifiedFee(); + const CAmount bump_fee{std::max(bump_fee_with_ancestors, bump_fee_individual)}; + Assume(bump_fee >= 0); + for (const auto& outpoint : outpoints) { + m_bump_fees.emplace(outpoint, bump_fee); + } + } + } + return m_bump_fees; +} + +std::optional<CAmount> MiniMiner::CalculateTotalBumpFees(const CFeeRate& target_feerate) +{ + if (!m_ready_to_calculate) return std::nullopt; + // Build a block template until the target feerate is hit. + BuildMockTemplate(target_feerate); + + // All remaining ancestors that are not part of m_in_block must be bumped, but no other relatives + std::set<MockEntryMap::iterator, IteratorComparator> ancestors; + std::set<MockEntryMap::iterator, IteratorComparator> to_process; + for (const auto& [txid, outpoints] : m_requested_outpoints_by_txid) { + // Skip any ancestors that already have a miner score higher than the target feerate + // (already "made it" into the block) + if (m_in_block.count(txid)) continue; + auto iter = m_entries_by_txid.find(txid); + if (iter == m_entries_by_txid.end()) continue; + to_process.insert(iter); + ancestors.insert(iter); + } + + std::set<uint256> has_been_processed; + while (!to_process.empty()) { + auto iter = to_process.begin(); + const CTransaction& tx = (*iter)->second.GetTx(); + for (const auto& input : tx.vin) { + if (auto parent_it{m_entries_by_txid.find(input.prevout.hash)}; parent_it != m_entries_by_txid.end()) { + if (!has_been_processed.count(input.prevout.hash)) { + to_process.insert(parent_it); + } + ancestors.insert(parent_it); + } + } + has_been_processed.insert(tx.GetHash()); + to_process.erase(iter); + } + const auto ancestor_package_size = std::accumulate(ancestors.cbegin(), ancestors.cend(), int64_t{0}, + [](int64_t sum, const auto it) {return sum + it->second.GetTxSize();}); + const auto ancestor_package_fee = std::accumulate(ancestors.cbegin(), ancestors.cend(), CAmount{0}, + [](CAmount sum, const auto it) {return sum + it->second.GetModifiedFee();}); + return target_feerate.GetFee(ancestor_package_size) - ancestor_package_fee; +} +} // namespace node diff --git a/src/node/mini_miner.h b/src/node/mini_miner.h new file mode 100644 index 0000000000..db07e6d1bf --- /dev/null +++ b/src/node/mini_miner.h @@ -0,0 +1,121 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_MINI_MINER_H +#define BITCOIN_NODE_MINI_MINER_H + +#include <txmempool.h> + +#include <memory> +#include <optional> +#include <stdint.h> + +namespace node { + +// Container for tracking updates to ancestor feerate as we include ancestors in the "block" +class MiniMinerMempoolEntry +{ + const CAmount fee_individual; + const CTransactionRef tx; + const int64_t vsize_individual; + +// This class must be constructed while holding mempool.cs. After construction, the object's +// methods can be called without holding that lock. +public: + CAmount fee_with_ancestors; + int64_t vsize_with_ancestors; + explicit MiniMinerMempoolEntry(CTxMemPool::txiter entry) : + fee_individual{entry->GetModifiedFee()}, + tx{entry->GetSharedTx()}, + vsize_individual(entry->GetTxSize()), + fee_with_ancestors{entry->GetModFeesWithAncestors()}, + vsize_with_ancestors(entry->GetSizeWithAncestors()) + { } + + CAmount GetModifiedFee() const { return fee_individual; } + CAmount GetModFeesWithAncestors() const { return fee_with_ancestors; } + int64_t GetTxSize() const { return vsize_individual; } + int64_t GetSizeWithAncestors() const { return vsize_with_ancestors; } + const CTransaction& GetTx() const LIFETIMEBOUND { return *tx; } +}; + +// Comparator needed for std::set<MockEntryMap::iterator> +struct IteratorComparator +{ + template<typename I> + bool operator()(const I& a, const I& b) const + { + return &(*a) < &(*b); + } +}; + +/** A minimal version of BlockAssembler. Allows us to run the mining algorithm on a subset of + * mempool transactions, ignoring consensus rules, to calculate mining scores. */ +class MiniMiner +{ + // When true, a caller may use CalculateBumpFees(). Becomes false if we failed to retrieve + // mempool entries (i.e. cluster size too large) or bump fees have already been calculated. + bool m_ready_to_calculate{true}; + + // Set once per lifetime, fill in during initialization. + // txids of to-be-replaced transactions + std::set<uint256> m_to_be_replaced; + + // If multiple argument outpoints correspond to the same transaction, cache them together in + // a single entry indexed by txid. Then we can just work with txids since all outpoints from + // the same tx will have the same bumpfee. Excludes non-mempool transactions. + std::map<uint256, std::vector<COutPoint>> m_requested_outpoints_by_txid; + + // What we're trying to calculate. + std::map<COutPoint, CAmount> m_bump_fees; + + // The constructed block template + std::set<uint256> m_in_block; + + // Information on the current status of the block + CAmount m_total_fees{0}; + int32_t m_total_vsize{0}; + + /** Main data structure holding the entries, can be indexed by txid */ + std::map<uint256, MiniMinerMempoolEntry> m_entries_by_txid; + using MockEntryMap = decltype(m_entries_by_txid); + + /** Vector of entries, can be sorted by ancestor feerate. */ + std::vector<MockEntryMap::iterator> m_entries; + + /** Map of txid to its descendants. Should be inclusive. */ + std::map<uint256, std::vector<MockEntryMap::iterator>> m_descendant_set_by_txid; + + /** Consider this ancestor package "mined" so remove all these entries from our data structures. */ + void DeleteAncestorPackage(const std::set<MockEntryMap::iterator, IteratorComparator>& ancestors); + + /** Perform some checks. */ + void SanityCheck() const; + +public: + /** Returns true if CalculateBumpFees may be called, false if not. */ + bool IsReadyToCalculate() const { return m_ready_to_calculate; } + + /** Build a block template until the target feerate is hit. */ + void BuildMockTemplate(const CFeeRate& target_feerate); + + /** Returns set of txids in the block template if one has been constructed. */ + std::set<uint256> GetMockTemplateTxids() const { return m_in_block; } + + MiniMiner(const CTxMemPool& mempool, const std::vector<COutPoint>& outpoints); + + /** Construct a new block template and, for each outpoint corresponding to a transaction that + * did not make it into the block, calculate the cost of bumping those transactions (and their + * ancestors) to the minimum feerate. Returns a map from outpoint to bump fee, or an empty map + * if they cannot be calculated. */ + std::map<COutPoint, CAmount> CalculateBumpFees(const CFeeRate& target_feerate); + + /** Construct a new block template and, calculate the cost of bumping all transactions that did + * not make it into the block to the target feerate. Returns the total bump fee, or std::nullopt + * if it cannot be calculated. */ + std::optional<CAmount> CalculateTotalBumpFees(const CFeeRate& target_feerate); +}; +} // namespace node + +#endif // BITCOIN_NODE_MINI_MINER_H diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index c7c8493f0c..026c8084dd 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -122,7 +122,7 @@ TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef t return TransactionError::OK; } -CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock) +CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, uint256& hashBlock, const BlockManager& blockman) { if (mempool && !block_index) { CTransactionRef ptx = mempool->get(hash); @@ -143,7 +143,7 @@ CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMe } if (block_index) { CBlock block; - if (ReadBlockFromDisk(block, block_index, consensusParams)) { + if (blockman.ReadBlockFromDisk(block, *block_index)) { for (const auto& tx : block.vtx) { if (tx->GetHash() == hash) { hashBlock = block_index->GetBlockHash(); diff --git a/src/node/transaction.h b/src/node/transaction.h index 45f174f13c..168273594c 100644 --- a/src/node/transaction.h +++ b/src/node/transaction.h @@ -16,6 +16,7 @@ struct Params; } namespace node { +class BlockManager; struct NodeContext; /** Maximum fee rate for sendrawtransaction and testmempoolaccept RPC calls. @@ -53,11 +54,10 @@ static const CFeeRate DEFAULT_MAX_RAW_TX_FEE_RATE{COIN / 10}; * @param[in] block_index The block to read from disk, or nullptr * @param[in] mempool If provided, check mempool for tx * @param[in] hash The txid - * @param[in] consensusParams The params * @param[out] hashBlock The block hash, if the tx was found via -txindex or block_index * @returns The tx if found, otherwise nullptr */ -CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock); +CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, uint256& hashBlock, const BlockManager& blockman); } // namespace node #endif // BITCOIN_NODE_TRANSACTION_H diff --git a/src/node/txreconciliation.cpp b/src/node/txreconciliation.cpp index 9938759074..d62046daaa 100644 --- a/src/node/txreconciliation.cpp +++ b/src/node/txreconciliation.cpp @@ -4,9 +4,9 @@ #include <node/txreconciliation.h> +#include <common/system.h> #include <logging.h> #include <util/check.h> -#include <util/system.h> #include <unordered_map> #include <variant> diff --git a/src/node/utxo_snapshot.cpp b/src/node/utxo_snapshot.cpp index 3dae46fb84..036a25d0a5 100644 --- a/src/node/utxo_snapshot.cpp +++ b/src/node/utxo_snapshot.cpp @@ -4,7 +4,6 @@ #include <node/utxo_snapshot.h> -#include <common/args.h> #include <logging.h> #include <streams.h> #include <sync.h> @@ -82,10 +81,10 @@ std::optional<uint256> ReadSnapshotBaseBlockhash(fs::path chaindir) return base_blockhash; } -std::optional<fs::path> FindSnapshotChainstateDir() +std::optional<fs::path> FindSnapshotChainstateDir(const fs::path& data_dir) { fs::path possible_dir = - gArgs.GetDataDirNet() / fs::u8path(strprintf("chainstate%s", SNAPSHOT_CHAINSTATE_SUFFIX)); + data_dir / fs::u8path(strprintf("chainstate%s", SNAPSHOT_CHAINSTATE_SUFFIX)); if (fs::exists(possible_dir)) { return possible_dir; diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h index 44ddd77dc3..a6dd3f3f13 100644 --- a/src/node/utxo_snapshot.h +++ b/src/node/utxo_snapshot.h @@ -67,7 +67,7 @@ constexpr std::string_view SNAPSHOT_CHAINSTATE_SUFFIX = "_snapshot"; //! Return a path to the snapshot-based chainstate dir, if one exists. -std::optional<fs::path> FindSnapshotChainstateDir(); +std::optional<fs::path> FindSnapshotChainstateDir(const fs::path& data_dir); } // namespace node diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 6121224979..ae226f7011 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -6,6 +6,7 @@ #include <policy/fees.h> #include <clientversion.h> +#include <common/system.h> #include <consensus/amount.h> #include <kernel/mempool_entry.h> #include <logging.h> @@ -19,7 +20,6 @@ #include <uint256.h> #include <util/fs.h> #include <util/serfloat.h> -#include <util/system.h> #include <util/time.h> #include <algorithm> diff --git a/src/policy/policy.h b/src/policy/policy.h index 394fb34230..9135cae91c 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -24,7 +24,7 @@ static constexpr unsigned int DEFAULT_BLOCK_MAX_WEIGHT{MAX_BLOCK_WEIGHT - 4000}; /** Default for -blockmintxfee, which sets the minimum feerate for a transaction in blocks created by mining code **/ static constexpr unsigned int DEFAULT_BLOCK_MIN_TX_FEE{1000}; /** The maximum weight for transactions we're willing to relay/mine */ -static constexpr unsigned int MAX_STANDARD_TX_WEIGHT{400000}; +static constexpr int32_t MAX_STANDARD_TX_WEIGHT{400000}; /** The minimum non-witness size for transactions we're willing to relay/mine: one larger than 64 */ static constexpr unsigned int MIN_STANDARD_TX_NONWITNESS_SIZE{65}; /** Maximum number of signature check operations in an IsStandard() P2SH script */ diff --git a/src/protocol.cpp b/src/protocol.cpp index 5725813b7e..5ecaabec36 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -5,7 +5,7 @@ #include <protocol.h> -#include <util/system.h> +#include <common/system.h> #include <atomic> diff --git a/src/protocol.h b/src/protocol.h index cbcd400fef..ac4545c311 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -134,7 +134,8 @@ extern const char* GETADDR; /** * The mempool message requests the TXIDs of transactions that the receiving * node has verified as valid but which have not yet appeared in a block. - * @since protocol version 60002. + * @since protocol version 60002 as described by BIP35. + * Only available with service bit NODE_BLOOM, see also BIP111. */ extern const char* MEMPOOL; /** @@ -278,8 +279,6 @@ enum ServiceFlags : uint64_t { // set by all Bitcoin Core non pruned nodes, and is unset by SPV clients or other light clients. NODE_NETWORK = (1 << 0), // NODE_BLOOM means the node is capable and willing to handle bloom-filtered connections. - // Bitcoin Core nodes used to support this by default, without advertising this bit, - // but no longer do as of protocol version 70011 (= NO_BLOOM_VERSION) NODE_BLOOM = (1 << 2), // NODE_WITNESS indicates that a node can be asked for blocks and transactions including // witness data. diff --git a/src/qt/addresstablemodel.cpp b/src/qt/addresstablemodel.cpp index 0d0f1a4d15..e4689e4389 100644 --- a/src/qt/addresstablemodel.cpp +++ b/src/qt/addresstablemodel.cpp @@ -53,15 +53,14 @@ struct AddressTableEntryLessThan }; /* Determine address type from address purpose */ -static AddressTableEntry::Type translateTransactionType(wallet::AddressPurpose purpose, bool isMine) +constexpr AddressTableEntry::Type translateTransactionType(wallet::AddressPurpose purpose, bool isMine) { // "refund" addresses aren't shown, and change addresses aren't returned by getAddresses at all. switch (purpose) { case wallet::AddressPurpose::SEND: return AddressTableEntry::Sending; case wallet::AddressPurpose::RECEIVE: return AddressTableEntry::Receiving; case wallet::AddressPurpose::REFUND: return AddressTableEntry::Hidden; - // No default case to allow for compiler to warn - } + } // no default case, so the compiler can warn about missing cases assert(false); } diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 2f00009596..8f45af9485 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -9,8 +9,10 @@ #include <qt/bitcoin.h> #include <chainparams.h> +#include <node/context.h> #include <common/args.h> #include <common/init.h> +#include <common/system.h> #include <init.h> #include <interfaces/handler.h> #include <interfaces/init.h> @@ -32,7 +34,6 @@ #include <uint256.h> #include <util/exception.h> #include <util/string.h> -#include <util/system.h> #include <util/threadnames.h> #include <util/translation.h> #include <validation.h> @@ -397,9 +398,7 @@ void BitcoinApplication::initializeResult(bool success, interfaces::BlockAndHead { qDebug() << __func__ << ": Initialization result: " << success; - // Set exit result. - returnValue = success ? EXIT_SUCCESS : EXIT_FAILURE; - if(success) { + if (success) { delete m_splash; m_splash = nullptr; @@ -601,7 +600,7 @@ int GuiMain(int argc, char* argv[]) PaymentServer::ipcParseCommandLine(argc, argv); #endif - QScopedPointer<const NetworkStyle> networkStyle(NetworkStyle::instantiate(Params().NetworkIDString())); + QScopedPointer<const NetworkStyle> networkStyle(NetworkStyle::instantiate(Params().GetChainType())); assert(!networkStyle.isNull()); // Allow for separate UI settings for testnets QApplication::setApplicationName(networkStyle->getAppName()); @@ -653,7 +652,6 @@ int GuiMain(int argc, char* argv[]) app.InitPruneSetting(prune_MiB); } - int rv = EXIT_SUCCESS; try { app.createWindow(networkStyle.data()); @@ -666,10 +664,9 @@ int GuiMain(int argc, char* argv[]) WinShutdownMonitor::registerShutdownBlockReason(QObject::tr("%1 didn't yet exit safely…").arg(PACKAGE_NAME), (HWND)app.getMainWinId()); #endif app.exec(); - rv = app.getReturnValue(); } else { // A dialog with detailed error will have been shown by InitError() - rv = EXIT_FAILURE; + return EXIT_FAILURE; } } catch (const std::exception& e) { PrintExceptionContinue(&e, "Runaway exception"); @@ -678,5 +675,5 @@ int GuiMain(int argc, char* argv[]) PrintExceptionContinue(nullptr, "Runaway exception"); app.handleRunawayException(QString::fromStdString(app.node().getWarnings().translated)); } - return rv; + return app.node().getExitStatus(); } diff --git a/src/qt/bitcoin.h b/src/qt/bitcoin.h index 9174e23de6..9622c9d57d 100644 --- a/src/qt/bitcoin.h +++ b/src/qt/bitcoin.h @@ -62,9 +62,6 @@ public: /// Request core initialization void requestInitialize(); - /// Get process return value - int getReturnValue() const { return returnValue; } - /// Get window identifier of QMainWindow (BitcoinGUI) WId getMainWinId() const; @@ -104,7 +101,6 @@ private: PaymentServer* paymentServer{nullptr}; WalletController* m_wallet_controller{nullptr}; #endif - int returnValue{0}; const PlatformStyle* platformStyle{nullptr}; std::unique_ptr<QWidget> shutdownWindow; SplashScreen* m_splash = nullptr; diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index d26ef52eb4..f201d8fa01 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -30,16 +30,17 @@ #include <qt/macdockiconhandler.h> #endif -#include <functional> #include <chain.h> #include <chainparams.h> +#include <common/system.h> #include <interfaces/handler.h> #include <interfaces/node.h> #include <node/interface_ui.h> -#include <util/system.h> #include <util/translation.h> #include <validation.h> +#include <functional> + #include <QAction> #include <QActionGroup> #include <QApplication> diff --git a/src/qt/clientmodel.cpp b/src/qt/clientmodel.cpp index 837e5f4fed..ff7405d139 100644 --- a/src/qt/clientmodel.cpp +++ b/src/qt/clientmodel.cpp @@ -12,11 +12,11 @@ #include <clientversion.h> #include <common/args.h> +#include <common/system.h> #include <interfaces/handler.h> #include <interfaces/node.h> #include <net.h> #include <netbase.h> -#include <util/system.h> #include <util/threadnames.h> #include <util/time.h> #include <validation.h> @@ -28,8 +28,8 @@ #include <QThread> #include <QTimer> -static int64_t nLastHeaderTipUpdateNotification = 0; -static int64_t nLastBlockTipUpdateNotification = 0; +static SteadyClock::time_point g_last_header_tip_update_notification{}; +static SteadyClock::time_point g_last_block_tip_update_notification{}; ClientModel::ClientModel(interfaces::Node& node, OptionsModel *_optionsModel, QObject *parent) : QObject(parent), @@ -222,9 +222,9 @@ void ClientModel::TipChanged(SynchronizationState sync_state, interfaces::BlockT // Throttle GUI notifications about (a) blocks during initial sync, and (b) both blocks and headers during reindex. const bool throttle = (sync_state != SynchronizationState::POST_INIT && synctype == SyncType::BLOCK_SYNC) || sync_state == SynchronizationState::INIT_REINDEX; - const int64_t now = throttle ? GetTimeMillis() : 0; - int64_t& nLastUpdateNotification = synctype != SyncType::BLOCK_SYNC ? nLastHeaderTipUpdateNotification : nLastBlockTipUpdateNotification; - if (throttle && now < nLastUpdateNotification + count_milliseconds(MODEL_UPDATE_DELAY)) { + const auto now{throttle ? SteadyClock::now() : SteadyClock::time_point{}}; + auto& nLastUpdateNotification = synctype != SyncType::BLOCK_SYNC ? g_last_header_tip_update_notification : g_last_block_tip_update_notification; + if (throttle && now < nLastUpdateNotification + MODEL_UPDATE_DELAY) { return; } diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index e1b1ae12e9..50d3c2e559 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -412,8 +412,7 @@ void CoinControlDialog::updateLabels(CCoinControl& m_coin_control, WalletModel * unsigned int nQuantity = 0; bool fWitness = false; - std::vector<COutPoint> vCoinControl; - m_coin_control.ListSelected(vCoinControl); + auto vCoinControl{m_coin_control.ListSelected()}; size_t i = 0; for (const auto& out : model->wallet().getCoins(vCoinControl)) { diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index dc7daed4bc..8d8328aad8 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -21,6 +21,7 @@ #include <protocol.h> #include <script/script.h> #include <script/standard.h> +#include <util/chaintype.h> #include <util/exception.h> #include <util/fs.h> #include <util/fs_helpers.h> @@ -503,12 +504,12 @@ bool LabelOutOfFocusEventFilter::eventFilter(QObject* watched, QEvent* event) #ifdef WIN32 fs::path static StartupShortcutPath() { - std::string chain = gArgs.GetChainName(); - if (chain == CBaseChainParams::MAIN) + ChainType chain = gArgs.GetChainType(); + if (chain == ChainType::MAIN) return GetSpecialFolderPath(CSIDL_STARTUP) / "Bitcoin.lnk"; - if (chain == CBaseChainParams::TESTNET) // Remove this special case when CBaseChainParams::TESTNET = "testnet4" + if (chain == ChainType::TESTNET) // Remove this special case when testnet CBaseChainParams::DataDir() is incremented to "testnet4" return GetSpecialFolderPath(CSIDL_STARTUP) / "Bitcoin (testnet).lnk"; - return GetSpecialFolderPath(CSIDL_STARTUP) / fs::u8path(strprintf("Bitcoin (%s).lnk", chain)); + return GetSpecialFolderPath(CSIDL_STARTUP) / fs::u8path(strprintf("Bitcoin (%s).lnk", ChainTypeToString(chain))); } bool GetStartOnSystemStartup() @@ -541,7 +542,7 @@ bool SetStartOnSystemStartup(bool fAutoStart) // Start client minimized QString strArgs = "-min"; // Set -testnet /-regtest options - strArgs += QString::fromStdString(strprintf(" -chain=%s", gArgs.GetChainName())); + strArgs += QString::fromStdString(strprintf(" -chain=%s", gArgs.GetChainTypeString())); // Set the path to the shortcut target psl->SetPath(pszExePath); @@ -586,10 +587,10 @@ fs::path static GetAutostartDir() fs::path static GetAutostartFilePath() { - std::string chain = gArgs.GetChainName(); - if (chain == CBaseChainParams::MAIN) + ChainType chain = gArgs.GetChainType(); + if (chain == ChainType::MAIN) return GetAutostartDir() / "bitcoin.desktop"; - return GetAutostartDir() / fs::u8path(strprintf("bitcoin-%s.desktop", chain)); + return GetAutostartDir() / fs::u8path(strprintf("bitcoin-%s.desktop", ChainTypeToString(chain))); } bool GetStartOnSystemStartup() @@ -629,15 +630,15 @@ bool SetStartOnSystemStartup(bool fAutoStart) std::ofstream optionFile{GetAutostartFilePath(), std::ios_base::out | std::ios_base::trunc}; if (!optionFile.good()) return false; - std::string chain = gArgs.GetChainName(); + ChainType chain = gArgs.GetChainType(); // Write a bitcoin.desktop file to the autostart directory: optionFile << "[Desktop Entry]\n"; optionFile << "Type=Application\n"; - if (chain == CBaseChainParams::MAIN) + if (chain == ChainType::MAIN) optionFile << "Name=Bitcoin\n"; else - optionFile << strprintf("Name=Bitcoin (%s)\n", chain); - optionFile << "Exec=" << pszExePath << strprintf(" -min -chain=%s\n", chain); + optionFile << strprintf("Name=Bitcoin (%s)\n", ChainTypeToString(chain)); + optionFile << "Exec=" << pszExePath << strprintf(" -min -chain=%s\n", ChainTypeToString(chain)); optionFile << "Terminal=false\n"; optionFile << "Hidden=false\n"; optionFile.close(); diff --git a/src/qt/intro.cpp b/src/qt/intro.cpp index a54ba19354..f86b167076 100644 --- a/src/qt/intro.cpp +++ b/src/qt/intro.cpp @@ -9,6 +9,7 @@ #include <chainparams.h> #include <qt/intro.h> #include <qt/forms/ui_intro.h> +#include <util/chaintype.h> #include <util/fs.h> #include <qt/guiconstants.h> @@ -219,7 +220,7 @@ bool Intro::showIfNeeded(bool& did_show_intro, int64_t& prune_MiB) { /* Use selectParams here to guarantee Params() can be used by node interface */ try { - SelectParams(gArgs.GetChainName()); + SelectParams(gArgs.GetChainType()); } catch (const std::exception&) { return false; } diff --git a/src/qt/networkstyle.cpp b/src/qt/networkstyle.cpp index b789e6a958..b6314f5533 100644 --- a/src/qt/networkstyle.cpp +++ b/src/qt/networkstyle.cpp @@ -6,21 +6,21 @@ #include <qt/guiconstants.h> -#include <chainparamsbase.h> #include <tinyformat.h> +#include <util/chaintype.h> #include <QApplication> static const struct { - const char *networkId; + const ChainType networkId; const char *appName; const int iconColorHueShift; const int iconColorSaturationReduction; } network_styles[] = { - {"main", QAPP_APP_NAME_DEFAULT, 0, 0}, - {"test", QAPP_APP_NAME_TESTNET, 70, 30}, - {"signet", QAPP_APP_NAME_SIGNET, 35, 15}, - {"regtest", QAPP_APP_NAME_REGTEST, 160, 30}, + {ChainType::MAIN, QAPP_APP_NAME_DEFAULT, 0, 0}, + {ChainType::TESTNET, QAPP_APP_NAME_TESTNET, 70, 30}, + {ChainType::SIGNET, QAPP_APP_NAME_SIGNET, 35, 15}, + {ChainType::REGTEST, QAPP_APP_NAME_REGTEST, 160, 30}, }; // titleAddText needs to be const char* for tr() @@ -77,9 +77,9 @@ NetworkStyle::NetworkStyle(const QString &_appName, const int iconColorHueShift, trayAndWindowIcon = QIcon(pixmap.scaled(QSize(256,256))); } -const NetworkStyle* NetworkStyle::instantiate(const std::string& networkId) +const NetworkStyle* NetworkStyle::instantiate(const ChainType networkId) { - std::string titleAddText = networkId == CBaseChainParams::MAIN ? "" : strprintf("[%s]", networkId); + std::string titleAddText = networkId == ChainType::MAIN ? "" : strprintf("[%s]", ChainTypeToString(networkId)); for (const auto& network_style : network_styles) { if (networkId == network_style.networkId) { return new NetworkStyle( diff --git a/src/qt/networkstyle.h b/src/qt/networkstyle.h index a73e3e2625..dd2aee3eb3 100644 --- a/src/qt/networkstyle.h +++ b/src/qt/networkstyle.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_QT_NETWORKSTYLE_H #define BITCOIN_QT_NETWORKSTYLE_H +#include <util/chaintype.h> + #include <QIcon> #include <QPixmap> #include <QString> @@ -14,7 +16,7 @@ class NetworkStyle { public: /** Get style associated with provided network id, or 0 if not known */ - static const NetworkStyle* instantiate(const std::string& networkId); + static const NetworkStyle* instantiate(const ChainType networkId); const QString &getAppName() const { return appName; } const QIcon &getAppIcon() const { return appIcon; } diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 6dec4b2e42..512fce473d 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -15,11 +15,11 @@ #include <qt/guiutil.h> #include <qt/optionsmodel.h> +#include <common/system.h> #include <interfaces/node.h> -#include <validation.h> // for DEFAULT_SCRIPTCHECK_THREADS and MAX_SCRIPTCHECK_THREADS #include <netbase.h> -#include <txdb.h> // for -dbcache defaults -#include <util/system.h> +#include <txdb.h> +#include <validation.h> #include <chrono> @@ -406,9 +406,8 @@ void OptionsDialog::updateProxyValidationState() void OptionsDialog::updateDefaultProxyNets() { - CNetAddr ui_proxy_netaddr; - LookupHost(ui->proxyIp->text().toStdString(), ui_proxy_netaddr, /*fAllowLookup=*/false); - const CService ui_proxy{ui_proxy_netaddr, ui->proxyPort->text().toUShort()}; + const std::optional<CNetAddr> ui_proxy_netaddr{LookupHost(ui->proxyIp->text().toStdString(), /*fAllowLookup=*/false)}; + const CService ui_proxy{ui_proxy_netaddr.value_or(CNetAddr{}), ui->proxyPort->text().toUShort()}; Proxy proxy; bool has_proxy; diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index 87a7e703b8..c1563fe1e2 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -60,7 +60,7 @@ static const char* SettingName(OptionsModel::OptionID option) } /** Call node.updateRwSetting() with Bitcoin 22.x workaround. */ -static void UpdateRwSetting(interfaces::Node& node, OptionsModel::OptionID option, const std::string& suffix, const util::SettingsValue& value) +static void UpdateRwSetting(interfaces::Node& node, OptionsModel::OptionID option, const std::string& suffix, const common::SettingsValue& value) { if (value.isNum() && (option == OptionsModel::DatabaseCache || @@ -81,14 +81,14 @@ static void UpdateRwSetting(interfaces::Node& node, OptionsModel::OptionID optio } //! Convert enabled/size values to bitcoin -prune setting. -static util::SettingsValue PruneSetting(bool prune_enabled, int prune_size_gb) +static common::SettingsValue PruneSetting(bool prune_enabled, int prune_size_gb) { assert(!prune_enabled || prune_size_gb >= 1); // PruneSizeGB and ParsePruneSizeGB never return less return prune_enabled ? PruneGBtoMiB(prune_size_gb) : 0; } //! Get pruning enabled value to show in GUI from bitcoin -prune setting. -static bool PruneEnabled(const util::SettingsValue& prune_setting) +static bool PruneEnabled(const common::SettingsValue& prune_setting) { // -prune=1 setting is manual pruning mode, so disabled for purposes of the gui return SettingToInt(prune_setting, 0) > 1; @@ -96,7 +96,7 @@ static bool PruneEnabled(const util::SettingsValue& prune_setting) //! Get pruning size value to show in GUI from bitcoin -prune setting. If //! pruning is not enabled, just show default recommended pruning size (2GB). -static int PruneSizeGB(const util::SettingsValue& prune_setting) +static int PruneSizeGB(const common::SettingsValue& prune_setting) { int value = SettingToInt(prune_setting, 0); return value > 1 ? PruneMiBtoGB(value) : DEFAULT_PRUNE_TARGET_GB; @@ -311,8 +311,8 @@ static QString GetDefaultProxyAddress() void OptionsModel::SetPruneTargetGB(int prune_target_gb) { - const util::SettingsValue cur_value = node().getPersistentSetting("prune"); - const util::SettingsValue new_value = PruneSetting(prune_target_gb > 0, prune_target_gb); + const common::SettingsValue cur_value = node().getPersistentSetting("prune"); + const common::SettingsValue new_value = PruneSetting(prune_target_gb > 0, prune_target_gb); // Force setting to take effect. It is still safe to change the value at // this point because this function is only called after the intro screen is @@ -331,7 +331,7 @@ void OptionsModel::SetPruneTargetGB(int prune_target_gb) // Keep previous pruning size, if pruning was disabled. if (PruneEnabled(cur_value)) { - UpdateRwSetting(node(), Prune, "-prev", PruneEnabled(new_value) ? util::SettingsValue{} : cur_value); + UpdateRwSetting(node(), Prune, "-prev", PruneEnabled(new_value) ? common::SettingsValue{} : cur_value); } } @@ -457,7 +457,7 @@ QVariant OptionsModel::getOption(OptionID option, const std::string& suffix) con bool OptionsModel::setOption(OptionID option, const QVariant& value, const std::string& suffix) { auto changed = [&] { return value.isValid() && value != getOption(option, suffix); }; - auto update = [&](const util::SettingsValue& value) { return UpdateRwSetting(node(), option, suffix, value); }; + auto update = [&](const common::SettingsValue& value) { return UpdateRwSetting(node(), option, suffix, value); }; bool successful = true; /* set to false on parse error */ QSettings settings; diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 76500a4a36..90aae0219e 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -10,6 +10,7 @@ #include <qt/forms/ui_debugwindow.h> #include <chainparams.h> +#include <common/system.h> #include <interfaces/node.h> #include <qt/bantablemodel.h> #include <qt/clientmodel.h> @@ -21,7 +22,6 @@ #include <rpc/server.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <util/threadnames.h> #include <univalue.h> @@ -249,7 +249,7 @@ bool RPCConsole::RPCParseCommandLine(interfaces::Node* node, std::string &strRes subelement = lastResult[parsed.value()]; } else if (lastResult.isObject()) - subelement = find_value(lastResult, curarg); + subelement = lastResult.find_value(curarg); else throw std::runtime_error("Invalid result query"); //no array or object: abort lastResult = subelement; @@ -448,8 +448,8 @@ void RPCExecutor::request(const QString &command, const WalletModel* wallet_mode { try // Nice formatting for standard-format error { - int code = find_value(objError, "code").getInt<int>(); - std::string message = find_value(objError, "message").get_str(); + int code = objError.find_value("code").getInt<int>(); + std::string message = objError.find_value("message").get_str(); Q_EMIT reply(RPCConsole::CMD_ERROR, QString::fromStdString(message) + " (code " + QString::number(code) + ")"); } catch (const std::runtime_error&) // raised when converting to invalid type, i.e. missing code or message @@ -744,7 +744,7 @@ void RPCConsole::setClientModel(ClientModel *model, int bestblock_height, int64_ ui->dataDir->setText(model->dataDir()); ui->blocksDir->setText(model->blocksDir()); ui->startupTime->setText(model->formatClientStartupTime()); - ui->networkName->setText(QString::fromStdString(Params().NetworkIDString())); + ui->networkName->setText(QString::fromStdString(Params().GetChainTypeString())); //Setup autocomplete and attach it QStringList wordList; diff --git a/src/qt/splashscreen.cpp b/src/qt/splashscreen.cpp index 096f8a0ded..8872f8be32 100644 --- a/src/qt/splashscreen.cpp +++ b/src/qt/splashscreen.cpp @@ -9,13 +9,13 @@ #include <qt/splashscreen.h> #include <clientversion.h> +#include <common/system.h> #include <interfaces/handler.h> #include <interfaces/node.h> #include <interfaces/wallet.h> #include <qt/guiutil.h> #include <qt/networkstyle.h> #include <qt/walletmodel.h> -#include <util/system.h> #include <util/translation.h> #include <functional> diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index 5706964cc9..f17a6b7f74 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -19,6 +19,7 @@ #include <key.h> #include <key_io.h> #include <wallet/wallet.h> +#include <wallet/test/util.h> #include <walletinitinterface.h> #include <chrono> @@ -31,7 +32,7 @@ using wallet::AddWallet; using wallet::CWallet; -using wallet::CreateMockWalletDatabase; +using wallet::CreateMockableWalletDatabase; using wallet::RemoveWallet; using wallet::WALLET_FLAG_DESCRIPTORS; using wallet::WalletContext; @@ -75,7 +76,7 @@ void TestAddAddressesToSendBook(interfaces::Node& node) auto wallet_loader = interfaces::MakeWalletLoader(*test.m_node.chain, *Assert(test.m_node.args)); test.m_node.wallet_loader = wallet_loader.get(); node.setContext(&test.m_node); - const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockableWalletDatabase()); wallet->LoadWallet(); wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); { diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index 000bbe65be..e918e84184 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -73,7 +73,7 @@ void AppTests::appTests() qRegisterMetaType<interfaces::BlockAndHeaderTipInfo>("interfaces::BlockAndHeaderTipInfo"); m_app.parameterSetup(); QVERIFY(m_app.createOptionsModel(/*resetSettings=*/true)); - QScopedPointer<const NetworkStyle> style(NetworkStyle::instantiate(Params().NetworkIDString())); + QScopedPointer<const NetworkStyle> style(NetworkStyle::instantiate(Params().GetChainType())); m_app.setupPlatformStyle(); m_app.createWindow(style.data()); connect(&m_app, &BitcoinApplication::windowShown, this, &AppTests::guiTests); diff --git a/src/qt/test/optiontests.cpp b/src/qt/test/optiontests.cpp index 0838e21678..e5a5179d87 100644 --- a/src/qt/test/optiontests.cpp +++ b/src/qt/test/optiontests.cpp @@ -18,13 +18,13 @@ OptionTests::OptionTests(interfaces::Node& node) : m_node(node) { - gArgs.LockSettings([&](util::Settings& s) { m_previous_settings = s; }); + gArgs.LockSettings([&](common::Settings& s) { m_previous_settings = s; }); } void OptionTests::init() { // reset args - gArgs.LockSettings([&](util::Settings& s) { s = m_previous_settings; }); + gArgs.LockSettings([&](common::Settings& s) { s = m_previous_settings; }); gArgs.ClearPathCache(); } @@ -76,14 +76,14 @@ void OptionTests::integerGetArgBug() // Test regression https://github.com/bitcoin/bitcoin/issues/24457. Ensure // that setting integer prune value doesn't cause an exception to be thrown // in the OptionsModel constructor - gArgs.LockSettings([&](util::Settings& settings) { + gArgs.LockSettings([&](common::Settings& settings) { settings.forced_settings.erase("prune"); settings.rw_settings["prune"] = 3814; }); gArgs.WriteSettingsFile(); bilingual_str error; QVERIFY(OptionsModel{m_node}.Init(error)); - gArgs.LockSettings([&](util::Settings& settings) { + gArgs.LockSettings([&](common::Settings& settings) { settings.rw_settings.erase("prune"); }); gArgs.WriteSettingsFile(); @@ -95,7 +95,7 @@ void OptionTests::parametersInteraction() // It was fixed via https://github.com/bitcoin-core/gui/pull/568. // With fListen=false in ~/.config/Bitcoin/Bitcoin-Qt.conf and all else left as default, // bitcoin-qt should set both -listen and -listenonion to false and start successfully. - gArgs.LockSettings([&](util::Settings& s) { + gArgs.LockSettings([&](common::Settings& s) { s.forced_settings.erase("listen"); s.forced_settings.erase("listenonion"); }); diff --git a/src/qt/test/optiontests.h b/src/qt/test/optiontests.h index 0c458c97a6..2d7b94933f 100644 --- a/src/qt/test/optiontests.h +++ b/src/qt/test/optiontests.h @@ -5,9 +5,9 @@ #ifndef BITCOIN_QT_TEST_OPTIONTESTS_H #define BITCOIN_QT_TEST_OPTIONTESTS_H +#include <common/settings.h> #include <qt/optionsmodel.h> #include <univalue.h> -#include <util/settings.h> #include <QObject> @@ -26,7 +26,7 @@ private Q_SLOTS: private: interfaces::Node& m_node; - util::Settings m_previous_settings; + common::Settings m_previous_settings; }; #endif // BITCOIN_QT_TEST_OPTIONTESTS_H diff --git a/src/qt/test/rpcnestedtests.cpp b/src/qt/test/rpcnestedtests.cpp index 669a05fe0f..72e8055425 100644 --- a/src/qt/test/rpcnestedtests.cpp +++ b/src/qt/test/rpcnestedtests.cpp @@ -4,12 +4,12 @@ #include <qt/test/rpcnestedtests.h> +#include <common/system.h> #include <interfaces/node.h> -#include <rpc/server.h> #include <qt/rpcconsole.h> +#include <rpc/server.h> #include <test/util/setup_common.h> #include <univalue.h> -#include <util/system.h> #include <QTest> diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index 2d069f76a0..e45fc1ced8 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -14,6 +14,7 @@ #include <qt/test/rpcnestedtests.h> #include <qt/test/uritests.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> #ifdef ENABLE_WALLET #include <qt/test/addressbooktests.h> @@ -57,7 +58,7 @@ int main(int argc, char* argv[]) // // All tests must use their own testing setup (if needed). fs::create_directories([] { - BasicTestingSetup dummy{CBaseChainParams::REGTEST}; + BasicTestingSetup dummy{ChainType::REGTEST}; return gArgs.GetDataDirNet() / "blocks"; }()); @@ -70,6 +71,9 @@ int main(int argc, char* argv[]) gArgs.ForceSetArg("-upnp", "0"); gArgs.ForceSetArg("-natpmp", "0"); + std::string error; + if (!gArgs.ReadConfigFiles(error, true)) QWARN(error.c_str()); + // Prefer the "minimal" platform for the test instead of the normal default // platform ("xcb", "windows", or "cocoa") so tests can't unintentionally // interfere with any background GUIs and don't require extra resources. diff --git a/src/qt/test/util.cpp b/src/qt/test/util.cpp index c5ed20d967..d91b90fbba 100644 --- a/src/qt/test/util.cpp +++ b/src/qt/test/util.cpp @@ -2,6 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <qt/test/util.h> + #include <chrono> #include <QApplication> diff --git a/src/qt/test/util.h b/src/qt/test/util.h index 13170c89ea..a1788ca74e 100644 --- a/src/qt/test/util.h +++ b/src/qt/test/util.h @@ -7,6 +7,8 @@ #include <chrono> +#include <qglobal.h> + QT_BEGIN_NAMESPACE class QString; QT_END_NAMESPACE diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 072718fe15..5f789e400e 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -26,6 +26,7 @@ #include <qt/walletmodel.h> #include <test/util/setup_common.h> #include <validation.h> +#include <wallet/test/util.h> #include <wallet/wallet.h> #include <chrono> @@ -46,7 +47,7 @@ using wallet::AddWallet; using wallet::CWallet; -using wallet::CreateMockWalletDatabase; +using wallet::CreateMockableWalletDatabase; using wallet::RemoveWallet; using wallet::WALLET_FLAG_DESCRIPTORS; using wallet::WALLET_FLAG_DISABLE_PRIVATE_KEYS; @@ -189,7 +190,7 @@ void SyncUpWallet(const std::shared_ptr<CWallet>& wallet, interfaces::Node& node std::shared_ptr<CWallet> SetupLegacyWatchOnlyWallet(interfaces::Node& node, TestChain100Setup& test) { - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockWalletDatabase()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockableWalletDatabase()); wallet->LoadWallet(); { LOCK(wallet->cs_wallet); @@ -207,7 +208,7 @@ std::shared_ptr<CWallet> SetupLegacyWatchOnlyWallet(interfaces::Node& node, Test std::shared_ptr<CWallet> SetupDescriptorsWallet(interfaces::Node& node, TestChain100Setup& test) { - std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockWalletDatabase()); + std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(node.context()->chain.get(), "", CreateMockableWalletDatabase()); wallet->LoadWallet(); LOCK(wallet->cs_wallet); wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index 2461a06930..fa110cfbc9 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -13,12 +13,12 @@ #include <qt/paymentserver.h> #include <qt/transactionrecord.h> +#include <common/system.h> #include <consensus/consensus.h> #include <interfaces/node.h> #include <interfaces/wallet.h> #include <key_io.h> #include <policy/policy.h> -#include <util/system.h> #include <validation.h> #include <wallet/types.h> diff --git a/src/random.cpp b/src/random.cpp index f4c51574cc..54500e6cc6 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -28,14 +28,10 @@ #include <sys/time.h> #endif -#ifdef HAVE_SYS_GETRANDOM -#include <sys/syscall.h> -#include <linux/random.h> -#endif -#if defined(HAVE_GETENTROPY_RAND) && defined(MAC_OSX) -#include <unistd.h> +#if defined(HAVE_GETRANDOM) || (defined(HAVE_GETENTROPY_RAND) && defined(MAC_OSX)) #include <sys/random.h> #endif + #ifdef HAVE_SYSCTL_ARND #include <sys/sysctl.h> #endif @@ -252,7 +248,7 @@ static void Strengthen(const unsigned char (&seed)[32], SteadyClock::duration du /** Fallback: get 32 bytes of system entropy from /dev/urandom. The most * compatible way to get cryptographic randomness on UNIX-ish platforms. */ -static void GetDevURandom(unsigned char *ent32) +[[maybe_unused]] static void GetDevURandom(unsigned char *ent32) { int f = open("/dev/urandom", O_RDONLY); if (f == -1) { @@ -285,23 +281,14 @@ void GetOSRand(unsigned char *ent32) RandFailure(); } CryptReleaseContext(hProvider, 0); -#elif defined(HAVE_SYS_GETRANDOM) +#elif defined(HAVE_GETRANDOM) /* Linux. From the getrandom(2) man page: * "If the urandom source has been initialized, reads of up to 256 bytes * will always return as many bytes as requested and will not be * interrupted by signals." */ - int rv = syscall(SYS_getrandom, ent32, NUM_OS_RANDOM_BYTES, 0); - if (rv != NUM_OS_RANDOM_BYTES) { - if (rv < 0 && errno == ENOSYS) { - /* Fallback for kernel <3.17: the return value will be -1 and errno - * ENOSYS if the syscall is not available, in that case fall back - * to /dev/urandom. - */ - GetDevURandom(ent32); - } else { - RandFailure(); - } + if (getrandom(ent32, NUM_OS_RANDOM_BYTES, 0) != NUM_OS_RANDOM_BYTES) { + RandFailure(); } #elif defined(__OpenBSD__) /* OpenBSD. From the arc4random(3) man page: @@ -311,16 +298,10 @@ void GetOSRand(unsigned char *ent32) The function call is always successful. */ arc4random_buf(ent32, NUM_OS_RANDOM_BYTES); - // Silence a compiler warning about unused function. - (void)GetDevURandom; #elif defined(HAVE_GETENTROPY_RAND) && defined(MAC_OSX) - /* getentropy() is available on macOS 10.12 and later. - */ if (getentropy(ent32, NUM_OS_RANDOM_BYTES) != 0) { RandFailure(); } - // Silence a compiler warning about unused function. - (void)GetDevURandom; #elif defined(HAVE_SYSCTL_ARND) /* FreeBSD, NetBSD and similar. It is possible for the call to return less * bytes than requested, so need to read in a loop. @@ -334,8 +315,6 @@ void GetOSRand(unsigned char *ent32) } have += len; } while (have < NUM_OS_RANDOM_BYTES); - // Silence a compiler warning about unused function. - (void)GetDevURandom; #else /* Fall back to /dev/urandom if there is no specific method implemented to * get system entropy for this OS. diff --git a/src/rest.cpp b/src/rest.cpp index e46406f1ad..ba149c1a9e 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -24,8 +24,8 @@ #include <streams.h> #include <sync.h> #include <txmempool.h> +#include <util/any.h> #include <util/check.h> -#include <util/system.h> #include <validation.h> #include <version.h> @@ -36,7 +36,6 @@ using node::GetTransaction; using node::NodeContext; -using node::ReadBlockFromDisk; static const size_t MAX_GETUTXOS_OUTPOINTS = 15; //allow a max of 15 outpoints to be queried at once static constexpr unsigned int MAX_REST_HEADERS_RESULTS = 2000; @@ -311,7 +310,7 @@ static bool rest_block(const std::any& context, } - if (!ReadBlockFromDisk(block, pblockindex, chainman.GetParams().GetConsensus())) { + if (!chainman.m_blockman.ReadBlockFromDisk(block, *pblockindex)) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } @@ -628,7 +627,7 @@ static bool rest_deploymentinfo(const std::any& context, HTTPRequest* req, const return RESTERR(req, HTTP_BAD_REQUEST, "Block not found"); } - jsonRequest.params.pushKV("blockhash", hash_str); + jsonRequest.params.push_back(hash_str); } req->WriteHeader("Content-Type", "application/json"); @@ -716,7 +715,7 @@ static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string const NodeContext* const node = GetNodeContext(context, req); if (!node) return false; uint256 hashBlock = uint256(); - const CTransactionRef tx = GetTransaction(/*block_index=*/nullptr, node->mempool.get(), hash, Params().GetConsensus(), hashBlock); + const CTransactionRef tx = GetTransaction(/*block_index=*/nullptr, node->mempool.get(), hash, hashBlock, node->chainman->m_blockman); if (!tx) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 11484a9d8d..ee3237638e 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -58,9 +58,7 @@ using kernel::CoinStatsHashType; using node::BlockManager; using node::NodeContext; -using node::ReadBlockFromDisk; using node::SnapshotMetadata; -using node::UndoReadFromDisk; struct CUpdatedBlock { @@ -183,7 +181,7 @@ UniValue blockToJSON(BlockManager& blockman, const CBlock& block, const CBlockIn case TxVerbosity::SHOW_DETAILS_AND_PREVOUT: CBlockUndo blockUndo; const bool is_not_pruned{WITH_LOCK(::cs_main, return !blockman.IsBlockPruned(blockindex))}; - const bool have_undo{is_not_pruned && UndoReadFromDisk(blockUndo, blockindex)}; + const bool have_undo{is_not_pruned && blockman.UndoReadFromDisk(blockUndo, *blockindex)}; for (size_t i = 0; i < block.vtx.size(); ++i) { const CTransactionRef& tx = block.vtx.at(i); @@ -430,7 +428,7 @@ static RPCHelpMan getblockfrompeer() "getblockfrompeer", "Attempt to fetch block from a given peer.\n\n" "We must have the header for this block, e.g. using submitheader.\n" - "Subsequent calls for the same block and a new peer will cause the response from the previous peer to be ignored.\n" + "Subsequent calls for the same block may cause the response from the previous peer to be ignored.\n" "Peers generally ignore requests for a stale block that they never fully verified, or one that is more than a month old.\n" "When a peer does not respond with a block, we will disconnect.\n" "Note: The block could be re-pruned as soon as it is received.\n\n" @@ -587,7 +585,7 @@ static CBlock GetBlockChecked(BlockManager& blockman, const CBlockIndex* pblocki } } - if (!ReadBlockFromDisk(block, pblockindex, Params().GetConsensus())) { + if (!blockman.ReadBlockFromDisk(block, *pblockindex)) { // Block not found on disk. This could be because we have the block // header in our index but not yet have the block or did not accept the // block. Or if the block was pruned right after we released the lock above. @@ -611,7 +609,7 @@ static CBlockUndo GetUndoChecked(BlockManager& blockman, const CBlockIndex* pblo } } - if (!UndoReadFromDisk(blockUndo, pblockindex)) { + if (!blockman.UndoReadFromDisk(blockUndo, *pblockindex)) { throw JSONRPCError(RPC_MISC_ERROR, "Can't read undo data from disk"); } @@ -1125,7 +1123,7 @@ static RPCHelpMan verifychain() LOCK(cs_main); Chainstate& active_chainstate = chainman.ActiveChainstate(); - return CVerifyDB().VerifyDB( + return CVerifyDB(chainman.GetNotifications()).VerifyDB( active_chainstate, chainman.GetParams().GetConsensus(), active_chainstate.CoinsTip(), check_level, check_depth) == VerifyDBResult::SUCCESS; }, }; @@ -1256,7 +1254,7 @@ RPCHelpMan getblockchaininfo() const CBlockIndex& tip{*CHECK_NONFATAL(active_chainstate.m_chain.Tip())}; const int height{tip.nHeight}; UniValue obj(UniValue::VOBJ); - obj.pushKV("chain", chainman.GetParams().NetworkIDString()); + obj.pushKV("chain", chainman.GetParams().GetChainTypeString()); obj.pushKV("blocks", height); obj.pushKV("headers", chainman.m_best_header ? chainman.m_best_header->nHeight : -1); obj.pushKV("bestblockhash", tip.GetBlockHash().GetHex()); diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index f3c19003ff..edc0fb05d7 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -101,6 +101,11 @@ static const CRPCConvertParam vRPCConvertParams[] = { "listunspent", 2, "addresses" }, { "listunspent", 3, "include_unsafe" }, { "listunspent", 4, "query_options" }, + { "listunspent", 4, "minimumAmount" }, + { "listunspent", 4, "maximumAmount" }, + { "listunspent", 4, "maximumCount" }, + { "listunspent", 4, "minimumSumAmount" }, + { "listunspent", 4, "include_immature_coinbase" }, { "getblock", 1, "verbosity" }, { "getblock", 1, "verbose" }, { "getblockheader", 1, "verbose" }, @@ -124,15 +129,45 @@ static const CRPCConvertParam vRPCConvertParams[] = { "submitpackage", 0, "package" }, { "combinerawtransaction", 0, "txs" }, { "fundrawtransaction", 1, "options" }, + { "fundrawtransaction", 1, "add_inputs"}, + { "fundrawtransaction", 1, "include_unsafe"}, + { "fundrawtransaction", 1, "minconf"}, + { "fundrawtransaction", 1, "maxconf"}, + { "fundrawtransaction", 1, "changePosition"}, + { "fundrawtransaction", 1, "includeWatching"}, + { "fundrawtransaction", 1, "lockUnspents"}, + { "fundrawtransaction", 1, "fee_rate"}, + { "fundrawtransaction", 1, "feeRate"}, + { "fundrawtransaction", 1, "subtractFeeFromOutputs"}, + { "fundrawtransaction", 1, "input_weights"}, + { "fundrawtransaction", 1, "conf_target"}, + { "fundrawtransaction", 1, "replaceable"}, + { "fundrawtransaction", 1, "solving_data"}, { "fundrawtransaction", 2, "iswitness" }, { "walletcreatefundedpsbt", 0, "inputs" }, { "walletcreatefundedpsbt", 1, "outputs" }, { "walletcreatefundedpsbt", 2, "locktime" }, { "walletcreatefundedpsbt", 3, "options" }, + { "walletcreatefundedpsbt", 3, "add_inputs"}, + { "walletcreatefundedpsbt", 3, "include_unsafe"}, + { "walletcreatefundedpsbt", 3, "minconf"}, + { "walletcreatefundedpsbt", 3, "maxconf"}, + { "walletcreatefundedpsbt", 3, "changePosition"}, + { "walletcreatefundedpsbt", 3, "includeWatching"}, + { "walletcreatefundedpsbt", 3, "lockUnspents"}, + { "walletcreatefundedpsbt", 3, "fee_rate"}, + { "walletcreatefundedpsbt", 3, "feeRate"}, + { "walletcreatefundedpsbt", 3, "subtractFeeFromOutputs"}, + { "walletcreatefundedpsbt", 3, "conf_target"}, + { "walletcreatefundedpsbt", 3, "replaceable"}, + { "walletcreatefundedpsbt", 3, "solving_data"}, { "walletcreatefundedpsbt", 4, "bip32derivs" }, { "walletprocesspsbt", 1, "sign" }, { "walletprocesspsbt", 3, "bip32derivs" }, { "walletprocesspsbt", 4, "finalize" }, + { "descriptorprocesspsbt", 1, "descriptors"}, + { "descriptorprocesspsbt", 3, "bip32derivs" }, + { "descriptorprocesspsbt", 4, "finalize" }, { "createpsbt", 0, "inputs" }, { "createpsbt", 1, "outputs" }, { "createpsbt", 2, "locktime" }, @@ -154,18 +189,49 @@ static const CRPCConvertParam vRPCConvertParams[] = { "send", 1, "conf_target" }, { "send", 3, "fee_rate"}, { "send", 4, "options" }, + { "send", 4, "add_inputs"}, + { "send", 4, "include_unsafe"}, + { "send", 4, "minconf"}, + { "send", 4, "maxconf"}, + { "send", 4, "add_to_wallet"}, + { "send", 4, "change_position"}, + { "send", 4, "fee_rate"}, + { "send", 4, "include_watching"}, + { "send", 4, "inputs"}, + { "send", 4, "locktime"}, + { "send", 4, "lock_unspents"}, + { "send", 4, "psbt"}, + { "send", 4, "subtract_fee_from_outputs"}, + { "send", 4, "conf_target"}, + { "send", 4, "replaceable"}, + { "send", 4, "solving_data"}, { "sendall", 0, "recipients" }, { "sendall", 1, "conf_target" }, { "sendall", 3, "fee_rate"}, { "sendall", 4, "options" }, + { "sendall", 4, "add_to_wallet"}, + { "sendall", 4, "fee_rate"}, + { "sendall", 4, "include_watching"}, + { "sendall", 4, "inputs"}, + { "sendall", 4, "locktime"}, + { "sendall", 4, "lock_unspents"}, + { "sendall", 4, "psbt"}, + { "sendall", 4, "send_max"}, + { "sendall", 4, "minconf"}, + { "sendall", 4, "maxconf"}, + { "sendall", 4, "conf_target"}, + { "sendall", 4, "replaceable"}, + { "sendall", 4, "solving_data"}, { "simulaterawtransaction", 0, "rawtxs" }, { "simulaterawtransaction", 1, "options" }, + { "simulaterawtransaction", 1, "include_watchonly"}, { "importprivkey", 2, "rescan" }, { "importaddress", 2, "rescan" }, { "importaddress", 3, "p2sh" }, { "importpubkey", 2, "rescan" }, { "importmulti", 0, "requests" }, { "importmulti", 1, "options" }, + { "importmulti", 1, "rescan" }, { "importdescriptors", 0, "requests" }, { "listdescriptors", 0, "private" }, { "verifychain", 0, "checklevel" }, @@ -189,7 +255,15 @@ static const CRPCConvertParam vRPCConvertParams[] = { "getmempooldescendants", 1, "verbose" }, { "gettxspendingprevout", 0, "outputs" }, { "bumpfee", 1, "options" }, + { "bumpfee", 1, "conf_target"}, + { "bumpfee", 1, "fee_rate"}, + { "bumpfee", 1, "replaceable"}, + { "bumpfee", 1, "outputs"}, { "psbtbumpfee", 1, "options" }, + { "psbtbumpfee", 1, "conf_target"}, + { "psbtbumpfee", 1, "fee_rate"}, + { "psbtbumpfee", 1, "replaceable"}, + { "psbtbumpfee", 1, "outputs"}, { "logging", 0, "include" }, { "logging", 1, "exclude" }, { "disconnectnode", 1, "nodeid" }, @@ -223,6 +297,14 @@ static const CRPCConvertParam vRPCConvertParams[] = }; // clang-format on +/** Parse string to UniValue or throw runtime_error if string contains invalid JSON */ +static UniValue Parse(std::string_view raw) +{ + UniValue parsed; + if (!parsed.read(raw)) throw std::runtime_error(tfm::format("Error parsing JSON: %s", raw)); + return parsed; +} + class CRPCConvertTable { private: @@ -235,13 +317,13 @@ public: /** Return arg_value as UniValue, and first parse it if it is a non-string parameter */ UniValue ArgToUniValue(std::string_view arg_value, const std::string& method, int param_idx) { - return members.count({method, param_idx}) > 0 ? ParseNonRFCJSONValue(arg_value) : arg_value; + return members.count({method, param_idx}) > 0 ? Parse(arg_value) : arg_value; } /** Return arg_value as UniValue, and first parse it if it is a non-string parameter */ UniValue ArgToUniValue(std::string_view arg_value, const std::string& method, const std::string& param_name) { - return membersByName.count({method, param_name}) > 0 ? ParseNonRFCJSONValue(arg_value) : arg_value; + return membersByName.count({method, param_name}) > 0 ? Parse(arg_value) : arg_value; } }; @@ -255,16 +337,6 @@ CRPCConvertTable::CRPCConvertTable() static CRPCConvertTable rpcCvtTable; -/** Non-RFC4627 JSON parser, accepts internal values (such as numbers, true, false, null) - * as well as objects and arrays. - */ -UniValue ParseNonRFCJSONValue(std::string_view raw) -{ - UniValue parsed; - if (!parsed.read(raw)) throw std::runtime_error(tfm::format("Error parsing JSON: %s", raw)); - return parsed; -} - UniValue RPCConvertValues(const std::string &strMethod, const std::vector<std::string> &strParams) { UniValue params(UniValue::VARR); diff --git a/src/rpc/client.h b/src/rpc/client.h index 3c5c4fc4d6..b67cd27fdf 100644 --- a/src/rpc/client.h +++ b/src/rpc/client.h @@ -17,9 +17,4 @@ UniValue RPCConvertValues(const std::string& strMethod, const std::vector<std::s /** Convert named arguments to command-specific RPC representation */ UniValue RPCConvertNamedValues(const std::string& strMethod, const std::vector<std::string>& strParams); -/** Non-RFC4627 JSON parser, accepts internal values (such as numbers, true, false, null) - * as well as objects and arrays. - */ -UniValue ParseNonRFCJSONValue(std::string_view raw); - #endif // BITCOIN_RPC_CLIENT_H diff --git a/src/rpc/external_signer.cpp b/src/rpc/external_signer.cpp index 1e139c9990..310eec5f15 100644 --- a/src/rpc/external_signer.cpp +++ b/src/rpc/external_signer.cpp @@ -2,14 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <chainparamsbase.h> #include <common/args.h> +#include <common/system.h> #include <external_signer.h> #include <rpc/protocol.h> #include <rpc/server.h> #include <rpc/util.h> #include <util/strencodings.h> -#include <util/system.h> #include <string> #include <vector> @@ -43,7 +42,7 @@ static RPCHelpMan enumeratesigners() { const std::string command = gArgs.GetArg("-signer", ""); if (command == "") throw JSONRPCError(RPC_MISC_ERROR, "Error: restart bitcoind with -signer=<cmd>"); - const std::string chain = gArgs.GetChainName(); + const std::string chain = gArgs.GetChainTypeString(); UniValue signers_res = UniValue::VARR; try { std::vector<ExternalSigner> signers; diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 927b4ce1fc..89c403b6f5 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -638,7 +638,7 @@ static RPCHelpMan gettxspendingprevout() }, /*fAllowNull=*/false, /*fStrict=*/true); const uint256 txid(ParseHashO(o, "txid")); - const int nOutput{find_value(o, "vout").getInt<int>()}; + const int nOutput{o.find_value("vout").getInt<int>()}; if (nOutput < 0) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, vout cannot be negative"); } diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index d55e20ba5e..074cecadd2 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -5,6 +5,7 @@ #include <chain.h> #include <chainparams.h> +#include <common/system.h> #include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/merkle.h> @@ -32,7 +33,6 @@ #include <univalue.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <util/translation.h> #include <validation.h> #include <validationinterface.h> @@ -435,7 +435,7 @@ static RPCHelpMan getmininginfo() obj.pushKV("difficulty", (double)GetDifficulty(active_chain.Tip())); obj.pushKV("networkhashps", getnetworkhashps().HandleRequest(request)); obj.pushKV("pooledtx", (uint64_t)mempool.size()); - obj.pushKV("chain", chainman.GetParams().NetworkIDString()); + obj.pushKV("chain", chainman.GetParams().GetChainTypeString()); obj.pushKV("warnings", GetWarnings(false).original); return obj; }, @@ -480,6 +480,40 @@ static RPCHelpMan prioritisetransaction() }; } +static RPCHelpMan getprioritisedtransactions() +{ + return RPCHelpMan{"getprioritisedtransactions", + "Returns a map of all user-created (see prioritisetransaction) fee deltas by txid, and whether the tx is present in mempool.", + {}, + RPCResult{ + RPCResult::Type::OBJ_DYN, "prioritisation-map", "prioritisation keyed by txid", + { + {RPCResult::Type::OBJ, "txid", "", { + {RPCResult::Type::NUM, "fee_delta", "transaction fee delta in satoshis"}, + {RPCResult::Type::BOOL, "in_mempool", "whether this transaction is currently in mempool"}, + }} + }, + }, + RPCExamples{ + HelpExampleCli("getprioritisedtransactions", "") + + HelpExampleRpc("getprioritisedtransactions", "") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + NodeContext& node = EnsureAnyNodeContext(request.context); + CTxMemPool& mempool = EnsureMemPool(node); + UniValue rpc_result{UniValue::VOBJ}; + for (const auto& delta_info : mempool.GetPrioritisedTransactions()) { + UniValue result_inner{UniValue::VOBJ}; + result_inner.pushKV("fee_delta", delta_info.delta); + result_inner.pushKV("in_mempool", delta_info.in_mempool); + rpc_result.pushKV(delta_info.txid.GetHex(), result_inner); + } + return rpc_result; + }, + }; +} + // NOTE: Assumes a conclusive result; if result is inconclusive, it must be handled by caller static UniValue BIP22ValidationResult(const BlockValidationState& state) @@ -612,7 +646,7 @@ static RPCHelpMan getblocktemplate() if (!request.params[0].isNull()) { const UniValue& oparam = request.params[0].get_obj(); - const UniValue& modeval = find_value(oparam, "mode"); + const UniValue& modeval = oparam.find_value("mode"); if (modeval.isStr()) strMode = modeval.get_str(); else if (modeval.isNull()) @@ -621,11 +655,11 @@ static RPCHelpMan getblocktemplate() } else throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid mode"); - lpval = find_value(oparam, "longpollid"); + lpval = oparam.find_value("longpollid"); if (strMode == "proposal") { - const UniValue& dataval = find_value(oparam, "data"); + const UniValue& dataval = oparam.find_value("data"); if (!dataval.isStr()) throw JSONRPCError(RPC_TYPE_ERROR, "Missing data String key for proposal"); @@ -652,7 +686,7 @@ static RPCHelpMan getblocktemplate() return BIP22ValidationResult(state); } - const UniValue& aClientRules = find_value(oparam, "rules"); + const UniValue& aClientRules = oparam.find_value("rules"); if (aClientRules.isArray()) { for (unsigned int i = 0; i < aClientRules.size(); ++i) { const UniValue& v = aClientRules[i]; @@ -1048,6 +1082,7 @@ void RegisterMiningRPCCommands(CRPCTable& t) {"mining", &getnetworkhashps}, {"mining", &getmininginfo}, {"mining", &prioritisetransaction}, + {"mining", &getprioritisedtransactions}, {"mining", &getblocktemplate}, {"mining", &submitblock}, {"mining", &submitheader}, diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 7ffa777ef4..a2a46ef32f 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -21,6 +21,7 @@ #include <rpc/util.h> #include <sync.h> #include <timedata.h> +#include <util/chaintype.h> #include <util/strencodings.h> #include <util/string.h> #include <util/time.h> @@ -354,7 +355,7 @@ static RPCHelpMan addconnection() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - if (Params().NetworkIDString() != CBaseChainParams::REGTEST) { + if (Params().GetChainType() != ChainType::REGTEST) { throw std::runtime_error("addconnection is for regression testing (-regtest mode) only."); } @@ -512,15 +513,15 @@ static RPCHelpMan getaddednodeinfo() static RPCHelpMan getnettotals() { return RPCHelpMan{"getnettotals", - "\nReturns information about network traffic, including bytes in, bytes out,\n" - "and current time.\n", - {}, + "Returns information about network traffic, including bytes in, bytes out,\n" + "and current system time.", + {}, RPCResult{ RPCResult::Type::OBJ, "", "", { {RPCResult::Type::NUM, "totalbytesrecv", "Total bytes received"}, {RPCResult::Type::NUM, "totalbytessent", "Total bytes sent"}, - {RPCResult::Type::NUM_TIME, "timemillis", "Current " + UNIX_EPOCH_TIME + " in milliseconds"}, + {RPCResult::Type::NUM_TIME, "timemillis", "Current system " + UNIX_EPOCH_TIME + " in milliseconds"}, {RPCResult::Type::OBJ, "uploadtarget", "", { {RPCResult::Type::NUM, "timeframe", "Length of the measuring timeframe in seconds"}, @@ -544,7 +545,7 @@ static RPCHelpMan getnettotals() UniValue obj(UniValue::VOBJ); obj.pushKV("totalbytesrecv", connman.GetTotalBytesRecv()); obj.pushKV("totalbytessent", connman.GetTotalBytesSent()); - obj.pushKV("timemillis", GetTimeMillis()); + obj.pushKV("timemillis", TicksSinceEpoch<std::chrono::milliseconds>(SystemClock::now())); UniValue outboundLimit(UniValue::VOBJ); outboundLimit.pushKV("timeframe", count_seconds(connman.GetMaxOutboundTimeframe())); @@ -712,9 +713,10 @@ static RPCHelpMan setban() isSubnet = true; if (!isSubnet) { - CNetAddr resolved; - LookupHost(request.params[0].get_str(), resolved, false); - netAddr = resolved; + const std::optional<CNetAddr> addr{LookupHost(request.params[0].get_str(), false)}; + if (addr.has_value()) { + netAddr = addr.value(); + } } else LookupSubNet(request.params[0].get_str(), subNet); @@ -942,11 +944,11 @@ static RPCHelpMan addpeeraddress() const bool tried{request.params[2].isNull() ? false : request.params[2].get_bool()}; UniValue obj(UniValue::VOBJ); - CNetAddr net_addr; + std::optional<CNetAddr> net_addr{LookupHost(addr_string, false)}; bool success{false}; - if (LookupHost(addr_string, net_addr, false)) { - CService service{net_addr, port}; + if (net_addr.has_value()) { + CService service{net_addr.value(), port}; CAddress address{MaybeFlipIPv6toCJDNS(service), ServiceFlags{NODE_NETWORK | NODE_WITNESS}}; address.nTime = Now<NodeSeconds>(); // The source address is set equal to the address. This is equivalent to the peer diff --git a/src/rpc/node.cpp b/src/rpc/node.cpp index 5918bc6e38..45d46d223b 100644 --- a/src/rpc/node.cpp +++ b/src/rpc/node.cpp @@ -19,9 +19,9 @@ #include <rpc/util.h> #include <scheduler.h> #include <univalue.h> +#include <util/any.h> #include <util/check.h> #include <util/syscall_sandbox.h> -#include <util/system.h> #include <stdint.h> #ifdef HAVE_MALLOC_INFO @@ -163,7 +163,7 @@ static RPCHelpMan getmemoryinfo() { {"mode", RPCArg::Type::STR, RPCArg::Default{"stats"}, "determines what kind of information is returned.\n" " - \"stats\" returns general statistics about memory usage in the daemon.\n" - " - \"mallocinfo\" returns an XML string describing low-level heap state (only available if compiled with glibc 2.10+)."}, + " - \"mallocinfo\" returns an XML string describing low-level heap state (only available if compiled with glibc)."}, }, { RPCResult{"mode \"stats\"", diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 4a918cbd42..eb0200ccf5 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -51,8 +51,6 @@ using node::FindCoins; using node::GetTransaction; using node::NodeContext; using node::PSBTAnalysis; -using node::ReadBlockFromDisk; -using node::UndoReadFromDisk; static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry, Chainstate& active_chainstate, const CTxUndo* txundo = nullptr, @@ -172,8 +170,9 @@ static std::vector<RPCArg> CreateTxDoc() }; } -// Update PSBT with information from the mempool, the UTXO set, the txindex, and the provided descriptors -PartiallySignedTransaction ProcessPSBT(const std::string& psbt_string, const std::any& context, const HidingSigningProvider& provider) +// Update PSBT with information from the mempool, the UTXO set, the txindex, and the provided descriptors. +// Optionally, sign the inputs that we can using information from the descriptors. +PartiallySignedTransaction ProcessPSBT(const std::string& psbt_string, const std::any& context, const HidingSigningProvider& provider, int sighash_type, bool finalize) { // Unserialize the transactions PartiallySignedTransaction psbtx; @@ -242,9 +241,10 @@ PartiallySignedTransaction ProcessPSBT(const std::string& psbt_string, const std } // Update script/keypath information using descriptor data. - // Note that SignPSBTInput does a lot more than just constructing ECDSA signatures - // we don't actually care about those here, in fact. - SignPSBTInput(provider, psbtx, /*index=*/i, &txdata, /*sighash=*/1); + // Note that SignPSBTInput does a lot more than just constructing ECDSA signatures. + // We only actually care about those if our signing provider doesn't hide private + // information, as is the case with `descriptorprocesspsbt` + SignPSBTInput(provider, psbtx, /*index=*/i, &txdata, sighash_type, /*out_sigdata=*/nullptr, finalize); } // Update script/keypath information using descriptor data. @@ -362,7 +362,7 @@ static RPCHelpMan getrawtransaction() } uint256 hash_block; - const CTransactionRef tx = GetTransaction(blockindex, node.mempool.get(), hash, chainman.GetConsensus(), hash_block); + const CTransactionRef tx = GetTransaction(blockindex, node.mempool.get(), hash, hash_block, chainman.m_blockman); if (!tx) { std::string errmsg; if (blockindex) { @@ -406,7 +406,7 @@ static RPCHelpMan getrawtransaction() if (tx->IsCoinBase() || !blockindex || is_block_pruned || - !(UndoReadFromDisk(blockUndo, blockindex) && ReadBlockFromDisk(block, blockindex, Params().GetConsensus()))) { + !(chainman.m_blockman.UndoReadFromDisk(blockUndo, *blockindex) && chainman.m_blockman.ReadBlockFromDisk(block, *blockindex))) { TxToJSON(*tx, hash_block, result, chainman.ActiveChainstate()); return result; } @@ -1697,7 +1697,9 @@ static RPCHelpMan utxoupdatepsbt() const PartiallySignedTransaction& psbtx = ProcessPSBT( request.params[0].get_str(), request.context, - HidingSigningProvider(&provider, /*hide_secret=*/true, /*hide_origin=*/false)); + HidingSigningProvider(&provider, /*hide_secret=*/true, /*hide_origin=*/false), + /*sighash_type=*/SIGHASH_ALL, + /*finalize=*/false); CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); ssTx << psbtx; @@ -1916,6 +1918,82 @@ static RPCHelpMan analyzepsbt() }; } +RPCHelpMan descriptorprocesspsbt() +{ + return RPCHelpMan{"descriptorprocesspsbt", + "\nUpdate all segwit inputs in a PSBT with information from output descriptors, the UTXO set or the mempool. \n" + "Then, sign the inputs we are able to with information from the output descriptors. ", + { + {"psbt", RPCArg::Type::STR, RPCArg::Optional::NO, "The transaction base64 string"}, + {"descriptors", RPCArg::Type::ARR, RPCArg::Optional::NO, "An array of either strings or objects", { + {"", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "An output descriptor"}, + {"", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "An object with an output descriptor and extra information", { + {"desc", RPCArg::Type::STR, RPCArg::Optional::NO, "An output descriptor"}, + {"range", RPCArg::Type::RANGE, RPCArg::Default{1000}, "Up to what index HD chains should be explored (either end or [begin,end])"}, + }}, + }}, + {"sighashtype", RPCArg::Type::STR, RPCArg::Default{"DEFAULT for Taproot, ALL otherwise"}, "The signature hash type to sign with if not specified by the PSBT. Must be one of\n" + " \"DEFAULT\"\n" + " \"ALL\"\n" + " \"NONE\"\n" + " \"SINGLE\"\n" + " \"ALL|ANYONECANPAY\"\n" + " \"NONE|ANYONECANPAY\"\n" + " \"SINGLE|ANYONECANPAY\""}, + {"bip32derivs", RPCArg::Type::BOOL, RPCArg::Default{true}, "Include BIP 32 derivation paths for public keys if we know them"}, + {"finalize", RPCArg::Type::BOOL, RPCArg::Default{true}, "Also finalize inputs if possible"}, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR, "psbt", "The base64-encoded partially signed transaction"}, + {RPCResult::Type::BOOL, "complete", "If the transaction has a complete set of signatures"}, + } + }, + RPCExamples{ + HelpExampleCli("descriptorprocesspsbt", "\"psbt\" \"[\\\"descriptor1\\\", \\\"descriptor2\\\"]\"") + + HelpExampleCli("descriptorprocesspsbt", "\"psbt\" \"[{\\\"desc\\\":\\\"mydescriptor\\\", \\\"range\\\":21}]\"") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue +{ + // Add descriptor information to a signing provider + FlatSigningProvider provider; + + auto descs = request.params[1].get_array(); + for (size_t i = 0; i < descs.size(); ++i) { + EvalDescriptorStringOrObject(descs[i], provider, /*expand_priv=*/true); + } + + int sighash_type = ParseSighashString(request.params[2]); + bool bip32derivs = request.params[3].isNull() ? true : request.params[3].get_bool(); + bool finalize = request.params[4].isNull() ? true : request.params[4].get_bool(); + + const PartiallySignedTransaction& psbtx = ProcessPSBT( + request.params[0].get_str(), + request.context, + HidingSigningProvider(&provider, /*hide_secret=*/false, !bip32derivs), + sighash_type, + finalize); + + // Check whether or not all of the inputs are now signed + bool complete = true; + for (const auto& input : psbtx.inputs) { + complete &= PSBTInputSigned(input); + } + + CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + ssTx << psbtx; + + UniValue result(UniValue::VOBJ); + + result.pushKV("psbt", EncodeBase64(ssTx)); + result.pushKV("complete", complete); + + return result; +}, + }; +} + void RegisterRawTransactionRPCCommands(CRPCTable& t) { static const CRPCCommand commands[]{ @@ -1931,6 +2009,7 @@ void RegisterRawTransactionRPCCommands(CRPCTable& t) {"rawtransactions", &createpsbt}, {"rawtransactions", &converttopsbt}, {"rawtransactions", &utxoupdatepsbt}, + {"rawtransactions", &descriptorprocesspsbt}, {"rawtransactions", &joinpsbts}, {"rawtransactions", &analyzepsbt}, }; diff --git a/src/rpc/rawtransaction_util.cpp b/src/rpc/rawtransaction_util.cpp index 3ba930f84f..3a6fa39e4d 100644 --- a/src/rpc/rawtransaction_util.cpp +++ b/src/rpc/rawtransaction_util.cpp @@ -36,7 +36,7 @@ void AddInputs(CMutableTransaction& rawTx, const UniValue& inputs_in, std::optio uint256 txid = ParseHashO(o, "txid"); - const UniValue& vout_v = find_value(o, "vout"); + const UniValue& vout_v = o.find_value("vout"); if (!vout_v.isNum()) throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, missing vout key"); int nOutput = vout_v.getInt<int>(); @@ -54,7 +54,7 @@ void AddInputs(CMutableTransaction& rawTx, const UniValue& inputs_in, std::optio } // set the sequence number if passed in the parameters object - const UniValue& sequenceObj = find_value(o, "sequence"); + const UniValue& sequenceObj = o.find_value("sequence"); if (sequenceObj.isNum()) { int64_t seqNr64 = sequenceObj.getInt<int64_t>(); if (seqNr64 < 0 || seqNr64 > CTxIn::SEQUENCE_FINAL) { @@ -187,7 +187,7 @@ void ParsePrevouts(const UniValue& prevTxsUnival, FillableSigningProvider* keyst uint256 txid = ParseHashO(prevOut, "txid"); - int nOut = find_value(prevOut, "vout").getInt<int>(); + int nOut = prevOut.find_value("vout").getInt<int>(); if (nOut < 0) { throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "vout cannot be negative"); } @@ -208,7 +208,7 @@ void ParsePrevouts(const UniValue& prevTxsUnival, FillableSigningProvider* keyst newcoin.out.scriptPubKey = scriptPubKey; newcoin.out.nValue = MAX_MONEY; if (prevOut.exists("amount")) { - newcoin.out.nValue = AmountFromValue(find_value(prevOut, "amount")); + newcoin.out.nValue = AmountFromValue(prevOut.find_value("amount")); } newcoin.nHeight = 1; coins[out] = std::move(newcoin); @@ -223,8 +223,8 @@ void ParsePrevouts(const UniValue& prevTxsUnival, FillableSigningProvider* keyst {"redeemScript", UniValueType(UniValue::VSTR)}, {"witnessScript", UniValueType(UniValue::VSTR)}, }, true); - UniValue rs = find_value(prevOut, "redeemScript"); - UniValue ws = find_value(prevOut, "witnessScript"); + const UniValue& rs{prevOut.find_value("redeemScript")}; + const UniValue& ws{prevOut.find_value("witnessScript")}; if (rs.isNull() && ws.isNull()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Missing redeemScript/witnessScript"); } diff --git a/src/rpc/request.cpp b/src/rpc/request.cpp index ad91ed0f23..4c67da8b70 100644 --- a/src/rpc/request.cpp +++ b/src/rpc/request.cpp @@ -88,7 +88,7 @@ bool GenerateAuthCookie(std::string *cookie_out) std::string cookie = COOKIEAUTH_USER + ":" + HexStr(rand_pwd); /** the umask determines what permissions are used to create this file - - * these are set to 0077 in util/system.cpp. + * these are set to 0077 in common/system.cpp. */ std::ofstream file; fs::path filepath_tmp = GetAuthCookieFile(true); @@ -165,10 +165,10 @@ void JSONRPCRequest::parse(const UniValue& valRequest) const UniValue& request = valRequest.get_obj(); // Parse id now so errors from here on will have the id - id = find_value(request, "id"); + id = request.find_value("id"); // Parse method - UniValue valMethod = find_value(request, "method"); + const UniValue& valMethod{request.find_value("method")}; if (valMethod.isNull()) throw JSONRPCError(RPC_INVALID_REQUEST, "Missing method"); if (!valMethod.isStr()) @@ -181,7 +181,7 @@ void JSONRPCRequest::parse(const UniValue& valRequest) LogPrint(BCLog::RPC, "ThreadRPCServer method=%s user=%s\n", SanitizeString(strMethod), this->authUser); // Parse params - UniValue valParams = find_value(request, "params"); + const UniValue& valParams{request.find_value("params")}; if (valParams.isArray() || valParams.isObject()) params = valParams; else if (valParams.isNull()) diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index 354f949002..d39efcb245 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -6,13 +6,13 @@ #include <rpc/server.h> #include <common/args.h> +#include <common/system.h> #include <logging.h> #include <rpc/util.h> #include <shutdown.h> #include <sync.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <util/time.h> #include <boost/signals2/signal.hpp> @@ -392,7 +392,7 @@ std::string JSONRPCExecBatch(const JSONRPCRequest& jreq, const UniValue& vReq) * Process named arguments into a vector of positional arguments, based on the * passed-in specification for the RPC call's arguments. */ -static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, const std::vector<std::string>& argNames) +static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, const std::vector<std::pair<std::string, bool>>& argNames) { JSONRPCRequest out = in; out.params = UniValue(UniValue::VARR); @@ -417,7 +417,9 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c // "args" parameter, if present. int hole = 0; int initial_hole_size = 0; - for (const std::string &argNamePattern: argNames) { + const std::string* initial_param = nullptr; + UniValue options{UniValue::VOBJ}; + for (const auto& [argNamePattern, named_only]: argNames) { std::vector<std::string> vargNames = SplitString(argNamePattern, '|'); auto fr = argsIn.end(); for (const std::string & argName : vargNames) { @@ -426,7 +428,22 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c break; } } - if (fr != argsIn.end()) { + + // Handle named-only parameters by pushing them into a temporary options + // object, and then pushing the accumulated options as the next + // positional argument. + if (named_only) { + if (fr != argsIn.end()) { + if (options.exists(fr->first)) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter " + fr->first + " specified multiple times"); + } + options.__pushKV(fr->first, *fr->second); + argsIn.erase(fr); + } + continue; + } + + if (!options.empty() || fr != argsIn.end()) { for (int i = 0; i < hole; ++i) { // Fill hole between specified parameters with JSON nulls, // but not at the end (for backwards compatibility with calls @@ -434,12 +451,26 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c out.params.push_back(UniValue()); } hole = 0; - out.params.push_back(*fr->second); - argsIn.erase(fr); + if (!initial_param) initial_param = &argNamePattern; } else { hole += 1; if (out.params.empty()) initial_hole_size = hole; } + + // If named input parameter "fr" is present, push it onto out.params. If + // options are present, push them onto out.params. If both are present, + // throw an error. + if (fr != argsIn.end()) { + if (!options.empty()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter " + fr->first + " conflicts with parameter " + options.getKeys().front()); + } + out.params.push_back(*fr->second); + argsIn.erase(fr); + } + if (!options.empty()) { + out.params.push_back(std::move(options)); + options = UniValue{UniValue::VOBJ}; + } } // If leftover "args" param was found, use it as a source of positional // arguments and add named arguments after. This is a convenience for @@ -447,9 +478,8 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c // arguments as described in doc/JSON-RPC-interface.md#parameter-passing auto positional_args{argsIn.extract("args")}; if (positional_args && positional_args.mapped()->isArray()) { - const bool has_named_arguments{initial_hole_size < (int)argNames.size()}; - if (initial_hole_size < (int)positional_args.mapped()->size() && has_named_arguments) { - throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter " + argNames[initial_hole_size] + " specified twice both as positional and named argument"); + if (initial_hole_size < (int)positional_args.mapped()->size() && initial_param) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter " + *initial_param + " specified twice both as positional and named argument"); } // Assign positional_args to out.params and append named_args after. UniValue named_args{std::move(out.params)}; diff --git a/src/rpc/server.h b/src/rpc/server.h index 01e8556050..24658ddb8b 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -95,7 +95,7 @@ public: using Actor = std::function<bool(const JSONRPCRequest& request, UniValue& result, bool last_handler)>; //! Constructor taking Actor callback supporting multiple handlers. - CRPCCommand(std::string category, std::string name, Actor actor, std::vector<std::string> args, intptr_t unique_id) + CRPCCommand(std::string category, std::string name, Actor actor, std::vector<std::pair<std::string, bool>> args, intptr_t unique_id) : category(std::move(category)), name(std::move(name)), actor(std::move(actor)), argNames(std::move(args)), unique_id(unique_id) { @@ -115,7 +115,16 @@ public: std::string category; std::string name; Actor actor; - std::vector<std::string> argNames; + //! List of method arguments and whether they are named-only. Incoming RPC + //! requests contain a "params" field that can either be an array containing + //! unnamed arguments or an object containing named arguments. The + //! "argNames" vector is used in the latter case to transform the params + //! object into an array. Each argument in "argNames" gets mapped to a + //! unique position in the array, based on the order it is listed, unless + //! the argument is a named-only argument with argNames[x].second set to + //! true. Named-only arguments are combined into a JSON object that is + //! appended after other arguments, see transformNamedArguments for details. + std::vector<std::pair<std::string, bool>> argNames; intptr_t unique_id; }; diff --git a/src/rpc/server_util.cpp b/src/rpc/server_util.cpp index 13d007b496..1d4afb3758 100644 --- a/src/rpc/server_util.cpp +++ b/src/rpc/server_util.cpp @@ -11,7 +11,7 @@ #include <rpc/protocol.h> #include <rpc/request.h> #include <txmempool.h> -#include <util/system.h> +#include <util/any.h> #include <validation.h> #include <any> diff --git a/src/rpc/txoutproof.cpp b/src/rpc/txoutproof.cpp index 24b5d04115..d74959cecc 100644 --- a/src/rpc/txoutproof.cpp +++ b/src/rpc/txoutproof.cpp @@ -18,7 +18,6 @@ #include <validation.h> using node::GetTransaction; -using node::ReadBlockFromDisk; static RPCHelpMan gettxoutproof() { @@ -85,7 +84,7 @@ static RPCHelpMan gettxoutproof() } if (pblockindex == nullptr) { - const CTransactionRef tx = GetTransaction(/*block_index=*/nullptr, /*mempool=*/nullptr, *setTxids.begin(), chainman.GetConsensus(), hashBlock); + const CTransactionRef tx = GetTransaction(/*block_index=*/nullptr, /*mempool=*/nullptr, *setTxids.begin(), hashBlock, chainman.m_blockman); if (!tx || hashBlock.IsNull()) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not yet in block"); } @@ -98,7 +97,7 @@ static RPCHelpMan gettxoutproof() } CBlock block; - if (!ReadBlockFromDisk(block, pblockindex, chainman.GetConsensus())) { + if (!chainman.m_blockman.ReadBlockFromDisk(block, *pblockindex)) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Can't read block from disk"); } diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 1f3f37d0a0..19e14f88df 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -37,7 +37,7 @@ void RPCTypeCheckObj(const UniValue& o, bool fStrict) { for (const auto& t : typesExpected) { - const UniValue& v = find_value(o, t.first); + const UniValue& v = o.find_value(t.first); if (!fAllowNull && v.isNull()) throw JSONRPCError(RPC_TYPE_ERROR, strprintf("Missing %s", t.first)); @@ -81,7 +81,7 @@ uint256 ParseHashV(const UniValue& v, std::string strName) } uint256 ParseHashO(const UniValue& o, std::string strKey) { - return ParseHashV(find_value(o, strKey), strKey); + return ParseHashV(o.find_value(strKey), strKey); } std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName) { @@ -94,7 +94,7 @@ std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName) } std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey) { - return ParseHexV(find_value(o, strKey), strKey); + return ParseHexV(o.find_value(strKey), strKey); } namespace { @@ -389,7 +389,8 @@ struct Sections { case RPCArg::Type::NUM: case RPCArg::Type::AMOUNT: case RPCArg::Type::RANGE: - case RPCArg::Type::BOOL: { + case RPCArg::Type::BOOL: + case RPCArg::Type::OBJ_NAMED_PARAMS: { if (is_top_level_arg) return; // Nothing more to do for non-recursive types on first recursion auto left = indent; if (arg.m_opts.type_str.size() != 0 && push_name) { @@ -485,12 +486,32 @@ RPCHelpMan::RPCHelpMan(std::string name, std::string description, std::vector<RP m_results{std::move(results)}, m_examples{std::move(examples)} { - std::set<std::string> named_args; + // Map of parameter names and types just used to check whether the names are + // unique. Parameter names always need to be unique, with the exception that + // there can be pairs of POSITIONAL and NAMED parameters with the same name. + enum ParamType { POSITIONAL = 1, NAMED = 2, NAMED_ONLY = 4 }; + std::map<std::string, int> param_names; + for (const auto& arg : m_args) { std::vector<std::string> names = SplitString(arg.m_names, '|'); // Should have unique named arguments for (const std::string& name : names) { - CHECK_NONFATAL(named_args.insert(name).second); + auto& param_type = param_names[name]; + CHECK_NONFATAL(!(param_type & POSITIONAL)); + CHECK_NONFATAL(!(param_type & NAMED_ONLY)); + param_type |= POSITIONAL; + } + if (arg.m_type == RPCArg::Type::OBJ_NAMED_PARAMS) { + for (const auto& inner : arg.m_inner) { + std::vector<std::string> inner_names = SplitString(inner.m_names, '|'); + for (const std::string& inner_name : inner_names) { + auto& param_type = param_names[inner_name]; + CHECK_NONFATAL(!(param_type & POSITIONAL) || inner.m_opts.also_positional); + CHECK_NONFATAL(!(param_type & NAMED)); + CHECK_NONFATAL(!(param_type & NAMED_ONLY)); + param_type |= inner.m_opts.also_positional ? NAMED : NAMED_ONLY; + } + } } // Default value type should match argument type only when defined if (arg.m_fallback.index() == 2) { @@ -605,12 +626,17 @@ bool RPCHelpMan::IsValidNumArgs(size_t num_args) const return num_required_args <= num_args && num_args <= m_args.size(); } -std::vector<std::string> RPCHelpMan::GetArgNames() const +std::vector<std::pair<std::string, bool>> RPCHelpMan::GetArgNames() const { - std::vector<std::string> ret; + std::vector<std::pair<std::string, bool>> ret; ret.reserve(m_args.size()); for (const auto& arg : m_args) { - ret.emplace_back(arg.m_names); + if (arg.m_type == RPCArg::Type::OBJ_NAMED_PARAMS) { + for (const auto& inner : arg.m_inner) { + ret.emplace_back(inner.m_names, /*named_only=*/true); + } + } + ret.emplace_back(arg.m_names, /*named_only=*/false); } return ret; } @@ -642,20 +668,31 @@ std::string RPCHelpMan::ToString() const // Arguments Sections sections; + Sections named_only_sections; for (size_t i{0}; i < m_args.size(); ++i) { const auto& arg = m_args.at(i); if (arg.m_opts.hidden) break; // Any arg that follows is also hidden - if (i == 0) ret += "\nArguments:\n"; - // Push named argument name and description sections.m_sections.emplace_back(::ToString(i + 1) + ". " + arg.GetFirstName(), arg.ToDescriptionString(/*is_named_arg=*/true)); sections.m_max_pad = std::max(sections.m_max_pad, sections.m_sections.back().m_left.size()); // Recursively push nested args sections.Push(arg); + + // Push named-only argument sections + if (arg.m_type == RPCArg::Type::OBJ_NAMED_PARAMS) { + for (const auto& arg_inner : arg.m_inner) { + named_only_sections.PushSection({arg_inner.GetFirstName(), arg_inner.ToDescriptionString(/*is_named_arg=*/true)}); + named_only_sections.Push(arg_inner); + } + } } + + if (!sections.m_sections.empty()) ret += "\nArguments:\n"; ret += sections.ToString(); + if (!named_only_sections.m_sections.empty()) ret += "\nNamed Arguments:\n"; + ret += named_only_sections.ToString(); // Result ret += m_results.ToDescriptionString(); @@ -669,17 +706,30 @@ std::string RPCHelpMan::ToString() const UniValue RPCHelpMan::GetArgMap() const { UniValue arr{UniValue::VARR}; + + auto push_back_arg_info = [&arr](const std::string& rpc_name, int pos, const std::string& arg_name, const RPCArg::Type& type) { + UniValue map{UniValue::VARR}; + map.push_back(rpc_name); + map.push_back(pos); + map.push_back(arg_name); + map.push_back(type == RPCArg::Type::STR || + type == RPCArg::Type::STR_HEX); + arr.push_back(map); + }; + for (int i{0}; i < int(m_args.size()); ++i) { const auto& arg = m_args.at(i); std::vector<std::string> arg_names = SplitString(arg.m_names, '|'); for (const auto& arg_name : arg_names) { - UniValue map{UniValue::VARR}; - map.push_back(m_name); - map.push_back(i); - map.push_back(arg_name); - map.push_back(arg.m_type == RPCArg::Type::STR || - arg.m_type == RPCArg::Type::STR_HEX); - arr.push_back(map); + push_back_arg_info(m_name, i, arg_name, arg.m_type); + if (arg.m_type == RPCArg::Type::OBJ_NAMED_PARAMS) { + for (const auto& inner : arg.m_inner) { + std::vector<std::string> inner_names = SplitString(inner.m_names, '|'); + for (const std::string& inner_name : inner_names) { + push_back_arg_info(m_name, i, inner_name, inner.m_type); + } + } + } } } return arr; @@ -708,6 +758,7 @@ static std::optional<UniValue::VType> ExpectedType(RPCArg::Type type) return UniValue::VBOOL; } case Type::OBJ: + case Type::OBJ_NAMED_PARAMS: case Type::OBJ_USER_KEYS: { return UniValue::VOBJ; } @@ -781,6 +832,7 @@ std::string RPCArg::ToDescriptionString(bool is_named_arg) const break; } case Type::OBJ: + case Type::OBJ_NAMED_PARAMS: case Type::OBJ_USER_KEYS: { ret += "json object"; break; @@ -809,6 +861,7 @@ std::string RPCArg::ToDescriptionString(bool is_named_arg) const } // no default case, so the compiler can warn about missing cases } ret += ")"; + if (m_type == Type::OBJ_NAMED_PARAMS) ret += " Options object that can be used to pass named arguments, listed below."; ret += m_description.empty() ? "" : " " + m_description; return ret; } @@ -1054,6 +1107,7 @@ std::string RPCArg::ToStringObj(const bool oneline) const } return res + "...]"; case Type::OBJ: + case Type::OBJ_NAMED_PARAMS: case Type::OBJ_USER_KEYS: // Currently unused, so avoid writing dead code NONFATAL_UNREACHABLE(); @@ -1077,6 +1131,7 @@ std::string RPCArg::ToString(const bool oneline) const return GetFirstName(); } case Type::OBJ: + case Type::OBJ_NAMED_PARAMS: case Type::OBJ_USER_KEYS: { const std::string res = Join(m_inner, ",", [&](const RPCArg& i) { return i.ToStringObj(oneline); }); if (m_type == Type::OBJ) { @@ -1126,17 +1181,17 @@ std::pair<int64_t, int64_t> ParseDescriptorRange(const UniValue& value) return {low, high}; } -std::vector<CScript> EvalDescriptorStringOrObject(const UniValue& scanobject, FlatSigningProvider& provider) +std::vector<CScript> EvalDescriptorStringOrObject(const UniValue& scanobject, FlatSigningProvider& provider, const bool expand_priv) { std::string desc_str; std::pair<int64_t, int64_t> range = {0, 1000}; if (scanobject.isStr()) { desc_str = scanobject.get_str(); } else if (scanobject.isObject()) { - UniValue desc_uni = find_value(scanobject, "desc"); + const UniValue& desc_uni{scanobject.find_value("desc")}; if (desc_uni.isNull()) throw JSONRPCError(RPC_INVALID_PARAMETER, "Descriptor needs to be provided in scan object"); desc_str = desc_uni.get_str(); - UniValue range_uni = find_value(scanobject, "range"); + const UniValue& range_uni{scanobject.find_value("range")}; if (!range_uni.isNull()) { range = ParseDescriptorRange(range_uni); } @@ -1159,6 +1214,9 @@ std::vector<CScript> EvalDescriptorStringOrObject(const UniValue& scanobject, Fl if (!desc->Expand(i, provider, scripts, provider)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, strprintf("Cannot derive script without private keys: '%s'", desc_str)); } + if (expand_priv) { + desc->ExpandPrivate(/*pos=*/i, provider, /*out=*/provider); + } std::move(scripts.begin(), scripts.end(), std::back_inserter(ret)); } return ret; diff --git a/src/rpc/util.h b/src/rpc/util.h index bb5c30a2f4..4cba5a9818 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -110,7 +110,7 @@ UniValue JSONRPCTransactionError(TransactionError terr, const std::string& err_s std::pair<int64_t, int64_t> ParseDescriptorRange(const UniValue& value); /** Evaluate a descriptor given as a string, or as a {"desc":...,"range":...} object, with default range of 1000. */ -std::vector<CScript> EvalDescriptorStringOrObject(const UniValue& scanobject, FlatSigningProvider& provider); +std::vector<CScript> EvalDescriptorStringOrObject(const UniValue& scanobject, FlatSigningProvider& provider, const bool expand_priv = false); /** Returns, given services flags, a list of humanly readable (known) network services */ UniValue GetServicesNames(ServiceFlags services); @@ -130,6 +130,15 @@ struct RPCArgOptions { std::string oneline_description{}; //!< Should be empty unless it is supposed to override the auto-generated summary line std::vector<std::string> type_str{}; //!< Should be empty unless it is supposed to override the auto-generated type strings. Vector length is either 0 or 2, m_opts.type_str.at(0) will override the type of the value in a key-value pair, m_opts.type_str.at(1) will override the type in the argument description. bool hidden{false}; //!< For testing only + bool also_positional{false}; //!< If set allows a named-parameter field in an OBJ_NAMED_PARAM options object + //!< to have the same name as a top-level parameter. By default the RPC + //!< framework disallows this, because if an RPC request passes the value by + //!< name, it is assigned to top-level parameter position, not to the options + //!< position, defeating the purpose of using OBJ_NAMED_PARAMS instead OBJ for + //!< that option. But sometimes it makes sense to allow less-commonly used + //!< options to be passed by name only, and more commonly used options to be + //!< passed by name or position, so the RPC framework allows this as long as + //!< methods set the also_positional flag and read values from both positions. }; struct RPCArg { @@ -139,6 +148,13 @@ struct RPCArg { STR, NUM, BOOL, + OBJ_NAMED_PARAMS, //!< Special type that behaves almost exactly like + //!< OBJ, defining an options object with a list of + //!< pre-defined keys. The only difference between OBJ + //!< and OBJ_NAMED_PARAMS is that OBJ_NAMED_PARMS + //!< also allows the keys to be passed as top-level + //!< named parameters, as a more convenient way to pass + //!< options to the RPC method without nesting them. OBJ_USER_KEYS, //!< Special type where the user must set the keys e.g. to define multiple addresses; as opposed to e.g. an options object where the keys are predefined AMOUNT, //!< Special type representing a floating point amount (can be either NUM or STR) STR_HEX, //!< Special type that is a STR with only hex chars @@ -183,7 +199,7 @@ struct RPCArg { m_description{std::move(description)}, m_opts{std::move(opts)} { - CHECK_NONFATAL(type != Type::ARR && type != Type::OBJ && type != Type::OBJ_USER_KEYS); + CHECK_NONFATAL(type != Type::ARR && type != Type::OBJ && type != Type::OBJ_NAMED_PARAMS && type != Type::OBJ_USER_KEYS); } RPCArg( @@ -200,7 +216,7 @@ struct RPCArg { m_description{std::move(description)}, m_opts{std::move(opts)} { - CHECK_NONFATAL(type == Type::ARR || type == Type::OBJ || type == Type::OBJ_USER_KEYS); + CHECK_NONFATAL(type == Type::ARR || type == Type::OBJ || type == Type::OBJ_NAMED_PARAMS || type == Type::OBJ_USER_KEYS); } bool IsOptional() const; @@ -369,7 +385,8 @@ public: UniValue GetArgMap() const; /** If the supplied number of args is neither too small nor too high */ bool IsValidNumArgs(size_t num_args) const; - std::vector<std::string> GetArgNames() const; + //! Return list of arguments and whether they are named-only. + std::vector<std::pair<std::string, bool>> GetArgNames() const; const std::string m_name; diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 0951504ee0..b8ade1684a 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -197,7 +197,9 @@ public: /** Get the descriptor string form including private data (if available in arg). */ virtual bool ToPrivateString(const SigningProvider& arg, std::string& out) const = 0; - /** Get the descriptor string form with the xpub at the last hardened derivation */ + /** Get the descriptor string form with the xpub at the last hardened derivation, + * and always use h for hardened derivation. + */ virtual bool ToNormalizedString(const SigningProvider& arg, std::string& out, const DescriptorCache* cache = nullptr) const = 0; /** Derive a private key, if private data is available in arg. */ @@ -208,14 +210,15 @@ class OriginPubkeyProvider final : public PubkeyProvider { KeyOriginInfo m_origin; std::unique_ptr<PubkeyProvider> m_provider; + bool m_apostrophe; - std::string OriginString() const + std::string OriginString(bool normalized=false) const { - return HexStr(m_origin.fingerprint) + FormatHDKeypath(m_origin.path); + return HexStr(m_origin.fingerprint) + FormatHDKeypath(m_origin.path, /*apostrophe=*/!normalized && m_apostrophe); } public: - OriginPubkeyProvider(uint32_t exp_index, KeyOriginInfo info, std::unique_ptr<PubkeyProvider> provider) : PubkeyProvider(exp_index), m_origin(std::move(info)), m_provider(std::move(provider)) {} + OriginPubkeyProvider(uint32_t exp_index, KeyOriginInfo info, std::unique_ptr<PubkeyProvider> provider, bool apostrophe) : PubkeyProvider(exp_index), m_origin(std::move(info)), m_provider(std::move(provider)), m_apostrophe(apostrophe) {} bool GetPubKey(int pos, const SigningProvider& arg, CPubKey& key, KeyOriginInfo& info, const DescriptorCache* read_cache = nullptr, DescriptorCache* write_cache = nullptr) const override { if (!m_provider->GetPubKey(pos, arg, key, info, read_cache, write_cache)) return false; @@ -242,9 +245,9 @@ public: // and append that to our own origin string. if (sub[0] == '[') { sub = sub.substr(9); - ret = "[" + OriginString() + std::move(sub); + ret = "[" + OriginString(/*normalized=*/true) + std::move(sub); } else { - ret = "[" + OriginString() + "]" + std::move(sub); + ret = "[" + OriginString(/*normalized=*/true) + "]" + std::move(sub); } return true; } @@ -312,6 +315,8 @@ class BIP32PubkeyProvider final : public PubkeyProvider CExtPubKey m_root_extkey; KeyPath m_path; DeriveType m_derive; + // Whether ' or h is used in harded derivation + bool m_apostrophe; bool GetExtKey(const SigningProvider& arg, CExtKey& ret) const { @@ -348,7 +353,7 @@ class BIP32PubkeyProvider final : public PubkeyProvider } public: - BIP32PubkeyProvider(uint32_t exp_index, const CExtPubKey& extkey, KeyPath path, DeriveType derive) : PubkeyProvider(exp_index), m_root_extkey(extkey), m_path(std::move(path)), m_derive(derive) {} + BIP32PubkeyProvider(uint32_t exp_index, const CExtPubKey& extkey, KeyPath path, DeriveType derive, bool apostrophe) : PubkeyProvider(exp_index), m_root_extkey(extkey), m_path(std::move(path)), m_derive(derive), m_apostrophe(apostrophe) {} bool IsRange() const override { return m_derive != DeriveType::NO; } size_t GetSize() const override { return 33; } bool GetPubKey(int pos, const SigningProvider& arg, CPubKey& key_out, KeyOriginInfo& final_info_out, const DescriptorCache* read_cache = nullptr, DescriptorCache* write_cache = nullptr) const override @@ -416,31 +421,36 @@ public: return true; } - std::string ToString() const override + std::string ToString(bool normalized) const { - std::string ret = EncodeExtPubKey(m_root_extkey) + FormatHDKeypath(m_path); + const bool use_apostrophe = !normalized && m_apostrophe; + std::string ret = EncodeExtPubKey(m_root_extkey) + FormatHDKeypath(m_path, /*apostrophe=*/use_apostrophe); if (IsRange()) { ret += "/*"; - if (m_derive == DeriveType::HARDENED) ret += '\''; + if (m_derive == DeriveType::HARDENED) ret += use_apostrophe ? '\'' : 'h'; } return ret; } + std::string ToString() const override + { + return ToString(/*normalized=*/false); + } bool ToPrivateString(const SigningProvider& arg, std::string& out) const override { CExtKey key; if (!GetExtKey(arg, key)) return false; - out = EncodeExtKey(key) + FormatHDKeypath(m_path); + out = EncodeExtKey(key) + FormatHDKeypath(m_path, /*apostrophe=*/m_apostrophe); if (IsRange()) { out += "/*"; - if (m_derive == DeriveType::HARDENED) out += '\''; + if (m_derive == DeriveType::HARDENED) out += m_apostrophe ? '\'' : 'h'; } return true; } bool ToNormalizedString(const SigningProvider& arg, std::string& out, const DescriptorCache* cache) const override { - // For hardened derivation type, just return the typical string, nothing to normalize if (m_derive == DeriveType::HARDENED) { - out = ToString(); + out = ToString(/*normalized=*/true); + return true; } // Step backwards to find the last hardened step in the path @@ -1049,15 +1059,27 @@ enum class ParseScriptContext { P2TR, //!< Inside tr() (either internal key, or BIP342 script leaf) }; -/** Parse a key path, being passed a split list of elements (the first element is ignored). */ -[[nodiscard]] bool ParseKeyPath(const std::vector<Span<const char>>& split, KeyPath& out, std::string& error) +/** + * Parse a key path, being passed a split list of elements (the first element is ignored). + * + * @param[in] split BIP32 path string, using either ' or h for hardened derivation + * @param[out] out the key path + * @param[out] apostrophe only updated if hardened derivation is found + * @param[out] error parsing error message + * @returns false if parsing failed + **/ +[[nodiscard]] bool ParseKeyPath(const std::vector<Span<const char>>& split, KeyPath& out, bool& apostrophe, std::string& error) { for (size_t i = 1; i < split.size(); ++i) { Span<const char> elem = split[i]; bool hardened = false; - if (elem.size() > 0 && (elem[elem.size() - 1] == '\'' || elem[elem.size() - 1] == 'h')) { - elem = elem.first(elem.size() - 1); - hardened = true; + if (elem.size() > 0) { + const char last = elem[elem.size() - 1]; + if (last == '\'' || last == 'h') { + elem = elem.first(elem.size() - 1); + hardened = true; + apostrophe = last == '\''; + } } uint32_t p; if (!ParseUInt32(std::string(elem.begin(), elem.end()), &p)) { @@ -1073,7 +1095,7 @@ enum class ParseScriptContext { } /** Parse a public key that excludes origin information. */ -std::unique_ptr<PubkeyProvider> ParsePubkeyInner(uint32_t key_exp_index, const Span<const char>& sp, ParseScriptContext ctx, FlatSigningProvider& out, std::string& error) +std::unique_ptr<PubkeyProvider> ParsePubkeyInner(uint32_t key_exp_index, const Span<const char>& sp, ParseScriptContext ctx, FlatSigningProvider& out, bool& apostrophe, std::string& error) { using namespace spanparsing; @@ -1130,15 +1152,16 @@ std::unique_ptr<PubkeyProvider> ParsePubkeyInner(uint32_t key_exp_index, const S split.pop_back(); type = DeriveType::UNHARDENED; } else if (split.back() == Span{"*'"}.first(2) || split.back() == Span{"*h"}.first(2)) { + apostrophe = split.back() == Span{"*'"}.first(2); split.pop_back(); type = DeriveType::HARDENED; } - if (!ParseKeyPath(split, path, error)) return nullptr; + if (!ParseKeyPath(split, path, apostrophe, error)) return nullptr; if (extkey.key.IsValid()) { extpubkey = extkey.Neuter(); out.keys.emplace(extpubkey.pubkey.GetID(), extkey.key); } - return std::make_unique<BIP32PubkeyProvider>(key_exp_index, extpubkey, std::move(path), type); + return std::make_unique<BIP32PubkeyProvider>(key_exp_index, extpubkey, std::move(path), type, apostrophe); } /** Parse a public key including origin information (if enabled). */ @@ -1151,7 +1174,11 @@ std::unique_ptr<PubkeyProvider> ParsePubkey(uint32_t key_exp_index, const Span<c error = "Multiple ']' characters found for a single pubkey"; return nullptr; } - if (origin_split.size() == 1) return ParsePubkeyInner(key_exp_index, origin_split[0], ctx, out, error); + // This is set if either the origin or path suffix contains a hardened derivation. + bool apostrophe = false; + if (origin_split.size() == 1) { + return ParsePubkeyInner(key_exp_index, origin_split[0], ctx, out, apostrophe, error); + } if (origin_split[0].empty() || origin_split[0][0] != '[') { error = strprintf("Key origin start '[ character expected but not found, got '%c' instead", origin_split[0].empty() ? /** empty, implies split char */ ']' : origin_split[0][0]); @@ -1172,10 +1199,10 @@ std::unique_ptr<PubkeyProvider> ParsePubkey(uint32_t key_exp_index, const Span<c static_assert(sizeof(info.fingerprint) == 4, "Fingerprint must be 4 bytes"); assert(fpr_bytes.size() == 4); std::copy(fpr_bytes.begin(), fpr_bytes.end(), info.fingerprint); - if (!ParseKeyPath(slash_split, info.path, error)) return nullptr; - auto provider = ParsePubkeyInner(key_exp_index, origin_split[1], ctx, out, error); + if (!ParseKeyPath(slash_split, info.path, apostrophe, error)) return nullptr; + auto provider = ParsePubkeyInner(key_exp_index, origin_split[1], ctx, out, apostrophe, error); if (!provider) return nullptr; - return std::make_unique<OriginPubkeyProvider>(key_exp_index, std::move(info), std::move(provider)); + return std::make_unique<OriginPubkeyProvider>(key_exp_index, std::move(info), std::move(provider), apostrophe); } std::unique_ptr<PubkeyProvider> InferPubkey(const CPubKey& pubkey, ParseScriptContext, const SigningProvider& provider) @@ -1183,7 +1210,7 @@ std::unique_ptr<PubkeyProvider> InferPubkey(const CPubKey& pubkey, ParseScriptCo std::unique_ptr<PubkeyProvider> key_provider = std::make_unique<ConstPubkeyProvider>(0, pubkey, false); KeyOriginInfo info; if (provider.GetKeyOrigin(pubkey.GetID(), info)) { - return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider)); + return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider), /*apostrophe=*/false); } return key_provider; } @@ -1196,7 +1223,7 @@ std::unique_ptr<PubkeyProvider> InferXOnlyPubkey(const XOnlyPubKey& xkey, ParseS std::unique_ptr<PubkeyProvider> key_provider = std::make_unique<ConstPubkeyProvider>(0, pubkey, true); KeyOriginInfo info; if (provider.GetKeyOriginByXOnly(xkey, info)) { - return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider)); + return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider), /*apostrophe=*/false); } return key_provider; } diff --git a/src/script/sigcache.cpp b/src/script/sigcache.cpp index e52e8cd309..7c6c282cc4 100644 --- a/src/script/sigcache.cpp +++ b/src/script/sigcache.cpp @@ -5,11 +5,11 @@ #include <script/sigcache.h> +#include <common/system.h> #include <logging.h> #include <pubkey.h> #include <random.h> #include <uint256.h> -#include <util/system.h> #include <cuckoocache.h> diff --git a/src/shutdown.cpp b/src/shutdown.cpp index 2fffc0663c..d70017d734 100644 --- a/src/shutdown.cpp +++ b/src/shutdown.cpp @@ -11,6 +11,7 @@ #include <logging.h> #include <node/interface_ui.h> +#include <util/check.h> #include <util/tokenpipe.h> #include <warnings.h> @@ -20,6 +21,8 @@ #include <condition_variable> #endif +static std::atomic<int>* g_exit_status{nullptr}; + bool AbortNode(const std::string& strMessage, bilingual_str user_message) { SetMiscWarning(Untranslated(strMessage)); @@ -28,6 +31,7 @@ bool AbortNode(const std::string& strMessage, bilingual_str user_message) user_message = _("A fatal internal error occurred, see debug.log for details"); } InitError(user_message); + Assert(g_exit_status)->store(EXIT_FAILURE); StartShutdown(); return false; } @@ -44,8 +48,9 @@ static TokenPipeEnd g_shutdown_r; static TokenPipeEnd g_shutdown_w; #endif -bool InitShutdownState() +bool InitShutdownState(std::atomic<int>& exit_status) { + g_exit_status = &exit_status; #ifndef WIN32 std::optional<TokenPipe> pipe = TokenPipe::Make(); if (!pipe) return false; diff --git a/src/shutdown.h b/src/shutdown.h index 07a8315788..c119bee96f 100644 --- a/src/shutdown.h +++ b/src/shutdown.h @@ -8,13 +8,15 @@ #include <util/translation.h> // For bilingual_str +#include <atomic> + /** Abort with a message */ bool AbortNode(const std::string& strMessage, bilingual_str user_message = bilingual_str{}); /** Initialize shutdown state. This must be called before using either StartShutdown(), * AbortShutdown() or WaitForShutdown(). Calling ShutdownRequested() is always safe. */ -bool InitShutdownState(); +bool InitShutdownState(std::atomic<int>& exit_status); /** Request shutdown of the application. */ void StartShutdown(); diff --git a/src/signet.cpp b/src/signet.cpp index b76b1e342f..b73d82bb2e 100644 --- a/src/signet.cpp +++ b/src/signet.cpp @@ -8,6 +8,7 @@ #include <cstdint> #include <vector> +#include <common/system.h> #include <consensus/merkle.h> #include <consensus/params.h> #include <consensus/validation.h> @@ -22,7 +23,6 @@ #include <streams.h> #include <uint256.h> #include <util/strencodings.h> -#include <util/system.h> static constexpr uint8_t SIGNET_HEADER[4] = {0xec, 0xc7, 0xda, 0xa2}; diff --git a/src/streams.h b/src/streams.h index 8788343809..03df20b5db 100644 --- a/src/streams.h +++ b/src/streams.h @@ -293,14 +293,6 @@ public: vch.insert(vch.end(), src.begin(), src.end()); } - template<typename Stream> - void Serialize(Stream& s) const - { - // Special case: stream << stream concatenates like stream += stream - if (!vch.empty()) - s.write(MakeByteSpan(vch)); - } - template<typename T> DataStream& operator<<(const T& obj) { @@ -756,15 +748,25 @@ public: } //! search for a given byte in the stream, and remain positioned on it - void FindByte(uint8_t ch) + void FindByte(std::byte byte) { + // For best performance, avoid mod operation within the loop. + size_t buf_offset{size_t(m_read_pos % uint64_t(vchBuf.size()))}; while (true) { - if (m_read_pos == nSrcPos) + if (m_read_pos == nSrcPos) { + // No more bytes available; read from the file into the buffer, + // setting nSrcPos to one beyond the end of the new data. + // Throws exception if end-of-file reached. Fill(); - if (vchBuf[m_read_pos % vchBuf.size()] == std::byte{ch}) { - break; } - m_read_pos++; + const size_t len{std::min<size_t>(vchBuf.size() - buf_offset, nSrcPos - m_read_pos)}; + const auto it_start{vchBuf.begin() + buf_offset}; + const auto it_find{std::find(it_start, it_start + len, byte)}; + const size_t inc{size_t(std::distance(it_start, it_find))}; + m_read_pos += inc; + if (inc < len) break; + buf_offset += inc; + if (buf_offset >= vchBuf.size()) buf_offset = 0; } } }; diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index 54d923e4a4..328e7f81a0 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -33,16 +33,16 @@ static int32_t GetCheckRatio(const NodeContext& node_ctx) static CNetAddr ResolveIP(const std::string& ip) { - CNetAddr addr; - BOOST_CHECK_MESSAGE(LookupHost(ip, addr, false), strprintf("failed to resolve: %s", ip)); - return addr; + const std::optional<CNetAddr> addr{LookupHost(ip, false)}; + BOOST_CHECK_MESSAGE(addr.has_value(), strprintf("failed to resolve: %s", ip)); + return addr.value_or(CNetAddr{}); } static CService ResolveService(const std::string& ip, uint16_t port = 0) { - CService serv; - BOOST_CHECK_MESSAGE(Lookup(ip, serv, port, false), strprintf("failed to resolve: %s:%i", ip, port)); - return serv; + const std::optional<CService> serv{Lookup(ip, port, false)}; + BOOST_CHECK_MESSAGE(serv.has_value(), strprintf("failed to resolve: %s:%i", ip, port)); + return serv.value_or(CService{}); } @@ -948,18 +948,23 @@ BOOST_AUTO_TEST_CASE(load_addrman) { AddrMan addrman{EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)}; - CService addr1, addr2, addr3; - BOOST_CHECK(Lookup("250.7.1.1", addr1, 8333, false)); - BOOST_CHECK(Lookup("250.7.2.2", addr2, 9999, false)); - BOOST_CHECK(Lookup("250.7.3.3", addr3, 9999, false)); - BOOST_CHECK(Lookup("250.7.3.3"s, addr3, 9999, false)); - BOOST_CHECK(!Lookup("250.7.3.3\0example.com"s, addr3, 9999, false)); + std::optional<CService> addr1, addr2, addr3, addr4; + addr1 = Lookup("250.7.1.1", 8333, false); + BOOST_CHECK(addr1.has_value()); + addr2 = Lookup("250.7.2.2", 9999, false); + BOOST_CHECK(addr2.has_value()); + addr3 = Lookup("250.7.3.3", 9999, false); + BOOST_CHECK(addr3.has_value()); + addr3 = Lookup("250.7.3.3"s, 9999, false); + BOOST_CHECK(addr3.has_value()); + addr4 = Lookup("250.7.3.3\0example.com"s, 9999, false); + BOOST_CHECK(!addr4.has_value()); // Add three addresses to new table. - CService source; - BOOST_CHECK(Lookup("252.5.1.1", source, 8333, false)); - std::vector<CAddress> addresses{CAddress(addr1, NODE_NONE), CAddress(addr2, NODE_NONE), CAddress(addr3, NODE_NONE)}; - BOOST_CHECK(addrman.Add(addresses, source)); + const std::optional<CService> source{Lookup("252.5.1.1", 8333, false)}; + BOOST_CHECK(source.has_value()); + std::vector<CAddress> addresses{CAddress(addr1.value(), NODE_NONE), CAddress(addr2.value(), NODE_NONE), CAddress(addr3.value(), NODE_NONE)}; + BOOST_CHECK(addrman.Add(addresses, source.value())); BOOST_CHECK(addrman.Size() == 3); // Test that the de-serialization does not throw an exception. @@ -1004,12 +1009,12 @@ static CDataStream MakeCorruptPeersDat() int nUBuckets = ADDRMAN_NEW_BUCKET_COUNT ^ (1 << 30); s << nUBuckets; - CService serv; - BOOST_CHECK(Lookup("252.1.1.1", serv, 7777, false)); - CAddress addr = CAddress(serv, NODE_NONE); - CNetAddr resolved; - BOOST_CHECK(LookupHost("252.2.2.2", resolved, false)); - AddrInfo info = AddrInfo(addr, resolved); + const std::optional<CService> serv{Lookup("252.1.1.1", 7777, false)}; + BOOST_REQUIRE(serv.has_value()); + CAddress addr = CAddress(serv.value(), NODE_NONE); + std::optional<CNetAddr> resolved{LookupHost("252.2.2.2", false)}; + BOOST_REQUIRE(resolved.has_value()); + AddrInfo info = AddrInfo(addr, resolved.value()); s << info; return s; diff --git a/src/test/allocator_tests.cpp b/src/test/allocator_tests.cpp index f74e50a890..8c0af6f26f 100644 --- a/src/test/allocator_tests.cpp +++ b/src/test/allocator_tests.cpp @@ -2,8 +2,8 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <support/lockedpool.h> -#include <util/system.h> #include <limits> #include <memory> diff --git a/src/test/argsman_tests.cpp b/src/test/argsman_tests.cpp index 6a0925f0bb..0b789e7f5c 100644 --- a/src/test/argsman_tests.cpp +++ b/src/test/argsman_tests.cpp @@ -8,6 +8,7 @@ #include <test/util/setup_common.h> #include <test/util/str.h> #include <univalue.h> +#include <util/chaintype.h> #include <util/fs.h> #include <util/strencodings.h> @@ -84,7 +85,7 @@ class CheckValueTest : public TestChain100Setup { public: struct Expect { - util::SettingsValue setting; + common::SettingsValue setting; bool default_string = false; bool default_int = false; bool default_bool = false; @@ -94,7 +95,7 @@ public: std::optional<std::vector<std::string>> list_value; const char* error = nullptr; - explicit Expect(util::SettingsValue s) : setting(std::move(s)) {} + explicit Expect(common::SettingsValue s) : setting(std::move(s)) {} Expect& DefaultString() { default_string = true; return *this; } Expect& DefaultInt() { default_int = true; return *this; } Expect& DefaultBool() { default_bool = true; return *this; } @@ -254,7 +255,7 @@ BOOST_AUTO_TEST_CASE(util_ParseInvalidParameters) BOOST_CHECK(!test.ParseParameters(2, (char**)argv, error)); BOOST_CHECK_EQUAL(error, "Invalid parameter -unregistered"); - // Make sure registered parameters prefixed with a chain name trigger errors. + // Make sure registered parameters prefixed with a chain type trigger errors. // (Previously, they were accepted and ignored.) argv[1] = "-test.registered"; BOOST_CHECK(!test.ParseParameters(2, (char**)argv, error)); @@ -580,7 +581,7 @@ BOOST_AUTO_TEST_CASE(util_ReadConfigStream) test_args.SetNetworkOnlyArg("-ccc"); test_args.SetNetworkOnlyArg("-h"); - test_args.SelectConfigNetwork(CBaseChainParams::MAIN); + test_args.SelectConfigNetwork(ChainTypeToString(ChainType::MAIN)); BOOST_CHECK(test_args.GetArg("-d", "xxx") == "e"); BOOST_CHECK(test_args.GetArgs("-ccc").size() == 2); BOOST_CHECK(test_args.GetArg("-h", "xxx") == "0"); @@ -637,7 +638,7 @@ BOOST_AUTO_TEST_CASE(util_GetArg) BOOST_CHECK_EQUAL(testArgs.GetArg("pritest4", "default"), "b"); } -BOOST_AUTO_TEST_CASE(util_GetChainName) +BOOST_AUTO_TEST_CASE(util_GetChainTypeString) { TestArgsManager test_args; const auto testnet = std::make_pair("-testnet", ArgsManager::ALLOW_ANY); @@ -655,39 +656,39 @@ BOOST_AUTO_TEST_CASE(util_GetChainName) std::string error; BOOST_CHECK(test_args.ParseParameters(0, (char**)argv_testnet, error)); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "main"); + BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "main"); BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_testnet, error)); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_regtest, error)); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "regtest"); + BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "regtest"); BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_test_no_reg, error)); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_both, error)); - BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); + BOOST_CHECK_THROW(test_args.GetChainTypeString(), std::runtime_error); BOOST_CHECK(test_args.ParseParameters(0, (char**)argv_testnet, error)); test_args.ReadConfigString(testnetconf); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_testnet, error)); test_args.ReadConfigString(testnetconf); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_regtest, error)); test_args.ReadConfigString(testnetconf); - BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); + BOOST_CHECK_THROW(test_args.GetChainTypeString(), std::runtime_error); BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_test_no_reg, error)); test_args.ReadConfigString(testnetconf); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_both, error)); test_args.ReadConfigString(testnetconf); - BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); + BOOST_CHECK_THROW(test_args.GetChainTypeString(), std::runtime_error); // check setting the network to test (and thus making // [test] regtest=1 potentially relevant) doesn't break things @@ -695,23 +696,23 @@ BOOST_AUTO_TEST_CASE(util_GetChainName) BOOST_CHECK(test_args.ParseParameters(0, (char**)argv_testnet, error)); test_args.ReadConfigString(testnetconf); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_testnet, error)); test_args.ReadConfigString(testnetconf); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_regtest, error)); test_args.ReadConfigString(testnetconf); - BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); + BOOST_CHECK_THROW(test_args.GetChainTypeString(), std::runtime_error); BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_test_no_reg, error)); test_args.ReadConfigString(testnetconf); - BOOST_CHECK_EQUAL(test_args.GetChainName(), "test"); + BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_both, error)); test_args.ReadConfigString(testnetconf); - BOOST_CHECK_THROW(test_args.GetChainName(), std::runtime_error); + BOOST_CHECK_THROW(test_args.GetChainTypeString(), std::runtime_error); } // Test different ways settings can be merged, and verify results. This test can @@ -755,8 +756,8 @@ struct ArgsMergeTestingSetup : public BasicTestingSetup { ForEachNoDup(conf_actions, SET, SECTION_NEGATE, [&] { for (bool soft_set : {false, true}) { for (bool force_set : {false, true}) { - for (const std::string& section : {CBaseChainParams::MAIN, CBaseChainParams::TESTNET, CBaseChainParams::SIGNET}) { - for (const std::string& network : {CBaseChainParams::MAIN, CBaseChainParams::TESTNET, CBaseChainParams::SIGNET}) { + for (const std::string& section : {ChainTypeToString(ChainType::MAIN), ChainTypeToString(ChainType::TESTNET), ChainTypeToString(ChainType::SIGNET)}) { + for (const std::string& network : {ChainTypeToString(ChainType::MAIN), ChainTypeToString(ChainType::TESTNET), ChainTypeToString(ChainType::SIGNET)}) { for (bool net_specific : {false, true}) { fn(arg_actions, conf_actions, soft_set, force_set, section, network, net_specific); } @@ -913,7 +914,7 @@ BOOST_FIXTURE_TEST_CASE(util_ArgsMerge, ArgsMergeTestingSetup) BOOST_CHECK_EQUAL(out_sha_hex, "d1e436c1cd510d0ec44d5205d4b4e3bee6387d316e0075c58206cb16603f3d82"); } -// Similar test as above, but for ArgsManager::GetChainName function. +// Similar test as above, but for ArgsManager::GetChainTypeString function. struct ChainMergeTestingSetup : public BasicTestingSetup { static constexpr int MAX_ACTIONS = 2; @@ -982,7 +983,7 @@ BOOST_FIXTURE_TEST_CASE(util_ChainMerge, ChainMergeTestingSetup) desc += " || "; try { - desc += parser.GetChainName(); + desc += parser.GetChainTypeString(); } catch (const std::runtime_error& e) { desc += "error: "; desc += e.what(); @@ -1021,14 +1022,14 @@ BOOST_AUTO_TEST_CASE(util_ReadWriteSettings) // Test writing setting. TestArgsManager args1; args1.ForceSetArg("-datadir", fs::PathToString(m_path_root)); - args1.LockSettings([&](util::Settings& settings) { settings.rw_settings["name"] = "value"; }); + args1.LockSettings([&](common::Settings& settings) { settings.rw_settings["name"] = "value"; }); args1.WriteSettingsFile(); // Test reading setting. TestArgsManager args2; args2.ForceSetArg("-datadir", fs::PathToString(m_path_root)); args2.ReadSettingsFile(); - args2.LockSettings([&](util::Settings& settings) { BOOST_CHECK_EQUAL(settings.rw_settings["name"].get_str(), "value"); }); + args2.LockSettings([&](common::Settings& settings) { BOOST_CHECK_EQUAL(settings.rw_settings["name"].get_str(), "value"); }); // Test error logging, and remove previously written setting. { diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index a572bb02b9..1ff5d6cf59 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -19,6 +19,7 @@ #include <boost/test/unit_test.hpp> using node::BlockAssembler; +using node::BlockManager; using node::CBlockTemplate; BOOST_AUTO_TEST_SUITE(blockfilter_index_tests) @@ -29,10 +30,10 @@ struct BuildChainTestingSetup : public TestChain100Setup { }; static bool CheckFilterLookups(BlockFilterIndex& filter_index, const CBlockIndex* block_index, - uint256& last_header) + uint256& last_header, const BlockManager& blockman) { BlockFilter expected_filter; - if (!ComputeFilter(filter_index.GetFilterType(), block_index, expected_filter)) { + if (!ComputeFilter(filter_index.GetFilterType(), *block_index, expected_filter, blockman)) { BOOST_ERROR("ComputeFilter failed on block " << block_index->nHeight); return false; } @@ -141,10 +142,10 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) BOOST_REQUIRE(filter_index.Start()); // Allow filter index to catch up with the block index. - constexpr int64_t timeout_ms = 10 * 1000; - int64_t time_start = GetTimeMillis(); + constexpr auto timeout{10s}; + const auto time_start{SteadyClock::now()}; while (!filter_index.BlockUntilSyncedToCurrentChain()) { - BOOST_REQUIRE(time_start + timeout_ms > GetTimeMillis()); + BOOST_REQUIRE(time_start + timeout > SteadyClock::now()); UninterruptibleSleep(std::chrono::milliseconds{100}); } @@ -155,7 +156,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) for (block_index = m_node.chainman->ActiveChain().Genesis(); block_index != nullptr; block_index = m_node.chainman->ActiveChain().Next(block_index)) { - CheckFilterLookups(filter_index, block_index, last_header); + CheckFilterLookups(filter_index, block_index, last_header, m_node.chainman->m_blockman); } } @@ -189,7 +190,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); - CheckFilterLookups(filter_index, block_index, chainA_last_header); + CheckFilterLookups(filter_index, block_index, chainA_last_header, m_node.chainman->m_blockman); } // Reorg to chain B. @@ -207,7 +208,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); - CheckFilterLookups(filter_index, block_index, chainB_last_header); + CheckFilterLookups(filter_index, block_index, chainB_last_header, m_node.chainman->m_blockman); } // Check that filters for stale blocks on A can be retrieved. @@ -221,7 +222,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); - CheckFilterLookups(filter_index, block_index, chainA_last_header); + CheckFilterLookups(filter_index, block_index, chainA_last_header, m_node.chainman->m_blockman); } // Reorg back to chain A. @@ -241,14 +242,14 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) block_index = m_node.chainman->m_blockman.LookupBlockIndex(chainA[i]->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); - CheckFilterLookups(filter_index, block_index, chainA_last_header); + CheckFilterLookups(filter_index, block_index, chainA_last_header, m_node.chainman->m_blockman); { LOCK(cs_main); block_index = m_node.chainman->m_blockman.LookupBlockIndex(chainB[i]->GetHash()); } BOOST_CHECK(filter_index.BlockUntilSyncedToCurrentChain()); - CheckFilterLookups(filter_index, block_index, chainB_last_header); + CheckFilterLookups(filter_index, block_index, chainB_last_header, m_node.chainman->m_blockman); } // Test lookups for a range of filters/hashes. diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp index 2118f476cd..f094766886 100644 --- a/src/test/blockmanager_tests.cpp +++ b/src/test/blockmanager_tests.cpp @@ -5,6 +5,7 @@ #include <chainparams.h> #include <node/blockstorage.h> #include <node/context.h> +#include <util/chaintype.h> #include <validation.h> #include <boost/test/unit_test.hpp> @@ -13,31 +14,34 @@ using node::BlockManager; using node::BLOCK_SERIALIZATION_HEADER_SIZE; using node::MAX_BLOCKFILE_SIZE; -using node::OpenBlockFile; // use BasicTestingSetup here for the data directory configuration, setup, and cleanup BOOST_FIXTURE_TEST_SUITE(blockmanager_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos) { - const auto params {CreateChainParams(ArgsManager{}, CBaseChainParams::MAIN)}; - BlockManager blockman{{}}; + const auto params {CreateChainParams(ArgsManager{}, ChainType::MAIN)}; + const BlockManager::Options blockman_opts{ + .chainparams = *params, + .blocks_dir = m_args.GetBlocksDirPath(), + }; + BlockManager blockman{blockman_opts}; CChain chain {}; // simulate adding a genesis block normally - BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, *params, nullptr).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); + BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, nullptr).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); // simulate what happens during reindex // simulate a well-formed genesis block being found at offset 8 in the blk00000.dat file // the block is found at offset 8 because there is an 8 byte serialization header // consisting of 4 magic bytes + 4 length bytes before each block in a well-formed blk file. FlatFilePos pos{0, BLOCK_SERIALIZATION_HEADER_SIZE}; - BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, *params, &pos).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); + BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, &pos).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); // now simulate what happens after reindex for the first new block processed // the actual block contents don't matter, just that it's a block. // verify that the write position is at offset 0x12d. // this is a check to make sure that https://github.com/bitcoin/bitcoin/issues/21379 does not recur // 8 bytes (for serialization header) + 285 (for serialized genesis block) = 293 // add another 8 bytes for the second block's serialization header and we get 293 + 8 = 301 - FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1, chain, *params, nullptr)}; + FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1, chain, nullptr)}; BOOST_CHECK_EQUAL(actual.nPos, BLOCK_SERIALIZATION_HEADER_SIZE + ::GetSerializeSize(params->GenesisBlock(), CLIENT_VERSION) + BLOCK_SERIALIZATION_HEADER_SIZE); } @@ -63,13 +67,13 @@ BOOST_FIXTURE_TEST_CASE(blockmanager_scan_unlink_already_pruned_files, TestChain // Check that the file is not unlinked after ScanAndUnlinkAlreadyPrunedFiles // if m_have_pruned is not yet set WITH_LOCK(chainman->GetMutex(), blockman.ScanAndUnlinkAlreadyPrunedFiles()); - BOOST_CHECK(!AutoFile(OpenBlockFile(pos, true)).IsNull()); + BOOST_CHECK(!AutoFile(blockman.OpenBlockFile(pos, true)).IsNull()); // Check that the file is unlinked after ScanAndUnlinkAlreadyPrunedFiles // once m_have_pruned is set blockman.m_have_pruned = true; WITH_LOCK(chainman->GetMutex(), blockman.ScanAndUnlinkAlreadyPrunedFiles()); - BOOST_CHECK(AutoFile(OpenBlockFile(pos, true)).IsNull()); + BOOST_CHECK(AutoFile(blockman.OpenBlockFile(pos, true)).IsNull()); // Check that calling with already pruned files doesn't cause an error WITH_LOCK(chainman->GetMutex(), blockman.ScanAndUnlinkAlreadyPrunedFiles()); @@ -79,7 +83,7 @@ BOOST_FIXTURE_TEST_CASE(blockmanager_scan_unlink_already_pruned_files, TestChain BOOST_CHECK_NE(old_tip, new_tip); const int new_file_number{WITH_LOCK(chainman->GetMutex(), return new_tip->GetBlockPos().nFile)}; const FlatFilePos new_pos(new_file_number, 0); - BOOST_CHECK(!AutoFile(OpenBlockFile(new_pos, true)).IsNull()); + BOOST_CHECK(!AutoFile(blockman.OpenBlockFile(new_pos, true)).IsNull()); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 5d4c5eea0e..93c0412593 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -5,6 +5,7 @@ #include <common/bloom.h> #include <clientversion.h> +#include <common/system.h> #include <key.h> #include <key_io.h> #include <merkleblock.h> @@ -16,7 +17,6 @@ #include <test/util/setup_common.h> #include <uint256.h> #include <util/strencodings.h> -#include <util/system.h> #include <vector> diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index 9011e703e8..cb3831071a 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -7,6 +7,7 @@ #include <sync.h> #include <test/util/random.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> #include <util/time.h> #include <boost/test/unit_test.hpp> @@ -27,9 +28,9 @@ struct NoLockLoggingTestingSetup : public TestingSetup { NoLockLoggingTestingSetup() #ifdef DEBUG_LOCKCONTENTION - : TestingSetup{CBaseChainParams::MAIN, /*extra_args=*/{"-debugexclude=lock"}} {} + : TestingSetup{ChainType::MAIN, /*extra_args=*/{"-debugexclude=lock"}} {} #else - : TestingSetup{CBaseChainParams::MAIN} {} + : TestingSetup{ChainType::MAIN} {} #endif }; diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index c4b2b4c63b..7a7790e2ae 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -127,7 +127,7 @@ std::set<std::pair<CPubKey, KeyOriginInfo>> GetKeyOriginData(const FlatSigningPr return ret; } -void DoCheck(const std::string& prv, const std::string& pub, const std::string& norm_pub, int flags, +void DoCheck(std::string prv, std::string pub, const std::string& norm_pub, int flags, const std::vector<std::vector<std::string>>& scripts, const std::optional<OutputType>& type, const std::set<std::vector<uint32_t>>& paths = ONLY_EMPTY, bool replace_apostrophe_with_h_in_prv=false, bool replace_apostrophe_with_h_in_pub=false, uint32_t spender_nlocktime=0, uint32_t spender_nsequence=CTxIn::SEQUENCE_FINAL, @@ -141,16 +141,14 @@ void DoCheck(const std::string& prv, const std::string& pub, const std::string& std::unique_ptr<Descriptor> parse_pub; // Check that parsing succeeds. if (replace_apostrophe_with_h_in_prv) { - parse_priv = Parse(UseHInsteadOfApostrophe(prv), keys_priv, error); - } else { - parse_priv = Parse(prv, keys_priv, error); + prv = UseHInsteadOfApostrophe(prv); } + parse_priv = Parse(prv, keys_priv, error); BOOST_CHECK_MESSAGE(parse_priv, error); if (replace_apostrophe_with_h_in_pub) { - parse_pub = Parse(UseHInsteadOfApostrophe(pub), keys_pub, error); - } else { - parse_pub = Parse(pub, keys_pub, error); + pub = UseHInsteadOfApostrophe(pub); } + parse_pub = Parse(pub, keys_pub, error); BOOST_CHECK_MESSAGE(parse_pub, error); // Check that the correct OutputType is inferred @@ -171,19 +169,19 @@ void DoCheck(const std::string& prv, const std::string& pub, const std::string& if (!(flags & MISSING_PRIVKEYS)) { std::string prv1; BOOST_CHECK(parse_priv->ToPrivateString(keys_priv, prv1)); - BOOST_CHECK(EqualDescriptor(prv, prv1)); + BOOST_CHECK_MESSAGE(EqualDescriptor(prv, prv1), "Private ser: " + prv1 + " Private desc: " + prv); BOOST_CHECK(!parse_priv->ToPrivateString(keys_pub, prv1)); BOOST_CHECK(parse_pub->ToPrivateString(keys_priv, prv1)); - BOOST_CHECK(EqualDescriptor(prv, prv1)); + BOOST_CHECK_MESSAGE(EqualDescriptor(prv, prv1), "Private ser: " + prv1 + " Private desc: " + prv); BOOST_CHECK(!parse_pub->ToPrivateString(keys_pub, prv1)); } // Check that private can produce the normalized descriptors std::string norm1; BOOST_CHECK(parse_priv->ToNormalizedString(keys_priv, norm1)); - BOOST_CHECK(EqualDescriptor(norm1, norm_pub)); + BOOST_CHECK_MESSAGE(EqualDescriptor(norm1, norm_pub), "priv->ToNormalizedString(): " + norm1 + " Norm. desc: " + norm_pub); BOOST_CHECK(parse_pub->ToNormalizedString(keys_priv, norm1)); - BOOST_CHECK(EqualDescriptor(norm1, norm_pub)); + BOOST_CHECK_MESSAGE(EqualDescriptor(norm1, norm_pub), "pub->ToNormalizedString(): " + norm1 + " Norm. desc: " + norm_pub); // Check whether IsRange on both returns the expected result BOOST_CHECK_EQUAL(parse_pub->IsRange(), (flags & RANGE) != 0); @@ -357,32 +355,13 @@ void Check(const std::string& prv, const std::string& pub, const std::string& no const std::set<std::vector<uint32_t>>& paths = ONLY_EMPTY, uint32_t spender_nlocktime=0, uint32_t spender_nsequence=CTxIn::SEQUENCE_FINAL, std::map<std::vector<uint8_t>, std::vector<uint8_t>> preimages={}) { - bool found_apostrophes_in_prv = false; - bool found_apostrophes_in_pub = false; - // Do not replace apostrophes with 'h' in prv and pub DoCheck(prv, pub, norm_pub, flags, scripts, type, paths, /*replace_apostrophe_with_h_in_prv=*/false, /*replace_apostrophe_with_h_in_pub=*/false, /*spender_nlocktime=*/spender_nlocktime, /*spender_nsequence=*/spender_nsequence, /*preimages=*/preimages); - // Replace apostrophes with 'h' in prv but not in pub, if apostrophes are found in prv - if (prv.find('\'') != std::string::npos) { - found_apostrophes_in_prv = true; - DoCheck(prv, pub, norm_pub, flags, scripts, type, paths, /*replace_apostrophe_with_h_in_prv=*/true, - /*replace_apostrophe_with_h_in_pub=*/false, /*spender_nlocktime=*/spender_nlocktime, - /*spender_nsequence=*/spender_nsequence, /*preimages=*/preimages); - } - - // Replace apostrophes with 'h' in pub but not in prv, if apostrophes are found in pub - if (pub.find('\'') != std::string::npos) { - found_apostrophes_in_pub = true; - DoCheck(prv, pub, norm_pub, flags, scripts, type, paths, /*replace_apostrophe_with_h_in_prv=*/false, - /*replace_apostrophe_with_h_in_pub=*/true, /*spender_nlocktime=*/spender_nlocktime, - /*spender_nsequence=*/spender_nsequence, /*preimages=*/preimages); - } - // Replace apostrophes with 'h' both in prv and in pub, if apostrophes are found in both - if (found_apostrophes_in_prv && found_apostrophes_in_pub) { + if (prv.find('\'') != std::string::npos && pub.find('\'') != std::string::npos) { DoCheck(prv, pub, norm_pub, flags, scripts, type, paths, /*replace_apostrophe_with_h_in_prv=*/true, /*replace_apostrophe_with_h_in_pub=*/true, /*spender_nlocktime=*/spender_nlocktime, /*spender_nsequence=*/spender_nsequence, /*preimages=*/preimages); @@ -398,12 +377,12 @@ BOOST_AUTO_TEST_CASE(descriptor_test) // Basic single-key compressed Check("combo(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "combo(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "combo(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"2103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bdac","76a9149a1c78a507689f6f54b847ad1cef1e614ee23f1e88ac","00149a1c78a507689f6f54b847ad1cef1e614ee23f1e","a91484ab21b1b2fd065d4504ff693d832434b6108d7b87"}}, std::nullopt); Check("pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"2103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bdac"}}, std::nullopt); - Check("pkh([deadbeef/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh([deadbeef/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "pkh([deadbeef/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"76a9149a1c78a507689f6f54b847ad1cef1e614ee23f1e88ac"}}, OutputType::LEGACY, {{1,0x80000002UL,3,0x80000004UL}}); + Check("pkh([deadbeef/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh([deadbeef/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "pkh([deadbeef/1/2h/3/4h]03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"76a9149a1c78a507689f6f54b847ad1cef1e614ee23f1e88ac"}}, OutputType::LEGACY, {{1,0x80000002UL,3,0x80000004UL}}); Check("wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"00149a1c78a507689f6f54b847ad1cef1e614ee23f1e"}}, OutputType::BECH32); Check("sh(wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1))", "sh(wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", "sh(wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd))", SIGNABLE, {{"a91484ab21b1b2fd065d4504ff693d832434b6108d7b87"}}, OutputType::P2SH_SEGWIT); Check("tr(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "tr(a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "tr(a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE | XONLY_KEYS, {{"512077aab6e066f8a7419c5ab714c12c67d25007ed55a43cadcacb4d7a970a093f11"}}, OutputType::BECH32M); CheckUnparsable("sh(wpkh(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY2))", "sh(wpkh(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5))", "wpkh(): Pubkey '03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5' is invalid"); // Invalid pubkey - CheckUnparsable("pkh(deadbeef/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh(deadbeef/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "pkh(): Key origin start '[ character expected but not found, got 'd' instead"); // Missing start bracket in key origin + CheckUnparsable("pkh(deadbeef/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh(deadbeef/1/2h/3/4h]03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "pkh(): Key origin start '[ character expected but not found, got 'd' instead"); // Missing start bracket in key origin CheckUnparsable("pkh([deadbeef]/1/2'/3/4']L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "pkh([deadbeef]/1/2'/3/4']03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "pkh(): Multiple ']' characters found for a single pubkey"); // Multiple end brackets in key origin // Basic single-key uncompressed @@ -426,10 +405,10 @@ BOOST_AUTO_TEST_CASE(descriptor_test) // Versions with BIP32 derivations Check("combo([01234567]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc)", "combo([01234567]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL)", "combo([01234567]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL)", SIGNABLE, {{"2102d2b36900396c9282fa14628566582f206a5dd0bcc8d5e892611806cafb0301f0ac","76a91431a507b815593dfc51ffc7245ae7e5aee304246e88ac","001431a507b815593dfc51ffc7245ae7e5aee304246e","a9142aafb926eb247cb18240a7f4c07983ad1f37922687"}}, std::nullopt); Check("pk(xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0)", "pk(xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0)", "pk(xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0)", DEFAULT, {{"210379e45b3cf75f9c5f9befd8e9506fb962f6a9d185ac87001ec44a8d3df8d4a9e3ac"}}, std::nullopt, {{0}}); - Check("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0)", "pkh([bd16bee5/2147483647']xpub69H7F5dQzmVd3vPuLKtcXJziMEQByuDidnX3YdwgtNsecY5HRGtAAQC5mXTt4dsv9RzyjgDjAQs9VGVV6ydYCHnprc9vvaA5YtqWyL6hyds/0)", HARDENED, {{"76a914ebdc90806a9c4356c1c88e42216611e1cb4c1c1788ac"}}, OutputType::LEGACY, {{0xFFFFFFFFUL,0}}); + Check("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0)", "pkh([bd16bee5/2147483647h]xpub69H7F5dQzmVd3vPuLKtcXJziMEQByuDidnX3YdwgtNsecY5HRGtAAQC5mXTt4dsv9RzyjgDjAQs9VGVV6ydYCHnprc9vvaA5YtqWyL6hyds/0)", HARDENED, {{"76a914ebdc90806a9c4356c1c88e42216611e1cb4c1c1788ac"}}, OutputType::LEGACY, {{0xFFFFFFFFUL,0}}); - Check("wpkh([ffffffff/13']xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*)", "wpkh([ffffffff/13']xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*)", "wpkh([ffffffff/13']xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*)", RANGE, {{"0014326b2249e3a25d5dc60935f044ee835d090ba859"},{"0014af0bd98abc2f2cae66e36896a39ffe2d32984fb7"},{"00141fa798efd1cbf95cebf912c031b8a4a6e9fb9f27"}}, OutputType::BECH32, {{0x8000000DUL, 1, 2, 0}, {0x8000000DUL, 1, 2, 1}, {0x8000000DUL, 1, 2, 2}}); - Check("sh(wpkh(xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "sh(wpkh(xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "sh(wpkh(xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", RANGE | HARDENED | DERIVE_HARDENED, {{"a9149a4d9901d6af519b2a23d4a2f51650fcba87ce7b87"},{"a914bed59fc0024fae941d6e20a3b44a109ae740129287"},{"a9148483aa1116eb9c05c482a72bada4b1db24af654387"}}, OutputType::P2SH_SEGWIT, {{10, 20, 30, 40, 0x80000000UL}, {10, 20, 30, 40, 0x80000001UL}, {10, 20, 30, 40, 0x80000002UL}}); + Check("wpkh([ffffffff/13']xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*)", "wpkh([ffffffff/13']xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*)", "wpkh([ffffffff/13h]xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*)", RANGE, {{"0014326b2249e3a25d5dc60935f044ee835d090ba859"},{"0014af0bd98abc2f2cae66e36896a39ffe2d32984fb7"},{"00141fa798efd1cbf95cebf912c031b8a4a6e9fb9f27"}}, OutputType::BECH32, {{0x8000000DUL, 1, 2, 0}, {0x8000000DUL, 1, 2, 1}, {0x8000000DUL, 1, 2, 2}}); + Check("sh(wpkh(xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "sh(wpkh(xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "sh(wpkh(xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*h))", RANGE | HARDENED | DERIVE_HARDENED, {{"a9149a4d9901d6af519b2a23d4a2f51650fcba87ce7b87"},{"a914bed59fc0024fae941d6e20a3b44a109ae740129287"},{"a9148483aa1116eb9c05c482a72bada4b1db24af654387"}}, OutputType::P2SH_SEGWIT, {{10, 20, 30, 40, 0x80000000UL}, {10, 20, 30, 40, 0x80000001UL}, {10, 20, 30, 40, 0x80000002UL}}); Check("combo(xprvA2JDeKCSNNZky6uBCviVfJSKyQ1mDYahRjijr5idH2WwLsEd4Hsb2Tyh8RfQMuPh7f7RtyzTtdrbdqqsunu5Mm3wDvUAKRHSC34sJ7in334/*)", "combo(xpub6FHa3pjLCk84BayeJxFW2SP4XRrFd1JYnxeLeU8EqN3vDfZmbqBqaGJAyiLjTAwm6ZLRQUMv1ZACTj37sR62cfN7fe5JnJ7dh8zL4fiyLHV/*)", "combo(xpub6FHa3pjLCk84BayeJxFW2SP4XRrFd1JYnxeLeU8EqN3vDfZmbqBqaGJAyiLjTAwm6ZLRQUMv1ZACTj37sR62cfN7fe5JnJ7dh8zL4fiyLHV/*)", RANGE, {{"2102df12b7035bdac8e3bab862a3a83d06ea6b17b6753d52edecba9be46f5d09e076ac","76a914f90e3178ca25f2c808dc76624032d352fdbdfaf288ac","0014f90e3178ca25f2c808dc76624032d352fdbdfaf2","a91408f3ea8c68d4a7585bf9e8bda226723f70e445f087"},{"21032869a233c9adff9a994e4966e5b821fd5bac066da6c3112488dc52383b4a98ecac","76a914a8409d1b6dfb1ed2a3e8aa5e0ef2ff26b15b75b788ac","0014a8409d1b6dfb1ed2a3e8aa5e0ef2ff26b15b75b7","a91473e39884cb71ae4e5ac9739e9225026c99763e6687"}}, std::nullopt, {{0}, {1}}); Check("tr(xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/0/*,pk(xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/1/*))", "tr(xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/0/*,pk(xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/1/*))", "tr(xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/0/*,pk(xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/1/*))", XONLY_KEYS | RANGE, {{"512078bc707124daa551b65af74de2ec128b7525e10f374dc67b64e00ce0ab8b3e12"}, {"512001f0a02a17808c20134b78faab80ef93ffba82261ccef0a2314f5d62b6438f11"}, {"512021024954fcec88237a9386fce80ef2ced5f1e91b422b26c59ccfc174c8d1ad25"}}, OutputType::BECH32M, {{0, 0}, {0, 1}, {0, 2}, {1, 0}, {1, 1}, {1, 2}}); // Mixed xpubs and const pubkeys @@ -440,23 +419,23 @@ BOOST_AUTO_TEST_CASE(descriptor_test) CheckUnparsable("combo([012345678]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc)", "combo([012345678]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL)", "combo(): Fingerprint is not 4 bytes (9 characters instead of 8 characters)"); // Too long key fingerprint CheckUnparsable("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483648)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483648)", "pkh(): Key path value 2147483648 is out of range"); // BIP 32 path element overflow CheckUnparsable("pkh(xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/1aa)", "pkh(xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/1aa)", "pkh(): Key path value '1aa' is not a valid uint32"); // Path is not valid uint - Check("pkh([01234567/10/20]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0)", "pkh([01234567/10/20]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0)", "pkh([01234567/10/20/2147483647']xpub69H7F5dQzmVd3vPuLKtcXJziMEQByuDidnX3YdwgtNsecY5HRGtAAQC5mXTt4dsv9RzyjgDjAQs9VGVV6ydYCHnprc9vvaA5YtqWyL6hyds/0)", HARDENED, {{"76a914ebdc90806a9c4356c1c88e42216611e1cb4c1c1788ac"}}, OutputType::LEGACY, {{10, 20, 0xFFFFFFFFUL, 0}}); + Check("pkh([01234567/10/20]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0)", "pkh([01234567/10/20]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0)", "pkh([01234567/10/20/2147483647h]xpub69H7F5dQzmVd3vPuLKtcXJziMEQByuDidnX3YdwgtNsecY5HRGtAAQC5mXTt4dsv9RzyjgDjAQs9VGVV6ydYCHnprc9vvaA5YtqWyL6hyds/0)", HARDENED, {{"76a914ebdc90806a9c4356c1c88e42216611e1cb4c1c1788ac"}}, OutputType::LEGACY, {{10, 20, 0xFFFFFFFFUL, 0}}); // Multisig constructions Check("multi(1,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "multi(1,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "multi(1,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", SIGNABLE, {{"512103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd4104a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea23552ae"}}, std::nullopt); Check("sortedmulti(1,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "sortedmulti(1,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "sortedmulti(1,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", SIGNABLE, {{"512103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd4104a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea23552ae"}}, std::nullopt); Check("sortedmulti(1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "sortedmulti(1,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", "sortedmulti(1,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", SIGNABLE, {{"512103a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd4104a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea23552ae"}}, std::nullopt); - Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, OutputType::LEGACY, {{0x8000006FUL,222},{0}}); + Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", "sh(multi(2,[00000000/111h/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, OutputType::LEGACY, {{0x8000006FUL,222},{0}}); Check("sortedmulti(2,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/*,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0/0/*)", "sortedmulti(2,xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/*,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0/0/*)", "sortedmulti(2,xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL/*,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0/0/*)", RANGE, {{"5221025d5fc65ebb8d44a5274b53bac21ff8307fec2334a32df05553459f8b1f7fe1b62102fbd47cc8034098f0e6a94c6aeee8528abf0a2153a5d8e46d325b7284c046784652ae"}, {"52210264fd4d1f5dea8ded94c61e9641309349b62f27fbffe807291f664e286bfbe6472103f4ece6dfccfa37b211eb3d0af4d0c61dba9ef698622dc17eecdf764beeb005a652ae"}, {"5221022ccabda84c30bad578b13c89eb3b9544ce149787e5b538175b1d1ba259cbb83321024d902e1a2fc7a8755ab5b694c575fce742c48d9ff192e63df5193e4c7afe1f9c52ae"}}, std::nullopt, {{0}, {1}, {2}, {0, 0, 0}, {0, 0, 1}, {0, 0, 2}}); - Check("wsh(multi(2,xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "wsh(multi(2,[bd16bee5/2147483647']xpub69H7F5dQzmVd3vPuLKtcXJziMEQByuDidnX3YdwgtNsecY5HRGtAAQC5mXTt4dsv9RzyjgDjAQs9VGVV6ydYCHnprc9vvaA5YtqWyL6hyds/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", HARDENED | RANGE | DERIVE_HARDENED, {{"0020b92623201f3bb7c3771d45b2ad1d0351ea8fbf8cfe0a0e570264e1075fa1948f"},{"002036a08bbe4923af41cf4316817c93b8d37e2f635dd25cfff06bd50df6ae7ea203"},{"0020a96e7ab4607ca6b261bfe3245ffda9c746b28d3f59e83d34820ec0e2b36c139c"}}, OutputType::BECH32, {{0xFFFFFFFFUL,0}, {1,2,0}, {1,2,1}, {1,2,2}, {10, 20, 30, 40, 0x80000000UL}, {10, 20, 30, 40, 0x80000001UL}, {10, 20, 30, 40, 0x80000002UL}}); + Check("wsh(multi(2,xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "wsh(multi(2,[bd16bee5/2147483647h]xpub69H7F5dQzmVd3vPuLKtcXJziMEQByuDidnX3YdwgtNsecY5HRGtAAQC5mXTt4dsv9RzyjgDjAQs9VGVV6ydYCHnprc9vvaA5YtqWyL6hyds/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*h))", HARDENED | RANGE | DERIVE_HARDENED, {{"0020b92623201f3bb7c3771d45b2ad1d0351ea8fbf8cfe0a0e570264e1075fa1948f"},{"002036a08bbe4923af41cf4316817c93b8d37e2f635dd25cfff06bd50df6ae7ea203"},{"0020a96e7ab4607ca6b261bfe3245ffda9c746b28d3f59e83d34820ec0e2b36c139c"}}, OutputType::BECH32, {{0xFFFFFFFFUL,0}, {1,2,0}, {1,2,1}, {1,2,2}, {10, 20, 30, 40, 0x80000000UL}, {10, 20, 30, 40, 0x80000001UL}, {10, 20, 30, 40, 0x80000002UL}}); Check("sh(wsh(multi(16,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f,L1okJGHGn1kFjdXHKxXjwVVtmCMR2JA5QsbKCSpSb7ReQjezKeoD,KxDCNSST75HFPaW5QKpzHtAyaCQC7p9Vo3FYfi2u4dXD1vgMiboK,L5edQjFtnkcf5UWURn6UuuoFrabgDQUHdheKCziwN42aLwS3KizU,KzF8UWFcEC7BYTq8Go1xVimMkDmyNYVmXV5PV7RuDicvAocoPB8i,L3nHUboKG2w4VSJ5jYZ5CBM97oeK6YuKvfZxrefdShECcjEYKMWZ,KyjHo36dWkYhimKmVVmQTq3gERv3pnqA4xFCpvUgbGDJad7eS8WE,KwsfyHKRUTZPQtysN7M3tZ4GXTnuov5XRgjdF2XCG8faAPmFruRF,KzCUbGhN9LJhdeFfL9zQgTJMjqxdBKEekRGZX24hXdgCNCijkkap,KzgpMBwwsDLwkaC5UrmBgCYaBD2WgZ7PBoGYXR8KT7gCA9UTN5a3,KyBXTPy4T7YG4q9tcAM3LkvfRpD1ybHMvcJ2ehaWXaSqeGUxEdkP,KzJDe9iwJRPtKP2F2AoN6zBgzS7uiuAwhWCfGdNeYJ3PC1HNJ8M8,L1xbHrxynrqLKkoYc4qtoQPx6uy5qYXR5ZDYVYBSRmCV5piU3JG9)))","sh(wsh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232)))", "sh(wsh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232)))", SIGNABLE, {{"a9147fc63e13dc25e8a95a3cee3d9a714ac3afd96f1e87"}}, OutputType::P2SH_SEGWIT); Check("tr(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,pk(KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy))", "tr(a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,pk(669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0))", "tr(a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,pk(669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0))", SIGNABLE | XONLY_KEYS, {{"512017cf18db381d836d8923b1bdb246cfcd818da1a9f0e6e7907f187f0b2f937754"}}, OutputType::BECH32M); CheckUnparsable("sh(multi(16,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGNz6YCCQtYvFzMtrC6D3tKTKdBBboMrLTsjr2NYVBwapCkn7Mr,KxogYhiNfwxuswvXV66eFyKcCpm7dZ7TqHVqujHAVUjJxyivxQ9X,L2BUNduTSyZwZjwNHynQTF14mv2uz2NRq5n5sYWTb4FkkmqgEE9f,L1okJGHGn1kFjdXHKxXjwVVtmCMR2JA5QsbKCSpSb7ReQjezKeoD,KxDCNSST75HFPaW5QKpzHtAyaCQC7p9Vo3FYfi2u4dXD1vgMiboK,L5edQjFtnkcf5UWURn6UuuoFrabgDQUHdheKCziwN42aLwS3KizU,KzF8UWFcEC7BYTq8Go1xVimMkDmyNYVmXV5PV7RuDicvAocoPB8i,L3nHUboKG2w4VSJ5jYZ5CBM97oeK6YuKvfZxrefdShECcjEYKMWZ,KyjHo36dWkYhimKmVVmQTq3gERv3pnqA4xFCpvUgbGDJad7eS8WE,KwsfyHKRUTZPQtysN7M3tZ4GXTnuov5XRgjdF2XCG8faAPmFruRF,KzCUbGhN9LJhdeFfL9zQgTJMjqxdBKEekRGZX24hXdgCNCijkkap,KzgpMBwwsDLwkaC5UrmBgCYaBD2WgZ7PBoGYXR8KT7gCA9UTN5a3,KyBXTPy4T7YG4q9tcAM3LkvfRpD1ybHMvcJ2ehaWXaSqeGUxEdkP,KzJDe9iwJRPtKP2F2AoN6zBgzS7uiuAwhWCfGdNeYJ3PC1HNJ8M8,L1xbHrxynrqLKkoYc4qtoQPx6uy5qYXR5ZDYVYBSRmCV5piU3JG9))","sh(multi(16,03669b8afcec803a0d323e9a17f3ea8e68e8abe5a278020a929adbec52421adbd0,0260b2003c386519fc9eadf2b5cf124dd8eea4c4e68d5e154050a9346ea98ce600,0362a74e399c39ed5593852a30147f2959b56bb827dfa3e60e464b02ccf87dc5e8,0261345b53de74a4d721ef877c255429961b7e43714171ac06168d7e08c542a8b8,02da72e8b46901a65d4374fe6315538d8f368557dda3a1dcf9ea903f3afe7314c8,0318c82dd0b53fd3a932d16e0ba9e278fcc937c582d5781be626ff16e201f72286,0297ccef1ef99f9d73dec9ad37476ddb232f1238aff877af19e72ba04493361009,02e502cfd5c3f972fe9a3e2a18827820638f96b6f347e54d63deb839011fd5765d,03e687710f0e3ebe81c1037074da939d409c0025f17eb86adb9427d28f0f7ae0e9,02c04d3a5274952acdbc76987f3184b346a483d43be40874624b29e3692c1df5af,02ed06e0f418b5b43a7ec01d1d7d27290fa15f75771cb69b642a51471c29c84acd,036d46073cbb9ffee90473f3da429abc8de7f8751199da44485682a989a4bebb24,02f5d1ff7c9029a80a4e36b9a5497027ef7f3e73384a4a94fbfe7c4e9164eec8bc,02e41deffd1b7cce11cde209a781adcffdabd1b91c0ba0375857a2bfd9302419f3,02d76625f7956a7fc505ab02556c23ee72d832f1bac391bcd2d3abce5710a13d06,0399eb0a5487515802dc14544cf10b3666623762fbed2ec38a3975716e2c29c232))", "P2SH script is too large, 547 bytes is larger than 520 bytes"); // P2SH does not fit 16 compressed pubkeys in a redeemscript - CheckUnparsable("wsh(multi(2,[aaaaaaaa][aaaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaa][aaaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multi: Multiple ']' characters found for a single pubkey"); // Double key origin descriptor - CheckUnparsable("wsh(multi(2,[aaaagaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaagaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multi: Fingerprint 'aaagaaaa' is not hex"); // Non hex fingerprint - CheckUnparsable("wsh(multi(2,[aaaaaaaa],xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaa],xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multi: No key provided"); // No public key with origin - CheckUnparsable("wsh(multi(2,[aaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multi: Fingerprint is not 4 bytes (7 characters instead of 8 characters)"); // Too short fingerprint - CheckUnparsable("wsh(multi(2,[aaaaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647'/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*'))", "Multi: Fingerprint is not 4 bytes (9 characters instead of 8 characters)"); // Too long fingerprint + CheckUnparsable("wsh(multi(2,[aaaaaaaa][aaaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaa][aaaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647h/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*h))", "Multi: Multiple ']' characters found for a single pubkey"); // Double key origin descriptor + CheckUnparsable("wsh(multi(2,[aaaagaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaagaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647h/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*h))", "Multi: Fingerprint 'aaagaaaa' is not hex"); // Non hex fingerprint + CheckUnparsable("wsh(multi(2,[aaaaaaaa],xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaa],xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*h))", "Multi: No key provided"); // No public key with origin + CheckUnparsable("wsh(multi(2,[aaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647h/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*h))", "Multi: Fingerprint is not 4 bytes (7 characters instead of 8 characters)"); // Too short fingerprint + CheckUnparsable("wsh(multi(2,[aaaaaaaaa]xprv9s21ZrQH143K31xYSDQpPDxsXRTUcvj2iNHm5NUtrGiGG5e2DtALGdso3pGz6ssrdK4PFmM8NSpSBHNqPqm55Qn3LqFtT2emdEXVYsCzC2U/2147483647'/0,xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/1/2/*,xprv9s21ZrQH143K3QTDL4LXw2F7HEK3wJUD2nW2nRk4stbPy6cq3jPPqjiChkVvvNKmPGJxWUtg6LnF5kejMRNNU3TGtRBeJgk33yuGBxrMPHi/10/20/30/40/*'))", "wsh(multi(2,[aaaaaaaaa]xpub661MyMwAqRbcFW31YEwpkMuc5THy2PSt5bDMsktWQcFF8syAmRUapSCGu8ED9W6oDMSgv6Zz8idoc4a6mr8BDzTJY47LJhkJ8UB7WEGuduB/2147483647h/0,xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/1/2/*,xpub661MyMwAqRbcFtXgS5sYJABqqG9YLmC4Q1Rdap9gSE8NqtwybGhePY2gZ29ESFjqJoCu1Rupje8YtGqsefD265TMg7usUDFdp6W1EGMcet8/10/20/30/40/*h))", "Multi: Fingerprint is not 4 bytes (9 characters instead of 8 characters)"); // Too long fingerprint CheckUnparsable("multi(a,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "multi(a,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "Multi threshold 'a' is not valid"); // Invalid threshold CheckUnparsable("multi(0,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "multi(0,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "Multisig threshold cannot be 0, must be at least 1"); // Threshold of 0 CheckUnparsable("multi(3,L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1,5KYZdUEo39z3FPrtuX2QbbwGnNP5zTd7yyr2SC1j299sBCnWjss)", "multi(3,03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd,04a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd5b8dec5235a0fa8722476c7709c02559e3aa73aa03918ba2d492eea75abea235)", "Multisig threshold cannot be larger than the number of keys; threshold is 3 but only 2 keys specified"); // Threshold larger than number of keys @@ -474,8 +453,8 @@ BOOST_AUTO_TEST_CASE(descriptor_test) CheckUnparsable("wsh(wsh(pk(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)))", "wsh(wsh(pk(03a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)))", "Can only have wsh() at top level or inside sh()"); // Cannot embed P2WSH inside P2WSH // Checksums - Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxfy", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5t", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5t", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, OutputType::LEGACY, {{0x8000006FUL,222},{0}}); - Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, OutputType::LEGACY, {{0x8000006FUL,222},{0}}); + Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxfy", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5t", "sh(multi(2,[00000000/111h/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#hgmsckna", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, OutputType::LEGACY, {{0x8000006FUL,222},{0}}); + Check("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", "sh(multi(2,[00000000/111h/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))", DEFAULT, {{"a91445a9a622a8b0a1269944be477640eedc447bbd8487"}}, OutputType::LEGACY, {{0x8000006FUL,222},{0}}); CheckUnparsable("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#", "Expected 8 character checksum, not 0 characters"); // Empty checksum CheckUnparsable("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxfyq", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5tq", "Expected 8 character checksum, not 9 characters"); // Too long checksum CheckUnparsable("sh(multi(2,[00000000/111'/222]xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0))#ggrsrxf", "sh(multi(2,[00000000/111'/222]xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0))#tjg09x5", "Expected 8 character checksum, not 7 characters"); // Too short checksum @@ -491,13 +470,12 @@ BOOST_AUTO_TEST_CASE(descriptor_test) Check( "rawtr(xprv9vHkqa6EV4sPZHYqZznhT2NPtPCjKuDKGY38FBWLvgaDx45zo9WQRUT3dKYnjwih2yJD9mkrocEZXo1ex8G81dwSM1fwqWpWkeS3v86pgKt/86'/1'/0'/1/*)#a5gn3t7k", "rawtr(xpub69H7F5d8KSRgmmdJg2KhpAK8SR3DjMwAdkxj3ZuxV27CprR9LgpeyGmXUbC6wb7ERfvrnKZjXoUmmDznezpbZb7ap6r1D3tgFxHmwMkQTPH/86'/1'/0'/1/*)#4ur3xhft", - "rawtr([5a61ff8e/86'/1'/0']xpub6DtZpc9PRL2B6pwoNGysmHAaBofDmWv5S6KQEKKGPKhf5fV62ywDtSziSApYVK3JnYY5KUSgiCwiXW5wtd8z7LNBxT9Mu5sEro8itdGfTeA/1/*)#llheyd9x", + "rawtr([5a61ff8e/86h/1h/0h]xpub6DtZpc9PRL2B6pwoNGysmHAaBofDmWv5S6KQEKKGPKhf5fV62ywDtSziSApYVK3JnYY5KUSgiCwiXW5wtd8z7LNBxT9Mu5sEro8itdGfTeA/1/*)#vwgx7hj9", RANGE | HARDENED | XONLY_KEYS, {{"51205172af752f057d543ce8e4a6f8dcf15548ec6be44041bfa93b72e191cfc8c1ee"}, {"51201b66f20b86f700c945ecb9ad9b0ad1662b73084e2bfea48bee02126350b8a5b1"}, {"512063e70f66d815218abcc2306aa930aaca07c5cde73b75127eb27b5e8c16b58a25"}}, OutputType::BECH32M, {{0x80000056, 0x80000001, 0x80000000, 1, 0}, {0x80000056, 0x80000001, 0x80000000, 1, 1}, {0x80000056, 0x80000001, 0x80000000, 1, 2}}); - Check( "rawtr(L4rK1yDtCWekvXuE6oXD9jCYfFNV2cWRpVuPLBcCU2z8TrisoyY1)", "rawtr(a34b99f22c790c4e36b2b3c2c35a36db06226e41c692fc82b8b56ac1c540c5bd)", diff --git a/src/test/fuzz/addrman.cpp b/src/test/fuzz/addrman.cpp index 5ad7a25c53..7218bb82d1 100644 --- a/src/test/fuzz/addrman.cpp +++ b/src/test/fuzz/addrman.cpp @@ -16,6 +16,7 @@ #include <test/util/setup_common.h> #include <time.h> #include <util/asmap.h> +#include <util/chaintype.h> #include <cassert> #include <cstdint> @@ -34,7 +35,7 @@ int32_t GetCheckRatio() void initialize_addrman() { - static const auto testing_setup = MakeNoLogFileContext<>(CBaseChainParams::REGTEST); + static const auto testing_setup = MakeNoLogFileContext<>(ChainType::REGTEST); g_setup = testing_setup.get(); } diff --git a/src/test/fuzz/block.cpp b/src/test/fuzz/block.cpp index c3e17724eb..e90dcc189a 100644 --- a/src/test/fuzz/block.cpp +++ b/src/test/fuzz/block.cpp @@ -11,6 +11,7 @@ #include <pubkey.h> #include <streams.h> #include <test/fuzz/fuzz.h> +#include <util/chaintype.h> #include <validation.h> #include <version.h> @@ -19,7 +20,7 @@ void initialize_block() { - SelectParams(CBaseChainParams::REGTEST); + SelectParams(ChainType::REGTEST); } FUZZ_TARGET_INIT(block, initialize_block) diff --git a/src/test/fuzz/buffered_file.cpp b/src/test/fuzz/buffered_file.cpp index 67cac8fa4e..2f7ce60c7f 100644 --- a/src/test/fuzz/buffered_file.cpp +++ b/src/test/fuzz/buffered_file.cpp @@ -53,7 +53,7 @@ FUZZ_TARGET(buffered_file) return; } try { - opt_buffered_file->FindByte(fuzzed_data_provider.ConsumeIntegral<uint8_t>()); + opt_buffered_file->FindByte(std::byte(fuzzed_data_provider.ConsumeIntegral<uint8_t>())); } catch (const std::ios_base::failure&) { } }, diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index 5843b80c0a..fc7e000dc7 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> -#include <chainparamsbase.h> #include <coins.h> #include <consensus/amount.h> #include <consensus/tx_check.h> diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index 0f3c850e66..f81658b832 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -4,7 +4,6 @@ #include <addrman.h> #include <chainparams.h> -#include <chainparamsbase.h> #include <common/args.h> #include <net.h> #include <netaddress.h> diff --git a/src/test/fuzz/descriptor_parse.cpp b/src/test/fuzz/descriptor_parse.cpp index 1f5601ca9f..12c22ef2ed 100644 --- a/src/test/fuzz/descriptor_parse.cpp +++ b/src/test/fuzz/descriptor_parse.cpp @@ -6,11 +6,12 @@ #include <pubkey.h> #include <script/descriptor.h> #include <test/fuzz/fuzz.h> +#include <util/chaintype.h> void initialize_descriptor_parse() { ECC_Start(); - SelectParams(CBaseChainParams::MAIN); + SelectParams(ChainType::MAIN); } FUZZ_TARGET_INIT(descriptor_parse, initialize_descriptor_parse) diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index 0bbfb206d5..44ba8bc254 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -14,14 +14,19 @@ #include <csignal> #include <cstdint> +#include <cstdio> +#include <cstdlib> +#include <cstring> #include <exception> #include <fstream> #include <functional> +#include <iostream> #include <map> #include <memory> #include <string> #include <tuple> #include <unistd.h> +#include <utility> #include <vector> const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; @@ -77,13 +82,13 @@ void initialize() return WrappedGetAddrInfo(name, false); }; - bool should_abort{false}; + bool should_exit{false}; if (std::getenv("PRINT_ALL_FUZZ_TARGETS_AND_ABORT")) { for (const auto& t : FuzzTargets()) { if (std::get<2>(t.second)) continue; std::cout << t.first << std::endl; } - should_abort = true; + should_exit = true; } if (const char* out_path = std::getenv("WRITE_ALL_FUZZ_TARGETS_AND_ABORT")) { std::cout << "Writing all fuzz target names to '" << out_path << "'." << std::endl; @@ -92,13 +97,23 @@ void initialize() if (std::get<2>(t.second)) continue; out_stream << t.first << std::endl; } - should_abort = true; + should_exit= true; + } + if (should_exit){ + std::exit(EXIT_SUCCESS); + } + if (const auto* env_fuzz{std::getenv("FUZZ")}) { + // To allow for easier fuzz executable binary modification, + static std::string g_copy{env_fuzz}; // create copy to avoid compiler optimizations, and + g_fuzz_target = g_copy.c_str(); // strip string after the first null-char. + } else { + std::cerr << "Must select fuzz target with the FUZZ env var." << std::endl; + std::cerr << "Hint: Set the PRINT_ALL_FUZZ_TARGETS_AND_ABORT=1 env var to see all compiled targets." << std::endl; + std::exit(EXIT_FAILURE); } - Assert(!should_abort); - g_fuzz_target = Assert(std::getenv("FUZZ")); const auto it = FuzzTargets().find(g_fuzz_target); if (it == FuzzTargets().end()) { - std::cerr << "No fuzzer for " << g_fuzz_target << "." << std::endl; + std::cerr << "No fuzz target compiled for " << g_fuzz_target << "." << std::endl; std::exit(EXIT_FAILURE); } Assert(!g_test_one_input); diff --git a/src/test/fuzz/headerssync.cpp b/src/test/fuzz/headerssync.cpp index 521afadb51..c1a187038b 100644 --- a/src/test/fuzz/headerssync.cpp +++ b/src/test/fuzz/headerssync.cpp @@ -6,6 +6,7 @@ #include <test/fuzz/util.h> #include <test/util/setup_common.h> #include <uint256.h> +#include <util/chaintype.h> #include <util/time.h> #include <validation.h> @@ -15,7 +16,7 @@ static void initialize_headers_sync_state_fuzz() { static const auto testing_setup = MakeNoLogFileContext<>( - /*chain_name=*/CBaseChainParams::MAIN); + /*chain_type=*/ChainType::MAIN); } void MakeHeadersContinuous( diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index ead877fe05..edb1dca457 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -4,6 +4,7 @@ #include <arith_uint256.h> #include <common/args.h> +#include <common/system.h> #include <compressor.h> #include <consensus/amount.h> #include <consensus/merkle.h> @@ -26,12 +27,12 @@ #include <test/fuzz/util.h> #include <uint256.h> #include <univalue.h> +#include <util/chaintype.h> #include <util/check.h> #include <util/moneystr.h> #include <util/overflow.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <version.h> #include <cassert> @@ -42,7 +43,7 @@ void initialize_integer() { - SelectParams(CBaseChainParams::REGTEST); + SelectParams(ChainType::REGTEST); } FUZZ_TARGET_INIT(integer, initialize_integer) diff --git a/src/test/fuzz/key.cpp b/src/test/fuzz/key.cpp index ea6883c08d..3eab2e20c0 100644 --- a/src/test/fuzz/key.cpp +++ b/src/test/fuzz/key.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> -#include <chainparamsbase.h> #include <key.h> #include <key_io.h> #include <outputtype.h> @@ -17,6 +16,7 @@ #include <script/standard.h> #include <streams.h> #include <test/fuzz/fuzz.h> +#include <util/chaintype.h> #include <util/strencodings.h> #include <cassert> @@ -28,7 +28,7 @@ void initialize_key() { ECC_Start(); - SelectParams(CBaseChainParams::REGTEST); + SelectParams(ChainType::REGTEST); } FUZZ_TARGET_INIT(key, initialize_key) diff --git a/src/test/fuzz/key_io.cpp b/src/test/fuzz/key_io.cpp index 29c6996365..a1c587a75b 100644 --- a/src/test/fuzz/key_io.cpp +++ b/src/test/fuzz/key_io.cpp @@ -5,6 +5,7 @@ #include <chainparams.h> #include <key_io.h> #include <test/fuzz/fuzz.h> +#include <util/chaintype.h> #include <cassert> #include <cstdint> @@ -14,7 +15,7 @@ void initialize_key_io() { ECC_Start(); - SelectParams(CBaseChainParams::MAIN); + SelectParams(ChainType::MAIN); } FUZZ_TARGET_INIT(key_io, initialize_key_io) diff --git a/src/test/fuzz/message.cpp b/src/test/fuzz/message.cpp index 63e24aacdd..8b7e3f11cc 100644 --- a/src/test/fuzz/message.cpp +++ b/src/test/fuzz/message.cpp @@ -7,6 +7,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <util/chaintype.h> #include <util/message.h> #include <util/strencodings.h> @@ -19,7 +20,7 @@ void initialize_message() { ECC_Start(); - SelectParams(CBaseChainParams::REGTEST); + SelectParams(ChainType::REGTEST); } FUZZ_TARGET_INIT(message, initialize_message) diff --git a/src/test/fuzz/mini_miner.cpp b/src/test/fuzz/mini_miner.cpp new file mode 100644 index 0000000000..2b371f6d5f --- /dev/null +++ b/src/test/fuzz/mini_miner.cpp @@ -0,0 +1,193 @@ +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/fuzz/util/mempool.h> +#include <test/util/script.h> +#include <test/util/setup_common.h> +#include <test/util/txmempool.h> +#include <test/util/mining.h> + +#include <node/mini_miner.h> +#include <node/miner.h> +#include <primitives/transaction.h> +#include <random.h> +#include <txmempool.h> + +#include <deque> +#include <vector> + +namespace { + +const TestingSetup* g_setup; +std::deque<COutPoint> g_available_coins; +void initialize_miner() +{ + static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); + g_setup = testing_setup.get(); + for (uint32_t i = 0; i < uint32_t{100}; ++i) { + g_available_coins.push_back(COutPoint{uint256::ZERO, i}); + } +} + +// Test that the MiniMiner can run with various outpoints and feerates. +FUZZ_TARGET_INIT(mini_miner, initialize_miner) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + CTxMemPool pool{CTxMemPool::Options{}}; + std::vector<COutPoint> outpoints; + std::deque<COutPoint> available_coins = g_available_coins; + LOCK2(::cs_main, pool.cs); + // Cluster size cannot exceed 500 + LIMITED_WHILE(!available_coins.empty(), 500) + { + CMutableTransaction mtx = CMutableTransaction(); + const size_t num_inputs = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, available_coins.size()); + const size_t num_outputs = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 50); + for (size_t n{0}; n < num_inputs; ++n) { + auto prevout = available_coins.front(); + mtx.vin.push_back(CTxIn(prevout, CScript())); + available_coins.pop_front(); + } + for (uint32_t n{0}; n < num_outputs; ++n) { + mtx.vout.push_back(CTxOut(100, P2WSH_OP_TRUE)); + } + CTransactionRef tx = MakeTransactionRef(mtx); + TestMemPoolEntryHelper entry; + const CAmount fee{ConsumeMoney(fuzzed_data_provider, /*max=*/MAX_MONEY/100000)}; + assert(MoneyRange(fee)); + pool.addUnchecked(entry.Fee(fee).FromTx(tx)); + + // All outputs are available to spend + for (uint32_t n{0}; n < num_outputs; ++n) { + if (fuzzed_data_provider.ConsumeBool()) { + available_coins.push_back(COutPoint{tx->GetHash(), n}); + } + } + + if (fuzzed_data_provider.ConsumeBool() && !tx->vout.empty()) { + // Add outpoint from this tx (may or not be spent by a later tx) + outpoints.push_back(COutPoint{tx->GetHash(), + (uint32_t)fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, tx->vout.size())}); + } else { + // Add some random outpoint (will be interpreted as confirmed or not yet submitted + // to mempool). + auto outpoint = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); + if (outpoint.has_value() && std::find(outpoints.begin(), outpoints.end(), *outpoint) == outpoints.end()) { + outpoints.push_back(*outpoint); + } + } + + } + + const CFeeRate target_feerate{CFeeRate{ConsumeMoney(fuzzed_data_provider, /*max=*/MAX_MONEY/1000)}}; + std::optional<CAmount> total_bumpfee; + CAmount sum_fees = 0; + { + node::MiniMiner mini_miner{pool, outpoints}; + assert(mini_miner.IsReadyToCalculate()); + const auto bump_fees = mini_miner.CalculateBumpFees(target_feerate); + for (const auto& outpoint : outpoints) { + auto it = bump_fees.find(outpoint); + assert(it != bump_fees.end()); + assert(it->second >= 0); + sum_fees += it->second; + } + assert(!mini_miner.IsReadyToCalculate()); + } + { + node::MiniMiner mini_miner{pool, outpoints}; + assert(mini_miner.IsReadyToCalculate()); + total_bumpfee = mini_miner.CalculateTotalBumpFees(target_feerate); + assert(total_bumpfee.has_value()); + assert(!mini_miner.IsReadyToCalculate()); + } + // Overlapping ancestry across multiple outpoints can only reduce the total bump fee. + assert (sum_fees >= *total_bumpfee); +} + +// Test that MiniMiner and BlockAssembler build the same block given the same transactions and constraints. +FUZZ_TARGET_INIT(mini_miner_selection, initialize_miner) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + CTxMemPool pool{CTxMemPool::Options{}}; + // Make a copy to preserve determinism. + std::deque<COutPoint> available_coins = g_available_coins; + std::vector<CTransactionRef> transactions; + + LOCK2(::cs_main, pool.cs); + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) + { + CMutableTransaction mtx = CMutableTransaction(); + assert(!available_coins.empty()); + const size_t num_inputs = std::min(size_t{2}, available_coins.size()); + const size_t num_outputs = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(2, 5); + for (size_t n{0}; n < num_inputs; ++n) { + auto prevout = available_coins.at(0); + mtx.vin.push_back(CTxIn(prevout, CScript())); + available_coins.pop_front(); + } + for (uint32_t n{0}; n < num_outputs; ++n) { + mtx.vout.push_back(CTxOut(100, P2WSH_OP_TRUE)); + } + CTransactionRef tx = MakeTransactionRef(mtx); + + // First 2 outputs are available to spend. The rest are added to outpoints to calculate bumpfees. + // There is no overlap between spendable coins and outpoints passed to MiniMiner because the + // MiniMiner interprets spent coins as to-be-replaced and excludes them. + for (uint32_t n{0}; n < num_outputs - 1; ++n) { + if (fuzzed_data_provider.ConsumeBool()) { + available_coins.push_front(COutPoint{tx->GetHash(), n}); + } else { + available_coins.push_back(COutPoint{tx->GetHash(), n}); + } + } + + // Stop if pool reaches DEFAULT_BLOCK_MAX_WEIGHT because BlockAssembler will stop when the + // block template reaches that, but the MiniMiner will keep going. + if (pool.GetTotalTxSize() + GetVirtualTransactionSize(*tx) >= DEFAULT_BLOCK_MAX_WEIGHT) break; + TestMemPoolEntryHelper entry; + const CAmount fee{ConsumeMoney(fuzzed_data_provider, /*max=*/MAX_MONEY/100000)}; + assert(MoneyRange(fee)); + pool.addUnchecked(entry.Fee(fee).FromTx(tx)); + transactions.push_back(tx); + } + std::vector<COutPoint> outpoints; + for (const auto& coin : g_available_coins) { + if (!pool.GetConflictTx(coin)) outpoints.push_back(coin); + } + for (const auto& tx : transactions) { + assert(pool.exists(GenTxid::Txid(tx->GetHash()))); + for (uint32_t n{0}; n < tx->vout.size(); ++n) { + COutPoint coin{tx->GetHash(), n}; + if (!pool.GetConflictTx(coin)) outpoints.push_back(coin); + } + } + const CFeeRate target_feerate{ConsumeMoney(fuzzed_data_provider, /*max=*/MAX_MONEY/100000)}; + + node::BlockAssembler::Options miner_options; + miner_options.blockMinFeeRate = target_feerate; + miner_options.nBlockMaxWeight = DEFAULT_BLOCK_MAX_WEIGHT; + miner_options.test_block_validity = false; + + node::BlockAssembler miner{g_setup->m_node.chainman->ActiveChainstate(), &pool, miner_options}; + node::MiniMiner mini_miner{pool, outpoints}; + assert(mini_miner.IsReadyToCalculate()); + + CScript spk_placeholder = CScript() << OP_0; + // Use BlockAssembler as oracle. BlockAssembler and MiniMiner should select the same + // transactions, stopping once packages do not meet target_feerate. + const auto blocktemplate{miner.CreateNewBlock(spk_placeholder)}; + mini_miner.BuildMockTemplate(target_feerate); + assert(!mini_miner.IsReadyToCalculate()); + auto mock_template_txids = mini_miner.GetMockTemplateTxids(); + // MiniMiner doesn't add a coinbase tx. + assert(mock_template_txids.count(blocktemplate->block.vtx[0]->GetHash()) == 0); + mock_template_txids.emplace(blocktemplate->block.vtx[0]->GetHash()); + assert(mock_template_txids.size() <= blocktemplate->block.vtx.size()); + assert(mock_template_txids.size() >= blocktemplate->block.vtx.size()); + assert(mock_template_txids.size() == blocktemplate->block.vtx.size()); + for (const auto& tx : blocktemplate->block.vtx) { + assert(mock_template_txids.count(tx->GetHash())); + } +} +} // namespace diff --git a/src/test/fuzz/net.cpp b/src/test/fuzz/net.cpp index 13b4638688..e090f13061 100644 --- a/src/test/fuzz/net.cpp +++ b/src/test/fuzz/net.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> -#include <chainparamsbase.h> #include <net.h> #include <net_permissions.h> #include <netaddress.h> @@ -16,6 +15,7 @@ #include <test/util/net.h> #include <test/util/setup_common.h> #include <util/asmap.h> +#include <util/chaintype.h> #include <cstdint> #include <optional> @@ -24,7 +24,7 @@ void initialize_net() { - static const auto testing_setup = MakeNoLogFileContext<>(CBaseChainParams::MAIN); + static const auto testing_setup = MakeNoLogFileContext<>(ChainType::MAIN); } FUZZ_TARGET_INIT(net, initialize_net) diff --git a/src/test/fuzz/netbase_dns_lookup.cpp b/src/test/fuzz/netbase_dns_lookup.cpp index 81e216b358..dcf500acc3 100644 --- a/src/test/fuzz/netbase_dns_lookup.cpp +++ b/src/test/fuzz/netbase_dns_lookup.cpp @@ -29,33 +29,29 @@ FUZZ_TARGET(netbase_dns_lookup) }; { - std::vector<CNetAddr> resolved_addresses; - if (LookupHost(name, resolved_addresses, max_results, allow_lookup, fuzzed_dns_lookup_function)) { - for (const CNetAddr& resolved_address : resolved_addresses) { - assert(!resolved_address.IsInternal()); - } + const std::vector<CNetAddr> resolved_addresses{LookupHost(name, max_results, allow_lookup, fuzzed_dns_lookup_function)}; + for (const CNetAddr& resolved_address : resolved_addresses) { + assert(!resolved_address.IsInternal()); } assert(resolved_addresses.size() <= max_results || max_results == 0); } { - CNetAddr resolved_address; - if (LookupHost(name, resolved_address, allow_lookup, fuzzed_dns_lookup_function)) { - assert(!resolved_address.IsInternal()); + const std::optional<CNetAddr> resolved_address{LookupHost(name, allow_lookup, fuzzed_dns_lookup_function)}; + if (resolved_address.has_value()) { + assert(!resolved_address.value().IsInternal()); } } { - std::vector<CService> resolved_services; - if (Lookup(name, resolved_services, default_port, allow_lookup, max_results, fuzzed_dns_lookup_function)) { - for (const CNetAddr& resolved_service : resolved_services) { - assert(!resolved_service.IsInternal()); - } + const std::vector<CService> resolved_services{Lookup(name, default_port, allow_lookup, max_results, fuzzed_dns_lookup_function)}; + for (const CNetAddr& resolved_service : resolved_services) { + assert(!resolved_service.IsInternal()); } assert(resolved_services.size() <= max_results || max_results == 0); } { - CService resolved_service; - if (Lookup(name, resolved_service, default_port, allow_lookup, fuzzed_dns_lookup_function)) { - assert(!resolved_service.IsInternal()); + const std::optional<CService> resolved_service{Lookup(name, default_port, allow_lookup, fuzzed_dns_lookup_function)}; + if (resolved_service.has_value()) { + assert(!resolved_service.value().IsInternal()); } } { diff --git a/src/test/fuzz/p2p_transport_serialization.cpp b/src/test/fuzz/p2p_transport_serialization.cpp index 96254aa222..ec3cdbff5a 100644 --- a/src/test/fuzz/p2p_transport_serialization.cpp +++ b/src/test/fuzz/p2p_transport_serialization.cpp @@ -9,6 +9,7 @@ #include <protocol.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <util/chaintype.h> #include <cassert> #include <cstdint> @@ -18,7 +19,7 @@ void initialize_p2p_transport_serialization() { - SelectParams(CBaseChainParams::REGTEST); + SelectParams(ChainType::REGTEST); } FUZZ_TARGET_INIT(p2p_transport_serialization, initialize_p2p_transport_serialization) diff --git a/src/test/fuzz/parse_hd_keypath.cpp b/src/test/fuzz/parse_hd_keypath.cpp index 411b70230a..9a2d9a73da 100644 --- a/src/test/fuzz/parse_hd_keypath.cpp +++ b/src/test/fuzz/parse_hd_keypath.cpp @@ -18,6 +18,6 @@ FUZZ_TARGET(parse_hd_keypath) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const std::vector<uint32_t> random_keypath = ConsumeRandomLengthIntegralVector<uint32_t>(fuzzed_data_provider); - (void)FormatHDKeypath(random_keypath); + (void)FormatHDKeypath(random_keypath, /*apostrophe=*/true); // WriteHDKeypath calls this with false (void)WriteHDKeypath(random_keypath); } diff --git a/src/test/fuzz/parse_univalue.cpp b/src/test/fuzz/parse_univalue.cpp index 16486f6b96..6d33c1a8cc 100644 --- a/src/test/fuzz/parse_univalue.cpp +++ b/src/test/fuzz/parse_univalue.cpp @@ -7,13 +7,14 @@ #include <rpc/client.h> #include <rpc/util.h> #include <test/fuzz/fuzz.h> +#include <util/chaintype.h> #include <limits> #include <string> void initialize_parse_univalue() { - SelectParams(CBaseChainParams::REGTEST); + SelectParams(ChainType::REGTEST); } FUZZ_TARGET_INIT(parse_univalue, initialize_parse_univalue) @@ -21,12 +22,9 @@ FUZZ_TARGET_INIT(parse_univalue, initialize_parse_univalue) const std::string random_string(buffer.begin(), buffer.end()); bool valid = true; const UniValue univalue = [&] { - try { - return ParseNonRFCJSONValue(random_string); - } catch (const std::runtime_error&) { - valid = false; - return UniValue{}; - } + UniValue uv; + if (!uv.read(random_string)) valid = false; + return valid ? uv : UniValue{}; }(); if (!valid) { return; diff --git a/src/test/fuzz/pow.cpp b/src/test/fuzz/pow.cpp index e5a3a6e68a..6d584c9f10 100644 --- a/src/test/fuzz/pow.cpp +++ b/src/test/fuzz/pow.cpp @@ -9,6 +9,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <util/chaintype.h> #include <util/check.h> #include <util/overflow.h> @@ -19,7 +20,7 @@ void initialize_pow() { - SelectParams(CBaseChainParams::MAIN); + SelectParams(ChainType::MAIN); } FUZZ_TARGET_INIT(pow, initialize_pow) diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 0a7924f226..744ff4701d 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -2,15 +2,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <banman.h> -#include <chainparams.h> #include <consensus/consensus.h> #include <net.h> #include <net_processing.h> +#include <primitives/transaction.h> #include <protocol.h> -#include <scheduler.h> #include <script/script.h> +#include <serialize.h> +#include <span.h> #include <streams.h> +#include <sync.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> @@ -19,45 +20,36 @@ #include <test/util/net.h> #include <test/util/setup_common.h> #include <test/util/validation.h> +#include <util/chaintype.h> +#include <util/check.h> +#include <util/time.h> +#include <validation.h> #include <validationinterface.h> #include <version.h> + #include <atomic> -#include <cassert> -#include <chrono> -#include <cstdint> -#include <iosfwd> +#include <cstdlib> #include <iostream> #include <memory> #include <string> +#include <string_view> +#include <vector> namespace { const TestingSetup* g_setup; +std::string_view LIMIT_TO_MESSAGE_TYPE{}; } // namespace -size_t& GetNumMsgTypes() -{ - static size_t g_num_msg_types{0}; - return g_num_msg_types; -} -#define FUZZ_TARGET_MSG(msg_type) \ - struct msg_type##_Count_Before_Main { \ - msg_type##_Count_Before_Main() \ - { \ - ++GetNumMsgTypes(); \ - } \ - } const static g_##msg_type##_count_before_main; \ - FUZZ_TARGET_INIT(process_message_##msg_type, initialize_process_message) \ - { \ - fuzz_target(buffer, #msg_type); \ - } - void initialize_process_message() { - Assert(GetNumMsgTypes() == getAllNetMessageTypes().size()); // If this fails, add or remove the message type below + if (const auto val{std::getenv("LIMIT_TO_MESSAGE_TYPE")}) { + LIMIT_TO_MESSAGE_TYPE = val; + Assert(std::count(getAllNetMessageTypes().begin(), getAllNetMessageTypes().end(), LIMIT_TO_MESSAGE_TYPE)); // Unknown message type passed + } static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>( - /*chain_name=*/CBaseChainParams::REGTEST, + /*chain_type=*/ChainType::REGTEST, /*extra_args=*/{"-txreconciliation"}); g_setup = testing_setup.get(); for (int i = 0; i < 2 * COINBASE_MATURITY; i++) { @@ -66,7 +58,7 @@ void initialize_process_message() SyncWithValidationInterfaceQueue(); } -void fuzz_target(FuzzBufferType buffer, const std::string& LIMIT_TO_MESSAGE_TYPE) +FUZZ_TARGET_INIT(process_message, initialize_process_message) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); @@ -100,40 +92,3 @@ void fuzz_target(FuzzBufferType buffer, const std::string& LIMIT_TO_MESSAGE_TYPE SyncWithValidationInterfaceQueue(); g_setup->m_node.connman->StopNodes(); } - -FUZZ_TARGET_INIT(process_message, initialize_process_message) { fuzz_target(buffer, ""); } -FUZZ_TARGET_MSG(addr); -FUZZ_TARGET_MSG(addrv2); -FUZZ_TARGET_MSG(block); -FUZZ_TARGET_MSG(blocktxn); -FUZZ_TARGET_MSG(cfcheckpt); -FUZZ_TARGET_MSG(cfheaders); -FUZZ_TARGET_MSG(cfilter); -FUZZ_TARGET_MSG(cmpctblock); -FUZZ_TARGET_MSG(feefilter); -FUZZ_TARGET_MSG(filteradd); -FUZZ_TARGET_MSG(filterclear); -FUZZ_TARGET_MSG(filterload); -FUZZ_TARGET_MSG(getaddr); -FUZZ_TARGET_MSG(getblocks); -FUZZ_TARGET_MSG(getblocktxn); -FUZZ_TARGET_MSG(getcfcheckpt); -FUZZ_TARGET_MSG(getcfheaders); -FUZZ_TARGET_MSG(getcfilters); -FUZZ_TARGET_MSG(getdata); -FUZZ_TARGET_MSG(getheaders); -FUZZ_TARGET_MSG(headers); -FUZZ_TARGET_MSG(inv); -FUZZ_TARGET_MSG(mempool); -FUZZ_TARGET_MSG(merkleblock); -FUZZ_TARGET_MSG(notfound); -FUZZ_TARGET_MSG(ping); -FUZZ_TARGET_MSG(pong); -FUZZ_TARGET_MSG(sendaddrv2); -FUZZ_TARGET_MSG(sendcmpct); -FUZZ_TARGET_MSG(sendheaders); -FUZZ_TARGET_MSG(sendtxrcncl); -FUZZ_TARGET_MSG(tx); -FUZZ_TARGET_MSG(verack); -FUZZ_TARGET_MSG(version); -FUZZ_TARGET_MSG(wtxidrelay); diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index 96339743ba..68d4e02a26 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -24,7 +24,7 @@ const TestingSetup* g_setup; void initialize_process_messages() { static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>( - /*chain_name=*/CBaseChainParams::REGTEST, + /*chain_type=*/ChainType::REGTEST, /*extra_args=*/{"-txreconciliation"}); g_setup = testing_setup.get(); for (int i = 0; i < 2 * COINBASE_MATURITY; i++) { diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 1c6140c66a..b1858a1800 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -23,6 +23,7 @@ #include <test/util/setup_common.h> #include <tinyformat.h> #include <univalue.h> +#include <util/chaintype.h> #include <util/strencodings.h> #include <util/string.h> #include <util/time.h> @@ -37,7 +38,7 @@ namespace { struct RPCFuzzTestingSetup : public TestingSetup { - RPCFuzzTestingSetup(const std::string& chain_name, const std::vector<const char*>& extra_args) : TestingSetup{chain_name, extra_args} + RPCFuzzTestingSetup(const ChainType chain_type, const std::vector<const char*>& extra_args) : TestingSetup{chain_type, extra_args} { } @@ -97,6 +98,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ "decoderawtransaction", "decodescript", "deriveaddresses", + "descriptorprocesspsbt", "disconnectnode", "echo", "echojson", @@ -134,6 +136,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ "getnetworkinfo", "getnodeaddresses", "getpeerinfo", + "getprioritisedtransactions", "getrawmempool", "getrawtransaction", "getrpcinfo", @@ -363,7 +366,7 @@ FUZZ_TARGET_INIT(rpc, initialize_rpc) try { rpc_testing_setup->CallRPC(rpc_command, arguments); } catch (const UniValue& json_rpc_error) { - const std::string error_msg{find_value(json_rpc_error, "message").get_str()}; + const std::string error_msg{json_rpc_error.find_value("message").get_str()}; // Once c++20 is allowed, starts_with can be used. // if (error_msg.starts_with("Internal bug detected")) { if (0 == error_msg.rfind("Internal bug detected", 0)) { diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index 1037dd934a..8a88c1107a 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -22,6 +22,7 @@ #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <univalue.h> +#include <util/chaintype.h> #include <algorithm> #include <cassert> @@ -32,7 +33,7 @@ void initialize_script() { - SelectParams(CBaseChainParams::REGTEST); + SelectParams(ChainType::REGTEST); } FUZZ_TARGET_INIT(script, initialize_script) diff --git a/src/test/fuzz/script_format.cpp b/src/test/fuzz/script_format.cpp index 9186746bcf..5aa0ea58ff 100644 --- a/src/test/fuzz/script_format.cpp +++ b/src/test/fuzz/script_format.cpp @@ -11,10 +11,11 @@ #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <univalue.h> +#include <util/chaintype.h> void initialize_script_format() { - SelectParams(CBaseChainParams::REGTEST); + SelectParams(ChainType::REGTEST); } FUZZ_TARGET_INIT(script_format, initialize_script_format) diff --git a/src/test/fuzz/script_sigcache.cpp b/src/test/fuzz/script_sigcache.cpp index de895cc69c..f332987987 100644 --- a/src/test/fuzz/script_sigcache.cpp +++ b/src/test/fuzz/script_sigcache.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> -#include <chainparamsbase.h> #include <key.h> #include <pubkey.h> #include <script/sigcache.h> diff --git a/src/test/fuzz/script_sign.cpp b/src/test/fuzz/script_sign.cpp index c78c22e6cc..8b62daf162 100644 --- a/src/test/fuzz/script_sign.cpp +++ b/src/test/fuzz/script_sign.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> -#include <chainparamsbase.h> #include <key.h> #include <psbt.h> #include <pubkey.h> @@ -14,6 +13,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <util/chaintype.h> #include <util/translation.h> #include <cassert> @@ -27,7 +27,7 @@ void initialize_script_sign() { ECC_Start(); - SelectParams(CBaseChainParams::REGTEST); + SelectParams(ChainType::REGTEST); } FUZZ_TARGET_INIT(script_sign, initialize_script_sign) diff --git a/src/test/fuzz/signet.cpp b/src/test/fuzz/signet.cpp index 303dcf13e3..e9af93c639 100644 --- a/src/test/fuzz/signet.cpp +++ b/src/test/fuzz/signet.cpp @@ -11,6 +11,7 @@ #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> #include <cstdint> #include <optional> @@ -18,7 +19,7 @@ void initialize_signet() { - static const auto testing_setup = MakeNoLogFileContext<>(CBaseChainParams::SIGNET); + static const auto testing_setup = MakeNoLogFileContext<>(ChainType::SIGNET); } FUZZ_TARGET_INIT(signet, initialize_signet) diff --git a/src/test/fuzz/socks5.cpp b/src/test/fuzz/socks5.cpp index 97f643db49..73235b7ced 100644 --- a/src/test/fuzz/socks5.cpp +++ b/src/test/fuzz/socks5.cpp @@ -14,12 +14,12 @@ #include <string> #include <vector> +extern std::chrono::milliseconds g_socks5_recv_timeout; + namespace { -int default_socks5_recv_timeout; +decltype(g_socks5_recv_timeout) default_socks5_recv_timeout; }; -extern int g_socks5_recv_timeout; - void initialize_socks5() { static const auto testing_setup = MakeNoLogFileContext<const BasicTestingSetup>(); @@ -35,7 +35,7 @@ FUZZ_TARGET_INIT(socks5, initialize_socks5) InterruptSocks5(fuzzed_data_provider.ConsumeBool()); // Set FUZZED_SOCKET_FAKE_LATENCY=1 to exercise recv timeout code paths. This // will slow down fuzzing. - g_socks5_recv_timeout = (fuzzed_data_provider.ConsumeBool() && std::getenv("FUZZED_SOCKET_FAKE_LATENCY") != nullptr) ? 1 : default_socks5_recv_timeout; + g_socks5_recv_timeout = (fuzzed_data_provider.ConsumeBool() && std::getenv("FUZZED_SOCKET_FAKE_LATENCY") != nullptr) ? 1ms : default_socks5_recv_timeout; FuzzedSock fuzzed_sock = ConsumeSock(fuzzed_data_provider); // This Socks5(...) fuzzing harness would have caught CVE-2017-18350 within // a few seconds of fuzzing. diff --git a/src/test/fuzz/string.cpp b/src/test/fuzz/string.cpp index 5de24a939d..e81efac6e0 100644 --- a/src/test/fuzz/string.cpp +++ b/src/test/fuzz/string.cpp @@ -5,6 +5,8 @@ #include <blockfilter.h> #include <clientversion.h> #include <common/args.h> +#include <common/settings.h> +#include <common/system.h> #include <common/url.h> #include <netbase.h> #include <outputtype.h> @@ -21,10 +23,8 @@ #include <test/fuzz/util.h> #include <util/error.h> #include <util/fees.h> -#include <util/settings.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <util/translation.h> #include <cassert> @@ -63,13 +63,9 @@ FUZZ_TARGET(string) (void)IsDeprecatedRPCEnabled(random_string_1); (void)Join(random_string_vector, random_string_1); (void)JSONRPCError(fuzzed_data_provider.ConsumeIntegral<int>(), random_string_1); - const util::Settings settings; + const common::Settings settings; (void)OnlyHasDefaultSectionSetting(settings, random_string_1, random_string_2); (void)ParseNetwork(random_string_1); - try { - (void)ParseNonRFCJSONValue(random_string_1); - } catch (const std::runtime_error&) { - } (void)ParseOutputType(random_string_1); (void)RemovePrefix(random_string_1, random_string_2); (void)ResolveErrMsg(random_string_1, random_string_2); diff --git a/src/test/fuzz/system.cpp b/src/test/fuzz/system.cpp index 935f0c21e1..73c01d9297 100644 --- a/src/test/fuzz/system.cpp +++ b/src/test/fuzz/system.cpp @@ -55,7 +55,7 @@ FUZZ_TARGET_INIT(system, initialize_system) [&] { const OptionsCategory options_category = fuzzed_data_provider.PickValueInArray<OptionsCategory>({OptionsCategory::OPTIONS, OptionsCategory::CONNECTION, OptionsCategory::WALLET, OptionsCategory::WALLET_DEBUG_TEST, OptionsCategory::ZMQ, OptionsCategory::DEBUG_TEST, OptionsCategory::CHAINPARAMS, OptionsCategory::NODE_RELAY, OptionsCategory::BLOCK_CREATION, OptionsCategory::RPC, OptionsCategory::GUI, OptionsCategory::COMMANDS, OptionsCategory::REGISTER_COMMANDS, OptionsCategory::HIDDEN}); // Avoid hitting: - // util/system.cpp:425: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. + // common/args.cpp:563: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. const std::string argument_name = GetArgumentName(fuzzed_data_provider.ConsumeRandomLengthString(16)); if (args_manager.GetArgFlags(argument_name) != std::nullopt) { return; @@ -64,7 +64,7 @@ FUZZ_TARGET_INIT(system, initialize_system) }, [&] { // Avoid hitting: - // util/system.cpp:425: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. + // common/args.cpp:563: void ArgsManager::AddArg(const std::string &, const std::string &, unsigned int, const OptionsCategory &): Assertion `ret.second' failed. const std::vector<std::string> names = ConsumeRandomLengthStringVector(fuzzed_data_provider); std::vector<std::string> hidden_arguments; for (const std::string& name : names) { @@ -108,7 +108,7 @@ FUZZ_TARGET_INIT(system, initialize_system) (void)args_manager.GetArgs(s1); (void)args_manager.GetBoolArg(s1, b); try { - (void)args_manager.GetChainName(); + (void)args_manager.GetChainTypeString(); } catch (const std::runtime_error&) { } (void)args_manager.GetHelpMessage(); diff --git a/src/test/fuzz/transaction.cpp b/src/test/fuzz/transaction.cpp index bacb178b44..7035c53d13 100644 --- a/src/test/fuzz/transaction.cpp +++ b/src/test/fuzz/transaction.cpp @@ -15,6 +15,7 @@ #include <streams.h> #include <test/fuzz/fuzz.h> #include <univalue.h> +#include <util/chaintype.h> #include <util/rbf.h> #include <validation.h> #include <version.h> @@ -23,7 +24,7 @@ void initialize_transaction() { - SelectParams(CBaseChainParams::REGTEST); + SelectParams(ChainType::REGTEST); } FUZZ_TARGET_INIT(transaction, initialize_transaction) diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index a055705575..b758c715ef 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -42,12 +42,12 @@ void initialize_tx_pool() g_setup = testing_setup.get(); for (int i = 0; i < 2 * COINBASE_MATURITY; ++i) { - CTxIn in = MineBlock(g_setup->m_node, P2WSH_OP_TRUE); + COutPoint prevout{MineBlock(g_setup->m_node, P2WSH_OP_TRUE)}; // Remember the txids to avoid expensive disk access later on auto& outpoints = i < COINBASE_MATURITY ? g_outpoints_coinbase_init_mature : g_outpoints_coinbase_init_immature; - outpoints.push_back(in.prevout); + outpoints.push_back(prevout); } SyncWithValidationInterfaceQueue(); } diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index c14f633029..5f2aff08da 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -6,7 +6,6 @@ #define BITCOIN_TEST_FUZZ_UTIL_H #include <arith_uint256.h> -#include <chainparamsbase.h> #include <coins.h> #include <compat/compat.h> #include <consensus/amount.h> diff --git a/src/test/fuzz/utxo_snapshot.cpp b/src/test/fuzz/utxo_snapshot.cpp index be34906025..b4ef0c7a3e 100644 --- a/src/test/fuzz/utxo_snapshot.cpp +++ b/src/test/fuzz/utxo_snapshot.cpp @@ -10,6 +10,7 @@ #include <test/fuzz/util.h> #include <test/util/mining.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> #include <util/fs.h> #include <validation.h> #include <validationinterface.h> @@ -22,7 +23,7 @@ const std::vector<std::shared_ptr<CBlock>>* g_chain; void initialize_chain() { - const auto params{CreateChainParams(ArgsManager{}, CBaseChainParams::REGTEST)}; + const auto params{CreateChainParams(ArgsManager{}, ChainType::REGTEST)}; static const auto chain{CreateBlockChain(2 * COINBASE_MATURITY, *params)}; g_chain = &chain; } diff --git a/src/test/fuzz/utxo_total_supply.cpp b/src/test/fuzz/utxo_total_supply.cpp new file mode 100644 index 0000000000..318797faf2 --- /dev/null +++ b/src/test/fuzz/utxo_total_supply.cpp @@ -0,0 +1,168 @@ +// Copyright (c) 2020 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <consensus/consensus.h> +#include <consensus/merkle.h> +#include <kernel/coinstats.h> +#include <node/miner.h> +#include <script/interpreter.h> +#include <streams.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/mining.h> +#include <test/util/setup_common.h> +#include <util/chaintype.h> +#include <validation.h> +#include <version.h> + +FUZZ_TARGET(utxo_total_supply) +{ + /** The testing setup that creates a chainman only (no chainstate) */ + ChainTestingSetup test_setup{ + ChainType::REGTEST, + { + "-testactivationheight=bip34@2", + }, + }; + // Create chainstate + test_setup.LoadVerifyActivateChainstate(); + auto& node{test_setup.m_node}; + auto& chainman{*Assert(test_setup.m_node.chainman)}; + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + + const auto ActiveHeight = [&]() { + LOCK(chainman.GetMutex()); + return chainman.ActiveHeight(); + }; + const auto PrepareNextBlock = [&]() { + // Use OP_FALSE to avoid BIP30 check from hitting early + auto block = PrepareBlock(node, CScript{} << OP_FALSE); + // Replace OP_FALSE with OP_TRUE + { + CMutableTransaction tx{*block->vtx.back()}; + tx.vout.at(0).scriptPubKey = CScript{} << OP_TRUE; + block->vtx.back() = MakeTransactionRef(tx); + } + return block; + }; + + /** The block template this fuzzer is working on */ + auto current_block = PrepareNextBlock(); + /** Append-only set of tx outpoints, entries are not removed when spent */ + std::vector<std::pair<COutPoint, CTxOut>> txos; + /** The utxo stats at the chain tip */ + kernel::CCoinsStats utxo_stats; + /** The total amount of coins in the utxo set */ + CAmount circulation{0}; + + + // Store the tx out in the txo map + const auto StoreLastTxo = [&]() { + // get last tx + const CTransaction& tx = *current_block->vtx.back(); + // get last out + const uint32_t i = tx.vout.size() - 1; + // store it + txos.emplace_back(COutPoint{tx.GetHash(), i}, tx.vout.at(i)); + if (current_block->vtx.size() == 1 && tx.vout.at(i).scriptPubKey[0] == OP_RETURN) { + // also store coinbase + const uint32_t i = tx.vout.size() - 2; + txos.emplace_back(COutPoint{tx.GetHash(), i}, tx.vout.at(i)); + } + }; + const auto AppendRandomTxo = [&](CMutableTransaction& tx) { + const auto& txo = txos.at(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, txos.size() - 1)); + tx.vin.emplace_back(txo.first); + tx.vout.emplace_back(txo.second.nValue, txo.second.scriptPubKey); // "Forward" coin with no fee + }; + const auto UpdateUtxoStats = [&]() { + LOCK(chainman.GetMutex()); + chainman.ActiveChainstate().ForceFlushStateToDisk(); + utxo_stats = std::move( + *Assert(kernel::ComputeUTXOStats(kernel::CoinStatsHashType::NONE, &chainman.ActiveChainstate().CoinsDB(), chainman.m_blockman, {}))); + // Check that miner can't print more money than they are allowed to + assert(circulation == utxo_stats.total_amount); + }; + + + // Update internal state to chain tip + StoreLastTxo(); + UpdateUtxoStats(); + assert(ActiveHeight() == 0); + // Get at which height we duplicate the coinbase + // Assuming that the fuzzer will mine relatively short chains (less than 200 blocks), we want the duplicate coinbase to be not too high. + // Up to 2000 seems reasonable. + int64_t duplicate_coinbase_height = fuzzed_data_provider.ConsumeIntegralInRange(0, 20 * COINBASE_MATURITY); + // Always pad with OP_0 at the end to avoid bad-cb-length error + const CScript duplicate_coinbase_script = CScript() << duplicate_coinbase_height << OP_0; + // Mine the first block with this duplicate + current_block = PrepareNextBlock(); + StoreLastTxo(); + + { + // Create duplicate (CScript should match exact format as in CreateNewBlock) + CMutableTransaction tx{*current_block->vtx.front()}; + tx.vin.at(0).scriptSig = duplicate_coinbase_script; + + // Mine block and create next block template + current_block->vtx.front() = MakeTransactionRef(tx); + } + current_block->hashMerkleRoot = BlockMerkleRoot(*current_block); + assert(!MineBlock(node, current_block).IsNull()); + circulation += GetBlockSubsidy(ActiveHeight(), Params().GetConsensus()); + + assert(ActiveHeight() == 1); + UpdateUtxoStats(); + current_block = PrepareNextBlock(); + StoreLastTxo(); + + // Limit to avoid timeout, but enough to cover duplicate_coinbase_height + // and CVE-2018-17144. + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes(), 2'000) + { + CallOneOf( + fuzzed_data_provider, + [&] { + // Append an input-output pair to the last tx in the current block + CMutableTransaction tx{*current_block->vtx.back()}; + AppendRandomTxo(tx); + current_block->vtx.back() = MakeTransactionRef(tx); + StoreLastTxo(); + }, + [&] { + // Append a tx to the list of txs in the current block + CMutableTransaction tx{}; + AppendRandomTxo(tx); + current_block->vtx.push_back(MakeTransactionRef(tx)); + StoreLastTxo(); + }, + [&] { + // Append the current block to the active chain + node::RegenerateCommitments(*current_block, chainman); + const bool was_valid = !MineBlock(node, current_block).IsNull(); + + const auto prev_utxo_stats = utxo_stats; + if (was_valid) { + if (duplicate_coinbase_height == ActiveHeight()) { + // we mined the duplicate coinbase + assert(current_block->vtx.at(0)->vin.at(0).scriptSig == duplicate_coinbase_script); + } + + circulation += GetBlockSubsidy(ActiveHeight(), Params().GetConsensus()); + } + + UpdateUtxoStats(); + + if (!was_valid) { + // utxo stats must not change + assert(prev_utxo_stats.hashSerialized == utxo_stats.hashSerialized); + } + + current_block = PrepareNextBlock(); + StoreLastTxo(); + }); + } +} diff --git a/src/test/fuzz/validation_load_mempool.cpp b/src/test/fuzz/validation_load_mempool.cpp index 817593cf6e..4e2ca6d903 100644 --- a/src/test/fuzz/validation_load_mempool.cpp +++ b/src/test/fuzz/validation_load_mempool.cpp @@ -4,7 +4,6 @@ #include <kernel/mempool_persist.h> -#include <chainparamsbase.h> #include <node/mempool_args.h> #include <node/mempool_persist_args.h> #include <test/fuzz/FuzzedDataProvider.h> diff --git a/src/test/fuzz/versionbits.cpp b/src/test/fuzz/versionbits.cpp index e6a19d6e91..b3df4dadd2 100644 --- a/src/test/fuzz/versionbits.cpp +++ b/src/test/fuzz/versionbits.cpp @@ -7,6 +7,7 @@ #include <common/args.h> #include <consensus/params.h> #include <primitives/block.h> +#include <util/chaintype.h> #include <versionbits.h> #include <test/fuzz/FuzzedDataProvider.h> @@ -104,7 +105,7 @@ std::unique_ptr<const CChainParams> g_params; void initialize() { // this is actually comparatively slow, so only do it once - g_params = CreateChainParams(ArgsManager{}, CBaseChainParams::MAIN); + g_params = CreateChainParams(ArgsManager{}, ChainType::MAIN); assert(g_params != nullptr); } diff --git a/src/test/getarg_tests.cpp b/src/test/getarg_tests.cpp index 715b6885f5..c73b675388 100644 --- a/src/test/getarg_tests.cpp +++ b/src/test/getarg_tests.cpp @@ -3,10 +3,10 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <common/args.h> +#include <common/settings.h> #include <logging.h> #include <test/util/setup_common.h> #include <univalue.h> -#include <util/settings.h> #include <util/strencodings.h> #include <limits> @@ -57,8 +57,8 @@ BOOST_AUTO_TEST_CASE(setting_args) ArgsManager args; SetupArgs(args, {{"-foo", ArgsManager::ALLOW_ANY}}); - auto set_foo = [&](const util::SettingsValue& value) { - args.LockSettings([&](util::Settings& settings) { + auto set_foo = [&](const common::SettingsValue& value) { + args.LockSettings([&](common::Settings& settings) { settings.rw_settings["foo"] = value; }); }; diff --git a/src/test/interfaces_tests.cpp b/src/test/interfaces_tests.cpp index d37fe35a11..68377e197f 100644 --- a/src/test/interfaces_tests.cpp +++ b/src/test/interfaces_tests.cpp @@ -98,7 +98,7 @@ BOOST_AUTO_TEST_CASE(findAncestorByHash) BOOST_AUTO_TEST_CASE(findCommonAncestor) { auto& chain = m_node.chain; - const CChain& active = WITH_LOCK(Assert(m_node.chainman)->GetMutex(), return Assert(m_node.chainman)->ActiveChain()); + const CChain& active{*WITH_LOCK(Assert(m_node.chainman)->GetMutex(), return &Assert(m_node.chainman)->ActiveChain())}; auto* orig_tip = active.Tip(); for (int i = 0; i < 10; ++i) { BlockValidationState state; diff --git a/src/test/key_io_tests.cpp b/src/test/key_io_tests.cpp index a400afee71..92bdbae3c4 100644 --- a/src/test/key_io_tests.cpp +++ b/src/test/key_io_tests.cpp @@ -10,6 +10,7 @@ #include <script/script.h> #include <test/util/json.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> #include <util/strencodings.h> #include <boost/test/unit_test.hpp> @@ -24,7 +25,7 @@ BOOST_AUTO_TEST_CASE(key_io_valid_parse) UniValue tests = read_json(std::string(json_tests::key_io_valid, json_tests::key_io_valid + sizeof(json_tests::key_io_valid))); CKey privkey; CTxDestination destination; - SelectParams(CBaseChainParams::MAIN); + SelectParams(ChainType::MAIN); for (unsigned int idx = 0; idx < tests.size(); idx++) { const UniValue& test = tests[idx]; @@ -36,11 +37,11 @@ BOOST_AUTO_TEST_CASE(key_io_valid_parse) std::string exp_base58string = test[0].get_str(); const std::vector<std::byte> exp_payload{ParseHex<std::byte>(test[1].get_str())}; const UniValue &metadata = test[2].get_obj(); - bool isPrivkey = find_value(metadata, "isPrivkey").get_bool(); - SelectParams(find_value(metadata, "chain").get_str()); - bool try_case_flip = find_value(metadata, "tryCaseFlip").isNull() ? false : find_value(metadata, "tryCaseFlip").get_bool(); + bool isPrivkey = metadata.find_value("isPrivkey").get_bool(); + SelectParams(ChainTypeFromString(metadata.find_value("chain").get_str()).value()); + bool try_case_flip = metadata.find_value("tryCaseFlip").isNull() ? false : metadata.find_value("tryCaseFlip").get_bool(); if (isPrivkey) { - bool isCompressed = find_value(metadata, "isCompressed").get_bool(); + bool isCompressed = metadata.find_value("isCompressed").get_bool(); // Must be valid private key privkey = DecodeSecret(exp_base58string); BOOST_CHECK_MESSAGE(privkey.IsValid(), "!IsValid:" + strTest); @@ -95,10 +96,10 @@ BOOST_AUTO_TEST_CASE(key_io_valid_gen) std::string exp_base58string = test[0].get_str(); std::vector<unsigned char> exp_payload = ParseHex(test[1].get_str()); const UniValue &metadata = test[2].get_obj(); - bool isPrivkey = find_value(metadata, "isPrivkey").get_bool(); - SelectParams(find_value(metadata, "chain").get_str()); + bool isPrivkey = metadata.find_value("isPrivkey").get_bool(); + SelectParams(ChainTypeFromString(metadata.find_value("chain").get_str()).value()); if (isPrivkey) { - bool isCompressed = find_value(metadata, "isCompressed").get_bool(); + bool isCompressed = metadata.find_value("isCompressed").get_bool(); CKey key; key.Set(exp_payload.begin(), exp_payload.end(), isCompressed); assert(key.IsValid()); @@ -113,7 +114,7 @@ BOOST_AUTO_TEST_CASE(key_io_valid_gen) } } - SelectParams(CBaseChainParams::MAIN); + SelectParams(ChainType::MAIN); } @@ -135,7 +136,7 @@ BOOST_AUTO_TEST_CASE(key_io_invalid) std::string exp_base58string = test[0].get_str(); // must be invalid as public and as private key - for (const auto& chain : { CBaseChainParams::MAIN, CBaseChainParams::TESTNET, CBaseChainParams::SIGNET, CBaseChainParams::REGTEST }) { + for (const auto& chain : {ChainType::MAIN, ChainType::TESTNET, ChainType::SIGNET, ChainType::REGTEST}) { SelectParams(chain); destination = DecodeDestination(exp_base58string); BOOST_CHECK_MESSAGE(!IsValidDestination(destination), "IsValid pubkey in mainnet:" + strTest); diff --git a/src/test/key_tests.cpp b/src/test/key_tests.cpp index ea5b94f3a5..8f11bf5db2 100644 --- a/src/test/key_tests.cpp +++ b/src/test/key_tests.cpp @@ -4,6 +4,7 @@ #include <key.h> +#include <common/system.h> #include <key_io.h> #include <streams.h> #include <test/util/random.h> @@ -11,7 +12,6 @@ #include <uint256.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <string> #include <vector> diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index 94e553a304..db58a0baec 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <policy/policy.h> #include <test/util/txmempool.h> #include <txmempool.h> -#include <util/system.h> #include <util/time.h> #include <test/util/setup_common.h> diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index cfab762307..94e3f27930 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <coins.h> +#include <common/system.h> #include <consensus/consensus.h> #include <consensus/merkle.h> #include <consensus/tx_verify.h> @@ -15,7 +16,6 @@ #include <txmempool.h> #include <uint256.h> #include <util/strencodings.h> -#include <util/system.h> #include <util/time.h> #include <validation.h> #include <versionbits.h> diff --git a/src/test/miniminer_tests.cpp b/src/test/miniminer_tests.cpp new file mode 100644 index 0000000000..b26f7dafe3 --- /dev/null +++ b/src/test/miniminer_tests.cpp @@ -0,0 +1,476 @@ +// Copyright (c) 2021 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <node/mini_miner.h> +#include <txmempool.h> +#include <util/time.h> + +#include <test/util/setup_common.h> +#include <test/util/txmempool.h> + +#include <boost/test/unit_test.hpp> +#include <optional> +#include <vector> + +BOOST_FIXTURE_TEST_SUITE(miniminer_tests, TestingSetup) + +static inline CTransactionRef make_tx(const std::vector<COutPoint>& inputs, size_t num_outputs) +{ + CMutableTransaction tx = CMutableTransaction(); + tx.vin.resize(inputs.size()); + tx.vout.resize(num_outputs); + for (size_t i = 0; i < inputs.size(); ++i) { + tx.vin[i].prevout = inputs[i]; + } + for (size_t i = 0; i < num_outputs; ++i) { + tx.vout[i].scriptPubKey = CScript() << OP_11 << OP_EQUAL; + // The actual input and output values of these transactions don't really + // matter, since all accounting will use the entries' cached fees. + tx.vout[i].nValue = COIN; + } + return MakeTransactionRef(tx); +} + +static inline bool sanity_check(const std::vector<CTransactionRef>& transactions, + const std::map<COutPoint, CAmount>& bumpfees) +{ + // No negative bumpfees. + for (const auto& [outpoint, fee] : bumpfees) { + if (fee < 0) return false; + if (fee == 0) continue; + auto outpoint_ = outpoint; // structured bindings can't be captured in C++17, so we need to use a variable + const bool found = std::any_of(transactions.cbegin(), transactions.cend(), [&](const auto& tx) { + return outpoint_.hash == tx->GetHash() && outpoint_.n < tx->vout.size(); + }); + if (!found) return false; + } + for (const auto& tx : transactions) { + // If tx has multiple outputs, they must all have the same bumpfee (if they exist). + if (tx->vout.size() > 1) { + std::set<CAmount> distinct_bumpfees; + for (size_t i{0}; i < tx->vout.size(); ++i) { + const auto bumpfee = bumpfees.find(COutPoint{tx->GetHash(), static_cast<uint32_t>(i)}); + if (bumpfee != bumpfees.end()) distinct_bumpfees.insert(bumpfee->second); + } + if (distinct_bumpfees.size() > 1) return false; + } + } + return true; +} + +template <typename Key, typename Value> +Value Find(const std::map<Key, Value>& map, const Key& key) +{ + auto it = map.find(key); + BOOST_CHECK_MESSAGE(it != map.end(), strprintf("Cannot find %s", key.ToString())); + return it->second; +} + +BOOST_FIXTURE_TEST_CASE(miniminer_1p1c, TestChain100Setup) +{ + CTxMemPool& pool = *Assert(m_node.mempool); + LOCK2(::cs_main, pool.cs); + TestMemPoolEntryHelper entry; + + const CAmount low_fee{CENT/2000}; + const CAmount normal_fee{CENT/200}; + const CAmount high_fee{CENT/10}; + + // Create a parent tx1 and child tx2 with normal fees: + const auto tx1 = make_tx({COutPoint{m_coinbase_txns[0]->GetHash(), 0}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx1)); + const auto tx2 = make_tx({COutPoint{tx1->GetHash(), 0}}, /*num_outputs=*/1); + pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx2)); + + // Create a low-feerate parent tx3 and high-feerate child tx4 (cpfp) + const auto tx3 = make_tx({COutPoint{m_coinbase_txns[1]->GetHash(), 0}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(low_fee).FromTx(tx3)); + const auto tx4 = make_tx({COutPoint{tx3->GetHash(), 0}}, /*num_outputs=*/1); + pool.addUnchecked(entry.Fee(high_fee).FromTx(tx4)); + + // Create a parent tx5 and child tx6 where both have low fees + const auto tx5 = make_tx({COutPoint{m_coinbase_txns[2]->GetHash(), 0}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(low_fee).FromTx(tx5)); + const auto tx6 = make_tx({COutPoint{tx5->GetHash(), 0}}, /*num_outputs=*/1); + pool.addUnchecked(entry.Fee(low_fee).FromTx(tx6)); + // Make tx6's modified fee much higher than its base fee. This should cause it to pass + // the fee-related checks despite being low-feerate. + pool.PrioritiseTransaction(tx6->GetHash(), CENT/100); + + // Create a high-feerate parent tx7, low-feerate child tx8 + const auto tx7 = make_tx({COutPoint{m_coinbase_txns[3]->GetHash(), 0}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(high_fee).FromTx(tx7)); + const auto tx8 = make_tx({COutPoint{tx7->GetHash(), 0}}, /*num_outputs=*/1); + pool.addUnchecked(entry.Fee(low_fee).FromTx(tx8)); + + std::vector<COutPoint> all_unspent_outpoints({ + COutPoint{tx1->GetHash(), 1}, + COutPoint{tx2->GetHash(), 0}, + COutPoint{tx3->GetHash(), 1}, + COutPoint{tx4->GetHash(), 0}, + COutPoint{tx5->GetHash(), 1}, + COutPoint{tx6->GetHash(), 0}, + COutPoint{tx7->GetHash(), 1}, + COutPoint{tx8->GetHash(), 0} + }); + for (const auto& outpoint : all_unspent_outpoints) BOOST_CHECK(!pool.isSpent(outpoint)); + + std::vector<COutPoint> all_spent_outpoints({ + COutPoint{tx1->GetHash(), 0}, + COutPoint{tx3->GetHash(), 0}, + COutPoint{tx5->GetHash(), 0}, + COutPoint{tx7->GetHash(), 0} + }); + for (const auto& outpoint : all_spent_outpoints) BOOST_CHECK(pool.GetConflictTx(outpoint) != nullptr); + + std::vector<COutPoint> all_parent_outputs({ + COutPoint{tx1->GetHash(), 0}, + COutPoint{tx1->GetHash(), 1}, + COutPoint{tx3->GetHash(), 0}, + COutPoint{tx3->GetHash(), 1}, + COutPoint{tx5->GetHash(), 0}, + COutPoint{tx5->GetHash(), 1}, + COutPoint{tx7->GetHash(), 0}, + COutPoint{tx7->GetHash(), 1} + }); + + + std::vector<CTransactionRef> all_transactions{tx1, tx2, tx3, tx4, tx5, tx6, tx7, tx8}; + struct TxDimensions { + int32_t vsize; CAmount mod_fee; CFeeRate feerate; + }; + std::map<uint256, TxDimensions> tx_dims; + for (const auto& tx : all_transactions) { + const auto it = pool.GetIter(tx->GetHash()).value(); + tx_dims.emplace(tx->GetHash(), TxDimensions{it->GetTxSize(), it->GetModifiedFee(), + CFeeRate(it->GetModifiedFee(), it->GetTxSize())}); + } + + const std::vector<CFeeRate> various_normal_feerates({CFeeRate(0), CFeeRate(500), CFeeRate(999), + CFeeRate(1000), CFeeRate(2000), CFeeRate(2500), + CFeeRate(3333), CFeeRate(7800), CFeeRate(11199), + CFeeRate(23330), CFeeRate(50000), CFeeRate(5*CENT)}); + + // All nonexistent entries have a bumpfee of zero, regardless of feerate + std::vector<COutPoint> nonexistent_outpoints({ COutPoint{GetRandHash(), 0}, COutPoint{GetRandHash(), 3} }); + for (const auto& outpoint : nonexistent_outpoints) BOOST_CHECK(!pool.isSpent(outpoint)); + for (const auto& feerate : various_normal_feerates) { + node::MiniMiner mini_miner(pool, nonexistent_outpoints); + BOOST_CHECK(mini_miner.IsReadyToCalculate()); + auto bump_fees = mini_miner.CalculateBumpFees(feerate); + BOOST_CHECK(!mini_miner.IsReadyToCalculate()); + BOOST_CHECK(sanity_check(all_transactions, bump_fees)); + BOOST_CHECK(bump_fees.size() == nonexistent_outpoints.size()); + for (const auto& outpoint: nonexistent_outpoints) { + auto it = bump_fees.find(outpoint); + BOOST_CHECK(it != bump_fees.end()); + BOOST_CHECK_EQUAL(it->second, 0); + } + } + + // Gather bump fees for all available UTXOs. + for (const auto& target_feerate : various_normal_feerates) { + node::MiniMiner mini_miner(pool, all_unspent_outpoints); + BOOST_CHECK(mini_miner.IsReadyToCalculate()); + auto bump_fees = mini_miner.CalculateBumpFees(target_feerate); + BOOST_CHECK(!mini_miner.IsReadyToCalculate()); + BOOST_CHECK(sanity_check(all_transactions, bump_fees)); + BOOST_CHECK_EQUAL(bump_fees.size(), all_unspent_outpoints.size()); + + // Check tx1 bumpfee: no other bumper. + const TxDimensions& tx1_dimensions = tx_dims.find(tx1->GetHash())->second; + CAmount bumpfee1 = Find(bump_fees, COutPoint{tx1->GetHash(), 1}); + if (target_feerate <= tx1_dimensions.feerate) { + BOOST_CHECK_EQUAL(bumpfee1, 0); + } else { + // Difference is fee to bump tx1 from current to target feerate. + BOOST_CHECK_EQUAL(bumpfee1, target_feerate.GetFee(tx1_dimensions.vsize) - tx1_dimensions.mod_fee); + } + + // Check tx3 bumpfee: assisted by tx4. + const TxDimensions& tx3_dimensions = tx_dims.find(tx3->GetHash())->second; + const TxDimensions& tx4_dimensions = tx_dims.find(tx4->GetHash())->second; + const CFeeRate tx3_feerate = CFeeRate(tx3_dimensions.mod_fee + tx4_dimensions.mod_fee, tx3_dimensions.vsize + tx4_dimensions.vsize); + CAmount bumpfee3 = Find(bump_fees, COutPoint{tx3->GetHash(), 1}); + if (target_feerate <= tx3_feerate) { + // As long as target feerate is below tx4's ancestor feerate, there is no bump fee. + BOOST_CHECK_EQUAL(bumpfee3, 0); + } else { + // Difference is fee to bump tx3 from current to target feerate, without tx4. + BOOST_CHECK_EQUAL(bumpfee3, target_feerate.GetFee(tx3_dimensions.vsize) - tx3_dimensions.mod_fee); + } + + // If tx6’s modified fees are sufficient for tx5 and tx6 to be picked + // into the block, our prospective new transaction would not need to + // bump tx5 when using tx5’s second output. If however even tx6’s + // modified fee (which essentially indicates "effective feerate") is + // not sufficient to bump tx5, using the second output of tx5 would + // require our transaction to bump tx5 from scratch since we evaluate + // transaction packages per ancestor sets and do not consider multiple + // children’s fees. + const TxDimensions& tx5_dimensions = tx_dims.find(tx5->GetHash())->second; + const TxDimensions& tx6_dimensions = tx_dims.find(tx6->GetHash())->second; + const CFeeRate tx5_feerate = CFeeRate(tx5_dimensions.mod_fee + tx6_dimensions.mod_fee, tx5_dimensions.vsize + tx6_dimensions.vsize); + CAmount bumpfee5 = Find(bump_fees, COutPoint{tx5->GetHash(), 1}); + if (target_feerate <= tx5_feerate) { + // As long as target feerate is below tx6's ancestor feerate, there is no bump fee. + BOOST_CHECK_EQUAL(bumpfee5, 0); + } else { + // Difference is fee to bump tx5 from current to target feerate, without tx6. + BOOST_CHECK_EQUAL(bumpfee5, target_feerate.GetFee(tx5_dimensions.vsize) - tx5_dimensions.mod_fee); + } + } + // Spent outpoints should usually not be requested as they would not be + // considered available. However, when they are explicitly requested, we + // can calculate their bumpfee to facilitate RBF-replacements + for (const auto& target_feerate : various_normal_feerates) { + node::MiniMiner mini_miner_all_spent(pool, all_spent_outpoints); + BOOST_CHECK(mini_miner_all_spent.IsReadyToCalculate()); + auto bump_fees_all_spent = mini_miner_all_spent.CalculateBumpFees(target_feerate); + BOOST_CHECK(!mini_miner_all_spent.IsReadyToCalculate()); + BOOST_CHECK_EQUAL(bump_fees_all_spent.size(), all_spent_outpoints.size()); + node::MiniMiner mini_miner_all_parents(pool, all_parent_outputs); + BOOST_CHECK(mini_miner_all_parents.IsReadyToCalculate()); + auto bump_fees_all_parents = mini_miner_all_parents.CalculateBumpFees(target_feerate); + BOOST_CHECK(!mini_miner_all_parents.IsReadyToCalculate()); + BOOST_CHECK_EQUAL(bump_fees_all_parents.size(), all_parent_outputs.size()); + for (auto& bump_fees : {bump_fees_all_parents, bump_fees_all_spent}) { + // For all_parents case, both outputs from the parent should have the same bump fee, + // even though only one of them is in a to-be-replaced transaction. + BOOST_CHECK(sanity_check(all_transactions, bump_fees)); + + // Check tx1 bumpfee: no other bumper. + const TxDimensions& tx1_dimensions = tx_dims.find(tx1->GetHash())->second; + CAmount it1_spent = Find(bump_fees, COutPoint{tx1->GetHash(), 0}); + if (target_feerate <= tx1_dimensions.feerate) { + BOOST_CHECK_EQUAL(it1_spent, 0); + } else { + // Difference is fee to bump tx1 from current to target feerate. + BOOST_CHECK_EQUAL(it1_spent, target_feerate.GetFee(tx1_dimensions.vsize) - tx1_dimensions.mod_fee); + } + + // Check tx3 bumpfee: no other bumper, because tx4 is to-be-replaced. + const TxDimensions& tx3_dimensions = tx_dims.find(tx3->GetHash())->second; + const CFeeRate tx3_feerate_unbumped = tx3_dimensions.feerate; + auto it3_spent = Find(bump_fees, COutPoint{tx3->GetHash(), 0}); + if (target_feerate <= tx3_feerate_unbumped) { + BOOST_CHECK_EQUAL(it3_spent, 0); + } else { + // Difference is fee to bump tx3 from current to target feerate, without tx4. + BOOST_CHECK_EQUAL(it3_spent, target_feerate.GetFee(tx3_dimensions.vsize) - tx3_dimensions.mod_fee); + } + + // Check tx5 bumpfee: no other bumper, because tx6 is to-be-replaced. + const TxDimensions& tx5_dimensions = tx_dims.find(tx5->GetHash())->second; + const CFeeRate tx5_feerate_unbumped = tx5_dimensions.feerate; + auto it5_spent = Find(bump_fees, COutPoint{tx5->GetHash(), 0}); + if (target_feerate <= tx5_feerate_unbumped) { + BOOST_CHECK_EQUAL(it5_spent, 0); + } else { + // Difference is fee to bump tx5 from current to target feerate, without tx6. + BOOST_CHECK_EQUAL(it5_spent, target_feerate.GetFee(tx5_dimensions.vsize) - tx5_dimensions.mod_fee); + } + } + } +} + +BOOST_FIXTURE_TEST_CASE(miniminer_overlap, TestChain100Setup) +{ + CTxMemPool& pool = *Assert(m_node.mempool); + LOCK2(::cs_main, pool.cs); + TestMemPoolEntryHelper entry; + + const CAmount low_fee{CENT/2000}; + const CAmount med_fee{CENT/200}; + const CAmount high_fee{CENT/10}; + + // Create 3 parents of different feerates, and 1 child spending from all 3. + const auto tx1 = make_tx({COutPoint{m_coinbase_txns[0]->GetHash(), 0}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(low_fee).FromTx(tx1)); + const auto tx2 = make_tx({COutPoint{m_coinbase_txns[1]->GetHash(), 0}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(med_fee).FromTx(tx2)); + const auto tx3 = make_tx({COutPoint{m_coinbase_txns[2]->GetHash(), 0}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(high_fee).FromTx(tx3)); + const auto tx4 = make_tx({COutPoint{tx1->GetHash(), 0}, COutPoint{tx2->GetHash(), 0}, COutPoint{tx3->GetHash(), 0}}, /*num_outputs=*/3); + pool.addUnchecked(entry.Fee(high_fee).FromTx(tx4)); + + // Create 1 grandparent and 1 parent, then 2 children. + const auto tx5 = make_tx({COutPoint{m_coinbase_txns[3]->GetHash(), 0}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(high_fee).FromTx(tx5)); + const auto tx6 = make_tx({COutPoint{tx5->GetHash(), 0}}, /*num_outputs=*/3); + pool.addUnchecked(entry.Fee(low_fee).FromTx(tx6)); + const auto tx7 = make_tx({COutPoint{tx6->GetHash(), 0}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(med_fee).FromTx(tx7)); + const auto tx8 = make_tx({COutPoint{tx6->GetHash(), 1}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(high_fee).FromTx(tx8)); + + std::vector<CTransactionRef> all_transactions{tx1, tx2, tx3, tx4, tx5, tx6, tx7, tx8}; + std::vector<int64_t> tx_vsizes; + tx_vsizes.reserve(all_transactions.size()); + for (const auto& tx : all_transactions) tx_vsizes.push_back(GetVirtualTransactionSize(*tx)); + + std::vector<COutPoint> all_unspent_outpoints({ + COutPoint{tx1->GetHash(), 1}, + COutPoint{tx2->GetHash(), 1}, + COutPoint{tx3->GetHash(), 1}, + COutPoint{tx4->GetHash(), 0}, + COutPoint{tx4->GetHash(), 1}, + COutPoint{tx4->GetHash(), 2}, + COutPoint{tx5->GetHash(), 1}, + COutPoint{tx6->GetHash(), 2}, + COutPoint{tx7->GetHash(), 0}, + COutPoint{tx8->GetHash(), 0} + }); + for (const auto& outpoint : all_unspent_outpoints) BOOST_CHECK(!pool.isSpent(outpoint)); + + const auto tx3_feerate = CFeeRate(high_fee, tx_vsizes[2]); + const auto tx4_feerate = CFeeRate(high_fee, tx_vsizes[3]); + // tx4's feerate is lower than tx3's. same fee, different weight. + BOOST_CHECK(tx3_feerate > tx4_feerate); + const auto tx4_anc_feerate = CFeeRate(low_fee + med_fee + high_fee, tx_vsizes[0] + tx_vsizes[1] + tx_vsizes[3]); + const auto tx5_feerate = CFeeRate(high_fee, tx_vsizes[4]); + const auto tx7_anc_feerate = CFeeRate(low_fee + med_fee, tx_vsizes[5] + tx_vsizes[6]); + const auto tx8_anc_feerate = CFeeRate(low_fee + high_fee, tx_vsizes[5] + tx_vsizes[7]); + BOOST_CHECK(tx5_feerate > tx7_anc_feerate); + BOOST_CHECK(tx5_feerate > tx8_anc_feerate); + + // Extremely high feerate: everybody's bumpfee is from their full ancestor set. + { + node::MiniMiner mini_miner(pool, all_unspent_outpoints); + const CFeeRate very_high_feerate(COIN); + BOOST_CHECK(tx4_anc_feerate < very_high_feerate); + BOOST_CHECK(mini_miner.IsReadyToCalculate()); + auto bump_fees = mini_miner.CalculateBumpFees(very_high_feerate); + BOOST_CHECK_EQUAL(bump_fees.size(), all_unspent_outpoints.size()); + BOOST_CHECK(!mini_miner.IsReadyToCalculate()); + BOOST_CHECK(sanity_check(all_transactions, bump_fees)); + const auto tx1_bumpfee = bump_fees.find(COutPoint{tx1->GetHash(), 1}); + BOOST_CHECK(tx1_bumpfee != bump_fees.end()); + BOOST_CHECK_EQUAL(tx1_bumpfee->second, very_high_feerate.GetFee(tx_vsizes[0]) - low_fee); + const auto tx4_bumpfee = bump_fees.find(COutPoint{tx4->GetHash(), 0}); + BOOST_CHECK(tx4_bumpfee != bump_fees.end()); + BOOST_CHECK_EQUAL(tx4_bumpfee->second, + very_high_feerate.GetFee(tx_vsizes[0] + tx_vsizes[1] + tx_vsizes[2] + tx_vsizes[3]) - (low_fee + med_fee + high_fee + high_fee)); + const auto tx7_bumpfee = bump_fees.find(COutPoint{tx7->GetHash(), 0}); + BOOST_CHECK(tx7_bumpfee != bump_fees.end()); + BOOST_CHECK_EQUAL(tx7_bumpfee->second, + very_high_feerate.GetFee(tx_vsizes[4] + tx_vsizes[5] + tx_vsizes[6]) - (high_fee + low_fee + med_fee)); + const auto tx8_bumpfee = bump_fees.find(COutPoint{tx8->GetHash(), 0}); + BOOST_CHECK(tx8_bumpfee != bump_fees.end()); + BOOST_CHECK_EQUAL(tx8_bumpfee->second, + very_high_feerate.GetFee(tx_vsizes[4] + tx_vsizes[5] + tx_vsizes[7]) - (high_fee + low_fee + high_fee)); + // Total fees: if spending multiple outputs from tx4 don't double-count fees. + node::MiniMiner mini_miner_total_tx4(pool, {COutPoint{tx4->GetHash(), 0}, COutPoint{tx4->GetHash(), 1}}); + BOOST_CHECK(mini_miner_total_tx4.IsReadyToCalculate()); + const auto tx4_bump_fee = mini_miner_total_tx4.CalculateTotalBumpFees(very_high_feerate); + BOOST_CHECK(!mini_miner_total_tx4.IsReadyToCalculate()); + BOOST_CHECK(tx4_bump_fee.has_value()); + BOOST_CHECK_EQUAL(tx4_bump_fee.value(), + very_high_feerate.GetFee(tx_vsizes[0] + tx_vsizes[1] + tx_vsizes[2] + tx_vsizes[3]) - (low_fee + med_fee + high_fee + high_fee)); + // Total fees: if spending both tx7 and tx8, don't double-count fees. + node::MiniMiner mini_miner_tx7_tx8(pool, {COutPoint{tx7->GetHash(), 0}, COutPoint{tx8->GetHash(), 0}}); + BOOST_CHECK(mini_miner_tx7_tx8.IsReadyToCalculate()); + const auto tx7_tx8_bumpfee = mini_miner_tx7_tx8.CalculateTotalBumpFees(very_high_feerate); + BOOST_CHECK(!mini_miner_tx7_tx8.IsReadyToCalculate()); + BOOST_CHECK(tx7_tx8_bumpfee.has_value()); + BOOST_CHECK_EQUAL(tx7_tx8_bumpfee.value(), + very_high_feerate.GetFee(tx_vsizes[4] + tx_vsizes[5] + tx_vsizes[6] + tx_vsizes[7]) - (high_fee + low_fee + med_fee + high_fee)); + } + // Feerate just below tx5: tx7 and tx8 have different bump fees. + { + const auto just_below_tx5 = CFeeRate(tx5_feerate.GetFeePerK() - 5); + node::MiniMiner mini_miner(pool, all_unspent_outpoints); + BOOST_CHECK(mini_miner.IsReadyToCalculate()); + auto bump_fees = mini_miner.CalculateBumpFees(just_below_tx5); + BOOST_CHECK(!mini_miner.IsReadyToCalculate()); + BOOST_CHECK_EQUAL(bump_fees.size(), all_unspent_outpoints.size()); + BOOST_CHECK(sanity_check(all_transactions, bump_fees)); + const auto tx7_bumpfee = bump_fees.find(COutPoint{tx7->GetHash(), 0}); + BOOST_CHECK(tx7_bumpfee != bump_fees.end()); + BOOST_CHECK_EQUAL(tx7_bumpfee->second, just_below_tx5.GetFee(tx_vsizes[5] + tx_vsizes[6]) - (low_fee + med_fee)); + const auto tx8_bumpfee = bump_fees.find(COutPoint{tx8->GetHash(), 0}); + BOOST_CHECK(tx8_bumpfee != bump_fees.end()); + BOOST_CHECK_EQUAL(tx8_bumpfee->second, just_below_tx5.GetFee(tx_vsizes[5] + tx_vsizes[7]) - (low_fee + high_fee)); + // Total fees: if spending both tx7 and tx8, don't double-count fees. + node::MiniMiner mini_miner_tx7_tx8(pool, {COutPoint{tx7->GetHash(), 0}, COutPoint{tx8->GetHash(), 0}}); + BOOST_CHECK(mini_miner_tx7_tx8.IsReadyToCalculate()); + const auto tx7_tx8_bumpfee = mini_miner_tx7_tx8.CalculateTotalBumpFees(just_below_tx5); + BOOST_CHECK(!mini_miner_tx7_tx8.IsReadyToCalculate()); + BOOST_CHECK(tx7_tx8_bumpfee.has_value()); + BOOST_CHECK_EQUAL(tx7_tx8_bumpfee.value(), just_below_tx5.GetFee(tx_vsizes[5] + tx_vsizes[6]) - (low_fee + med_fee)); + } + // Feerate between tx7 and tx8's ancestor feerates: don't need to bump tx6 because tx8 already does. + { + const auto just_above_tx7 = CFeeRate(med_fee + 10, tx_vsizes[6]); + BOOST_CHECK(just_above_tx7 <= CFeeRate(low_fee + high_fee, tx_vsizes[5] + tx_vsizes[7])); + node::MiniMiner mini_miner(pool, all_unspent_outpoints); + BOOST_CHECK(mini_miner.IsReadyToCalculate()); + auto bump_fees = mini_miner.CalculateBumpFees(just_above_tx7); + BOOST_CHECK(!mini_miner.IsReadyToCalculate()); + BOOST_CHECK_EQUAL(bump_fees.size(), all_unspent_outpoints.size()); + BOOST_CHECK(sanity_check(all_transactions, bump_fees)); + const auto tx7_bumpfee = bump_fees.find(COutPoint{tx7->GetHash(), 0}); + BOOST_CHECK(tx7_bumpfee != bump_fees.end()); + BOOST_CHECK_EQUAL(tx7_bumpfee->second, just_above_tx7.GetFee(tx_vsizes[6]) - (med_fee)); + const auto tx8_bumpfee = bump_fees.find(COutPoint{tx8->GetHash(), 0}); + BOOST_CHECK(tx8_bumpfee != bump_fees.end()); + BOOST_CHECK_EQUAL(tx8_bumpfee->second, 0); + } +} +BOOST_FIXTURE_TEST_CASE(calculate_cluster, TestChain100Setup) +{ + CTxMemPool& pool = *Assert(m_node.mempool); + LOCK2(cs_main, pool.cs); + + // Add chain of size 500 + TestMemPoolEntryHelper entry; + std::vector<uint256> chain_txids; + auto& lasttx = m_coinbase_txns[0]; + for (auto i{0}; i < 500; ++i) { + const auto tx = make_tx({COutPoint{lasttx->GetHash(), 0}}, /*num_outputs=*/1); + pool.addUnchecked(entry.Fee(CENT).FromTx(tx)); + chain_txids.push_back(tx->GetHash()); + lasttx = tx; + } + const auto cluster_500tx = pool.GatherClusters({lasttx->GetHash()}); + CTxMemPool::setEntries cluster_500tx_set{cluster_500tx.begin(), cluster_500tx.end()}; + BOOST_CHECK_EQUAL(cluster_500tx.size(), cluster_500tx_set.size()); + const auto vec_iters_500 = pool.GetIterVec(chain_txids); + for (const auto& iter : vec_iters_500) BOOST_CHECK(cluster_500tx_set.count(iter)); + + // GatherClusters stops at 500 transactions. + const auto tx_501 = make_tx({COutPoint{lasttx->GetHash(), 0}}, /*num_outputs=*/1); + pool.addUnchecked(entry.Fee(CENT).FromTx(tx_501)); + const auto cluster_501 = pool.GatherClusters({tx_501->GetHash()}); + BOOST_CHECK_EQUAL(cluster_501.size(), 0); + + // Zig Zag cluster: + // txp0 txp1 txp2 ... txp48 txp49 + // \ / \ / \ \ / + // txc0 txc1 txc2 ... txc48 + // Note that each transaction's ancestor size is 1 or 3, and each descendant size is 1, 2 or 3. + // However, all of these transactions are in the same cluster. + std::vector<uint256> zigzag_txids; + for (auto p{0}; p < 50; ++p) { + const auto txp = make_tx({COutPoint{GetRandHash(), 0}}, /*num_outputs=*/2); + pool.addUnchecked(entry.Fee(CENT).FromTx(txp)); + zigzag_txids.push_back(txp->GetHash()); + } + for (auto c{0}; c < 49; ++c) { + const auto txc = make_tx({COutPoint{zigzag_txids[c], 1}, COutPoint{zigzag_txids[c+1], 0}}, /*num_outputs=*/1); + pool.addUnchecked(entry.Fee(CENT).FromTx(txc)); + zigzag_txids.push_back(txc->GetHash()); + } + const auto vec_iters_zigzag = pool.GetIterVec(zigzag_txids); + // It doesn't matter which tx we calculate cluster for, everybody is in it. + const std::vector<size_t> indeces{0, 22, 72, zigzag_txids.size() - 1}; + for (const auto index : indeces) { + const auto cluster = pool.GatherClusters({zigzag_txids[index]}); + BOOST_CHECK_EQUAL(cluster.size(), zigzag_txids.size()); + CTxMemPool::setEntries clusterset{cluster.begin(), cluster.end()}; + BOOST_CHECK_EQUAL(cluster.size(), clusterset.size()); + for (const auto& iter : vec_iters_zigzag) BOOST_CHECK(clusterset.count(iter)); + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 631c213627..aa577f7b27 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -135,7 +135,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) CNetAddr addr; // IPv4, INADDR_ANY - BOOST_REQUIRE(LookupHost("0.0.0.0", addr, false)); + addr = LookupHost("0.0.0.0", false).value(); BOOST_REQUIRE(!addr.IsValid()); BOOST_REQUIRE(addr.IsIPv4()); @@ -144,7 +144,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_CHECK_EQUAL(addr.ToStringAddr(), "0.0.0.0"); // IPv4, INADDR_NONE - BOOST_REQUIRE(LookupHost("255.255.255.255", addr, false)); + addr = LookupHost("255.255.255.255", false).value(); BOOST_REQUIRE(!addr.IsValid()); BOOST_REQUIRE(addr.IsIPv4()); @@ -153,7 +153,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_CHECK_EQUAL(addr.ToStringAddr(), "255.255.255.255"); // IPv4, casual - BOOST_REQUIRE(LookupHost("12.34.56.78", addr, false)); + addr = LookupHost("12.34.56.78", false).value(); BOOST_REQUIRE(addr.IsValid()); BOOST_REQUIRE(addr.IsIPv4()); @@ -162,7 +162,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_CHECK_EQUAL(addr.ToStringAddr(), "12.34.56.78"); // IPv6, in6addr_any - BOOST_REQUIRE(LookupHost("::", addr, false)); + addr = LookupHost("::", false).value(); BOOST_REQUIRE(!addr.IsValid()); BOOST_REQUIRE(addr.IsIPv6()); @@ -171,7 +171,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) BOOST_CHECK_EQUAL(addr.ToStringAddr(), "::"); // IPv6, casual - BOOST_REQUIRE(LookupHost("1122:3344:5566:7788:9900:aabb:ccdd:eeff", addr, false)); + addr = LookupHost("1122:3344:5566:7788:9900:aabb:ccdd:eeff", false).value(); BOOST_REQUIRE(addr.IsValid()); BOOST_REQUIRE(addr.IsIPv6()); @@ -186,14 +186,14 @@ BOOST_AUTO_TEST_CASE(cnetaddr_basic) // id of "32", return the address as "fe80::1%32". const std::string link_local{"fe80::1"}; const std::string scoped_addr{link_local + "%32"}; - BOOST_REQUIRE(LookupHost(scoped_addr, addr, false)); + addr = LookupHost(scoped_addr, false).value(); BOOST_REQUIRE(addr.IsValid()); BOOST_REQUIRE(addr.IsIPv6()); BOOST_CHECK(!addr.IsBindAny()); BOOST_CHECK_EQUAL(addr.ToStringAddr(), scoped_addr); // Test that the delimiter "%" and default zone id of 0 can be omitted for the default scope. - BOOST_REQUIRE(LookupHost(link_local + "%0", addr, false)); + addr = LookupHost(link_local + "%0", false).value(); BOOST_REQUIRE(addr.IsValid()); BOOST_REQUIRE(addr.IsIPv6()); BOOST_CHECK(!addr.IsBindAny()); @@ -318,10 +318,9 @@ BOOST_AUTO_TEST_CASE(cnetaddr_tostring_canonical_ipv6) {"2001:db8:aaaa:bbbb:cccc:dddd:eeee:AaAa", "2001:db8:aaaa:bbbb:cccc:dddd:eeee:aaaa"}, }; for (const auto& [input_address, expected_canonical_representation_output] : canonical_representations_ipv6) { - CNetAddr net_addr; - BOOST_REQUIRE(LookupHost(input_address, net_addr, false)); - BOOST_REQUIRE(net_addr.IsIPv6()); - BOOST_CHECK_EQUAL(net_addr.ToStringAddr(), expected_canonical_representation_output); + const std::optional<CNetAddr> net_addr{LookupHost(input_address, false)}; + BOOST_REQUIRE(net_addr.value().IsIPv6()); + BOOST_CHECK_EQUAL(net_addr.value().ToStringAddr(), expected_canonical_representation_output); } } @@ -334,12 +333,12 @@ BOOST_AUTO_TEST_CASE(cnetaddr_serialize_v1) BOOST_CHECK_EQUAL(HexStr(s), "00000000000000000000000000000000"); s.clear(); - BOOST_REQUIRE(LookupHost("1.2.3.4", addr, false)); + addr = LookupHost("1.2.3.4", false).value(); s << addr; BOOST_CHECK_EQUAL(HexStr(s), "00000000000000000000ffff01020304"); s.clear(); - BOOST_REQUIRE(LookupHost("1a1b:2a2b:3a3b:4a4b:5a5b:6a6b:7a7b:8a8b", addr, false)); + addr = LookupHost("1a1b:2a2b:3a3b:4a4b:5a5b:6a6b:7a7b:8a8b", false).value(); s << addr; BOOST_CHECK_EQUAL(HexStr(s), "1a1b2a2b3a3b4a4b5a5b6a6b7a7b8a8b"); s.clear(); @@ -370,12 +369,12 @@ BOOST_AUTO_TEST_CASE(cnetaddr_serialize_v2) BOOST_CHECK_EQUAL(HexStr(s), "021000000000000000000000000000000000"); s.clear(); - BOOST_REQUIRE(LookupHost("1.2.3.4", addr, false)); + addr = LookupHost("1.2.3.4", false).value(); s << addr; BOOST_CHECK_EQUAL(HexStr(s), "010401020304"); s.clear(); - BOOST_REQUIRE(LookupHost("1a1b:2a2b:3a3b:4a4b:5a5b:6a6b:7a7b:8a8b", addr, false)); + addr = LookupHost("1a1b:2a2b:3a3b:4a4b:5a5b:6a6b:7a7b:8a8b", false).value(); s << addr; BOOST_CHECK_EQUAL(HexStr(s), "02101a1b2a2b3a3b4a4b5a5b6a6b7a7b8a8b"); s.clear(); diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 7e91819ddc..05953bfd10 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -24,9 +24,7 @@ BOOST_FIXTURE_TEST_SUITE(netbase_tests, BasicTestingSetup) static CNetAddr ResolveIP(const std::string& ip) { - CNetAddr addr; - LookupHost(ip, addr, false); - return addr; + return LookupHost(ip, false).value_or(CNetAddr{}); } static CSubNet ResolveSubNet(const std::string& subnet) @@ -477,11 +475,10 @@ BOOST_AUTO_TEST_CASE(netpermissions_test) BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters) { - CNetAddr addr; - BOOST_CHECK(LookupHost("127.0.0.1"s, addr, false)); - BOOST_CHECK(!LookupHost("127.0.0.1\0"s, addr, false)); - BOOST_CHECK(!LookupHost("127.0.0.1\0example.com"s, addr, false)); - BOOST_CHECK(!LookupHost("127.0.0.1\0example.com\0"s, addr, false)); + BOOST_CHECK(LookupHost("127.0.0.1"s, false).has_value()); + BOOST_CHECK(!LookupHost("127.0.0.1\0"s, false).has_value()); + BOOST_CHECK(!LookupHost("127.0.0.1\0example.com"s, false).has_value()); + BOOST_CHECK(!LookupHost("127.0.0.1\0example.com\0"s, false).has_value()); CSubNet ret; BOOST_CHECK(LookupSubNet("1.2.3.0/24"s, ret)); BOOST_CHECK(!LookupSubNet("1.2.3.0/24\0"s, ret)); diff --git a/src/test/pow_tests.cpp b/src/test/pow_tests.cpp index addc925bab..5bd14f22c6 100644 --- a/src/test/pow_tests.cpp +++ b/src/test/pow_tests.cpp @@ -7,6 +7,7 @@ #include <pow.h> #include <test/util/random.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> #include <boost/test/unit_test.hpp> @@ -15,7 +16,7 @@ BOOST_FIXTURE_TEST_SUITE(pow_tests, BasicTestingSetup) /* Test calculation of next difficulty target with no constraints applying */ BOOST_AUTO_TEST_CASE(get_next_work) { - const auto chainParams = CreateChainParams(*m_node.args, CBaseChainParams::MAIN); + const auto chainParams = CreateChainParams(*m_node.args, ChainType::MAIN); int64_t nLastRetargetTime = 1261130161; // Block #30240 CBlockIndex pindexLast; pindexLast.nHeight = 32255; @@ -34,7 +35,7 @@ BOOST_AUTO_TEST_CASE(get_next_work) /* Test the constraint on the upper bound for next work */ BOOST_AUTO_TEST_CASE(get_next_work_pow_limit) { - const auto chainParams = CreateChainParams(*m_node.args, CBaseChainParams::MAIN); + const auto chainParams = CreateChainParams(*m_node.args, ChainType::MAIN); int64_t nLastRetargetTime = 1231006505; // Block #0 CBlockIndex pindexLast; pindexLast.nHeight = 2015; @@ -48,7 +49,7 @@ BOOST_AUTO_TEST_CASE(get_next_work_pow_limit) /* Test the constraint on the lower bound for actual time taken */ BOOST_AUTO_TEST_CASE(get_next_work_lower_limit_actual) { - const auto chainParams = CreateChainParams(*m_node.args, CBaseChainParams::MAIN); + const auto chainParams = CreateChainParams(*m_node.args, ChainType::MAIN); int64_t nLastRetargetTime = 1279008237; // Block #66528 CBlockIndex pindexLast; pindexLast.nHeight = 68543; @@ -65,7 +66,7 @@ BOOST_AUTO_TEST_CASE(get_next_work_lower_limit_actual) /* Test the constraint on the upper bound for actual time taken */ BOOST_AUTO_TEST_CASE(get_next_work_upper_limit_actual) { - const auto chainParams = CreateChainParams(*m_node.args, CBaseChainParams::MAIN); + const auto chainParams = CreateChainParams(*m_node.args, ChainType::MAIN); int64_t nLastRetargetTime = 1263163443; // NOTE: Not an actual block time CBlockIndex pindexLast; pindexLast.nHeight = 46367; @@ -81,7 +82,7 @@ BOOST_AUTO_TEST_CASE(get_next_work_upper_limit_actual) BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_negative_target) { - const auto consensus = CreateChainParams(*m_node.args, CBaseChainParams::MAIN)->GetConsensus(); + const auto consensus = CreateChainParams(*m_node.args, ChainType::MAIN)->GetConsensus(); uint256 hash; unsigned int nBits; nBits = UintToArith256(consensus.powLimit).GetCompact(true); @@ -91,7 +92,7 @@ BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_negative_target) BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_overflow_target) { - const auto consensus = CreateChainParams(*m_node.args, CBaseChainParams::MAIN)->GetConsensus(); + const auto consensus = CreateChainParams(*m_node.args, ChainType::MAIN)->GetConsensus(); uint256 hash; unsigned int nBits{~0x00800000U}; hash.SetHex("0x1"); @@ -100,7 +101,7 @@ BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_overflow_target) BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_too_easy_target) { - const auto consensus = CreateChainParams(*m_node.args, CBaseChainParams::MAIN)->GetConsensus(); + const auto consensus = CreateChainParams(*m_node.args, ChainType::MAIN)->GetConsensus(); uint256 hash; unsigned int nBits; arith_uint256 nBits_arith = UintToArith256(consensus.powLimit); @@ -112,7 +113,7 @@ BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_too_easy_target) BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_biger_hash_than_target) { - const auto consensus = CreateChainParams(*m_node.args, CBaseChainParams::MAIN)->GetConsensus(); + const auto consensus = CreateChainParams(*m_node.args, ChainType::MAIN)->GetConsensus(); uint256 hash; unsigned int nBits; arith_uint256 hash_arith = UintToArith256(consensus.powLimit); @@ -124,7 +125,7 @@ BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_biger_hash_than_target) BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_zero_target) { - const auto consensus = CreateChainParams(*m_node.args, CBaseChainParams::MAIN)->GetConsensus(); + const auto consensus = CreateChainParams(*m_node.args, ChainType::MAIN)->GetConsensus(); uint256 hash; unsigned int nBits; arith_uint256 hash_arith{0}; @@ -135,7 +136,7 @@ BOOST_AUTO_TEST_CASE(CheckProofOfWork_test_zero_target) BOOST_AUTO_TEST_CASE(GetBlockProofEquivalentTime_test) { - const auto chainParams = CreateChainParams(*m_node.args, CBaseChainParams::MAIN); + const auto chainParams = CreateChainParams(*m_node.args, ChainType::MAIN); std::vector<CBlockIndex> blocks(10000); for (int i = 0; i < 10000; i++) { blocks[i].pprev = i ? &blocks[i - 1] : nullptr; @@ -155,9 +156,9 @@ BOOST_AUTO_TEST_CASE(GetBlockProofEquivalentTime_test) } } -void sanity_check_chainparams(const ArgsManager& args, std::string chainName) +void sanity_check_chainparams(const ArgsManager& args, ChainType chain_type) { - const auto chainParams = CreateChainParams(args, chainName); + const auto chainParams = CreateChainParams(args, chain_type); const auto consensus = chainParams->GetConsensus(); // hash genesis is correct @@ -184,22 +185,22 @@ void sanity_check_chainparams(const ArgsManager& args, std::string chainName) BOOST_AUTO_TEST_CASE(ChainParams_MAIN_sanity) { - sanity_check_chainparams(*m_node.args, CBaseChainParams::MAIN); + sanity_check_chainparams(*m_node.args, ChainType::MAIN); } BOOST_AUTO_TEST_CASE(ChainParams_REGTEST_sanity) { - sanity_check_chainparams(*m_node.args, CBaseChainParams::REGTEST); + sanity_check_chainparams(*m_node.args, ChainType::REGTEST); } BOOST_AUTO_TEST_CASE(ChainParams_TESTNET_sanity) { - sanity_check_chainparams(*m_node.args, CBaseChainParams::TESTNET); + sanity_check_chainparams(*m_node.args, ChainType::TESTNET); } BOOST_AUTO_TEST_CASE(ChainParams_SIGNET_sanity) { - sanity_check_chainparams(*m_node.args, CBaseChainParams::SIGNET); + sanity_check_chainparams(*m_node.args, ChainType::SIGNET); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/rbf_tests.cpp b/src/test/rbf_tests.cpp index 0ec253747b..10205cd641 100644 --- a/src/test/rbf_tests.cpp +++ b/src/test/rbf_tests.cpp @@ -1,11 +1,11 @@ // Copyright (c) 2021-2022 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <policy/rbf.h> #include <random.h> #include <test/util/txmempool.h> #include <txmempool.h> -#include <util/system.h> #include <util/time.h> #include <test/util/setup_common.h> diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 791c9ddf31..2f783a4b95 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -42,11 +42,11 @@ private: class RPCTestingSetup : public TestingSetup { public: - UniValue TransformParams(const UniValue& params, std::vector<std::string> arg_names) const; + UniValue TransformParams(const UniValue& params, std::vector<std::pair<std::string, bool>> arg_names) const; UniValue CallRPC(std::string args); }; -UniValue RPCTestingSetup::TransformParams(const UniValue& params, std::vector<std::string> arg_names) const +UniValue RPCTestingSetup::TransformParams(const UniValue& params, std::vector<std::pair<std::string, bool>> arg_names) const { UniValue transformed_params; CRPCTable table; @@ -75,7 +75,7 @@ UniValue RPCTestingSetup::CallRPC(std::string args) return result; } catch (const UniValue& objError) { - throw std::runtime_error(find_value(objError, "message").get_str()); + throw std::runtime_error(objError.find_value("message").get_str()); } } @@ -84,7 +84,7 @@ BOOST_FIXTURE_TEST_SUITE(rpc_tests, RPCTestingSetup) BOOST_AUTO_TEST_CASE(rpc_namedparams) { - const std::vector<std::string> arg_names{"arg1", "arg2", "arg3", "arg4", "arg5"}; + const std::vector<std::pair<std::string, bool>> arg_names{{"arg1", false}, {"arg2", false}, {"arg3", false}, {"arg4", false}, {"arg5", false}}; // Make sure named arguments are transformed into positional arguments in correct places separated by nulls BOOST_CHECK_EQUAL(TransformParams(JSON(R"({"arg2": 2, "arg4": 4})"), arg_names).write(), "[null,2,null,4]"); @@ -109,6 +109,28 @@ BOOST_AUTO_TEST_CASE(rpc_namedparams) BOOST_CHECK_EQUAL(TransformParams(JSON(R"([1,2,3,4,5,6,7,8,9,10])"), arg_names).write(), "[1,2,3,4,5,6,7,8,9,10]"); } +BOOST_AUTO_TEST_CASE(rpc_namedonlyparams) +{ + const std::vector<std::pair<std::string, bool>> arg_names{{"arg1", false}, {"arg2", false}, {"opt1", true}, {"opt2", true}, {"options", false}}; + + // Make sure optional parameters are really optional. + BOOST_CHECK_EQUAL(TransformParams(JSON(R"({"arg1": 1, "arg2": 2})"), arg_names).write(), "[1,2]"); + + // Make sure named-only parameters are passed as options. + BOOST_CHECK_EQUAL(TransformParams(JSON(R"({"arg1": 1, "arg2": 2, "opt1": 10, "opt2": 20})"), arg_names).write(), R"([1,2,{"opt1":10,"opt2":20}])"); + + // Make sure options can be passed directly. + BOOST_CHECK_EQUAL(TransformParams(JSON(R"({"arg1": 1, "arg2": 2, "options":{"opt1": 10, "opt2": 20}})"), arg_names).write(), R"([1,2,{"opt1":10,"opt2":20}])"); + + // Make sure options and named parameters conflict. + BOOST_CHECK_EXCEPTION(TransformParams(JSON(R"({"arg1": 1, "arg2": 2, "opt1": 10, "options":{"opt1": 10}})"), arg_names), UniValue, + HasJSON(R"({"code":-8,"message":"Parameter options conflicts with parameter opt1"})")); + + // Make sure options object specified through args array conflicts. + BOOST_CHECK_EXCEPTION(TransformParams(JSON(R"({"args": [1, 2, {"opt1": 10}], "opt2": 20})"), arg_names), UniValue, + HasJSON(R"({"code":-8,"message":"Parameter options specified twice both as positional and named argument"})")); +} + BOOST_AUTO_TEST_CASE(rpc_rawparams) { // Test raw transaction API argument handling @@ -130,9 +152,9 @@ BOOST_AUTO_TEST_CASE(rpc_rawparams) BOOST_CHECK_THROW(CallRPC("decoderawtransaction DEADBEEF"), std::runtime_error); std::string rawtx = "0100000001a15d57094aa7a21a28cb20b59aab8fc7d1149a3bdbcddba9c622e4f5f6a99ece010000006c493046022100f93bb0e7d8db7bd46e40132d1f8242026e045f03a0efe71bbb8e3f475e970d790221009337cd7f1f929f00cc6ff01f03729b069a7c21b59b1736ddfee5db5946c5da8c0121033b9b137ee87d5a812d6f506efdd37f0affa7ffc310711c06c7f3e097c9447c52ffffffff0100e1f505000000001976a9140389035a9225b3839e2bbf32d826a1e222031fd888ac00000000"; BOOST_CHECK_NO_THROW(r = CallRPC(std::string("decoderawtransaction ")+rawtx)); - BOOST_CHECK_EQUAL(find_value(r.get_obj(), "size").getInt<int>(), 193); - BOOST_CHECK_EQUAL(find_value(r.get_obj(), "version").getInt<int>(), 1); - BOOST_CHECK_EQUAL(find_value(r.get_obj(), "locktime").getInt<int>(), 0); + BOOST_CHECK_EQUAL(r.get_obj().find_value("size").getInt<int>(), 193); + BOOST_CHECK_EQUAL(r.get_obj().find_value("version").getInt<int>(), 1); + BOOST_CHECK_EQUAL(r.get_obj().find_value("locktime").getInt<int>(), 0); BOOST_CHECK_THROW(CallRPC(std::string("decoderawtransaction ")+rawtx+" extra"), std::runtime_error); BOOST_CHECK_NO_THROW(r = CallRPC(std::string("decoderawtransaction ")+rawtx+" false")); BOOST_CHECK_THROW(r = CallRPC(std::string("decoderawtransaction ")+rawtx+" false extra"), std::runtime_error); @@ -149,20 +171,20 @@ BOOST_AUTO_TEST_CASE(rpc_togglenetwork) UniValue r; r = CallRPC("getnetworkinfo"); - bool netState = find_value(r.get_obj(), "networkactive").get_bool(); + bool netState = r.get_obj().find_value("networkactive").get_bool(); BOOST_CHECK_EQUAL(netState, true); BOOST_CHECK_NO_THROW(CallRPC("setnetworkactive false")); r = CallRPC("getnetworkinfo"); - int numConnection = find_value(r.get_obj(), "connections").getInt<int>(); + int numConnection = r.get_obj().find_value("connections").getInt<int>(); BOOST_CHECK_EQUAL(numConnection, 0); - netState = find_value(r.get_obj(), "networkactive").get_bool(); + netState = r.get_obj().find_value("networkactive").get_bool(); BOOST_CHECK_EQUAL(netState, false); BOOST_CHECK_NO_THROW(CallRPC("setnetworkactive true")); r = CallRPC("getnetworkinfo"); - netState = find_value(r.get_obj(), "networkactive").get_bool(); + netState = r.get_obj().find_value("networkactive").get_bool(); BOOST_CHECK_EQUAL(netState, true); } @@ -180,9 +202,9 @@ BOOST_AUTO_TEST_CASE(rpc_rawsign) std::string privkey1 = "\"KzsXybp9jX64P5ekX1KUxRQ79Jht9uzW7LorgwE65i5rWACL6LQe\""; std::string privkey2 = "\"Kyhdf5LuKTRx4ge69ybABsiUAWjVRK4XGxAKk2FQLp2HjGMy87Z4\""; r = CallRPC(std::string("signrawtransactionwithkey ")+notsigned+" [] "+prevout); - BOOST_CHECK(find_value(r.get_obj(), "complete").get_bool() == false); + BOOST_CHECK(r.get_obj().find_value("complete").get_bool() == false); r = CallRPC(std::string("signrawtransactionwithkey ")+notsigned+" ["+privkey1+","+privkey2+"] "+prevout); - BOOST_CHECK(find_value(r.get_obj(), "complete").get_bool() == true); + BOOST_CHECK(r.get_obj().find_value("complete").get_bool() == true); } BOOST_AUTO_TEST_CASE(rpc_createraw_op_return) @@ -278,6 +300,7 @@ BOOST_AUTO_TEST_CASE(rpc_parse_monetary_values) BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.00000001000000")), 1LL); //should pass, cut trailing 0 BOOST_CHECK_THROW(AmountFromValue(ValueFromString("19e-9")), UniValue); //should fail BOOST_CHECK_EQUAL(AmountFromValue(ValueFromString("0.19e-6")), 19); //should pass, leading 0 is present + BOOST_CHECK_EXCEPTION(AmountFromValue(".19e-6"), UniValue, HasJSON(R"({"code":-3,"message":"Invalid amount"})")); //should fail, no leading 0 BOOST_CHECK_THROW(AmountFromValue(ValueFromString("92233720368.54775808")), UniValue); //overflow error BOOST_CHECK_THROW(AmountFromValue(ValueFromString("1e+11")), UniValue); //overflow error @@ -285,36 +308,6 @@ BOOST_AUTO_TEST_CASE(rpc_parse_monetary_values) BOOST_CHECK_THROW(AmountFromValue(ValueFromString("93e+9")), UniValue); //overflow error } -BOOST_AUTO_TEST_CASE(json_parse_errors) -{ - // Valid - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue("1.0").get_real(), 1.0); - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue("true").get_bool(), true); - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue("[false]")[0].get_bool(), false); - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue("{\"a\": true}")["a"].get_bool(), true); - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue("{\"1\": \"true\"}")["1"].get_str(), "true"); - // Valid, with leading or trailing whitespace - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue(" 1.0").get_real(), 1.0); - BOOST_CHECK_EQUAL(ParseNonRFCJSONValue("1.0 ").get_real(), 1.0); - - BOOST_CHECK_THROW(AmountFromValue(ParseNonRFCJSONValue(".19e-6")), std::runtime_error); //should fail, missing leading 0, therefore invalid JSON - BOOST_CHECK_EQUAL(AmountFromValue(ParseNonRFCJSONValue("0.00000000000000000000000000000000000001e+30 ")), 1); - // Invalid, initial garbage - BOOST_CHECK_THROW(ParseNonRFCJSONValue("[1.0"), std::runtime_error); - BOOST_CHECK_THROW(ParseNonRFCJSONValue("a1.0"), std::runtime_error); - // Invalid, trailing garbage - BOOST_CHECK_THROW(ParseNonRFCJSONValue("1.0sds"), std::runtime_error); - BOOST_CHECK_THROW(ParseNonRFCJSONValue("1.0]"), std::runtime_error); - // Invalid, keys have to be names - BOOST_CHECK_THROW(ParseNonRFCJSONValue("{1: \"true\"}"), std::runtime_error); - BOOST_CHECK_THROW(ParseNonRFCJSONValue("{true: 1}"), std::runtime_error); - BOOST_CHECK_THROW(ParseNonRFCJSONValue("{[1]: 1}"), std::runtime_error); - BOOST_CHECK_THROW(ParseNonRFCJSONValue("{{\"a\": \"a\"}: 1}"), std::runtime_error); - // BTC addresses should fail parsing - BOOST_CHECK_THROW(ParseNonRFCJSONValue("175tWpb8K1S7NmH4Zx6rewF9WQrcZv245W"), std::runtime_error); - BOOST_CHECK_THROW(ParseNonRFCJSONValue("3J98t1WpEZ73CNmQviecrnyiWrnqRhWNL"), std::runtime_error); -} - BOOST_AUTO_TEST_CASE(rpc_ban) { BOOST_CHECK_NO_THROW(CallRPC(std::string("clearbanned"))); @@ -325,7 +318,7 @@ BOOST_AUTO_TEST_CASE(rpc_ban) BOOST_CHECK_NO_THROW(r = CallRPC(std::string("listbanned"))); UniValue ar = r.get_array(); UniValue o1 = ar[0].get_obj(); - UniValue adr = find_value(o1, "address"); + UniValue adr = o1.find_value("address"); BOOST_CHECK_EQUAL(adr.get_str(), "127.0.0.0/32"); BOOST_CHECK_NO_THROW(CallRPC(std::string("setban 127.0.0.0 remove"))); BOOST_CHECK_NO_THROW(r = CallRPC(std::string("listbanned"))); @@ -336,8 +329,8 @@ BOOST_AUTO_TEST_CASE(rpc_ban) BOOST_CHECK_NO_THROW(r = CallRPC(std::string("listbanned"))); ar = r.get_array(); o1 = ar[0].get_obj(); - adr = find_value(o1, "address"); - int64_t banned_until{find_value(o1, "banned_until").getInt<int64_t>()}; + adr = o1.find_value("address"); + int64_t banned_until{o1.find_value("banned_until").getInt<int64_t>()}; BOOST_CHECK_EQUAL(adr.get_str(), "127.0.0.0/24"); BOOST_CHECK_EQUAL(banned_until, 9907731200); // absolute time check @@ -351,11 +344,11 @@ BOOST_AUTO_TEST_CASE(rpc_ban) BOOST_CHECK_NO_THROW(r = CallRPC(std::string("listbanned"))); ar = r.get_array(); o1 = ar[0].get_obj(); - adr = find_value(o1, "address"); - banned_until = find_value(o1, "banned_until").getInt<int64_t>(); - const int64_t ban_created{find_value(o1, "ban_created").getInt<int64_t>()}; - const int64_t ban_duration{find_value(o1, "ban_duration").getInt<int64_t>()}; - const int64_t time_remaining{find_value(o1, "time_remaining").getInt<int64_t>()}; + adr = o1.find_value("address"); + banned_until = o1.find_value("banned_until").getInt<int64_t>(); + const int64_t ban_created{o1.find_value("ban_created").getInt<int64_t>()}; + const int64_t ban_duration{o1.find_value("ban_duration").getInt<int64_t>()}; + const int64_t time_remaining{o1.find_value("time_remaining").getInt<int64_t>()}; BOOST_CHECK_EQUAL(adr.get_str(), "127.0.0.0/24"); BOOST_CHECK_EQUAL(banned_until, time_remaining_expected + now.count()); BOOST_CHECK_EQUAL(ban_duration, banned_until - ban_created); @@ -385,7 +378,7 @@ BOOST_AUTO_TEST_CASE(rpc_ban) BOOST_CHECK_NO_THROW(r = CallRPC(std::string("listbanned"))); ar = r.get_array(); o1 = ar[0].get_obj(); - adr = find_value(o1, "address"); + adr = o1.find_value("address"); BOOST_CHECK_EQUAL(adr.get_str(), "fe80::202:b3ff:fe1e:8329/128"); BOOST_CHECK_NO_THROW(CallRPC(std::string("clearbanned"))); @@ -393,7 +386,7 @@ BOOST_AUTO_TEST_CASE(rpc_ban) BOOST_CHECK_NO_THROW(r = CallRPC(std::string("listbanned"))); ar = r.get_array(); o1 = ar[0].get_obj(); - adr = find_value(o1, "address"); + adr = o1.find_value("address"); BOOST_CHECK_EQUAL(adr.get_str(), "2001:db8::/30"); BOOST_CHECK_NO_THROW(CallRPC(std::string("clearbanned"))); @@ -401,7 +394,7 @@ BOOST_AUTO_TEST_CASE(rpc_ban) BOOST_CHECK_NO_THROW(r = CallRPC(std::string("listbanned"))); ar = r.get_array(); o1 = ar[0].get_obj(); - adr = find_value(o1, "address"); + adr = o1.find_value("address"); BOOST_CHECK_EQUAL(adr.get_str(), "2001:4d48:ac57:400:cacf:e9ff:fe1d:9c63/128"); } @@ -506,6 +499,53 @@ BOOST_AUTO_TEST_CASE(rpc_getblockstats_calculate_percentiles_by_weight) } } +// Make sure errors are triggered appropriately if parameters have the same names. +BOOST_AUTO_TEST_CASE(check_dup_param_names) +{ + enum ParamType { POSITIONAL, NAMED, NAMED_ONLY }; + auto make_rpc = [](std::vector<std::tuple<std::string, ParamType>> param_names) { + std::vector<RPCArg> params; + std::vector<RPCArg> options; + auto push_options = [&] { if (!options.empty()) params.emplace_back(RPCArg{strprintf("options%i", params.size()), RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", std::move(options)}); }; + for (auto& [param_name, param_type] : param_names) { + if (param_type == POSITIONAL) { + push_options(); + params.emplace_back(std::move(param_name), RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "description"); + } else { + options.emplace_back(std::move(param_name), RPCArg::Type::NUM, RPCArg::Optional::OMITTED, "description", RPCArgOptions{.also_positional = param_type == NAMED}); + } + } + push_options(); + return RPCHelpMan{"method_name", "description", params, RPCResults{}, RPCExamples{""}}; + }; + + // No errors if parameter names are unique. + make_rpc({{"p1", POSITIONAL}, {"p2", POSITIONAL}}); + make_rpc({{"p1", POSITIONAL}, {"p2", NAMED}}); + make_rpc({{"p1", POSITIONAL}, {"p2", NAMED_ONLY}}); + make_rpc({{"p1", NAMED}, {"p2", POSITIONAL}}); + make_rpc({{"p1", NAMED}, {"p2", NAMED}}); + make_rpc({{"p1", NAMED}, {"p2", NAMED_ONLY}}); + make_rpc({{"p1", NAMED_ONLY}, {"p2", POSITIONAL}}); + make_rpc({{"p1", NAMED_ONLY}, {"p2", NAMED}}); + make_rpc({{"p1", NAMED_ONLY}, {"p2", NAMED_ONLY}}); + + // Error if parameters names are duplicates, unless one parameter is + // positional and the other is named and .also_positional is true. + BOOST_CHECK_THROW(make_rpc({{"p1", POSITIONAL}, {"p1", POSITIONAL}}), NonFatalCheckError); + make_rpc({{"p1", POSITIONAL}, {"p1", NAMED}}); + BOOST_CHECK_THROW(make_rpc({{"p1", POSITIONAL}, {"p1", NAMED_ONLY}}), NonFatalCheckError); + make_rpc({{"p1", NAMED}, {"p1", POSITIONAL}}); + BOOST_CHECK_THROW(make_rpc({{"p1", NAMED}, {"p1", NAMED}}), NonFatalCheckError); + BOOST_CHECK_THROW(make_rpc({{"p1", NAMED}, {"p1", NAMED_ONLY}}), NonFatalCheckError); + BOOST_CHECK_THROW(make_rpc({{"p1", NAMED_ONLY}, {"p1", POSITIONAL}}), NonFatalCheckError); + BOOST_CHECK_THROW(make_rpc({{"p1", NAMED_ONLY}, {"p1", NAMED}}), NonFatalCheckError); + BOOST_CHECK_THROW(make_rpc({{"p1", NAMED_ONLY}, {"p1", NAMED_ONLY}}), NonFatalCheckError); + + // Make sure duplicate aliases are detected too. + BOOST_CHECK_THROW(make_rpc({{"p1", POSITIONAL}, {"p2|p1", NAMED_ONLY}}), NonFatalCheckError); +} + BOOST_AUTO_TEST_CASE(help_example) { // test different argument types diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index d8898743b0..c89f2c1f5b 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -5,6 +5,7 @@ #include <test/data/script_tests.json.h> #include <test/data/bip341_wallet_vectors.json.h> +#include <common/system.h> #include <core_io.h> #include <key.h> #include <rpc/util.h> @@ -20,7 +21,6 @@ #include <test/util/transaction_utils.h> #include <util/fs.h> #include <util/strencodings.h> -#include <util/system.h> #if defined(HAVE_CONSENSUS_LIB) #include <script/bitcoinconsensus.h> diff --git a/src/test/settings_tests.cpp b/src/test/settings_tests.cpp index 604eb48df6..c24921bf9b 100644 --- a/src/test/settings_tests.cpp +++ b/src/test/settings_tests.cpp @@ -2,16 +2,16 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <util/settings.h> +#include <common/settings.h> #include <test/util/setup_common.h> #include <test/util/str.h> -#include <util/fs.h> - #include <boost/test/unit_test.hpp> #include <common/args.h> #include <univalue.h> +#include <util/chaintype.h> +#include <util/fs.h> #include <util/strencodings.h> #include <util/string.h> @@ -21,20 +21,20 @@ #include <system_error> #include <vector> -inline bool operator==(const util::SettingsValue& a, const util::SettingsValue& b) +inline bool operator==(const common::SettingsValue& a, const common::SettingsValue& b) { return a.write() == b.write(); } -inline std::ostream& operator<<(std::ostream& os, const util::SettingsValue& value) +inline std::ostream& operator<<(std::ostream& os, const common::SettingsValue& value) { os << value.write(); return os; } -inline std::ostream& operator<<(std::ostream& os, const std::pair<std::string, util::SettingsValue>& kv) +inline std::ostream& operator<<(std::ostream& os, const std::pair<std::string, common::SettingsValue>& kv) { - util::SettingsValue out(util::SettingsValue::VOBJ); + common::SettingsValue out(common::SettingsValue::VOBJ); out.__pushKV(kv.first, kv.second); os << out.write(); return os; @@ -60,7 +60,7 @@ BOOST_AUTO_TEST_CASE(ReadWrite) "null": null })"); - std::map<std::string, util::SettingsValue> expected{ + std::map<std::string, common::SettingsValue> expected{ {"string", "string"}, {"num", 5}, {"bool", true}, @@ -68,15 +68,15 @@ BOOST_AUTO_TEST_CASE(ReadWrite) }; // Check file read. - std::map<std::string, util::SettingsValue> values; + std::map<std::string, common::SettingsValue> values; std::vector<std::string> errors; - BOOST_CHECK(util::ReadSettings(path, values, errors)); + BOOST_CHECK(common::ReadSettings(path, values, errors)); BOOST_CHECK_EQUAL_COLLECTIONS(values.begin(), values.end(), expected.begin(), expected.end()); BOOST_CHECK(errors.empty()); // Check no errors if file doesn't exist. fs::remove(path); - BOOST_CHECK(util::ReadSettings(path, values, errors)); + BOOST_CHECK(common::ReadSettings(path, values, errors)); BOOST_CHECK(values.empty()); BOOST_CHECK(errors.empty()); @@ -85,29 +85,29 @@ BOOST_AUTO_TEST_CASE(ReadWrite) "dupe": "string", "dupe": "dupe" })"); - BOOST_CHECK(!util::ReadSettings(path, values, errors)); + BOOST_CHECK(!common::ReadSettings(path, values, errors)); std::vector<std::string> dup_keys = {strprintf("Found duplicate key dupe in settings file %s", fs::PathToString(path))}; BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), dup_keys.begin(), dup_keys.end()); BOOST_CHECK(values.empty()); // Check non-kv json files not allowed WriteText(path, R"("non-kv")"); - BOOST_CHECK(!util::ReadSettings(path, values, errors)); + BOOST_CHECK(!common::ReadSettings(path, values, errors)); std::vector<std::string> non_kv = {strprintf("Found non-object value \"non-kv\" in settings file %s", fs::PathToString(path))}; BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), non_kv.begin(), non_kv.end()); // Check invalid json not allowed WriteText(path, R"(invalid json)"); - BOOST_CHECK(!util::ReadSettings(path, values, errors)); + BOOST_CHECK(!common::ReadSettings(path, values, errors)); std::vector<std::string> fail_parse = {strprintf("Unable to parse settings file %s", fs::PathToString(path))}; BOOST_CHECK_EQUAL_COLLECTIONS(errors.begin(), errors.end(), fail_parse.begin(), fail_parse.end()); } //! Check settings struct contents against expected json strings. -static void CheckValues(const util::Settings& settings, const std::string& single_val, const std::string& list_val) +static void CheckValues(const common::Settings& settings, const std::string& single_val, const std::string& list_val) { - util::SettingsValue single_value = GetSetting(settings, "section", "name", false, false, false); - util::SettingsValue list_value(util::SettingsValue::VARR); + common::SettingsValue single_value = GetSetting(settings, "section", "name", false, false, false); + common::SettingsValue list_value(common::SettingsValue::VARR); for (const auto& item : GetSettingsList(settings, "section", "name", false)) { list_value.push_back(item); } @@ -118,7 +118,7 @@ static void CheckValues(const util::Settings& settings, const std::string& singl // Simple settings merge test case. BOOST_AUTO_TEST_CASE(Simple) { - util::Settings settings; + common::Settings settings; settings.command_line_options["name"].push_back("val1"); settings.command_line_options["name"].push_back("val2"); settings.ro_config["section"]["name"].push_back(2); @@ -126,7 +126,7 @@ BOOST_AUTO_TEST_CASE(Simple) // The last given arg takes precedence when specified via commandline. CheckValues(settings, R"("val2")", R"(["val1","val2",2])"); - util::Settings settings2; + common::Settings settings2; settings2.ro_config["section"]["name"].push_back("val2"); settings2.ro_config["section"]["name"].push_back("val3"); @@ -140,7 +140,7 @@ BOOST_AUTO_TEST_CASE(Simple) // its default value. BOOST_AUTO_TEST_CASE(NullOverride) { - util::Settings settings; + common::Settings settings; settings.command_line_options["name"].push_back("value"); BOOST_CHECK_EQUAL(R"("value")", GetSetting(settings, "section", "name", false, false, false).write().c_str()); settings.forced_settings["name"] = {}; @@ -190,16 +190,16 @@ BOOST_FIXTURE_TEST_CASE(Merge, MergeTestingSetup) if (!out_file) throw std::system_error(errno, std::generic_category(), "fopen failed"); } - const std::string& network = CBaseChainParams::MAIN; + const std::string& network = ChainTypeToString(ChainType::MAIN); ForEachMergeSetup([&](const ActionList& arg_actions, const ActionList& conf_actions, bool force_set, bool ignore_default_section_config) { std::string desc; int value_suffix = 0; - util::Settings settings; + common::Settings settings; const std::string& name = ignore_default_section_config ? "wallet" : "server"; auto push_values = [&](Action action, const char* value_prefix, const std::string& name_prefix, - std::vector<util::SettingsValue>& dest) { + std::vector<common::SettingsValue>& dest) { if (action == SET || action == SECTION_SET) { for (int i = 0; i < 2; ++i) { dest.push_back(value_prefix + ToString(++value_suffix)); @@ -225,7 +225,7 @@ BOOST_FIXTURE_TEST_CASE(Merge, MergeTestingSetup) } desc += " || "; - desc += GetSetting(settings, network, name, ignore_default_section_config, /*ignore_nonpersistent=*/false, /*get_chain_name=*/false).write(); + desc += GetSetting(settings, network, name, ignore_default_section_config, /*ignore_nonpersistent=*/false, /*get_chain_type=*/false).write(); desc += " |"; for (const auto& s : GetSettingsList(settings, network, name, ignore_default_section_config)) { desc += " "; diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index e2d11afa6a..68ef719c71 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <consensus/tx_check.h> #include <consensus/validation.h> #include <hash.h> @@ -14,7 +15,6 @@ #include <test/util/random.h> #include <test/util/setup_common.h> #include <util/strencodings.h> -#include <util/system.h> #include <version.h> #include <iostream> diff --git a/src/test/sock_tests.cpp b/src/test/sock_tests.cpp index 9e6f73745e..26ee724bf8 100644 --- a/src/test/sock_tests.cpp +++ b/src/test/sock_tests.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <compat/compat.h> #include <test/util/setup_common.h> #include <util/sock.h> -#include <util/system.h> #include <util/threadinterrupt.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index a0392570f1..55e4f200b1 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -463,7 +463,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) size_t find = currentPos + InsecureRandRange(8); if (find >= fileSize) find = fileSize - 1; - bf.FindByte(uint8_t(find)); + bf.FindByte(std::byte(find)); // The value at each offset is the offset. BOOST_CHECK_EQUAL(bf.GetPos(), find); currentPos = find; diff --git a/src/test/system_tests.cpp b/src/test/system_tests.cpp index c0a2566959..7ce350b84b 100644 --- a/src/test/system_tests.cpp +++ b/src/test/system_tests.cpp @@ -52,7 +52,7 @@ BOOST_AUTO_TEST_CASE(run_command) const UniValue result = RunCommandParseJSON("echo \"{\"success\": true}\""); #endif BOOST_CHECK(result.isObject()); - const UniValue& success = find_value(result, "success"); + const UniValue& success = result.find_value("success"); BOOST_CHECK(!success.isNull()); BOOST_CHECK_EQUAL(success.get_bool(), true); } @@ -106,7 +106,7 @@ BOOST_AUTO_TEST_CASE(run_command) { const UniValue result = RunCommandParseJSON("cat", "{\"success\": true}"); BOOST_CHECK(result.isObject()); - const UniValue& success = find_value(result, "success"); + const UniValue& success = result.find_value("success"); BOOST_CHECK(!success.isNull()); BOOST_CHECK_EQUAL(success.get_bool(), true); } diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index b9bfa65c0a..b666517ae2 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -32,10 +32,10 @@ BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) BOOST_REQUIRE(txindex.Start()); // Allow tx index to catch up with the block index. - constexpr int64_t timeout_ms = 10 * 1000; - int64_t time_start = GetTimeMillis(); + constexpr auto timeout{10s}; + const auto time_start{SteadyClock::now()}; while (!txindex.BlockUntilSyncedToCurrentChain()) { - BOOST_REQUIRE(time_start + timeout_ms > GetTimeMillis()); + BOOST_REQUIRE(time_start + timeout > SteadyClock::now()); UninterruptibleSleep(std::chrono::milliseconds{100}); } diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 8cdea3890e..8f8628169f 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -9,13 +9,14 @@ #include <script/standard.h> #include <test/util/setup_common.h> #include <txmempool.h> +#include <util/chaintype.h> #include <validation.h> #include <boost/test/unit_test.hpp> struct Dersig100Setup : public TestChain100Setup { Dersig100Setup() - : TestChain100Setup{CBaseChainParams::REGTEST, {"-testactivationheight=dersig@102"}} {} + : TestChain100Setup{ChainType::REGTEST, {"-testactivationheight=dersig@102"}} {} }; bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, diff --git a/src/test/util/blockfilter.cpp b/src/test/util/blockfilter.cpp index ec703c6a7b..a806844e34 100644 --- a/src/test/util/blockfilter.cpp +++ b/src/test/util/blockfilter.cpp @@ -8,20 +8,19 @@ #include <node/blockstorage.h> #include <validation.h> -using node::ReadBlockFromDisk; -using node::UndoReadFromDisk; +using node::BlockManager; -bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex* block_index, BlockFilter& filter) +bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex& block_index, BlockFilter& filter, const BlockManager& blockman) { LOCK(::cs_main); CBlock block; - if (!ReadBlockFromDisk(block, block_index->GetBlockPos(), Params().GetConsensus())) { + if (!blockman.ReadBlockFromDisk(block, block_index.GetBlockPos())) { return false; } CBlockUndo block_undo; - if (block_index->nHeight > 0 && !UndoReadFromDisk(block_undo, block_index)) { + if (block_index.nHeight > 0 && !blockman.UndoReadFromDisk(block_undo, block_index)) { return false; } diff --git a/src/test/util/blockfilter.h b/src/test/util/blockfilter.h index 79d11dcad8..789ce5d3aa 100644 --- a/src/test/util/blockfilter.h +++ b/src/test/util/blockfilter.h @@ -6,8 +6,12 @@ #define BITCOIN_TEST_UTIL_BLOCKFILTER_H #include <blockfilter.h> + class CBlockIndex; +namespace node { +class BlockManager; +} -bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex* block_index, BlockFilter& filter); +bool ComputeFilter(BlockFilterType filter_type, const CBlockIndex& block_index, BlockFilter& filter, const node::BlockManager& blockman); #endif // BITCOIN_TEST_UTIL_BLOCKFILTER_H diff --git a/src/test/util/mining.cpp b/src/test/util/mining.cpp index 0df1db84c4..51f4b89512 100644 --- a/src/test/util/mining.cpp +++ b/src/test/util/mining.cpp @@ -6,19 +6,22 @@ #include <chainparams.h> #include <consensus/merkle.h> +#include <consensus/validation.h> #include <key_io.h> #include <node/context.h> #include <pow.h> +#include <primitives/transaction.h> #include <script/standard.h> #include <test/util/script.h> #include <util/check.h> #include <validation.h> +#include <validationinterface.h> #include <versionbits.h> using node::BlockAssembler; using node::NodeContext; -CTxIn generatetoaddress(const NodeContext& node, const std::string& address) +COutPoint generatetoaddress(const NodeContext& node, const std::string& address) { const auto dest = DecodeDestination(address); assert(IsValidDestination(dest)); @@ -58,19 +61,52 @@ std::vector<std::shared_ptr<CBlock>> CreateBlockChain(size_t total_height, const return ret; } -CTxIn MineBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) +COutPoint MineBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey) { auto block = PrepareBlock(node, coinbase_scriptPubKey); + auto valid = MineBlock(node, block); + assert(!valid.IsNull()); + return valid; +} + +struct BlockValidationStateCatcher : public CValidationInterface { + const uint256 m_hash; + std::optional<BlockValidationState> m_state; + BlockValidationStateCatcher(const uint256& hash) + : m_hash{hash}, + m_state{} {} + +protected: + void BlockChecked(const CBlock& block, const BlockValidationState& state) override + { + if (block.GetHash() != m_hash) return; + m_state = state; + } +}; + +COutPoint MineBlock(const NodeContext& node, std::shared_ptr<CBlock>& block) +{ while (!CheckProofOfWork(block->GetHash(), block->nBits, Params().GetConsensus())) { ++block->nNonce; assert(block->nNonce); } - bool processed{Assert(node.chainman)->ProcessNewBlock(block, true, true, nullptr)}; - assert(processed); - - return CTxIn{block->vtx[0]->GetHash(), 0}; + auto& chainman{*Assert(node.chainman)}; + const auto old_height = WITH_LOCK(chainman.GetMutex(), return chainman.ActiveHeight()); + bool new_block; + BlockValidationStateCatcher bvsc{block->GetHash()}; + RegisterValidationInterface(&bvsc); + const bool processed{chainman.ProcessNewBlock(block, true, true, &new_block)}; + const bool duplicate{!new_block && processed}; + assert(!duplicate); + UnregisterValidationInterface(&bvsc); + SyncWithValidationInterfaceQueue(); + const bool was_valid{bvsc.m_state && bvsc.m_state->IsValid()}; + assert(old_height + was_valid == WITH_LOCK(chainman.GetMutex(), return chainman.ActiveHeight())); + + if (was_valid) return {block->vtx[0]->GetHash(), 0}; + return {}; } std::shared_ptr<CBlock> PrepareBlock(const NodeContext& node, const CScript& coinbase_scriptPubKey, diff --git a/src/test/util/mining.h b/src/test/util/mining.h index 70b1f7b3fb..3f071257f1 100644 --- a/src/test/util/mining.h +++ b/src/test/util/mining.h @@ -13,8 +13,8 @@ class CBlock; class CChainParams; +class COutPoint; class CScript; -class CTxIn; namespace node { struct NodeContext; } // namespace node @@ -23,7 +23,13 @@ struct NodeContext; std::vector<std::shared_ptr<CBlock>> CreateBlockChain(size_t total_height, const CChainParams& params); /** Returns the generated coin */ -CTxIn MineBlock(const node::NodeContext&, const CScript& coinbase_scriptPubKey); +COutPoint MineBlock(const node::NodeContext&, const CScript& coinbase_scriptPubKey); + +/** + * Returns the generated coin (or Null if the block was invalid). + * It is recommended to call RegenerateCommitments before mining the block to avoid merkle tree mismatches. + **/ +COutPoint MineBlock(const node::NodeContext&, std::shared_ptr<CBlock>& block); /** Prepare a block to be mined */ std::shared_ptr<CBlock> PrepareBlock(const node::NodeContext&, const CScript& coinbase_scriptPubKey); @@ -31,6 +37,6 @@ std::shared_ptr<CBlock> PrepareBlock(const node::NodeContext& node, const CScrip const node::BlockAssembler::Options& assembler_options); /** RPC-like helper function, returns the generated coin */ -CTxIn generatetoaddress(const node::NodeContext&, const std::string& address); +COutPoint generatetoaddress(const node::NodeContext&, const std::string& address); #endif // BITCOIN_TEST_UTIL_MINING_H diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index ddc1e7ab37..483404779e 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -9,6 +9,7 @@ #include <addrman.h> #include <banman.h> #include <chainparams.h> +#include <common/system.h> #include <common/url.h> #include <consensus/consensus.h> #include <consensus/params.h> @@ -23,6 +24,7 @@ #include <node/blockstorage.h> #include <node/chainstate.h> #include <node/context.h> +#include <node/kernel_notifications.h> #include <node/mempool_args.h> #include <node/miner.h> #include <node/validation_cache_args.h> @@ -42,9 +44,9 @@ #include <timedata.h> #include <txdb.h> #include <txmempool.h> +#include <util/chaintype.h> #include <util/strencodings.h> #include <util/string.h> -#include <util/system.h> #include <util/thread.h> #include <util/threadnames.h> #include <util/time.h> @@ -61,7 +63,9 @@ using kernel::ValidationCacheSizes; using node::ApplyArgsManOptions; using node::BlockAssembler; +using node::BlockManager; using node::CalculateCacheSizes; +using node::KernelNotifications; using node::LoadChainstate; using node::RegenerateCommitments; using node::VerifyLoadedChainstate; @@ -98,7 +102,7 @@ std::ostream& operator<<(std::ostream& os, const uint256& num) return os; } -BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args) +BasicTestingSetup::BasicTestingSetup(const ChainType chainType, const std::vector<const char*>& extra_args) : m_path_root{fs::temp_directory_path() / "test_common_" PACKAGE_NAME / g_insecure_rand_ctx_temp_path.rand256().ToString()}, m_args{} { @@ -132,7 +136,7 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve throw std::runtime_error{error}; } } - SelectParams(chainName); + SelectParams(chainType); SeedInsecureRand(); if (G_TEST_LOG_FUN) LogInstance().PushBackCallback(G_TEST_LOG_FUN); InitLogging(*m_node.args); @@ -153,6 +157,7 @@ BasicTestingSetup::BasicTestingSetup(const std::string& chainName, const std::ve noui_connect(); noui_connected = true; } + node::g_indexes_ready_to_sync = true; } BasicTestingSetup::~BasicTestingSetup() @@ -163,8 +168,8 @@ BasicTestingSetup::~BasicTestingSetup() gArgs.ClearArgs(); } -ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::vector<const char*>& extra_args) - : BasicTestingSetup(chainName, extra_args) +ChainTestingSetup::ChainTestingSetup(const ChainType chainType, const std::vector<const char*>& extra_args) + : BasicTestingSetup(chainType, extra_args) { const CChainParams& chainparams = Params(); @@ -179,13 +184,20 @@ ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::ve m_cache_sizes = CalculateCacheSizes(m_args); + m_node.notifications = std::make_unique<KernelNotifications>(); + const ChainstateManager::Options chainman_opts{ .chainparams = chainparams, .datadir = m_args.GetDataDirNet(), .adjusted_time_callback = GetAdjustedTime, .check_block_index = true, + .notifications = *m_node.notifications, + }; + const BlockManager::Options blockman_opts{ + .chainparams = chainman_opts.chainparams, + .blocks_dir = m_args.GetBlocksDirPath(), }; - m_node.chainman = std::make_unique<ChainstateManager>(chainman_opts, node::BlockManager::Options{}); + m_node.chainman = std::make_unique<ChainstateManager>(chainman_opts, blockman_opts); m_node.chainman->m_blockman.m_block_tree_db = std::make_unique<CBlockTreeDB>(DBParams{ .path = m_args.GetDataDirNet() / "blocks" / "index", .cache_bytes = static_cast<size_t>(m_cache_sizes.block_tree_db), @@ -211,7 +223,7 @@ ChainTestingSetup::~ChainTestingSetup() m_node.chainman.reset(); } -void TestingSetup::LoadVerifyActivateChainstate() +void ChainTestingSetup::LoadVerifyActivateChainstate() { auto& chainman{*Assert(m_node.chainman)}; node::ChainstateLoadOptions options; @@ -237,14 +249,14 @@ void TestingSetup::LoadVerifyActivateChainstate() } TestingSetup::TestingSetup( - const std::string& chainName, + const ChainType chainType, const std::vector<const char*>& extra_args, const bool coins_db_in_memory, const bool block_tree_db_in_memory) - : ChainTestingSetup(chainName, extra_args), - m_coins_db_in_memory(coins_db_in_memory), - m_block_tree_db_in_memory(block_tree_db_in_memory) + : ChainTestingSetup(chainType, extra_args) { + m_coins_db_in_memory = coins_db_in_memory; + m_block_tree_db_in_memory = block_tree_db_in_memory; // Ideally we'd move all the RPC tests to the functional testing framework // instead of unit tests, but for now we need these here. RegisterAllCoreRPCCommands(tableRPC); @@ -268,11 +280,11 @@ TestingSetup::TestingSetup( } TestChain100Setup::TestChain100Setup( - const std::string& chain_name, + const ChainType chain_type, const std::vector<const char*>& extra_args, const bool coins_db_in_memory, const bool block_tree_db_in_memory) - : TestingSetup{CBaseChainParams::REGTEST, extra_args, coins_db_in_memory, block_tree_db_in_memory} + : TestingSetup{ChainType::REGTEST, extra_args, coins_db_in_memory, block_tree_db_in_memory} { SetMockTime(1598887952); constexpr std::array<unsigned char, 32> vchKey = { diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 750f010fb0..b7429df02c 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -5,15 +5,15 @@ #ifndef BITCOIN_TEST_UTIL_SETUP_COMMON_H #define BITCOIN_TEST_UTIL_SETUP_COMMON_H -#include <chainparamsbase.h> #include <common/args.h> #include <key.h> #include <node/caches.h> -#include <node/context.h> +#include <node/context.h> // IWYU pragma: export #include <primitives/transaction.h> #include <pubkey.h> #include <random.h> #include <stdexcept> +#include <util/chaintype.h> #include <util/check.h> #include <util/fs.h> #include <util/string.h> @@ -80,7 +80,7 @@ static constexpr CAmount CENT{1000000}; struct BasicTestingSetup { node::NodeContext m_node; // keep as first member to be destructed last - explicit BasicTestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); + explicit BasicTestingSetup(const ChainType chainType = ChainType::MAIN, const std::vector<const char*>& extra_args = {}); ~BasicTestingSetup(); const fs::path m_path_root; @@ -93,21 +93,21 @@ struct BasicTestingSetup { */ struct ChainTestingSetup : public BasicTestingSetup { node::CacheSizes m_cache_sizes{}; + bool m_coins_db_in_memory{true}; + bool m_block_tree_db_in_memory{true}; - explicit ChainTestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); + explicit ChainTestingSetup(const ChainType chainType = ChainType::MAIN, const std::vector<const char*>& extra_args = {}); ~ChainTestingSetup(); + + // Supplies a chainstate, if one is needed + void LoadVerifyActivateChainstate(); }; /** Testing setup that configures a complete environment. */ struct TestingSetup : public ChainTestingSetup { - bool m_coins_db_in_memory{true}; - bool m_block_tree_db_in_memory{true}; - - void LoadVerifyActivateChainstate(); - explicit TestingSetup( - const std::string& chainName = CBaseChainParams::MAIN, + const ChainType chainType = ChainType::MAIN, const std::vector<const char*>& extra_args = {}, const bool coins_db_in_memory = true, const bool block_tree_db_in_memory = true); @@ -116,7 +116,7 @@ struct TestingSetup : public ChainTestingSetup { /** Identical to TestingSetup, but chain set to regtest */ struct RegTestingSetup : public TestingSetup { RegTestingSetup() - : TestingSetup{CBaseChainParams::REGTEST} {} + : TestingSetup{ChainType::REGTEST} {} }; class CBlock; @@ -128,7 +128,7 @@ class CScript; */ struct TestChain100Setup : public TestingSetup { TestChain100Setup( - const std::string& chain_name = CBaseChainParams::REGTEST, + const ChainType chain_type = ChainType::REGTEST, const std::vector<const char*>& extra_args = {}, const bool coins_db_in_memory = true, const bool block_tree_db_in_memory = true); @@ -206,7 +206,7 @@ struct TestChain100Setup : public TestingSetup { * be used in "hot loops", for example fuzzing or benchmarking. */ template <class T = const BasicTestingSetup> -std::unique_ptr<T> MakeNoLogFileContext(const std::string& chain_name = CBaseChainParams::REGTEST, const std::vector<const char*>& extra_args = {}) +std::unique_ptr<T> MakeNoLogFileContext(const ChainType chain_type = ChainType::REGTEST, const std::vector<const char*>& extra_args = {}) { const std::vector<const char*> arguments = Cat( { @@ -215,7 +215,7 @@ std::unique_ptr<T> MakeNoLogFileContext(const std::string& chain_name = CBaseCha }, extra_args); - return std::make_unique<T>(chain_name, arguments); + return std::make_unique<T>(chain_type, arguments); } CBlock getBlock13b8a(); diff --git a/src/test/util/txmempool.cpp b/src/test/util/txmempool.cpp index 1873cf5ec8..4797d9c310 100644 --- a/src/test/util/txmempool.cpp +++ b/src/test/util/txmempool.cpp @@ -22,8 +22,8 @@ CTxMemPool::Options MemPoolOptionsForTest(const NodeContext& node) // chainparams.DefaultConsistencyChecks for tests .check_ratio = 1, }; - const auto err{ApplyArgsManOptions(*node.args, ::Params(), mempool_opts)}; - Assert(!err); + const auto result{ApplyArgsManOptions(*node.args, ::Params(), mempool_opts)}; + Assert(result); return mempool_opts; } diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 812737429d..26677bfa55 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -1687,7 +1687,7 @@ BOOST_AUTO_TEST_CASE(message_hash) BOOST_AUTO_TEST_CASE(remove_prefix) { - BOOST_CHECK_EQUAL(RemovePrefix("./util/system.h", "./"), "util/system.h"); + BOOST_CHECK_EQUAL(RemovePrefix("./common/system.h", "./"), "common/system.h"); BOOST_CHECK_EQUAL(RemovePrefixView("foo", "foo"), ""); BOOST_CHECK_EQUAL(RemovePrefix("foo", "fo"), "o"); BOOST_CHECK_EQUAL(RemovePrefixView("foo", "f"), "oo"); diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index edc7e4b70a..b797de46af 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -4,6 +4,7 @@ // #include <chainparams.h> #include <consensus/validation.h> +#include <node/kernel_notifications.h> #include <node/utxo_snapshot.h> #include <random.h> #include <rpc/blockchain.h> @@ -22,6 +23,8 @@ #include <boost/test/unit_test.hpp> +using node::BlockManager; +using node::KernelNotifications; using node::SnapshotMetadata; BOOST_FIXTURE_TEST_SUITE(validation_chainstatemanager_tests, ChainTestingSetup) @@ -181,7 +184,7 @@ struct SnapshotTestSetup : TestChain100Setup { { LOCK(::cs_main); BOOST_CHECK(!chainman.IsSnapshotValidated()); - BOOST_CHECK(!node::FindSnapshotChainstateDir()); + BOOST_CHECK(!node::FindSnapshotChainstateDir(chainman.m_options.datadir)); } size_t initial_size; @@ -231,7 +234,7 @@ struct SnapshotTestSetup : TestChain100Setup { auto_infile >> coin; })); - BOOST_CHECK(!node::FindSnapshotChainstateDir()); + BOOST_CHECK(!node::FindSnapshotChainstateDir(chainman.m_options.datadir)); BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( this, [](AutoFile& auto_infile, SnapshotMetadata& metadata) { @@ -255,7 +258,7 @@ struct SnapshotTestSetup : TestChain100Setup { })); BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(this)); - BOOST_CHECK(fs::exists(*node::FindSnapshotChainstateDir())); + BOOST_CHECK(fs::exists(*node::FindSnapshotChainstateDir(chainman.m_options.datadir))); // Ensure our active chain is the snapshot chainstate. BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash->IsNull()); @@ -268,7 +271,7 @@ struct SnapshotTestSetup : TestChain100Setup { { LOCK(::cs_main); - fs::path found = *node::FindSnapshotChainstateDir(); + fs::path found = *node::FindSnapshotChainstateDir(chainman.m_options.datadir); // Note: WriteSnapshotBaseBlockhash() is implicitly tested above. BOOST_CHECK_EQUAL( @@ -376,15 +379,21 @@ struct SnapshotTestSetup : TestChain100Setup { LOCK(::cs_main); chainman.ResetChainstates(); BOOST_CHECK_EQUAL(chainman.GetAll().size(), 0); + m_node.notifications = std::make_unique<KernelNotifications>(); const ChainstateManager::Options chainman_opts{ .chainparams = ::Params(), - .datadir = m_args.GetDataDirNet(), + .datadir = chainman.m_options.datadir, .adjusted_time_callback = GetAdjustedTime, + .notifications = *m_node.notifications, + }; + const BlockManager::Options blockman_opts{ + .chainparams = chainman_opts.chainparams, + .blocks_dir = m_args.GetBlocksDirPath(), }; // For robustness, ensure the old manager is destroyed before creating a // new one. m_node.chainman.reset(); - m_node.chainman = std::make_unique<ChainstateManager>(chainman_opts, node::BlockManager::Options{}); + m_node.chainman = std::make_unique<ChainstateManager>(chainman_opts, blockman_opts); } return *Assert(m_node.chainman); } @@ -482,7 +491,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_init, SnapshotTestSetup) this->SetupSnapshot(); - fs::path snapshot_chainstate_dir = *node::FindSnapshotChainstateDir(); + fs::path snapshot_chainstate_dir = *node::FindSnapshotChainstateDir(chainman.m_options.datadir); BOOST_CHECK(fs::exists(snapshot_chainstate_dir)); BOOST_CHECK_EQUAL(snapshot_chainstate_dir, gArgs.GetDataDirNet() / "chainstate_snapshot"); @@ -556,7 +565,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_completion, SnapshotTestSetup SnapshotCompletionResult res; auto mock_shutdown = [](bilingual_str msg) {}; - fs::path snapshot_chainstate_dir = *node::FindSnapshotChainstateDir(); + fs::path snapshot_chainstate_dir = *node::FindSnapshotChainstateDir(chainman.m_options.datadir); BOOST_CHECK(fs::exists(snapshot_chainstate_dir)); BOOST_CHECK_EQUAL(snapshot_chainstate_dir, gArgs.GetDataDirNet() / "chainstate_snapshot"); diff --git a/src/test/validation_tests.cpp b/src/test/validation_tests.cpp index 0b4c491615..d00f2ff4d1 100644 --- a/src/test/validation_tests.cpp +++ b/src/test/validation_tests.cpp @@ -7,6 +7,7 @@ #include <net.h> #include <signet.h> #include <uint256.h> +#include <util/chaintype.h> #include <validation.h> #include <test/util/setup_common.h> @@ -41,7 +42,7 @@ static void TestBlockSubsidyHalvings(int nSubsidyHalvingInterval) BOOST_AUTO_TEST_CASE(block_subsidy_test) { - const auto chainParams = CreateChainParams(*m_node.args, CBaseChainParams::MAIN); + const auto chainParams = CreateChainParams(*m_node.args, ChainType::MAIN); TestBlockSubsidyHalvings(chainParams->GetConsensus()); // As in main TestBlockSubsidyHalvings(150); // As in regtest TestBlockSubsidyHalvings(1000); // Just another interval @@ -49,7 +50,7 @@ BOOST_AUTO_TEST_CASE(block_subsidy_test) BOOST_AUTO_TEST_CASE(subsidy_limit_test) { - const auto chainParams = CreateChainParams(*m_node.args, CBaseChainParams::MAIN); + const auto chainParams = CreateChainParams(*m_node.args, ChainType::MAIN); CAmount nSum = 0; for (int nHeight = 0; nHeight < 14000000; nHeight += 1000) { CAmount nSubsidy = GetBlockSubsidy(nHeight, chainParams->GetConsensus()); @@ -64,7 +65,7 @@ BOOST_AUTO_TEST_CASE(signet_parse_tests) { ArgsManager signet_argsman; signet_argsman.ForceSetArg("-signetchallenge", "51"); // set challenge to OP_TRUE - const auto signet_params = CreateChainParams(signet_argsman, CBaseChainParams::SIGNET); + const auto signet_params = CreateChainParams(signet_argsman, ChainType::SIGNET); CBlock block; BOOST_CHECK(signet_params->GetConsensus().signet_challenge == std::vector<uint8_t>{OP_TRUE}); CScript challenge{OP_TRUE}; @@ -124,7 +125,7 @@ BOOST_AUTO_TEST_CASE(signet_parse_tests) //! Test retrieval of valid assumeutxo values. BOOST_AUTO_TEST_CASE(test_assumeutxo) { - const auto params = CreateChainParams(*m_node.args, CBaseChainParams::REGTEST); + const auto params = CreateChainParams(*m_node.args, ChainType::REGTEST); // These heights don't have assumeutxo configurations associated, per the contents // of kernel/chainparams.cpp. diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index 80c00036e7..9e69992752 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -7,6 +7,7 @@ #include <consensus/params.h> #include <test/util/random.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> #include <versionbits.h> #include <boost/test/unit_test.hpp> @@ -418,8 +419,8 @@ BOOST_AUTO_TEST_CASE(versionbits_computeblockversion) // check that any deployment on any chain can conceivably reach both // ACTIVE and FAILED states in roughly the way we expect - for (const auto& chain_name : {CBaseChainParams::MAIN, CBaseChainParams::TESTNET, CBaseChainParams::SIGNET, CBaseChainParams::REGTEST}) { - const auto chainParams = CreateChainParams(*m_node.args, chain_name); + for (const auto& chain_type: {ChainType::MAIN, ChainType::TESTNET, ChainType::SIGNET, ChainType::REGTEST}) { + const auto chainParams = CreateChainParams(*m_node.args, chain_type); uint32_t chain_all_vbits{0}; for (int i = 0; i < (int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; ++i) { const auto dep = static_cast<Consensus::DeploymentPos>(i); @@ -440,7 +441,7 @@ BOOST_AUTO_TEST_CASE(versionbits_computeblockversion) // deployment that's not always/never active ArgsManager args; args.ForceSetArg("-vbparams", "testdummy:1199145601:1230767999"); // January 1, 2008 - December 31, 2008 - const auto chainParams = CreateChainParams(args, CBaseChainParams::REGTEST); + const auto chainParams = CreateChainParams(args, ChainType::REGTEST); check_computeblockversion(vbcache, chainParams->GetConsensus(), Consensus::DEPLOYMENT_TESTDUMMY); } @@ -450,7 +451,7 @@ BOOST_AUTO_TEST_CASE(versionbits_computeblockversion) // live deployment ArgsManager args; args.ForceSetArg("-vbparams", "testdummy:1199145601:1230767999:403200"); // January 1, 2008 - December 31, 2008, min act height 403200 - const auto chainParams = CreateChainParams(args, CBaseChainParams::REGTEST); + const auto chainParams = CreateChainParams(args, ChainType::REGTEST); check_computeblockversion(vbcache, chainParams->GetConsensus(), Consensus::DEPLOYMENT_TESTDUMMY); } } diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 92b55f9fc4..98d68f93e9 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -133,15 +133,15 @@ bool TorControlConnection::Connect(const std::string& tor_control_center, const Disconnect(); } - CService control_service; - if (!Lookup(tor_control_center, control_service, 9051, fNameLookup)) { + const std::optional<CService> control_service{Lookup(tor_control_center, 9051, fNameLookup)}; + if (!control_service.has_value()) { LogPrintf("tor: Failed to look up control center %s\n", tor_control_center); return false; } struct sockaddr_storage control_address; socklen_t control_address_len = sizeof(control_address); - if (!control_service.GetSockAddr(reinterpret_cast<struct sockaddr*>(&control_address), &control_address_len)) { + if (!control_service.value().GetSockAddr(reinterpret_cast<struct sockaddr*>(&control_address), &control_address_len)) { LogPrintf("tor: Error parsing socket address %s\n", tor_control_center); return false; } diff --git a/src/txdb.cpp b/src/txdb.cpp index 15351a4355..b2095bd4a3 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -31,22 +31,22 @@ static constexpr uint8_t DB_COINS{'c'}; static constexpr uint8_t DB_TXINDEX_BLOCK{'T'}; // uint8_t DB_TXINDEX{'t'} -std::optional<bilingual_str> CheckLegacyTxindex(CBlockTreeDB& block_tree_db) +util::Result<void> CheckLegacyTxindex(CBlockTreeDB& block_tree_db) { CBlockLocator ignored{}; if (block_tree_db.Read(DB_TXINDEX_BLOCK, ignored)) { - return _("The -txindex upgrade started by a previous version cannot be completed. Restart with the previous version or run a full -reindex."); + return util::Error{_("The -txindex upgrade started by a previous version cannot be completed. Restart with the previous version or run a full -reindex.")}; } bool txindex_legacy_flag{false}; block_tree_db.ReadFlag("txindex", txindex_legacy_flag); if (txindex_legacy_flag) { // Disable legacy txindex and warn once about occupied disk space if (!block_tree_db.WriteFlag("txindex", false)) { - return Untranslated("Failed to write block index db flag 'txindex'='0'"); + return util::Error{Untranslated("Failed to write block index db flag 'txindex'='0'")}; } - return _("The block index db contains a legacy 'txindex'. To clear the occupied disk space, run a full -reindex, otherwise ignore this error. This error message will not be displayed again."); + return util::Error{_("The block index db contains a legacy 'txindex'. To clear the occupied disk space, run a full -reindex, otherwise ignore this error. This error message will not be displayed again.")}; } - return std::nullopt; + return {}; } bool CCoinsViewDB::NeedsUpgrade() diff --git a/src/txdb.h b/src/txdb.h index 63c7152097..04d0ecb39f 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -11,7 +11,11 @@ #include <kernel/cs_main.h> #include <sync.h> #include <util/fs.h> +#include <util/result.h> +#include <cstddef> +#include <cstdint> +#include <functional> #include <memory> #include <optional> #include <string> @@ -20,11 +24,11 @@ class CBlockFileInfo; class CBlockIndex; +class COutPoint; class uint256; namespace Consensus { struct Params; }; -struct bilingual_str; //! -dbcache default (MiB) static const int64_t nDefaultDbCache = 450; @@ -98,6 +102,6 @@ public: EXCLUSIVE_LOCKS_REQUIRED(::cs_main); }; -std::optional<bilingual_str> CheckLegacyTxindex(CBlockTreeDB& block_tree_db); +[[nodiscard]] util::Result<void> CheckLegacyTxindex(CBlockTreeDB& block_tree_db); #endif // BITCOIN_TXDB_H diff --git a/src/txmempool.cpp b/src/txmempool.cpp index da875c271e..1286eba035 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -7,6 +7,7 @@ #include <chain.h> #include <coins.h> +#include <common/system.h> #include <consensus/consensus.h> #include <consensus/tx_verify.h> #include <consensus/validation.h> @@ -19,13 +20,13 @@ #include <util/moneystr.h> #include <util/overflow.h> #include <util/result.h> -#include <util/system.h> #include <util/time.h> #include <util/trace.h> #include <util/translation.h> #include <validationinterface.h> #include <cmath> +#include <numeric> #include <optional> #include <string_view> #include <utility> @@ -74,7 +75,7 @@ void CTxMemPool::UpdateForDescendants(txiter updateIt, cacheMap& cachedDescendan } // descendants now contains all in-mempool descendants of updateIt. // Update and add to cached descendant map - int64_t modifySize = 0; + int32_t modifySize = 0; CAmount modifyFee = 0; int64_t modifyCount = 0; for (const CTxMemPoolEntry& descendant : descendants) { @@ -90,7 +91,7 @@ void CTxMemPool::UpdateForDescendants(txiter updateIt, cacheMap& cachedDescendan // Don't directly remove the transaction here -- doing so would // invalidate iterators in cachedDescendants. Mark it for removal // by inserting into descendants_to_remove. - if (descendant.GetCountWithAncestors() > uint64_t(m_limits.ancestor_count) || descendant.GetSizeWithAncestors() > uint64_t(m_limits.ancestor_size_vbytes)) { + if (descendant.GetCountWithAncestors() > uint64_t(m_limits.ancestor_count) || descendant.GetSizeWithAncestors() > m_limits.ancestor_size_vbytes) { descendants_to_remove.insert(descendant.GetTx().GetHash()); } } @@ -277,8 +278,8 @@ void CTxMemPool::UpdateAncestorsOf(bool add, txiter it, setEntries &setAncestors for (const CTxMemPoolEntry& parent : parents) { UpdateChild(mapTx.iterator_to(parent), it, add); } - const int64_t updateCount = (add ? 1 : -1); - const int64_t updateSize = updateCount * it->GetTxSize(); + const int32_t updateCount = (add ? 1 : -1); + const int32_t updateSize{updateCount * it->GetTxSize()}; const CAmount updateFee = updateCount * it->GetModifiedFee(); for (txiter ancestorIt : setAncestors) { mapTx.modify(ancestorIt, [=](CTxMemPoolEntry& e) { e.UpdateDescendantState(updateSize, updateFee, updateCount); }); @@ -322,7 +323,7 @@ void CTxMemPool::UpdateForRemoveFromMempool(const setEntries &entriesToRemove, b setEntries setDescendants; CalculateDescendants(removeIt, setDescendants); setDescendants.erase(removeIt); // don't update state for self - int64_t modifySize = -((int64_t)removeIt->GetTxSize()); + int32_t modifySize = -removeIt->GetTxSize(); CAmount modifyFee = -removeIt->GetModifiedFee(); int modifySigOps = -removeIt->GetSigOpCost(); for (txiter dit : setDescendants) { @@ -364,21 +365,21 @@ void CTxMemPool::UpdateForRemoveFromMempool(const setEntries &entriesToRemove, b } } -void CTxMemPoolEntry::UpdateDescendantState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount) +void CTxMemPoolEntry::UpdateDescendantState(int32_t modifySize, CAmount modifyFee, int64_t modifyCount) { nSizeWithDescendants += modifySize; - assert(int64_t(nSizeWithDescendants) > 0); + assert(nSizeWithDescendants > 0); nModFeesWithDescendants = SaturatingAdd(nModFeesWithDescendants, modifyFee); - nCountWithDescendants += modifyCount; + nCountWithDescendants += uint64_t(modifyCount); assert(int64_t(nCountWithDescendants) > 0); } -void CTxMemPoolEntry::UpdateAncestorState(int64_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps) +void CTxMemPoolEntry::UpdateAncestorState(int32_t modifySize, CAmount modifyFee, int64_t modifyCount, int64_t modifySigOps) { nSizeWithAncestors += modifySize; - assert(int64_t(nSizeWithAncestors) > 0); + assert(nSizeWithAncestors > 0); nModFeesWithAncestors = SaturatingAdd(nModFeesWithAncestors, modifyFee); - nCountWithAncestors += modifyCount; + nCountWithAncestors += uint64_t(modifyCount); assert(int64_t(nCountWithAncestors) > 0); nSigOpCostWithAncestors += modifySigOps; assert(int(nSigOpCostWithAncestors) >= 0); @@ -698,7 +699,7 @@ void CTxMemPool::check(const CCoinsViewCache& active_coins_tip, int64_t spendhei // Verify ancestor state is correct. auto ancestors{AssumeCalculateMemPoolAncestors(__func__, *it, Limits::NoLimits())}; uint64_t nCountCheck = ancestors.size() + 1; - uint64_t nSizeCheck = it->GetTxSize(); + int32_t nSizeCheck = it->GetTxSize(); CAmount nFeesCheck = it->GetModifiedFee(); int64_t nSigOpCheck = it->GetSigOpCost(); @@ -719,7 +720,7 @@ void CTxMemPool::check(const CCoinsViewCache& active_coins_tip, int64_t spendhei // Check children against mapNextTx CTxMemPoolEntry::Children setChildrenCheck; auto iter = mapNextTx.lower_bound(COutPoint(it->GetTx().GetHash(), 0)); - uint64_t child_sizes = 0; + int32_t child_sizes{0}; for (; iter != mapNextTx.end() && iter->first->hash == it->GetTx().GetHash(); ++iter) { txiter childit = mapTx.find(iter->second->GetHash()); assert(childit != mapTx.end()); // mapNextTx points to in-mempool transactions @@ -755,11 +756,16 @@ void CTxMemPool::check(const CCoinsViewCache& active_coins_tip, int64_t spendhei bool CTxMemPool::CompareDepthAndScore(const uint256& hasha, const uint256& hashb, bool wtxid) { + /* Return `true` if hasha should be considered sooner than hashb. Namely when: + * a is not in the mempool, but b is + * both are in the mempool and a has fewer ancestors than b + * both are in the mempool and a has a higher score than b + */ LOCK(cs); - indexed_transaction_set::const_iterator i = wtxid ? get_iter_from_wtxid(hasha) : mapTx.find(hasha); - if (i == mapTx.end()) return false; indexed_transaction_set::const_iterator j = wtxid ? get_iter_from_wtxid(hashb) : mapTx.find(hashb); - if (j == mapTx.end()) return true; + if (j == mapTx.end()) return false; + indexed_transaction_set::const_iterator i = wtxid ? get_iter_from_wtxid(hasha) : mapTx.find(hasha); + if (i == mapTx.end()) return true; uint64_t counta = i->GetCountWithAncestors(); uint64_t countb = j->GetCountWithAncestors(); if (counta == countb) { @@ -870,8 +876,17 @@ void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeD } ++nTransactionsUpdated; } + if (delta == 0) { + mapDeltas.erase(hash); + LogPrintf("PrioritiseTransaction: %s (%sin mempool) delta cleared\n", hash.ToString(), it == mapTx.end() ? "not " : ""); + } else { + LogPrintf("PrioritiseTransaction: %s (%sin mempool) fee += %s, new delta=%s\n", + hash.ToString(), + it == mapTx.end() ? "not " : "", + FormatMoney(nFeeDelta), + FormatMoney(delta)); + } } - LogPrintf("PrioritiseTransaction: %s fee += %s\n", hash.ToString(), FormatMoney(nFeeDelta)); } void CTxMemPool::ApplyDelta(const uint256& hash, CAmount &nFeeDelta) const @@ -890,6 +905,22 @@ void CTxMemPool::ClearPrioritisation(const uint256& hash) mapDeltas.erase(hash); } +std::vector<CTxMemPool::delta_info> CTxMemPool::GetPrioritisedTransactions() const +{ + AssertLockNotHeld(cs); + LOCK(cs); + std::vector<delta_info> result; + result.reserve(mapDeltas.size()); + for (const auto& [txid, delta] : mapDeltas) { + const auto iter{mapTx.find(txid)}; + const bool in_mempool{iter != mapTx.end()}; + std::optional<CAmount> modified_fee; + if (in_mempool) modified_fee = iter->GetModifiedFee(); + result.emplace_back(delta_info{in_mempool, delta, modified_fee, txid}); + } + return result; +} + const CTransaction* CTxMemPool::GetConflictTx(const COutPoint& prevout) const { const auto it = mapNextTx.find(prevout); @@ -913,6 +944,19 @@ CTxMemPool::setEntries CTxMemPool::GetIterSet(const std::set<uint256>& hashes) c return ret; } +std::vector<CTxMemPool::txiter> CTxMemPool::GetIterVec(const std::vector<uint256>& txids) const +{ + AssertLockHeld(cs); + std::vector<txiter> ret; + ret.reserve(txids.size()); + for (const auto& txid : txids) { + const auto it{GetIter(txid)}; + if (!it) return {}; + ret.push_back(*it); + } + return ret; +} + bool CTxMemPool::HasNoInputsOf(const CTransaction &tx) const { for (unsigned int i = 0; i < tx.vin.size(); i++) @@ -1142,7 +1186,6 @@ void CTxMemPool::SetLoadTried(bool load_tried) m_load_tried = load_tried; } - std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept { switch (r) { @@ -1155,3 +1198,30 @@ std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept } assert(false); } + +std::vector<CTxMemPool::txiter> CTxMemPool::GatherClusters(const std::vector<uint256>& txids) const +{ + AssertLockHeld(cs); + std::vector<txiter> clustered_txs{GetIterVec(txids)}; + // Use epoch: visiting an entry means we have added it to the clustered_txs vector. It does not + // necessarily mean the entry has been processed. + WITH_FRESH_EPOCH(m_epoch); + for (const auto& it : clustered_txs) { + visited(it); + } + // i = index of where the list of entries to process starts + for (size_t i{0}; i < clustered_txs.size(); ++i) { + // DoS protection: if there are 500 or more entries to process, just quit. + if (clustered_txs.size() > 500) return {}; + const txiter& tx_iter = clustered_txs.at(i); + for (const auto& entries : {tx_iter->GetMemPoolParentsConst(), tx_iter->GetMemPoolChildrenConst()}) { + for (const CTxMemPoolEntry& entry : entries) { + const auto entry_it = mapTx.iterator_to(entry); + if (!visited(entry_it)) { + clustered_txs.push_back(entry_it); + } + } + } + } + return clustered_txs; +} diff --git a/src/txmempool.h b/src/txmempool.h index 2c3cb7e9db..846def02cd 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -33,8 +33,11 @@ #include <util/result.h> #include <boost/multi_index/hashed_index.hpp> +#include <boost/multi_index/identity.hpp> +#include <boost/multi_index/indexed_by.hpp> #include <boost/multi_index/ordered_index.hpp> #include <boost/multi_index/sequenced_index.hpp> +#include <boost/multi_index/tag.hpp> #include <boost/multi_index_container.hpp> class CBlockIndex; @@ -219,7 +222,7 @@ struct TxMempoolInfo CAmount fee; /** Virtual size of the transaction. */ - size_t vsize; + int32_t vsize; /** The fee delta. */ int64_t nFeeDelta; @@ -516,15 +519,35 @@ public: void ApplyDelta(const uint256& hash, CAmount &nFeeDelta) const EXCLUSIVE_LOCKS_REQUIRED(cs); void ClearPrioritisation(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs); + struct delta_info { + /** Whether this transaction is in the mempool. */ + const bool in_mempool; + /** The fee delta added using PrioritiseTransaction(). */ + const CAmount delta; + /** The modified fee (base fee + delta) of this entry. Only present if in_mempool=true. */ + std::optional<CAmount> modified_fee; + /** The prioritised transaction's txid. */ + const uint256 txid; + }; + /** Return a vector of all entries in mapDeltas with their corresponding delta_info. */ + std::vector<delta_info> GetPrioritisedTransactions() const EXCLUSIVE_LOCKS_REQUIRED(!cs); + /** Get the transaction in the pool that spends the same prevout */ const CTransaction* GetConflictTx(const COutPoint& prevout) const EXCLUSIVE_LOCKS_REQUIRED(cs); /** Returns an iterator to the given hash, if found */ std::optional<txiter> GetIter(const uint256& txid) const EXCLUSIVE_LOCKS_REQUIRED(cs); - /** Translate a set of hashes into a set of pool iterators to avoid repeated lookups */ + /** Translate a set of hashes into a set of pool iterators to avoid repeated lookups. + * Does not require that all of the hashes correspond to actual transactions in the mempool, + * only returns the ones that exist. */ setEntries GetIterSet(const std::set<uint256>& hashes) const EXCLUSIVE_LOCKS_REQUIRED(cs); + /** Translate a list of hashes into a list of mempool iterators to avoid repeated lookups. + * The nth element in txids becomes the nth element in the returned vector. If any of the txids + * don't actually exist in the mempool, returns an empty vector. */ + std::vector<txiter> GetIterVec(const std::vector<uint256>& txids) const EXCLUSIVE_LOCKS_REQUIRED(cs); + /** Remove a set of transactions from the mempool. * If a transaction is in this set, then all in-mempool descendants must * also be in the set, unless this transaction is being removed for being @@ -585,6 +608,12 @@ public: const Limits& limits, bool fSearchForParents = true) const EXCLUSIVE_LOCKS_REQUIRED(cs); + /** Collect the entire cluster of connected transactions for each transaction in txids. + * All txids must correspond to transaction entries in the mempool, otherwise this returns an + * empty vector. This call will also exit early and return an empty vector if it collects 500 or + * more transactions as a DoS protection. */ + std::vector<txiter> GatherClusters(const std::vector<uint256>& txids) const EXCLUSIVE_LOCKS_REQUIRED(cs); + /** Calculate all in-mempool ancestors of a set of transactions not already in the mempool and * check ancestor and descendant limits. Heuristics are used to estimate the ancestor and * descendant count of all entries if the package were to be added to the mempool. The limits diff --git a/src/txrequest.cpp b/src/txrequest.cpp index 96a3d2eeeb..dd042103bd 100644 --- a/src/txrequest.cpp +++ b/src/txrequest.cpp @@ -10,8 +10,12 @@ #include <random.h> #include <uint256.h> -#include <boost/multi_index_container.hpp> +#include <boost/multi_index/indexed_by.hpp> #include <boost/multi_index/ordered_index.hpp> +#include <boost/multi_index/sequenced_index.hpp> +#include <boost/multi_index/tag.hpp> +#include <boost/multi_index_container.hpp> +#include <boost/tuple/tuple.hpp> #include <chrono> #include <unordered_map> @@ -69,7 +73,7 @@ struct Announcement { const bool m_is_wtxid : 1; /** What state this announcement is in. - * This is a uint8_t instead of a State to silence a GCC warning in versions prior to 8.4 and 9.3. + * This is a uint8_t instead of a State to silence a GCC warning in versions prior to 9.3. * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61414 */ uint8_t m_state : 3; diff --git a/src/univalue/include/univalue.h b/src/univalue/include/univalue.h index d501c3fb69..004135ef97 100644 --- a/src/univalue/include/univalue.h +++ b/src/univalue/include/univalue.h @@ -123,7 +123,7 @@ public: const UniValue& get_array() const; enum VType type() const { return getType(); } - friend const UniValue& find_value( const UniValue& obj, const std::string& name); + const UniValue& find_value(std::string_view key) const; }; template <class It> @@ -201,6 +201,4 @@ static inline bool json_isspace(int ch) extern const UniValue NullUniValue; -const UniValue& find_value( const UniValue& obj, const std::string& name); - #endif // BITCOIN_UNIVALUE_INCLUDE_UNIVALUE_H diff --git a/src/univalue/lib/univalue.cpp b/src/univalue/lib/univalue.cpp index 5aa39edb75..c3d19caae0 100644 --- a/src/univalue/lib/univalue.cpp +++ b/src/univalue/lib/univalue.cpp @@ -230,12 +230,13 @@ const char *uvTypeName(UniValue::VType t) return nullptr; } -const UniValue& find_value(const UniValue& obj, const std::string& name) +const UniValue& UniValue::find_value(std::string_view key) const { - for (unsigned int i = 0; i < obj.keys.size(); i++) - if (obj.keys[i] == name) - return obj.values.at(i); - + for (unsigned int i = 0; i < keys.size(); ++i) { + if (keys[i] == key) { + return values.at(i); + } + } return NullUniValue; } diff --git a/src/univalue/test/object.cpp b/src/univalue/test/object.cpp index 5ddf300393..5fb973c67b 100644 --- a/src/univalue/test/object.cpp +++ b/src/univalue/test/object.cpp @@ -412,6 +412,33 @@ void univalue_readwrite() BOOST_CHECK_EQUAL(strJson1, v.write()); + // Valid + BOOST_CHECK(v.read("1.0") && (v.get_real() == 1.0)); + BOOST_CHECK(v.read("true") && v.get_bool()); + BOOST_CHECK(v.read("[false]") && !v[0].get_bool()); + BOOST_CHECK(v.read("{\"a\": true}") && v["a"].get_bool()); + BOOST_CHECK(v.read("{\"1\": \"true\"}") && (v["1"].get_str() == "true")); + // Valid, with leading or trailing whitespace + BOOST_CHECK(v.read(" 1.0") && (v.get_real() == 1.0)); + BOOST_CHECK(v.read("1.0 ") && (v.get_real() == 1.0)); + BOOST_CHECK(v.read("0.00000000000000000000000000000000000001e+30 ") && v.get_real() == 1e-8); + + BOOST_CHECK(!v.read(".19e-6")); //should fail, missing leading 0, therefore invalid JSON + // Invalid, initial garbage + BOOST_CHECK(!v.read("[1.0")); + BOOST_CHECK(!v.read("a1.0")); + // Invalid, trailing garbage + BOOST_CHECK(!v.read("1.0sds")); + BOOST_CHECK(!v.read("1.0]")); + // Invalid, keys have to be names + BOOST_CHECK(!v.read("{1: \"true\"}")); + BOOST_CHECK(!v.read("{true: 1}")); + BOOST_CHECK(!v.read("{[1]: 1}")); + BOOST_CHECK(!v.read("{{\"a\": \"a\"}: 1}")); + // BTC addresses should fail parsing + BOOST_CHECK(!v.read("175tWpb8K1S7NmH4Zx6rewF9WQrcZv245W")); + BOOST_CHECK(!v.read("3J98t1WpEZ73CNmQviecrnyiWrnqRhWNL")); + /* Check for (correctly reporting) a parsing error if the initial JSON construct is followed by more stuff. Note that whitespace is, of course, exempt. */ diff --git a/src/util/any.h b/src/util/any.h new file mode 100644 index 0000000000..4562c5bd8a --- /dev/null +++ b/src/util/any.h @@ -0,0 +1,26 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_ANY_H +#define BITCOIN_UTIL_ANY_H + +#include <any> + +namespace util { + +/** + * Helper function to access the contained object of a std::any instance. + * Returns a pointer to the object if passed instance has a value and the type + * matches, nullptr otherwise. + */ +template<typename T> +T* AnyPtr(const std::any& any) noexcept +{ + T* const* ptr = std::any_cast<T*>(&any); + return ptr ? *ptr : nullptr; +} + +} // namespace util + +#endif // BITCOIN_UTIL_ANY_H diff --git a/src/util/batchpriority.cpp b/src/util/batchpriority.cpp new file mode 100644 index 0000000000..c73aef1eb4 --- /dev/null +++ b/src/util/batchpriority.cpp @@ -0,0 +1,26 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <logging.h> +#include <util/syserror.h> + +#if (defined(__FreeBSD__) || defined(__OpenBSD__) || defined(__DragonFly__)) +#include <pthread.h> +#include <pthread_np.h> +#endif + +#ifndef WIN32 +#include <sched.h> +#endif + +void ScheduleBatchPriority() +{ +#ifdef SCHED_BATCH + const static sched_param param{}; + const int rc = pthread_setschedparam(pthread_self(), SCHED_BATCH, ¶m); + if (rc != 0) { + LogPrintf("Failed to pthread_setschedparam: %s\n", SysErrorString(rc)); + } +#endif +} diff --git a/src/util/batchpriority.h b/src/util/batchpriority.h new file mode 100644 index 0000000000..5ffc8dd684 --- /dev/null +++ b/src/util/batchpriority.h @@ -0,0 +1,15 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_BATCHPRIORITY_H +#define BITCOIN_UTIL_BATCHPRIORITY_H + +/** + * On platforms that support it, tell the kernel the calling thread is + * CPU-intensive and non-interactive. See SCHED_BATCH in sched(7) for details. + * + */ +void ScheduleBatchPriority(); + +#endif // BITCOIN_UTIL_BATCHPRIORITY_H diff --git a/src/util/bip32.cpp b/src/util/bip32.cpp index c0ad9257ce..4ab14bd704 100644 --- a/src/util/bip32.cpp +++ b/src/util/bip32.cpp @@ -51,17 +51,17 @@ bool ParseHDKeypath(const std::string& keypath_str, std::vector<uint32_t>& keypa return true; } -std::string FormatHDKeypath(const std::vector<uint32_t>& path) +std::string FormatHDKeypath(const std::vector<uint32_t>& path, bool apostrophe) { std::string ret; for (auto i : path) { ret += strprintf("/%i", (i << 1) >> 1); - if (i >> 31) ret += '\''; + if (i >> 31) ret += apostrophe ? '\'' : 'h'; } return ret; } -std::string WriteHDKeypath(const std::vector<uint32_t>& keypath) +std::string WriteHDKeypath(const std::vector<uint32_t>& keypath, bool apostrophe) { - return "m" + FormatHDKeypath(keypath); + return "m" + FormatHDKeypath(keypath, apostrophe); } diff --git a/src/util/bip32.h b/src/util/bip32.h index b720cb5638..ea5f192c07 100644 --- a/src/util/bip32.h +++ b/src/util/bip32.h @@ -13,7 +13,7 @@ [[nodiscard]] bool ParseHDKeypath(const std::string& keypath_str, std::vector<uint32_t>& keypath); /** Write HD keypaths as strings */ -std::string WriteHDKeypath(const std::vector<uint32_t>& keypath); -std::string FormatHDKeypath(const std::vector<uint32_t>& path); +std::string WriteHDKeypath(const std::vector<uint32_t>& keypath, bool apostrophe = false); +std::string FormatHDKeypath(const std::vector<uint32_t>& path, bool apostrophe = false); #endif // BITCOIN_UTIL_BIP32_H diff --git a/src/util/chaintype.cpp b/src/util/chaintype.cpp new file mode 100644 index 0000000000..8a199e352a --- /dev/null +++ b/src/util/chaintype.cpp @@ -0,0 +1,39 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <util/chaintype.h> + +#include <cassert> +#include <optional> +#include <string> + +std::string ChainTypeToString(ChainType chain) +{ + switch (chain) { + case ChainType::MAIN: + return "main"; + case ChainType::TESTNET: + return "test"; + case ChainType::SIGNET: + return "signet"; + case ChainType::REGTEST: + return "regtest"; + } + assert(false); +} + +std::optional<ChainType> ChainTypeFromString(std::string_view chain) +{ + if (chain == "main") { + return ChainType::MAIN; + } else if (chain == "test") { + return ChainType::TESTNET; + } else if (chain == "signet") { + return ChainType::SIGNET; + } else if (chain == "regtest") { + return ChainType::REGTEST; + } else { + return std::nullopt; + } +} diff --git a/src/util/chaintype.h b/src/util/chaintype.h new file mode 100644 index 0000000000..c73985df57 --- /dev/null +++ b/src/util/chaintype.h @@ -0,0 +1,22 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_CHAINTYPE_H +#define BITCOIN_UTIL_CHAINTYPE_H + +#include <optional> +#include <string> + +enum class ChainType { + MAIN, + TESTNET, + SIGNET, + REGTEST, +}; + +std::string ChainTypeToString(ChainType chain); + +std::optional<ChainType> ChainTypeFromString(std::string_view chain); + +#endif // BITCOIN_UTIL_CHAINTYPE_H diff --git a/src/util/fs.h b/src/util/fs.h index 886a30394e..8f79f6cba6 100644 --- a/src/util/fs.h +++ b/src/util/fs.h @@ -8,7 +8,7 @@ #include <tinyformat.h> #include <cstdio> -#include <filesystem> +#include <filesystem> // IWYU pragma: export #include <functional> #include <iomanip> #include <ios> diff --git a/src/util/insert.h b/src/util/insert.h new file mode 100644 index 0000000000..5332eca60a --- /dev/null +++ b/src/util/insert.h @@ -0,0 +1,24 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_INSERT_H +#define BITCOIN_UTIL_INSERT_H + +#include <set> + +namespace util { + +//! Simplification of std insertion +template <typename Tdst, typename Tsrc> +inline void insert(Tdst& dst, const Tsrc& src) { + dst.insert(dst.begin(), src.begin(), src.end()); +} +template <typename TsetT, typename Tsrc> +inline void insert(std::set<TsetT>& dst, const Tsrc& src) { + dst.insert(src.begin(), src.end()); +} + +} // namespace util + +#endif // BITCOIN_UTIL_INSERT_H diff --git a/src/util/result.h b/src/util/result.h index 972b1aada0..b99995c7e5 100644 --- a/src/util/result.h +++ b/src/util/result.h @@ -31,16 +31,19 @@ struct Error { //! `std::optional<T>` can be updated to return `util::Result<T>` and return //! error strings usually just replacing `return std::nullopt;` with `return //! util::Error{error_string};`. -template <class T> +template <class M> class Result { private: + using T = std::conditional_t<std::is_same_v<M, void>, std::monostate, M>; + std::variant<bilingual_str, T> m_variant; template <typename FT> friend bilingual_str ErrorString(const Result<FT>& result); public: + Result() : m_variant{std::in_place_index_t<1>{}, std::monostate{}} {} // constructor for void Result(T obj) : m_variant{std::in_place_index_t<1>{}, std::move(obj)} {} Result(Error error) : m_variant{std::in_place_index_t<0>{}, std::move(error.message)} {} diff --git a/src/util/sock.cpp b/src/util/sock.cpp index 53d20bdf19..c83869bc77 100644 --- a/src/util/sock.cpp +++ b/src/util/sock.cpp @@ -2,12 +2,12 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <compat/compat.h> #include <logging.h> #include <tinyformat.h> #include <util/sock.h> #include <util/syserror.h> -#include <util/system.h> #include <util/threadinterrupt.h> #include <util/time.h> diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 05e7b957c4..d792562735 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -17,8 +17,8 @@ #include <cstdint> #include <limits> #include <optional> -#include <string> -#include <string_view> +#include <string> // IWYU pragma: export +#include <string_view> // IWYU pragma: export #include <system_error> #include <type_traits> #include <vector> diff --git a/src/util/string.h b/src/util/string.h index fb93d2a80e..8b69d6ccae 100644 --- a/src/util/string.h +++ b/src/util/string.h @@ -12,8 +12,8 @@ #include <cstring> #include <locale> #include <sstream> -#include <string> -#include <string_view> +#include <string> // IWYU pragma: export +#include <string_view> // IWYU pragma: export #include <vector> void ReplaceAll(std::string& in_out, const std::string& search, const std::string& substitute); diff --git a/src/util/system.h b/src/util/system.h deleted file mode 100644 index e2fc3450f6..0000000000 --- a/src/util/system.h +++ /dev/null @@ -1,72 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_UTIL_SYSTEM_H -#define BITCOIN_UTIL_SYSTEM_H - -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif - -#include <compat/assumptions.h> -#include <compat/compat.h> - -#include <any> -#include <set> -#include <stdint.h> -#include <string> - -// Application startup time (used for uptime calculation) -int64_t GetStartupTime(); - -void SetupEnvironment(); -bool SetupNetworking(); -#ifndef WIN32 -std::string ShellEscape(const std::string& arg); -#endif -#if HAVE_SYSTEM -void runCommand(const std::string& strCommand); -#endif - -/** - * Return the number of cores available on the current system. - * @note This does count virtual cores, such as those provided by HyperThreading. - */ -int GetNumCores(); - -/** - * On platforms that support it, tell the kernel the calling thread is - * CPU-intensive and non-interactive. See SCHED_BATCH in sched(7) for details. - * - */ -void ScheduleBatchPriority(); - -namespace util { - -//! Simplification of std insertion -template <typename Tdst, typename Tsrc> -inline void insert(Tdst& dst, const Tsrc& src) { - dst.insert(dst.begin(), src.begin(), src.end()); -} -template <typename TsetT, typename Tsrc> -inline void insert(std::set<TsetT>& dst, const Tsrc& src) { - dst.insert(src.begin(), src.end()); -} - -/** - * Helper function to access the contained object of a std::any instance. - * Returns a pointer to the object if passed instance has a value and the type - * matches, nullptr otherwise. - */ -template<typename T> -T* AnyPtr(const std::any& any) noexcept -{ - T* const* ptr = std::any_cast<T*>(&any); - return ptr ? *ptr : nullptr; -} - -} // namespace util - -#endif // BITCOIN_UTIL_SYSTEM_H diff --git a/src/util/time.cpp b/src/util/time.cpp index fb9bc34931..5ca9d21f8d 100644 --- a/src/util/time.cpp +++ b/src/util/time.cpp @@ -78,14 +78,6 @@ NodeClock::time_point NodeClock::now() noexcept return time_point{ret}; }; -template <typename T> -static T GetSystemTime() -{ - const auto now = std::chrono::duration_cast<T>(std::chrono::system_clock::now().time_since_epoch()); - assert(now.count() > 0); - return now; -} - void SetMockTime(int64_t nMockTimeIn) { Assert(nMockTimeIn >= 0); @@ -102,11 +94,6 @@ std::chrono::seconds GetMockTime() return std::chrono::seconds(nMockTime.load(std::memory_order_relaxed)); } -int64_t GetTimeMillis() -{ - return int64_t{GetSystemTime<std::chrono::milliseconds>().count()}; -} - int64_t GetTime() { return GetTime<std::chrono::seconds>().count(); } std::string FormatISO8601DateTime(int64_t nTime) { diff --git a/src/util/time.h b/src/util/time.h index 8c6baeb12a..b6aab615ba 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -71,9 +71,6 @@ using MillisecondsDouble = std::chrono::duration<double, std::chrono::millisecon */ int64_t GetTime(); -/** Returns the system time (not mockable) */ -int64_t GetTimeMillis(); - /** * DEPRECATED * Use SetMockTime with chrono type diff --git a/src/validation.cpp b/src/validation.cpp index b2f4283e16..d9a0fce34f 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -11,7 +11,6 @@ #include <arith_uint256.h> #include <chain.h> #include <checkqueue.h> -#include <common/args.h> #include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/merkle.h> @@ -23,10 +22,10 @@ #include <hash.h> #include <kernel/chainparams.h> #include <kernel/mempool_entry.h> +#include <kernel/notifications_interface.h> #include <logging.h> #include <logging/timer.h> #include <node/blockstorage.h> -#include <node/interface_ui.h> #include <node/utxo_snapshot.h> #include <policy/policy.h> #include <policy/rbf.h> @@ -52,7 +51,6 @@ #include <util/moneystr.h> #include <util/rbf.h> #include <util/strencodings.h> -#include <util/system.h> #include <util/time.h> #include <util/trace.h> #include <util/translation.h> @@ -72,6 +70,7 @@ using kernel::CCoinsStats; using kernel::CoinStatsHashType; using kernel::ComputeUTXOStats; using kernel::LoadMempool; +using kernel::Notifications; using fsbridge::FopenFn; using node::BlockManager; @@ -79,10 +78,7 @@ using node::BlockMap; using node::CBlockIndexHeightOnlyComparator; using node::CBlockIndexWorkComparator; using node::fReindex; -using node::ReadBlockFromDisk; using node::SnapshotMetadata; -using node::UndoReadFromDisk; -using node::UnlinkPrunedFiles; /** Maximum kilobytes for transactions to store for processing during reorg */ static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; @@ -1642,26 +1638,6 @@ bool Chainstate::IsInitialBlockDownload() const return false; } -static void AlertNotify(const std::string& strMessage) -{ - uiInterface.NotifyAlertChanged(); -#if HAVE_SYSTEM - std::string strCmd = gArgs.GetArg("-alertnotify", ""); - if (strCmd.empty()) return; - - // Alert text should be plain ascii coming from a trusted source, but to - // be safe we first strip anything not in safeChars, then add single quotes around - // the whole string before passing it to the shell: - std::string singleQuote("'"); - std::string safeStatus = SanitizeString(strMessage); - safeStatus = singleQuote+safeStatus+singleQuote; - ReplaceAll(strCmd, "%s", safeStatus); - - std::thread t(runCommand, strCmd); - t.detach(); // thread runs free -#endif -} - void Chainstate::CheckForkWarningConditions() { AssertLockHeld(cs_main); @@ -1914,7 +1890,7 @@ DisconnectResult Chainstate::DisconnectBlock(const CBlock& block, const CBlockIn bool fClean = true; CBlockUndo blockUndo; - if (!UndoReadFromDisk(blockUndo, pindex)) { + if (!m_blockman.UndoReadFromDisk(blockUndo, *pindex)) { error("DisconnectBlock(): failure reading undo data"); return DISCONNECT_FAILED; } @@ -2017,8 +1993,6 @@ public: } }; -static std::array<ThresholdConditionCache, VERSIONBITS_NUM_BITS> warningcache GUARDED_BY(cs_main); - static unsigned int GetBlockScriptFlags(const CBlockIndex& block_index, const ChainstateManager& chainman) { const Consensus::Params& consensusparams = chainman.GetConsensus(); @@ -2374,7 +2348,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, if (fJustCheck) return true; - if (!m_blockman.WriteUndoDataForBlock(blockundo, state, pindex, params)) { + if (!m_blockman.WriteUndoDataForBlock(blockundo, state, *pindex)) { return false; } @@ -2523,7 +2497,7 @@ bool Chainstate::FlushStateToDisk( // Write blocks and block index to disk. if (fDoFullFlush || fPeriodicWrite) { // Ensure we can write block index - if (!CheckDiskSpace(gArgs.GetBlocksDirPath())) { + if (!CheckDiskSpace(m_blockman.m_opts.blocks_dir)) { return AbortNode(state, "Disk space is too low!", _("Disk space is too low!")); } { @@ -2545,7 +2519,7 @@ bool Chainstate::FlushStateToDisk( if (fFlushForPrune) { LOG_TIME_MILLIS_WITH_CATEGORY("unlink pruned files", BCLog::BENCH); - UnlinkPrunedFiles(setFilesToPrune); + m_blockman.UnlinkPrunedFiles(setFilesToPrune); } m_last_write = nNow; } @@ -2559,7 +2533,7 @@ bool Chainstate::FlushStateToDisk( // twice (once in the log, and once in the tables). This is already // an overestimation, as most will delete an existing entry or // overwrite one. Still, use a conservative safety factor of 2. - if (!CheckDiskSpace(gArgs.GetDataDirNet(), 48 * 2 * 2 * CoinsTip().GetCacheSize())) { + if (!CheckDiskSpace(m_chainman.m_options.datadir, 48 * 2 * 2 * CoinsTip().GetCacheSize())) { return AbortNode(state, "Disk space is too low!", _("Disk space is too low!")); } // Flush the chainstate (which may refer to block index entries). @@ -2602,16 +2576,6 @@ void Chainstate::PruneAndFlush() } } -static void DoWarning(const bilingual_str& warning) -{ - static bool fWarned = false; - SetMiscWarning(warning); - if (!fWarned) { - AlertNotify(warning.original); - fWarned = true; - } -} - /** Private helper function that concatenates warning messages. */ static void AppendWarning(bilingual_str& res, const bilingual_str& warn) { @@ -2674,11 +2638,11 @@ void Chainstate::UpdateTip(const CBlockIndex* pindexNew) const CBlockIndex* pindex = pindexNew; for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { WarningBitsConditionChecker checker(m_chainman, bit); - ThresholdState state = checker.GetStateFor(pindex, params.GetConsensus(), warningcache.at(bit)); + ThresholdState state = checker.GetStateFor(pindex, params.GetConsensus(), m_chainman.m_warningcache.at(bit)); if (state == ThresholdState::ACTIVE || state == ThresholdState::LOCKED_IN) { const bilingual_str warning = strprintf(_("Unknown new rules activated (versionbit %i)"), bit); if (state == ThresholdState::ACTIVE) { - DoWarning(warning); + m_chainman.GetNotifications().warning(warning); } else { AppendWarning(warning_messages, warning); } @@ -2709,7 +2673,7 @@ bool Chainstate::DisconnectTip(BlockValidationState& state, DisconnectedBlockTra // Read block from disk. std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); CBlock& block = *pblock; - if (!ReadBlockFromDisk(block, pindexDelete, m_chainman.GetConsensus())) { + if (!m_blockman.ReadBlockFromDisk(block, *pindexDelete)) { return error("DisconnectTip(): Failed to read block"); } // Apply the block atomically to the chain state. @@ -2763,7 +2727,6 @@ bool Chainstate::DisconnectTip(BlockValidationState& state, DisconnectedBlockTra return true; } -static SteadyClock::duration time_read_from_disk_total{}; static SteadyClock::duration time_connect_total{}; static SteadyClock::duration time_flush{}; static SteadyClock::duration time_chainstate{}; @@ -2826,7 +2789,7 @@ bool Chainstate::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, std::shared_ptr<const CBlock> pthisBlock; if (!pblock) { std::shared_ptr<CBlock> pblockNew = std::make_shared<CBlock>(); - if (!ReadBlockFromDisk(*pblockNew, pindexNew, m_chainman.GetConsensus())) { + if (!m_blockman.ReadBlockFromDisk(*pblockNew, *pindexNew)) { return AbortNode(state, "Failed to read block"); } pthisBlock = pblockNew; @@ -2837,12 +2800,11 @@ bool Chainstate::ConnectTip(BlockValidationState& state, CBlockIndex* pindexNew, const CBlock& blockConnecting = *pthisBlock; // Apply the block atomically to the chain state. const auto time_2{SteadyClock::now()}; - time_read_from_disk_total += time_2 - time_1; SteadyClock::time_point time_3; - LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms [%.2fs (%.2fms/blk)]\n", - Ticks<MillisecondsDouble>(time_2 - time_1), - Ticks<SecondsDouble>(time_read_from_disk_total), - Ticks<MillisecondsDouble>(time_read_from_disk_total) / num_blocks_total); + // When adding aggregate statistics in the future, keep in mind that + // num_blocks_total may be zero until the ConnectBlock() call below. + LogPrint(BCLog::BENCH, " - Load block from disk: %.2fms\n", + Ticks<MillisecondsDouble>(time_2 - time_1)); { CCoinsViewCache view(&CoinsTip()); bool rv = ConnectBlock(blockConnecting, state, pindexNew, view); @@ -3100,7 +3062,7 @@ static bool NotifyHeaderTip(Chainstate& chainstate) LOCKS_EXCLUDED(cs_main) { } // Send block tip changed notifications without cs_main if (fNotify) { - uiInterface.NotifyHeaderTip(GetSynchronizationState(fInitialBlockDownload), pindexHeader->nHeight, pindexHeader->nTime, false); + chainstate.m_chainman.GetNotifications().headerTip(GetSynchronizationState(fInitialBlockDownload), pindexHeader->nHeight, pindexHeader->nTime, false); } return fNotify; } @@ -3139,7 +3101,6 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< CBlockIndex *pindexMostWork = nullptr; CBlockIndex *pindexNewTip = nullptr; - int nStopAtHeight = gArgs.GetIntArg("-stopatheight", DEFAULT_STOPATHEIGHT); do { // Block until the validation queue drains. This should largely // never happen in normal operation, however may happen during @@ -3209,12 +3170,12 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< GetMainSignals().UpdatedBlockTip(pindexNewTip, pindexFork, fInitialDownload); // Always notify the UI if a new block tip was connected - uiInterface.NotifyBlockTip(GetSynchronizationState(fInitialDownload), pindexNewTip); + m_chainman.GetNotifications().blockTip(GetSynchronizationState(fInitialDownload), *pindexNewTip); } } // When we reach this point, we switched to a new tip (stored in pindexNewTip). - if (nStopAtHeight && pindexNewTip && pindexNewTip->nHeight >= nStopAtHeight) StartShutdown(); + if (m_chainman.StopAtHeight() && pindexNewTip && pindexNewTip->nHeight >= m_chainman.StopAtHeight()) StartShutdown(); if (WITH_LOCK(::cs_main, return m_disabled)) { // Background chainstate has reached the snapshot base block, so exit. @@ -3406,7 +3367,7 @@ bool Chainstate::InvalidateBlock(BlockValidationState& state, CBlockIndex* pinde // Only notify about a new block tip if the active chain was modified. if (pindex_was_in_chain) { - uiInterface.NotifyBlockTip(GetSynchronizationState(IsInitialBlockDownload()), to_mark_failed->pprev); + m_chainman.GetNotifications().blockTip(GetSynchronizationState(IsInitialBlockDownload()), *to_mark_failed->pprev); } return true; } @@ -3923,7 +3884,7 @@ void ChainstateManager::ReportHeadersPresync(const arith_uint256& work, int64_t m_last_presync_update = now; } bool initial_download = chainstate.IsInitialBlockDownload(); - uiInterface.NotifyHeaderTip(GetSynchronizationState(initial_download), height, timestamp, /*presync=*/true); + GetNotifications().headerTip(GetSynchronizationState(initial_download), height, timestamp, /*presync=*/true); if (initial_download) { const int64_t blocks_left{(GetTime() - timestamp) / GetConsensus().nPowTargetSpacing}; const double progress{100.0 * height / (height + blocks_left)}; @@ -4000,7 +3961,7 @@ bool Chainstate::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockV // Write block to history file if (fNewBlock) *fNewBlock = true; try { - FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, pindex->nHeight, m_chain, params, dbp)}; + FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, pindex->nHeight, m_chain, dbp)}; if (blockPos.IsNull()) { state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); return false; @@ -4148,14 +4109,15 @@ bool Chainstate::LoadChainTip() return true; } -CVerifyDB::CVerifyDB() +CVerifyDB::CVerifyDB(Notifications& notifications) + : m_notifications{notifications} { - uiInterface.ShowProgress(_("Verifying blocks…").translated, 0, false); + m_notifications.progress(_("Verifying blocks…"), 0, false); } CVerifyDB::~CVerifyDB() { - uiInterface.ShowProgress("", 100, false); + m_notifications.progress(bilingual_str{}, 100, false); } VerifyDBResult CVerifyDB::VerifyDB( @@ -4195,7 +4157,7 @@ VerifyDBResult CVerifyDB::VerifyDB( LogPrintf("Verification progress: %d%%\n", percentageDone); reportDone = percentageDone / 10; } - uiInterface.ShowProgress(_("Verifying blocks…").translated, percentageDone, false); + m_notifications.progress(_("Verifying blocks…"), percentageDone, false); if (pindex->nHeight <= chainstate.m_chain.Height() - nCheckDepth) { break; } @@ -4208,7 +4170,7 @@ VerifyDBResult CVerifyDB::VerifyDB( } CBlock block; // check level 0: read from disk - if (!ReadBlockFromDisk(block, pindex, consensus_params)) { + if (!chainstate.m_blockman.ReadBlockFromDisk(block, *pindex)) { LogPrintf("Verification error: ReadBlockFromDisk failed at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); return VerifyDBResult::CORRUPTED_BLOCK_DB; } @@ -4222,7 +4184,7 @@ VerifyDBResult CVerifyDB::VerifyDB( if (nCheckLevel >= 2 && pindex) { CBlockUndo undo; if (!pindex->GetUndoPos().IsNull()) { - if (!UndoReadFromDisk(undo, pindex)) { + if (!chainstate.m_blockman.UndoReadFromDisk(undo, *pindex)) { LogPrintf("Verification error: found bad undo data at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); return VerifyDBResult::CORRUPTED_BLOCK_DB; } @@ -4271,10 +4233,10 @@ VerifyDBResult CVerifyDB::VerifyDB( LogPrintf("Verification progress: %d%%\n", percentageDone); reportDone = percentageDone / 10; } - uiInterface.ShowProgress(_("Verifying blocks…").translated, percentageDone, false); + m_notifications.progress(_("Verifying blocks…"), percentageDone, false); pindex = chainstate.m_chain.Next(pindex); CBlock block; - if (!ReadBlockFromDisk(block, pindex, consensus_params)) { + if (!chainstate.m_blockman.ReadBlockFromDisk(block, *pindex)) { LogPrintf("Verification error: ReadBlockFromDisk failed at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); return VerifyDBResult::CORRUPTED_BLOCK_DB; } @@ -4303,7 +4265,7 @@ bool Chainstate::RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& in AssertLockHeld(cs_main); // TODO: merge with ConnectBlock CBlock block; - if (!ReadBlockFromDisk(block, pindex, m_chainman.GetConsensus())) { + if (!m_blockman.ReadBlockFromDisk(block, *pindex)) { return error("ReplayBlock(): ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } @@ -4330,7 +4292,7 @@ bool Chainstate::ReplayBlocks() if (hashHeads.empty()) return true; // We're already in a consistent state. if (hashHeads.size() != 2) return error("ReplayBlocks(): unknown inconsistent state"); - uiInterface.ShowProgress(_("Replaying blocks…").translated, 0, false); + m_chainman.GetNotifications().progress(_("Replaying blocks…"), 0, false); LogPrintf("Replaying blocks\n"); const CBlockIndex* pindexOld = nullptr; // Old tip during the interrupted flush. @@ -4355,7 +4317,7 @@ bool Chainstate::ReplayBlocks() while (pindexOld != pindexFork) { if (pindexOld->nHeight > 0) { // Never disconnect the genesis block. CBlock block; - if (!ReadBlockFromDisk(block, pindexOld, m_chainman.GetConsensus())) { + if (!m_blockman.ReadBlockFromDisk(block, *pindexOld)) { return error("RollbackBlock(): ReadBlockFromDisk() failed at %d, hash=%s", pindexOld->nHeight, pindexOld->GetBlockHash().ToString()); } LogPrintf("Rolling back %s (%i)\n", pindexOld->GetBlockHash().ToString(), pindexOld->nHeight); @@ -4377,13 +4339,13 @@ bool Chainstate::ReplayBlocks() const CBlockIndex& pindex{*Assert(pindexNew->GetAncestor(nHeight))}; LogPrintf("Rolling forward %s (%i)\n", pindex.GetBlockHash().ToString(), nHeight); - uiInterface.ShowProgress(_("Replaying blocks…").translated, (int) ((nHeight - nForkHeight) * 100.0 / (pindexNew->nHeight - nForkHeight)) , false); + m_chainman.GetNotifications().progress(_("Replaying blocks…"), (int)((nHeight - nForkHeight) * 100.0 / (pindexNew->nHeight - nForkHeight)), false); if (!RollforwardBlock(&pindex, cache)) return false; } cache.SetBestBlock(pindexNew->GetBlockHash()); cache.Flush(); - uiInterface.ShowProgress("", 100, false); + m_chainman.GetNotifications().progress(bilingual_str{}, 100, false); return true; } @@ -4418,7 +4380,7 @@ bool ChainstateManager::LoadBlockIndex() // Load block index from databases bool needs_init = fReindex; if (!fReindex) { - bool ret = m_blockman.LoadBlockIndexDB(GetConsensus()); + bool ret{m_blockman.LoadBlockIndexDB()}; if (!ret) return false; m_blockman.ScanAndUnlinkAlreadyPrunedFiles(); @@ -4522,7 +4484,7 @@ bool Chainstate::LoadGenesisBlock() try { const CBlock& block = params.GenesisBlock(); - FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, 0, m_chain, params, nullptr)}; + FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, 0, m_chain, nullptr)}; if (blockPos.IsNull()) { return error("%s: writing genesis block to disk failed", __func__); } @@ -4565,7 +4527,7 @@ void Chainstate::LoadExternalBlockFile( try { // locate a header unsigned char buf[CMessageHeader::MESSAGE_START_SIZE]; - blkdat.FindByte(params.MessageStart()[0]); + blkdat.FindByte(std::byte(params.MessageStart()[0])); nRewind = blkdat.GetPos() + 1; blkdat >> buf; if (memcmp(buf, params.MessageStart(), CMessageHeader::MESSAGE_START_SIZE)) { @@ -4593,6 +4555,9 @@ void Chainstate::LoadExternalBlockFile( // next block, but it's still possible to rewind to the start of the current block (without a disk read). nRewind = nBlockPos + nSize; blkdat.SkipTo(nRewind); + + std::shared_ptr<CBlock> pblock{}; // needs to remain available after the cs_main lock is released to avoid duplicate reads from disk + { LOCK(cs_main); // detect out of order blocks, and store them for later @@ -4610,7 +4575,7 @@ void Chainstate::LoadExternalBlockFile( if (!pindex || (pindex->nStatus & BLOCK_HAVE_DATA) == 0) { // This block can be processed immediately; rewind to its start, read and deserialize it. blkdat.SetPos(nBlockPos); - std::shared_ptr<CBlock> pblock{std::make_shared<CBlock>()}; + pblock = std::make_shared<CBlock>(); blkdat >> *pblock; nRewind = blkdat.GetPos(); @@ -4634,6 +4599,21 @@ void Chainstate::LoadExternalBlockFile( } } + if (m_blockman.IsPruneMode() && !fReindex && pblock) { + // must update the tip for pruning to work while importing with -loadblock. + // this is a tradeoff to conserve disk space at the expense of time + // spent updating the tip to be able to prune. + // otherwise, ActivateBestChain won't be called by the import process + // until after all of the block files are loaded. ActivateBestChain can be + // called by concurrent network message processing. but, that is not + // reliable for the purpose of pruning while importing. + BlockValidationState state; + if (!ActivateBestChain(state, pblock)) { + LogPrint(BCLog::REINDEX, "failed to activate chain (%s)\n", state.ToString()); + break; + } + } + NotifyHeaderTip(*this); if (!blocks_with_unknown_parent) continue; @@ -4648,7 +4628,7 @@ void Chainstate::LoadExternalBlockFile( while (range.first != range.second) { std::multimap<uint256, FlatFilePos>::iterator it = range.first; std::shared_ptr<CBlock> pblockrecursive = std::make_shared<CBlock>(); - if (ReadBlockFromDisk(*pblockrecursive, it->second, params.GetConsensus())) { + if (m_blockman.ReadBlockFromDisk(*pblockrecursive, it->second)) { LogPrint(BCLog::REINDEX, "%s: Processing out of order child %s of %s\n", __func__, pblockrecursive->GetHash().ToString(), head.ToString()); LOCK(cs_main); @@ -5124,7 +5104,7 @@ bool ChainstateManager::ActivateSnapshot( // PopulateAndValidateSnapshot can return (in error) before the leveldb datadir // has been created, so only attempt removal if we got that far. - if (auto snapshot_datadir = node::FindSnapshotChainstateDir()) { + if (auto snapshot_datadir = node::FindSnapshotChainstateDir(m_options.datadir)) { // We have to destruct leveldb::DB in order to release the db lock, otherwise // DestroyDB() (in DeleteCoinsDBFromDisk()) will fail. See `leveldb::~DBImpl()`. // Destructing the chainstate (and so resetting the coinsviews object) does this. @@ -5604,17 +5584,12 @@ ChainstateManager::~ChainstateManager() LOCK(::cs_main); m_versionbitscache.Clear(); - - // TODO: The warning cache should probably become non-global - for (auto& i : warningcache) { - i.clear(); - } } bool ChainstateManager::DetectSnapshotChainstate(CTxMemPool* mempool) { assert(!m_snapshot_chainstate); - std::optional<fs::path> path = node::FindSnapshotChainstateDir(); + std::optional<fs::path> path = node::FindSnapshotChainstateDir(m_options.datadir); if (!path) { return false; } diff --git a/src/validation.h b/src/validation.h index fd0e2115f7..cb5b291dab 100644 --- a/src/validation.h +++ b/src/validation.h @@ -65,8 +65,6 @@ struct Params; static const int MAX_SCRIPTCHECK_THREADS = 15; /** -par default (number of script-checking threads, 0 = auto) */ static const int DEFAULT_SCRIPTCHECK_THREADS = 0; -/** Default for -stopatheight */ -static const int DEFAULT_STOPATHEIGHT = 0; /** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ActiveChain().Tip() will not be pruned. */ static const unsigned int MIN_BLOCKS_TO_KEEP = 288; static const signed int DEFAULT_CHECKBLOCKS = 6; @@ -364,9 +362,13 @@ enum class VerifyDBResult { }; /** RAII wrapper for VerifyDB: Verify consistency of the block and coin databases */ -class CVerifyDB { +class CVerifyDB +{ +private: + kernel::Notifications& m_notifications; + public: - CVerifyDB(); + explicit CVerifyDB(kernel::Notifications& notifications); ~CVerifyDB(); [[nodiscard]] VerifyDBResult VerifyDB( Chainstate& chainstate, @@ -936,6 +938,8 @@ private: //! nullopt. std::optional<int> GetSnapshotBaseHeight() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + std::array<ThresholdConditionCache, VERSIONBITS_NUM_BITS> m_warningcache GUARDED_BY(::cs_main); + //! Return true if a chainstate is considered usable. //! //! This is false when a background validation chainstate has completed its @@ -955,6 +959,8 @@ public: bool ShouldCheckBlockIndex() const { return *Assert(m_options.check_block_index); } const arith_uint256& MinimumChainWork() const { return *Assert(m_options.minimum_chain_work); } const uint256& AssumedValidBlock() const { return *Assert(m_options.assumed_valid_block); } + kernel::Notifications& GetNotifications() const { return m_options.notifications; }; + int StopAtHeight() const { return m_options.stop_at_height; }; /** * Alias for ::cs_main. diff --git a/src/version.h b/src/version.h index ee646eefc3..611a670314 100644 --- a/src/version.h +++ b/src/version.h @@ -20,9 +20,6 @@ static const int MIN_PEER_PROTO_VERSION = 31800; //! BIP 0031, pong message, is enabled for all versions AFTER this one static const int BIP0031_VERSION = 60000; -//! "filter*" commands are disabled without NODE_BLOOM after and including this version -static const int NO_BLOOM_VERSION = 70011; - //! "sendheaders" command and announcing blocks with headers starts with this version static const int SENDHEADERS_VERSION = 70012; diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index fa6e56c6e4..68abdcd81e 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -668,14 +668,15 @@ void BerkeleyDatabase::ReloadDbEnv() env->ReloadDbEnv(); } -BerkeleyCursor::BerkeleyCursor(BerkeleyDatabase& database, BerkeleyBatch* batch) +BerkeleyCursor::BerkeleyCursor(BerkeleyDatabase& database, const BerkeleyBatch& batch, Span<const std::byte> prefix) + : m_key_prefix(prefix.begin(), prefix.end()) { if (!database.m_db.get()) { throw std::runtime_error(STR_INTERNAL_BUG("BerkeleyDatabase does not exist")); } // Transaction argument to cursor is only needed when using the cursor to // write to the database. Read-only cursors do not need a txn pointer. - int ret = database.m_db->cursor(batch ? batch->txn() : nullptr, &m_cursor, 0); + int ret = database.m_db->cursor(batch.txn(), &m_cursor, 0); if (ret != 0) { throw std::runtime_error(STR_INTERNAL_BUG(strprintf("BDB Cursor could not be created. Returned %d", ret))); } @@ -685,19 +686,30 @@ DatabaseCursor::Status BerkeleyCursor::Next(DataStream& ssKey, DataStream& ssVal { if (m_cursor == nullptr) return Status::FAIL; // Read at cursor - SafeDbt datKey; + SafeDbt datKey(m_key_prefix.data(), m_key_prefix.size()); SafeDbt datValue; - int ret = m_cursor->get(datKey, datValue, DB_NEXT); + int ret = -1; + if (m_first && !m_key_prefix.empty()) { + ret = m_cursor->get(datKey, datValue, DB_SET_RANGE); + } else { + ret = m_cursor->get(datKey, datValue, DB_NEXT); + } + m_first = false; if (ret == DB_NOTFOUND) { return Status::DONE; } - if (ret != 0 || datKey.get_data() == nullptr || datValue.get_data() == nullptr) { + if (ret != 0) { return Status::FAIL; } + Span<const std::byte> raw_key = {AsBytePtr(datKey.get_data()), datKey.get_size()}; + if (!m_key_prefix.empty() && std::mismatch(raw_key.begin(), raw_key.end(), m_key_prefix.begin(), m_key_prefix.end()).second != m_key_prefix.end()) { + return Status::DONE; + } + // Convert to streams ssKey.clear(); - ssKey.write({AsBytePtr(datKey.get_data()), datKey.get_size()}); + ssKey.write(raw_key); ssValue.clear(); ssValue.write({AsBytePtr(datValue.get_data()), datValue.get_size()}); return Status::MORE; @@ -713,7 +725,13 @@ BerkeleyCursor::~BerkeleyCursor() std::unique_ptr<DatabaseCursor> BerkeleyBatch::GetNewCursor() { if (!pdb) return nullptr; - return std::make_unique<BerkeleyCursor>(m_database); + return std::make_unique<BerkeleyCursor>(m_database, *this); +} + +std::unique_ptr<DatabaseCursor> BerkeleyBatch::GetNewPrefixCursor(Span<const std::byte> prefix) +{ + if (!pdb) return nullptr; + return std::make_unique<BerkeleyCursor>(m_database, *this, prefix); } bool BerkeleyBatch::TxnBegin() @@ -777,6 +795,7 @@ bool BerkeleyBatch::ReadKey(DataStream&& key, DataStream& value) SafeDbt datValue; int ret = pdb->get(activeTxn, datKey, datValue, 0); if (ret == 0 && datValue.get_data() != nullptr) { + value.clear(); value.write({AsBytePtr(datValue.get_data()), datValue.get_size()}); return true; } @@ -825,7 +844,7 @@ bool BerkeleyBatch::HasKey(DataStream&& key) bool BerkeleyBatch::ErasePrefix(Span<const std::byte> prefix) { if (!TxnBegin()) return false; - auto cursor{std::make_unique<BerkeleyCursor>(m_database, this)}; + auto cursor{std::make_unique<BerkeleyCursor>(m_database, *this)}; // const_cast is safe below even though prefix_key is an in/out parameter, // because we are not using the DB_DBT_USERMEM flag, so BDB will allocate // and return a different output data pointer diff --git a/src/wallet/bdb.h b/src/wallet/bdb.h index 4fac128bf1..8cc03692d6 100644 --- a/src/wallet/bdb.h +++ b/src/wallet/bdb.h @@ -7,10 +7,10 @@ #define BITCOIN_WALLET_BDB_H #include <clientversion.h> +#include <common/system.h> #include <serialize.h> #include <streams.h> #include <util/fs.h> -#include <util/system.h> #include <wallet/db.h> #include <atomic> @@ -190,9 +190,13 @@ class BerkeleyCursor : public DatabaseCursor { private: Dbc* m_cursor; + std::vector<std::byte> m_key_prefix; + bool m_first{true}; public: - explicit BerkeleyCursor(BerkeleyDatabase& database, BerkeleyBatch* batch=nullptr); + // Constructor for cursor for records matching the prefix + // To match all records, an empty prefix may be provided. + explicit BerkeleyCursor(BerkeleyDatabase& database, const BerkeleyBatch& batch, Span<const std::byte> prefix = {}); ~BerkeleyCursor() override; Status Next(DataStream& key, DataStream& value) override; @@ -229,6 +233,7 @@ public: void Close() override; std::unique_ptr<DatabaseCursor> GetNewCursor() override; + std::unique_ptr<DatabaseCursor> GetNewPrefixCursor(Span<const std::byte> prefix) override; bool TxnBegin() override; bool TxnCommit() override; bool TxnAbort() override; diff --git a/src/wallet/coincontrol.cpp b/src/wallet/coincontrol.cpp index ad2fab4b7d..2087119db9 100644 --- a/src/wallet/coincontrol.cpp +++ b/src/wallet/coincontrol.cpp @@ -11,4 +11,72 @@ CCoinControl::CCoinControl() { m_avoid_partial_spends = gArgs.GetBoolArg("-avoidpartialspends", DEFAULT_AVOIDPARTIALSPENDS); } + +bool CCoinControl::HasSelected() const +{ + return !m_selected_inputs.empty(); +} + +bool CCoinControl::IsSelected(const COutPoint& output) const +{ + return m_selected_inputs.count(output) > 0; +} + +bool CCoinControl::IsExternalSelected(const COutPoint& output) const +{ + return m_external_txouts.count(output) > 0; +} + +std::optional<CTxOut> CCoinControl::GetExternalOutput(const COutPoint& outpoint) const +{ + const auto ext_it = m_external_txouts.find(outpoint); + if (ext_it == m_external_txouts.end()) { + return std::nullopt; + } + + return std::make_optional(ext_it->second); +} + +void CCoinControl::Select(const COutPoint& output) +{ + m_selected_inputs.insert(output); +} + +void CCoinControl::SelectExternal(const COutPoint& outpoint, const CTxOut& txout) +{ + m_selected_inputs.insert(outpoint); + m_external_txouts.emplace(outpoint, txout); +} + +void CCoinControl::UnSelect(const COutPoint& output) +{ + m_selected_inputs.erase(output); +} + +void CCoinControl::UnSelectAll() +{ + m_selected_inputs.clear(); +} + +std::vector<COutPoint> CCoinControl::ListSelected() const +{ + return {m_selected_inputs.begin(), m_selected_inputs.end()}; +} + +void CCoinControl::SetInputWeight(const COutPoint& outpoint, int64_t weight) +{ + m_input_weights[outpoint] = weight; +} + +bool CCoinControl::HasInputWeight(const COutPoint& outpoint) const +{ + return m_input_weights.count(outpoint) > 0; +} + +int64_t CCoinControl::GetInputWeight(const COutPoint& outpoint) const +{ + auto it = m_input_weights.find(outpoint); + assert(it != m_input_weights.end()); + return it->second; +} } // namespace wallet diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index cb6f0a1635..7ff8fee5bc 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -13,9 +13,9 @@ #include <script/signingprovider.h> #include <script/standard.h> -#include <optional> #include <algorithm> #include <map> +#include <optional> #include <set> namespace wallet { @@ -63,76 +63,62 @@ public: CCoinControl(); - bool HasSelected() const - { - return (setSelected.size() > 0); - } - - bool IsSelected(const COutPoint& output) const - { - return (setSelected.count(output) > 0); - } - - bool IsExternalSelected(const COutPoint& output) const - { - return (m_external_txouts.count(output) > 0); - } - - bool GetExternalOutput(const COutPoint& outpoint, CTxOut& txout) const - { - const auto ext_it = m_external_txouts.find(outpoint); - if (ext_it == m_external_txouts.end()) { - return false; - } - txout = ext_it->second; - return true; - } - - void Select(const COutPoint& output) - { - setSelected.insert(output); - } - - void SelectExternal(const COutPoint& outpoint, const CTxOut& txout) - { - setSelected.insert(outpoint); - m_external_txouts.emplace(outpoint, txout); - } - - void UnSelect(const COutPoint& output) - { - setSelected.erase(output); - } - - void UnSelectAll() - { - setSelected.clear(); - } - - void ListSelected(std::vector<COutPoint>& vOutpoints) const - { - vOutpoints.assign(setSelected.begin(), setSelected.end()); - } - - void SetInputWeight(const COutPoint& outpoint, int64_t weight) - { - m_input_weights[outpoint] = weight; - } - - bool HasInputWeight(const COutPoint& outpoint) const - { - return m_input_weights.count(outpoint) > 0; - } - - int64_t GetInputWeight(const COutPoint& outpoint) const - { - auto it = m_input_weights.find(outpoint); - assert(it != m_input_weights.end()); - return it->second; - } + /** + * Returns true if there are pre-selected inputs. + */ + bool HasSelected() const; + /** + * Returns true if the given output is pre-selected. + */ + bool IsSelected(const COutPoint& output) const; + /** + * Returns true if the given output is selected as an external input. + */ + bool IsExternalSelected(const COutPoint& output) const; + /** + * Returns the external output for the given outpoint if it exists. + */ + std::optional<CTxOut> GetExternalOutput(const COutPoint& outpoint) const; + /** + * Lock-in the given output for spending. + * The output will be included in the transaction even if it's not the most optimal choice. + */ + void Select(const COutPoint& output); + /** + * Lock-in the given output as an external input for spending because it is not in the wallet. + * The output will be included in the transaction even if it's not the most optimal choice. + */ + void SelectExternal(const COutPoint& outpoint, const CTxOut& txout); + /** + * Unselects the given output. + */ + void UnSelect(const COutPoint& output); + /** + * Unselects all outputs. + */ + void UnSelectAll(); + /** + * List the selected inputs. + */ + std::vector<COutPoint> ListSelected() const; + /** + * Set an input's weight. + */ + void SetInputWeight(const COutPoint& outpoint, int64_t weight); + /** + * Returns true if the input weight is set. + */ + bool HasInputWeight(const COutPoint& outpoint) const; + /** + * Returns the input weight. + */ + int64_t GetInputWeight(const COutPoint& outpoint) const; private: - std::set<COutPoint> setSelected; + //! Selected inputs (inputs that will be used, regardless of whether they're optimal or not) + std::set<COutPoint> m_selected_inputs; + //! Map of external inputs to include in the transaction + //! These are not in the wallet, so we need to track them separately std::map<COutPoint, CTxOut> m_external_txouts; //! Map of COutPoints to the maximum weight for that input std::map<COutPoint, int64_t> m_input_weights; diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 832d24746f..bd74025f09 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -4,13 +4,13 @@ #include <wallet/coinselection.h> +#include <common/system.h> #include <consensus/amount.h> #include <consensus/consensus.h> #include <logging.h> #include <policy/feerate.h> #include <util/check.h> #include <util/moneystr.h> -#include <util/system.h> #include <numeric> #include <optional> diff --git a/src/wallet/coinselection.h b/src/wallet/coinselection.h index 723f5bbfb3..432d7d1431 100644 --- a/src/wallet/coinselection.h +++ b/src/wallet/coinselection.h @@ -11,8 +11,8 @@ #include <policy/feerate.h> #include <primitives/transaction.h> #include <random.h> -#include <util/system.h> #include <util/check.h> +#include <util/insert.h> #include <util/result.h> #include <optional> diff --git a/src/wallet/crypter.cpp b/src/wallet/crypter.cpp index cd414b3d44..e2799c2d05 100644 --- a/src/wallet/crypter.cpp +++ b/src/wallet/crypter.cpp @@ -4,9 +4,9 @@ #include <wallet/crypter.h> +#include <common/system.h> #include <crypto/aes.h> #include <crypto/sha512.h> -#include <util/system.h> #include <vector> diff --git a/src/wallet/db.h b/src/wallet/db.h index 6834ba6963..9d684225c3 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -113,6 +113,7 @@ public: virtual bool ErasePrefix(Span<const std::byte> prefix) = 0; virtual std::unique_ptr<DatabaseCursor> GetNewCursor() = 0; + virtual std::unique_ptr<DatabaseCursor> GetNewPrefixCursor(Span<const std::byte> prefix) = 0; virtual bool TxnBegin() = 0; virtual bool TxnCommit() = 0; virtual bool TxnAbort() = 0; @@ -174,51 +175,6 @@ public: virtual std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) = 0; }; -class DummyCursor : public DatabaseCursor -{ - Status Next(DataStream& key, DataStream& value) override { return Status::FAIL; } -}; - -/** RAII class that provides access to a DummyDatabase. Never fails. */ -class DummyBatch : public DatabaseBatch -{ -private: - bool ReadKey(DataStream&& key, DataStream& value) override { return true; } - bool WriteKey(DataStream&& key, DataStream&& value, bool overwrite = true) override { return true; } - bool EraseKey(DataStream&& key) override { return true; } - bool HasKey(DataStream&& key) override { return true; } - bool ErasePrefix(Span<const std::byte> prefix) override { return true; } - -public: - void Flush() override {} - void Close() override {} - - std::unique_ptr<DatabaseCursor> GetNewCursor() override { return std::make_unique<DummyCursor>(); } - bool TxnBegin() override { return true; } - bool TxnCommit() override { return true; } - bool TxnAbort() override { return true; } -}; - -/** A dummy WalletDatabase that does nothing and never fails. Only used by unit tests. - **/ -class DummyDatabase : public WalletDatabase -{ -public: - void Open() override {}; - void AddRef() override {} - void RemoveRef() override {} - bool Rewrite(const char* pszSkip=nullptr) override { return true; } - bool Backup(const std::string& strDest) const override { return true; } - void Close() override {} - void Flush() override {} - bool PeriodicFlush() override { return true; } - void IncrementUpdateCounter() override { ++nUpdateCounter; } - void ReloadDbEnv() override {} - std::string Filename() override { return "dummy"; } - std::string Format() override { return "dummy"; } - std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) override { return std::make_unique<DummyBatch>(); } -}; - enum class DatabaseFormat { BERKELEY, SQLITE, diff --git a/src/wallet/dump.cpp b/src/wallet/dump.cpp index 865af0bb23..44c93eed7c 100644 --- a/src/wallet/dump.cpp +++ b/src/wallet/dump.cpp @@ -255,11 +255,7 @@ bool CreateFromDump(const ArgsManager& args, const std::string& name, const fs:: std::vector<unsigned char> k = ParseHex(key); std::vector<unsigned char> v = ParseHex(value); - - DataStream ss_key{k}; - DataStream ss_value{v}; - - if (!batch->Write(ss_key, ss_value)) { + if (!batch->Write(Span{k}, Span{v})) { error = strprintf(_("Error: Unable to write record to new wallet")); ret = false; break; diff --git a/src/wallet/external_signer_scriptpubkeyman.cpp b/src/wallet/external_signer_scriptpubkeyman.cpp index 079e3baa4e..6d026d02c1 100644 --- a/src/wallet/external_signer_scriptpubkeyman.cpp +++ b/src/wallet/external_signer_scriptpubkeyman.cpp @@ -4,8 +4,8 @@ #include <chainparams.h> #include <common/args.h> +#include <common/system.h> #include <external_signer.h> -#include <util/system.h> #include <wallet/external_signer_scriptpubkeyman.h> #include <iostream> @@ -45,7 +45,7 @@ ExternalSigner ExternalSignerScriptPubKeyMan::GetExternalSigner() { const std::string command = gArgs.GetArg("-signer", ""); if (command == "") throw std::runtime_error(std::string(__func__) + ": restart bitcoind with -signer=<cmd>"); std::vector<ExternalSigner> signers; - ExternalSigner::Enumerate(command, signers, Params().NetworkIDString()); + ExternalSigner::Enumerate(command, signers, Params().GetChainTypeString()); if (signers.empty()) throw std::runtime_error(std::string(__func__) + ": No external signers found"); // TODO: add fingerprint argument instead of failing in case of multiple signers. if (signers.size() > 1) throw std::runtime_error(std::string(__func__) + ": More than one external signer found. Please connect only one at a time."); diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index b6b1fa1d3e..0e1f1f9847 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -2,13 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <common/system.h> #include <consensus/validation.h> #include <interfaces/chain.h> #include <policy/fees.h> #include <policy/policy.h> #include <util/moneystr.h> #include <util/rbf.h> -#include <util/system.h> #include <util/translation.h> #include <wallet/coincontrol.h> #include <wallet/feebumper.h> diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 2560dda87c..4cdfadbee2 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -62,7 +62,7 @@ bool VerifyWallets(WalletContext& context) options.require_existing = true; options.verify = false; if (MakeWalletDatabase("", options, status, error_string)) { - util::SettingsValue wallets(util::SettingsValue::VARR); + common::SettingsValue wallets(common::SettingsValue::VARR); wallets.push_back(""); // Default wallet name is "" // Pass write=false because no need to write file and probably // better not to. If unnamed wallet needs to be added next startup diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index a5be0739a9..0bd6a9670c 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -608,7 +608,9 @@ RPCHelpMan getaddressinfo() if (const std::unique_ptr<CKeyMetadata> meta = spk_man->GetMetadata(dest)) { ret.pushKV("timestamp", meta->nCreateTime); if (meta->has_key_origin) { - ret.pushKV("hdkeypath", WriteHDKeypath(meta->key_origin.path)); + // In legacy wallets hdkeypath has always used an apostrophe for + // hardened derivation. Perhaps some external tool depends on that. + ret.pushKV("hdkeypath", WriteHDKeypath(meta->key_origin.path, /*apostrophe=*/!desc_spk_man)); ret.pushKV("hdseedid", meta->hd_seed_id.GetHex()); ret.pushKV("hdmasterfingerprint", HexStr(meta->key_origin.fingerprint)); } diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 553bbfb62f..b93419292e 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -805,7 +805,7 @@ RPCHelpMan dumpwallet() } else { file << "change=1"; } - file << strprintf(" # addr=%s%s\n", strAddr, (metadata.has_key_origin ? " hdkeypath="+WriteHDKeypath(metadata.key_origin.path) : "")); + file << strprintf(" # addr=%s%s\n", strAddr, (metadata.has_key_origin ? " hdkeypath="+WriteHDKeypath(metadata.key_origin.path, /*apostrophe=*/true) : "")); } } file << "\n"; @@ -1298,7 +1298,7 @@ RPCHelpMan importmulti() }, }, RPCArgOptions{.oneline_description="\"requests\""}}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", { {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions after all imports."}, }, diff --git a/src/wallet/rpc/coins.cpp b/src/wallet/rpc/coins.cpp index 750ef69f6e..22f0f0b83c 100644 --- a/src/wallet/rpc/coins.cpp +++ b/src/wallet/rpc/coins.cpp @@ -320,7 +320,7 @@ RPCHelpMan lockunspent() }); const uint256 txid(ParseHashO(o, "txid")); - const int nOutput = find_value(o, "vout").getInt<int>(); + const int nOutput = o.find_value("vout").getInt<int>(); if (nOutput < 0) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, vout cannot be negative"); } @@ -513,7 +513,7 @@ RPCHelpMan listunspent() }, {"include_unsafe", RPCArg::Type::BOOL, RPCArg::Default{true}, "Include outputs that are not safe to spend\n" "See description of \"safe\" attribute below."}, - {"query_options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "JSON with query options", + {"query_options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", { {"minimumAmount", RPCArg::Type::AMOUNT, RPCArg::Default{FormatMoney(0)}, "Minimum value of each UTXO in " + CURRENCY_UNIT + ""}, {"maximumAmount", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"unlimited"}, "Maximum value of each UTXO in " + CURRENCY_UNIT + ""}, diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index 88ee6e96b0..feee643f26 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -453,9 +453,9 @@ RPCHelpMan settxfee() static std::vector<RPCArg> FundTxDoc(bool solving_data = true) { std::vector<RPCArg> args = { - {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks"}, + {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks", RPCArgOptions{.also_positional = true}}, {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, "The fee estimate mode, must be one of (case insensitive):\n" - "\"" + FeeModes("\"\n\"") + "\""}, + "\"" + FeeModes("\"\n\"") + "\"", RPCArgOptions{.also_positional = true}}, { "replaceable", RPCArg::Type::BOOL, RPCArg::DefaultHint{"wallet default"}, "Marks this transaction as BIP125-replaceable.\n" "Allows this transaction to be replaced by a transaction with higher fees" @@ -673,7 +673,7 @@ void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, for (const UniValue& input : options["input_weights"].get_array().getValues()) { uint256 txid = ParseHashO(input, "txid"); - const UniValue& vout_v = find_value(input, "vout"); + const UniValue& vout_v = input.find_value("vout"); if (!vout_v.isNum()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, missing vout key"); } @@ -682,7 +682,7 @@ void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, vout cannot be negative"); } - const UniValue& weight_v = find_value(input, "weight"); + const UniValue& weight_v = input.find_value("weight"); if (!weight_v.isNum()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, missing weight key"); } @@ -758,7 +758,7 @@ RPCHelpMan fundrawtransaction() "Only pay-to-pubkey, multisig, and P2SH versions thereof are currently supported for watch-only\n", { {"hexstring", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The hex string of the raw transaction"}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "for backward compatibility: passing in a true instead of an object will result in {\"includeWatching\":true}", + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "For backward compatibility: passing in a true instead of an object will result in {\"includeWatching\":true}", Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::Default{true}, "For a transaction with existing inputs, automatically include more if they are not enough."}, @@ -997,7 +997,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) "* WARNING: before version 0.21, fee_rate was in " + CURRENCY_UNIT + "/kvB. As of 0.21, fee_rate is in " + CURRENCY_ATOM + "/vB. *\n", { {"txid", RPCArg::Type::STR_HEX, RPCArg::Optional::NO, "The txid to be bumped"}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", { {"conf_target", RPCArg::Type::NUM, RPCArg::DefaultHint{"wallet -txconfirmtarget"}, "Confirmation target in blocks\n"}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, @@ -1187,7 +1187,7 @@ RPCHelpMan send() {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, "The fee estimate mode, must be one of (case insensitive):\n" "\"" + FeeModes("\"\n\"") + "\""}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::DefaultHint{"false when \"inputs\" are specified, true otherwise"},"Automatically include coins from the wallet to cover the target amount.\n"}, @@ -1200,7 +1200,7 @@ RPCHelpMan send() {"change_address", RPCArg::Type::STR, RPCArg::DefaultHint{"automatic"}, "The bitcoin address to receive the change"}, {"change_position", RPCArg::Type::NUM, RPCArg::DefaultHint{"random"}, "The index of the change output"}, {"change_type", RPCArg::Type::STR, RPCArg::DefaultHint{"set by -changetype"}, "The output type to use. Only valid if change_address is not specified. Options are \"legacy\", \"p2sh-segwit\", \"bech32\" and \"bech32m\"."}, - {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, + {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB.", RPCArgOptions{.also_positional = true}}, {"include_watching", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Also select inputs which are watch only.\n" "Only solvable inputs can be used. Watch-only destinations are solvable if the public key and/or output script was imported,\n" "e.g. with 'importpubkey' or 'importmulti' with the 'pubkeys' or 'desc' field."}, @@ -1302,11 +1302,11 @@ RPCHelpMan sendall() "\"" + FeeModes("\"\n\"") + "\""}, {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, { - "options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + "options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", Cat<std::vector<RPCArg>>( { {"add_to_wallet", RPCArg::Type::BOOL, RPCArg::Default{true}, "When false, returns the serialized transaction without broadcasting or adding it to the wallet"}, - {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB."}, + {"fee_rate", RPCArg::Type::AMOUNT, RPCArg::DefaultHint{"not set, fall back to wallet fee estimation"}, "Specify a fee rate in " + CURRENCY_ATOM + "/vB.", RPCArgOptions{.also_positional = true}}, {"include_watching", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Also select inputs which are watch-only.\n" "Only solvable inputs can be used. Watch-only destinations are solvable if the public key and/or output script was imported,\n" "e.g. with 'importpubkey' or 'importmulti' with the 'pubkeys' or 'desc' field."}, @@ -1635,7 +1635,7 @@ RPCHelpMan walletcreatefundedpsbt() OutputsDoc(), RPCArgOptions{.skip_type_check = true}}, {"locktime", RPCArg::Type::NUM, RPCArg::Default{0}, "Raw locktime. Non-0 value also locktime-activates inputs"}, - {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED, "", + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", Cat<std::vector<RPCArg>>( { {"add_inputs", RPCArg::Type::BOOL, RPCArg::DefaultHint{"false when \"inputs\" are specified, true otherwise"}, "Automatically include coins from the wallet to cover the target amount.\n"}, diff --git a/src/wallet/rpc/util.cpp b/src/wallet/rpc/util.cpp index 4ff44b84b0..06ec7db1bc 100644 --- a/src/wallet/rpc/util.cpp +++ b/src/wallet/rpc/util.cpp @@ -6,7 +6,7 @@ #include <common/url.h> #include <rpc/util.h> -#include <util/system.h> +#include <util/any.h> #include <util/translation.h> #include <wallet/context.h> #include <wallet/wallet.h> diff --git a/src/wallet/rpc/wallet.cpp b/src/wallet/rpc/wallet.cpp index d71359508d..340e4115af 100644 --- a/src/wallet/rpc/wallet.cpp +++ b/src/wallet/rpc/wallet.cpp @@ -123,7 +123,7 @@ static RPCHelpMan getwalletinfo() obj.pushKV("avoid_reuse", pwallet->IsWalletFlagSet(WALLET_FLAG_AVOID_REUSE)); if (pwallet->IsScanning()) { UniValue scanning(UniValue::VOBJ); - scanning.pushKV("duration", pwallet->ScanningDuration() / 1000); + scanning.pushKV("duration", Ticks<std::chrono::seconds>(pwallet->ScanningDuration())); scanning.pushKV("progress", pwallet->ScanningProgress()); obj.pushKV("scanning", scanning); } else { @@ -647,7 +647,7 @@ RPCHelpMan simulaterawtransaction() {"rawtx", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, }, }, - {"options", RPCArg::Type::OBJ_USER_KEYS, RPCArg::Optional::OMITTED, "Options", + {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", { {"include_watchonly", RPCArg::Type::BOOL, RPCArg::DefaultHint{"true for watch-only wallets, otherwise false"}, "Whether to include watch-only addresses (see RPC importaddress)"}, }, diff --git a/src/wallet/salvage.cpp b/src/wallet/salvage.cpp index 06c8c8bb37..e303310273 100644 --- a/src/wallet/salvage.cpp +++ b/src/wallet/salvage.cpp @@ -23,6 +23,52 @@ static bool KeyFilter(const std::string& type) return WalletBatch::IsKeyType(type) || type == DBKeys::HDCHAIN; } +class DummyCursor : public DatabaseCursor +{ + Status Next(DataStream& key, DataStream& value) override { return Status::FAIL; } +}; + +/** RAII class that provides access to a DummyDatabase. Never fails. */ +class DummyBatch : public DatabaseBatch +{ +private: + bool ReadKey(DataStream&& key, DataStream& value) override { return true; } + bool WriteKey(DataStream&& key, DataStream&& value, bool overwrite=true) override { return true; } + bool EraseKey(DataStream&& key) override { return true; } + bool HasKey(DataStream&& key) override { return true; } + bool ErasePrefix(Span<const std::byte> prefix) override { return true; } + +public: + void Flush() override {} + void Close() override {} + + std::unique_ptr<DatabaseCursor> GetNewCursor() override { return std::make_unique<DummyCursor>(); } + std::unique_ptr<DatabaseCursor> GetNewPrefixCursor(Span<const std::byte> prefix) override { return GetNewCursor(); } + bool TxnBegin() override { return true; } + bool TxnCommit() override { return true; } + bool TxnAbort() override { return true; } +}; + +/** A dummy WalletDatabase that does nothing and never fails. Only used by salvage. + **/ +class DummyDatabase : public WalletDatabase +{ +public: + void Open() override {}; + void AddRef() override {} + void RemoveRef() override {} + bool Rewrite(const char* pszSkip=nullptr) override { return true; } + bool Backup(const std::string& strDest) const override { return true; } + void Close() override {} + void Flush() override {} + bool PeriodicFlush() override { return true; } + void IncrementUpdateCounter() override { ++nUpdateCounter; } + void ReloadDbEnv() override {} + std::string Filename() override { return "dummy"; } + std::string Format() override { return "dummy"; } + std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) override { return std::make_unique<DummyBatch>(); } +}; + bool RecoverDatabaseFile(const ArgsManager& args, const fs::path& file_path, bilingual_str& error, std::vector<bilingual_str>& warnings) { DatabaseOptions options; @@ -135,7 +181,7 @@ bool RecoverDatabaseFile(const ArgsManager& args, const fs::path& file_path, bil } DbTxn* ptxn = env->TxnBegin(); - CWallet dummyWallet(nullptr, "", CreateDummyWalletDatabase()); + CWallet dummyWallet(nullptr, "", std::make_unique<DummyDatabase>()); for (KeyValPair& row : salvagedData) { /* Filter for only private key type KV pairs to be added to the salvaged wallet */ diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 8c34c45e49..796b7f11c5 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -710,6 +710,8 @@ void LegacyScriptPubKeyMan::UpdateTimeFirstKey(int64_t nCreateTime) } else if (!nTimeFirstKey || nCreateTime < nTimeFirstKey) { nTimeFirstKey = nCreateTime; } + + NotifyFirstKeyTimeChanged(this, nTimeFirstKey); } bool LegacyScriptPubKeyMan::LoadKey(const CKey& key, const CPubKey &pubkey) @@ -1555,7 +1557,7 @@ bool LegacyScriptPubKeyMan::AddKeyOriginWithDB(WalletBatch& batch, const CPubKey std::copy(info.fingerprint, info.fingerprint + 4, mapKeyMetadata[pubkey.GetID()].key_origin.fingerprint); mapKeyMetadata[pubkey.GetID()].key_origin.path = info.path; mapKeyMetadata[pubkey.GetID()].has_key_origin = true; - mapKeyMetadata[pubkey.GetID()].hdKeypath = WriteHDKeypath(info.path); + mapKeyMetadata[pubkey.GetID()].hdKeypath = WriteHDKeypath(info.path, /*apostrophe=*/true); return batch.WriteKeyMetadata(mapKeyMetadata[pubkey.GetID()], pubkey, true); } @@ -1821,7 +1823,7 @@ std::optional<MigrationData> LegacyScriptPubKeyMan::MigrateToDescriptor() // Make the combo descriptor std::string xpub = EncodeExtPubKey(master_key.Neuter()); - std::string desc_str = "combo(" + xpub + "/0'/" + ToString(i) + "'/*')"; + std::string desc_str = "combo(" + xpub + "/0h/" + ToString(i) + "h/*h)"; FlatSigningProvider keys; std::string error; std::unique_ptr<Descriptor> desc = Parse(desc_str, keys, error, false); @@ -2264,20 +2266,20 @@ bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_ std::string desc_suffix = "/*)"; switch (addr_type) { case OutputType::LEGACY: { - desc_prefix = "pkh(" + xpub + "/44'"; + desc_prefix = "pkh(" + xpub + "/44h"; break; } case OutputType::P2SH_SEGWIT: { - desc_prefix = "sh(wpkh(" + xpub + "/49'"; + desc_prefix = "sh(wpkh(" + xpub + "/49h"; desc_suffix += ")"; break; } case OutputType::BECH32: { - desc_prefix = "wpkh(" + xpub + "/84'"; + desc_prefix = "wpkh(" + xpub + "/84h"; break; } case OutputType::BECH32M: { - desc_prefix = "tr(" + xpub + "/86'"; + desc_prefix = "tr(" + xpub + "/86h"; break; } case OutputType::UNKNOWN: { @@ -2290,13 +2292,13 @@ bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_ // Mainnet derives at 0', testnet and regtest derive at 1' if (Params().IsTestChain()) { - desc_prefix += "/1'"; + desc_prefix += "/1h"; } else { - desc_prefix += "/0'"; + desc_prefix += "/0h"; } std::string internal_path = internal ? "/1" : "/0"; - std::string desc_str = desc_prefix + "/0'" + internal_path + desc_suffix; + std::string desc_str = desc_prefix + "/0h" + internal_path + desc_suffix; // Make the descriptor FlatSigningProvider keys; @@ -2736,6 +2738,8 @@ void DescriptorScriptPubKeyMan::UpdateWalletDescriptor(WalletDescriptor& descrip m_map_script_pub_keys.clear(); m_max_cached_index = -1; m_wallet_descriptor = descriptor; + + NotifyFirstKeyTimeChanged(this, m_wallet_descriptor.creation_time); } bool DescriptorScriptPubKeyMan::CanUpdateToWalletDescriptor(const WalletDescriptor& descriptor, std::string& error) diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 22b67c88e9..bf35c776ae 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_WALLET_SCRIPTPUBKEYMAN_H #define BITCOIN_WALLET_SCRIPTPUBKEYMAN_H +#include <logging.h> #include <psbt.h> #include <script/descriptor.h> #include <script/signingprovider.h> @@ -27,6 +28,8 @@ enum class OutputType; struct bilingual_str; namespace wallet { +struct MigrationData; + // Wallet storage things that ScriptPubKeyMans need in order to be able to store things to the wallet database. // It provides access to things that are part of the entire wallet and not specific to a ScriptPubKeyMan such as // wallet flags, wallet version, encryption keys, encryption status, and the database itself. This allows a @@ -256,6 +259,9 @@ public: /** Keypool has new keys */ boost::signals2::signal<void ()> NotifyCanGetAddressesChanged; + + /** Birth time changed */ + boost::signals2::signal<void (const ScriptPubKeyMan* spkm, int64_t new_birth_time)> NotifyFirstKeyTimeChanged; }; /** OutputTypes supported by the LegacyScriptPubKeyMan */ @@ -658,6 +664,18 @@ public: void UpgradeDescriptorCache(); }; + +/** struct containing information needed for migrating legacy wallets to descriptor wallets */ +struct MigrationData +{ + CExtKey master_key; + std::vector<std::pair<std::string, int64_t>> watch_descs; + std::vector<std::pair<std::string, int64_t>> solvable_descs; + std::vector<std::unique_ptr<DescriptorScriptPubKeyMan>> desc_spkms; + std::shared_ptr<CWallet> watchonly_wallet{nullptr}; + std::shared_ptr<CWallet> solvable_wallet{nullptr}; +}; + } // namespace wallet #endif // BITCOIN_WALLET_SCRIPTPUBKEYMAN_H diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 748f40dce8..99c6582f9c 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -4,6 +4,7 @@ #include <algorithm> #include <common/args.h> +#include <common/system.h> #include <consensus/amount.h> #include <consensus/validation.h> #include <interfaces/chain.h> @@ -15,7 +16,6 @@ #include <util/fees.h> #include <util/moneystr.h> #include <util/rbf.h> -#include <util/system.h> #include <util/trace.h> #include <util/translation.h> #include <wallet/coincontrol.h> @@ -52,9 +52,7 @@ int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* wallet, TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *wallet, const std::vector<CTxOut>& txouts, const CCoinControl* coin_control) { CMutableTransaction txNew(tx); - if (!wallet->DummySignTx(txNew, txouts, coin_control)) { - return TxSize{-1, -1}; - } + if (!wallet->DummySignTx(txNew, txouts, coin_control)) return TxSize{-1, -1}; CTransaction ctx(txNew); int64_t vsize = GetVirtualTransactionSize(ctx); int64_t weight = GetTransactionWeight(ctx); @@ -72,11 +70,9 @@ TxSize CalculateMaximumSignedTxSize(const CTransaction &tx, const CWallet *walle assert(input.prevout.n < mi->second.tx->vout.size()); txouts.emplace_back(mi->second.tx->vout.at(input.prevout.n)); } else if (coin_control) { - CTxOut txout; - if (!coin_control->GetExternalOutput(input.prevout, txout)) { - return TxSize{-1, -1}; - } - txouts.emplace_back(txout); + const auto& txout{coin_control->GetExternalOutput(input.prevout)}; + if (!txout) return TxSize{-1, -1}; + txouts.emplace_back(*txout); } else { return TxSize{-1, -1}; } @@ -163,10 +159,8 @@ util::Result<PreSelectedInputs> FetchSelectedInputs(const CWallet& wallet, const const CoinSelectionParams& coin_selection_params) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { PreSelectedInputs result; - std::vector<COutPoint> vPresetInputs; - coin_control.ListSelected(vPresetInputs); const bool can_grind_r = wallet.CanGrindR(); - for (const COutPoint& outpoint : vPresetInputs) { + for (const COutPoint& outpoint : coin_control.ListSelected()) { int input_bytes = -1; CTxOut txout; if (auto ptr_wtx = wallet.GetWalletTx(outpoint.hash)) { @@ -178,9 +172,12 @@ util::Result<PreSelectedInputs> FetchSelectedInputs(const CWallet& wallet, const input_bytes = CalculateMaximumSignedInputSize(txout, &wallet, &coin_control); } else { // The input is external. We did not find the tx in mapWallet. - if (!coin_control.GetExternalOutput(outpoint, txout)) { + const auto out{coin_control.GetExternalOutput(outpoint)}; + if (!out) { return util::Error{strprintf(_("Not found pre-selected input %s"), outpoint.ToString())}; } + + txout = *out; } if (input_bytes == -1) { diff --git a/src/wallet/sqlite.cpp b/src/wallet/sqlite.cpp index 77e8a4e9c1..8d7fe97bb1 100644 --- a/src/wallet/sqlite.cpp +++ b/src/wallet/sqlite.cpp @@ -9,6 +9,7 @@ #include <logging.h> #include <sync.h> #include <util/fs_helpers.h> +#include <util/check.h> #include <util/strencodings.h> #include <util/translation.h> #include <wallet/db.h> @@ -34,12 +35,31 @@ static void ErrorLogCallback(void* arg, int code, const char* msg) LogPrintf("SQLite Error. Code: %d. Message: %s\n", code, msg); } +static int TraceSqlCallback(unsigned code, void* context, void* param1, void* param2) +{ + auto* db = static_cast<SQLiteDatabase*>(context); + if (code == SQLITE_TRACE_STMT) { + auto* stmt = static_cast<sqlite3_stmt*>(param1); + // To be conservative and avoid leaking potentially secret information + // in the log file, only expand statements that query the database, not + // statements that update the database. + char* expanded{sqlite3_stmt_readonly(stmt) ? sqlite3_expanded_sql(stmt) : nullptr}; + LogPrintf("[%s] SQLite Statement: %s\n", db->Filename(), expanded ? expanded : sqlite3_sql(stmt)); + if (expanded) sqlite3_free(expanded); + } + return SQLITE_OK; +} + static bool BindBlobToStatement(sqlite3_stmt* stmt, int index, Span<const std::byte> blob, const std::string& description) { - int res = sqlite3_bind_blob(stmt, index, blob.data(), blob.size(), SQLITE_STATIC); + // Pass a pointer to the empty string "" below instead of passing the + // blob.data() pointer if the blob.data() pointer is null. Passing a null + // data pointer to bind_blob would cause sqlite to bind the SQL NULL value + // instead of the empty blob value X'', which would mess up SQL comparisons. + int res = sqlite3_bind_blob(stmt, index, blob.data() ? static_cast<const void*>(blob.data()) : "", blob.size(), SQLITE_STATIC); if (res != SQLITE_OK) { LogPrintf("Unable to bind %s to statement: %s\n", description, sqlite3_errstr(res)); sqlite3_clear_bindings(stmt); @@ -235,6 +255,13 @@ void SQLiteDatabase::Open() if (ret != SQLITE_OK) { throw std::runtime_error(strprintf("SQLiteDatabase: Failed to enable extended result codes: %s\n", sqlite3_errstr(ret))); } + // Trace SQL statements if tracing is enabled with -debug=walletdb -loglevel=walletdb:trace + if (LogAcceptCategory(BCLog::WALLETDB, BCLog::Level::Trace)) { + ret = sqlite3_trace_v2(m_db, SQLITE_TRACE_STMT, TraceSqlCallback, this); + if (ret != SQLITE_OK) { + LogPrintf("Failed to enable SQL tracing for %s\n", Filename()); + } + } } if (sqlite3_db_readonly(m_db, "main") != 0) { @@ -409,6 +436,7 @@ bool SQLiteBatch::ReadKey(DataStream&& key, DataStream& value) // Leftmost column in result is index 0 const std::byte* data{AsBytePtr(sqlite3_column_blob(m_read_stmt, 0))}; size_t data_size(sqlite3_column_bytes(m_read_stmt, 0)); + value.clear(); value.write({data, data_size}); sqlite3_clear_bindings(m_read_stmt); @@ -495,6 +523,9 @@ DatabaseCursor::Status SQLiteCursor::Next(DataStream& key, DataStream& value) return Status::FAIL; } + key.clear(); + value.clear(); + // Leftmost column in result is index 0 const std::byte* key_data{AsBytePtr(sqlite3_column_blob(m_cursor_stmt, 0))}; size_t key_data_size(sqlite3_column_bytes(m_cursor_stmt, 0)); @@ -507,6 +538,7 @@ DatabaseCursor::Status SQLiteCursor::Next(DataStream& key, DataStream& value) SQLiteCursor::~SQLiteCursor() { + sqlite3_clear_bindings(m_cursor_stmt); sqlite3_reset(m_cursor_stmt); int res = sqlite3_finalize(m_cursor_stmt); if (res != SQLITE_OK) { @@ -530,6 +562,48 @@ std::unique_ptr<DatabaseCursor> SQLiteBatch::GetNewCursor() return cursor; } +std::unique_ptr<DatabaseCursor> SQLiteBatch::GetNewPrefixCursor(Span<const std::byte> prefix) +{ + if (!m_database.m_db) return nullptr; + + // To get just the records we want, the SQL statement does a comparison of the binary data + // where the data must be greater than or equal to the prefix, and less than + // the prefix incremented by one (when interpreted as an integer) + std::vector<std::byte> start_range(prefix.begin(), prefix.end()); + std::vector<std::byte> end_range(prefix.begin(), prefix.end()); + auto it = end_range.rbegin(); + for (; it != end_range.rend(); ++it) { + if (*it == std::byte(std::numeric_limits<unsigned char>::max())) { + *it = std::byte(0); + continue; + } + *it = std::byte(std::to_integer<unsigned char>(*it) + 1); + break; + } + if (it == end_range.rend()) { + // If the prefix is all 0xff bytes, clear end_range as we won't need it + end_range.clear(); + } + + auto cursor = std::make_unique<SQLiteCursor>(start_range, end_range); + if (!cursor) return nullptr; + + const char* stmt_text = end_range.empty() ? "SELECT key, value FROM main WHERE key >= ?" : + "SELECT key, value FROM main WHERE key >= ? AND key < ?"; + int res = sqlite3_prepare_v2(m_database.m_db, stmt_text, -1, &cursor->m_cursor_stmt, nullptr); + if (res != SQLITE_OK) { + throw std::runtime_error(strprintf( + "SQLiteDatabase: Failed to setup cursor SQL statement: %s\n", sqlite3_errstr(res))); + } + + if (!BindBlobToStatement(cursor->m_cursor_stmt, 1, cursor->m_prefix_range_start, "prefix_start")) return nullptr; + if (!end_range.empty()) { + if (!BindBlobToStatement(cursor->m_cursor_stmt, 2, cursor->m_prefix_range_end, "prefix_end")) return nullptr; + } + + return cursor; +} + bool SQLiteBatch::TxnBegin() { if (!m_database.m_db || sqlite3_get_autocommit(m_database.m_db) == 0) return false; diff --git a/src/wallet/sqlite.h b/src/wallet/sqlite.h index d9de40569b..0378bbb8d6 100644 --- a/src/wallet/sqlite.h +++ b/src/wallet/sqlite.h @@ -15,12 +15,21 @@ struct bilingual_str; namespace wallet { class SQLiteDatabase; +/** RAII class that provides a database cursor */ class SQLiteCursor : public DatabaseCursor { public: sqlite3_stmt* m_cursor_stmt{nullptr}; + // Copies of the prefix things for the prefix cursor. + // Prevents SQLite from accessing temp variables for the prefix things. + std::vector<std::byte> m_prefix_range_start; + std::vector<std::byte> m_prefix_range_end; explicit SQLiteCursor() {} + explicit SQLiteCursor(std::vector<std::byte> start_range, std::vector<std::byte> end_range) + : m_prefix_range_start(std::move(start_range)), + m_prefix_range_end(std::move(end_range)) + {} ~SQLiteCursor() override; Status Next(DataStream& key, DataStream& value) override; @@ -57,6 +66,7 @@ public: void Close() override; std::unique_ptr<DatabaseCursor> GetNewCursor() override; + std::unique_ptr<DatabaseCursor> GetNewPrefixCursor(Span<const std::byte> prefix) override; bool TxnBegin() override; bool TxnCommit() override; bool TxnAbort() override; diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 7f66179517..b1d67c1432 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -12,6 +12,7 @@ #include <wallet/coincontrol.h> #include <wallet/coinselection.h> #include <wallet/spend.h> +#include <wallet/test/util.h> #include <wallet/test/wallet_test_fixture.h> #include <wallet/wallet.h> @@ -176,7 +177,7 @@ inline std::vector<OutputGroup>& KnapsackGroupOutputs(const CoinsResult& availab static std::unique_ptr<CWallet> NewWallet(const node::NodeContext& m_node, const std::string& wallet_name = "") { - std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), wallet_name, CreateMockWalletDatabase()); + std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), wallet_name, CreateMockableWalletDatabase()); BOOST_CHECK(wallet->LoadWallet() == DBErrors::LOAD_OK); LOCK(wallet->cs_wallet); wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -431,7 +432,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) CAmount selection_target = 16 * CENT; const auto& no_res = SelectCoinsBnB(GroupCoins(available_coins.All(), /*subtract_fee_outputs*/true), selection_target, /*cost_of_change=*/0, MAX_STANDARD_TX_WEIGHT); - BOOST_ASSERT(!no_res); + BOOST_REQUIRE(!no_res); BOOST_CHECK(util::ErrorString(no_res).original.find("The inputs size exceeds the maximum weight") != std::string::npos); // Now add same coin value with a good size and check that it gets selected diff --git a/src/wallet/test/db_tests.cpp b/src/wallet/test/db_tests.cpp index 7761308bbc..4cda35ed8d 100644 --- a/src/wallet/test/db_tests.cpp +++ b/src/wallet/test/db_tests.cpp @@ -6,13 +6,56 @@ #include <test/util/setup_common.h> #include <util/fs.h> +#include <util/translation.h> +#ifdef USE_BDB #include <wallet/bdb.h> +#endif +#ifdef USE_SQLITE +#include <wallet/sqlite.h> +#endif +#include <wallet/test/util.h> +#include <wallet/walletutil.h> // for WALLET_FLAG_DESCRIPTORS #include <fstream> #include <memory> #include <string> +inline std::ostream& operator<<(std::ostream& os, const std::pair<const SerializeData, SerializeData>& kv) +{ + Span key{kv.first}, value{kv.second}; + os << "(\"" << std::string_view{reinterpret_cast<const char*>(key.data()), key.size()} << "\", \"" + << std::string_view{reinterpret_cast<const char*>(key.data()), key.size()} << "\")"; + return os; +} + namespace wallet { + +static Span<const std::byte> StringBytes(std::string_view str) +{ + return AsBytes<const char>({str.data(), str.size()}); +} + +static SerializeData StringData(std::string_view str) +{ + auto bytes = StringBytes(str); + return SerializeData{bytes.begin(), bytes.end()}; +} + +static void CheckPrefix(DatabaseBatch& batch, Span<const std::byte> prefix, MockableData expected) +{ + std::unique_ptr<DatabaseCursor> cursor = batch.GetNewPrefixCursor(prefix); + MockableData actual; + while (true) { + DataStream key, value; + DatabaseCursor::Status status = cursor->Next(key, value); + if (status == DatabaseCursor::Status::DONE) break; + BOOST_CHECK(status == DatabaseCursor::Status::MORE); + BOOST_CHECK( + actual.emplace(SerializeData(key.begin(), key.end()), SerializeData(value.begin(), value.end())).second); + } + BOOST_CHECK_EQUAL_COLLECTIONS(actual.begin(), actual.end(), expected.begin(), expected.end()); +} + BOOST_FIXTURE_TEST_SUITE(db_tests, BasicTestingSetup) static std::shared_ptr<BerkeleyEnvironment> GetWalletEnv(const fs::path& path, fs::path& database_filename) @@ -78,5 +121,90 @@ BOOST_AUTO_TEST_CASE(getwalletenv_g_dbenvs_free_instance) BOOST_CHECK(env_2_a == env_2_b); } +static std::vector<std::unique_ptr<WalletDatabase>> TestDatabases(const fs::path& path_root) +{ + std::vector<std::unique_ptr<WalletDatabase>> dbs; + DatabaseOptions options; + DatabaseStatus status; + bilingual_str error; +#ifdef USE_BDB + dbs.emplace_back(MakeBerkeleyDatabase(path_root / "bdb", options, status, error)); +#endif +#ifdef USE_SQLITE + dbs.emplace_back(MakeSQLiteDatabase(path_root / "sqlite", options, status, error)); +#endif + dbs.emplace_back(CreateMockableWalletDatabase()); + return dbs; +} + +BOOST_AUTO_TEST_CASE(db_cursor_prefix_range_test) +{ + // Test each supported db + for (const auto& database : TestDatabases(m_path_root)) { + BOOST_ASSERT(database); + + std::vector<std::string> prefixes = {"", "FIRST", "SECOND", "P\xfe\xff", "P\xff\x01", "\xff\xff"}; + + // Write elements to it + std::unique_ptr<DatabaseBatch> handler = database->MakeBatch(); + for (unsigned int i = 0; i < 10; i++) { + for (const auto& prefix : prefixes) { + BOOST_CHECK(handler->Write(std::make_pair(prefix, i), i)); + } + } + + // Now read all the items by prefix and verify that each element gets parsed correctly + for (const auto& prefix : prefixes) { + DataStream s_prefix; + s_prefix << prefix; + std::unique_ptr<DatabaseCursor> cursor = handler->GetNewPrefixCursor(s_prefix); + DataStream key; + DataStream value; + for (int i = 0; i < 10; i++) { + DatabaseCursor::Status status = cursor->Next(key, value); + BOOST_ASSERT(status == DatabaseCursor::Status::MORE); + + std::string key_back; + unsigned int i_back; + key >> key_back >> i_back; + BOOST_CHECK_EQUAL(key_back, prefix); + + unsigned int value_back; + value >> value_back; + BOOST_CHECK_EQUAL(value_back, i_back); + } + + // Let's now read it once more, it should return DONE + BOOST_CHECK(cursor->Next(key, value) == DatabaseCursor::Status::DONE); + } + } +} + +// Lower level DatabaseBase::GetNewPrefixCursor test, to cover cases that aren't +// covered in the higher level test above. The higher level test uses +// serialized strings which are prefixed with string length, so it doesn't test +// truly empty prefixes or prefixes that begin with \xff +BOOST_AUTO_TEST_CASE(db_cursor_prefix_byte_test) +{ + const MockableData::value_type + e{StringData(""), StringData("e")}, + p{StringData("prefix"), StringData("p")}, + ps{StringData("prefixsuffix"), StringData("ps")}, + f{StringData("\xff"), StringData("f")}, + fs{StringData("\xffsuffix"), StringData("fs")}, + ff{StringData("\xff\xff"), StringData("ff")}, + ffs{StringData("\xff\xffsuffix"), StringData("ffs")}; + for (const auto& database : TestDatabases(m_path_root)) { + std::unique_ptr<DatabaseBatch> batch = database->MakeBatch(); + for (const auto& [k, v] : {e, p, ps, f, fs, ff, ffs}) { + batch->Write(MakeUCharSpan(k), MakeUCharSpan(v)); + } + CheckPrefix(*batch, StringBytes(""), {e, p, ps, f, fs, ff, ffs}); + CheckPrefix(*batch, StringBytes("prefix"), {p, ps}); + CheckPrefix(*batch, StringBytes("\xff"), {f, fs, ff, ffs}); + CheckPrefix(*batch, StringBytes("\xff\xff"), {ff, ffs}); + } +} + BOOST_AUTO_TEST_SUITE_END() } // namespace wallet diff --git a/src/wallet/test/fuzz/notifications.cpp b/src/wallet/test/fuzz/notifications.cpp index de381a5ec9..f4b69f7403 100644 --- a/src/wallet/test/fuzz/notifications.cpp +++ b/src/wallet/test/fuzz/notifications.cpp @@ -141,6 +141,10 @@ FUZZ_TARGET_INIT(wallet_notifications, initialize_setup) info.prev_hash = &block.hashPrevBlock; info.height = chain.size(); info.data = █ + // Ensure that no blocks are skipped by the wallet by setting the chain's accumulated + // time to the maximum value. This ensures that the wallet's birth time is always + // earlier than this maximum time. + info.chain_time_max = std::numeric_limits<unsigned int>::max(); a.wallet->blockConnected(info); b.wallet->blockConnected(info); // Store the coins for the next block diff --git a/src/wallet/test/group_outputs_tests.cpp b/src/wallet/test/group_outputs_tests.cpp index 283e87989c..e6b25cc216 100644 --- a/src/wallet/test/group_outputs_tests.cpp +++ b/src/wallet/test/group_outputs_tests.cpp @@ -6,6 +6,7 @@ #include <wallet/coinselection.h> #include <wallet/spend.h> +#include <wallet/test/util.h> #include <wallet/wallet.h> #include <boost/test/unit_test.hpp> @@ -17,7 +18,7 @@ static int nextLockTime = 0; static std::shared_ptr<CWallet> NewWallet(const node::NodeContext& m_node) { - std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", CreateMockWalletDatabase()); + std::unique_ptr<CWallet> wallet = std::make_unique<CWallet>(m_node.chain.get(), "", CreateMockableWalletDatabase()); wallet->LoadWallet(); LOCK(wallet->cs_wallet); wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); diff --git a/src/wallet/test/init_test_fixture.cpp b/src/wallet/test/init_test_fixture.cpp index 0adc63876c..5bdf36ec19 100644 --- a/src/wallet/test/init_test_fixture.cpp +++ b/src/wallet/test/init_test_fixture.cpp @@ -4,6 +4,7 @@ #include <common/args.h> #include <univalue.h> +#include <util/chaintype.h> #include <util/check.h> #include <util/fs.h> @@ -13,7 +14,7 @@ #include <wallet/test/init_test_fixture.h> namespace wallet { -InitWalletDirTestingSetup::InitWalletDirTestingSetup(const std::string& chainName) : BasicTestingSetup(chainName) +InitWalletDirTestingSetup::InitWalletDirTestingSetup(const ChainType chainType) : BasicTestingSetup(chainType) { m_wallet_loader = MakeWalletLoader(*m_node.chain, m_args); diff --git a/src/wallet/test/init_test_fixture.h b/src/wallet/test/init_test_fixture.h index df5819fd1d..ac7bb8997c 100644 --- a/src/wallet/test/init_test_fixture.h +++ b/src/wallet/test/init_test_fixture.h @@ -9,11 +9,12 @@ #include <interfaces/wallet.h> #include <node/context.h> #include <test/util/setup_common.h> +#include <util/chaintype.h> namespace wallet { struct InitWalletDirTestingSetup: public BasicTestingSetup { - explicit InitWalletDirTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); + explicit InitWalletDirTestingSetup(const ChainType chain_type = ChainType::MAIN); ~InitWalletDirTestingSetup(); void SetWalletDir(const fs::path& walletdir_path); diff --git a/src/wallet/test/ismine_tests.cpp b/src/wallet/test/ismine_tests.cpp index f6f2d423e4..fd0718fbb9 100644 --- a/src/wallet/test/ismine_tests.cpp +++ b/src/wallet/test/ismine_tests.cpp @@ -10,6 +10,7 @@ #include <test/util/setup_common.h> #include <wallet/types.h> #include <wallet/wallet.h> +#include <wallet/test/util.h> #include <boost/test/unit_test.hpp> @@ -55,7 +56,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK compressed - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForRawPubKey(pubkeys[0]); @@ -74,7 +75,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK compressed - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "pk(" + EncodeSecret(keys[0]) + ")"; auto spk_manager = CreateDescriptor(keystore, desc_str, true); @@ -86,7 +87,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK uncompressed - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForRawPubKey(uncompressedPubkey); @@ -105,7 +106,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PK uncompressed - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "pk(" + EncodeSecret(uncompressedKey) + ")"; auto spk_manager = CreateDescriptor(keystore, desc_str, true); @@ -117,7 +118,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PKH compressed - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForDestination(PKHash(pubkeys[0])); @@ -136,7 +137,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PKH compressed - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "pkh(" + EncodeSecret(keys[0]) + ")"; auto spk_manager = CreateDescriptor(keystore, desc_str, true); @@ -148,7 +149,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PKH uncompressed - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); scriptPubKey = GetScriptForDestination(PKHash(uncompressedPubkey)); @@ -167,7 +168,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2PKH uncompressed - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "pkh(" + EncodeSecret(uncompressedKey) + ")"; auto spk_manager = CreateDescriptor(keystore, desc_str, true); @@ -179,7 +180,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2SH - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -206,7 +207,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2SH - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "sh(pkh(" + EncodeSecret(keys[0]) + "))"; auto spk_manager = CreateDescriptor(keystore, desc_str, true); @@ -219,7 +220,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2SH inside P2SH (invalid) - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -238,7 +239,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2SH inside P2SH (invalid) - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "sh(sh(" + EncodeSecret(keys[0]) + "))"; auto spk_manager = CreateDescriptor(keystore, desc_str, false); @@ -247,7 +248,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2SH inside P2WSH (invalid) - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -266,7 +267,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2SH inside P2WSH (invalid) - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "wsh(sh(" + EncodeSecret(keys[0]) + "))"; auto spk_manager = CreateDescriptor(keystore, desc_str, false); @@ -275,7 +276,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH inside P2WSH (invalid) - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -292,7 +293,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH inside P2WSH (invalid) - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "wsh(wpkh(" + EncodeSecret(keys[0]) + "))"; auto spk_manager = CreateDescriptor(keystore, desc_str, false); @@ -301,7 +302,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2WSH inside P2WSH (invalid) - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -320,7 +321,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // (P2PKH inside) P2WSH inside P2WSH (invalid) - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "wsh(wsh(" + EncodeSecret(keys[0]) + "))"; auto spk_manager = CreateDescriptor(keystore, desc_str, false); @@ -329,7 +330,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH compressed - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -345,7 +346,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH compressed - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "wpkh(" + EncodeSecret(keys[0]) + ")"; auto spk_manager = CreateDescriptor(keystore, desc_str, true); @@ -357,7 +358,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH uncompressed - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); @@ -378,7 +379,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WPKH uncompressed (invalid) - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "wpkh(" + EncodeSecret(uncompressedKey) + ")"; auto spk_manager = CreateDescriptor(keystore, desc_str, false); @@ -387,7 +388,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // scriptPubKey multisig - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -422,7 +423,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // scriptPubKey multisig - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "multi(2, " + EncodeSecret(uncompressedKey) + ", " + EncodeSecret(keys[1]) + ")"; auto spk_manager = CreateDescriptor(keystore, desc_str, true); @@ -434,7 +435,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2SH multisig - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); @@ -457,7 +458,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2SH multisig - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "sh(multi(2, " + EncodeSecret(uncompressedKey) + ", " + EncodeSecret(keys[1]) + "))"; @@ -471,7 +472,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig with compressed keys - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -500,7 +501,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig with compressed keys - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "wsh(multi(2, " + EncodeSecret(keys[0]) + ", " + EncodeSecret(keys[1]) + "))"; @@ -514,7 +515,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig with uncompressed key - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(uncompressedKey)); @@ -543,7 +544,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig with uncompressed key (invalid) - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "wsh(multi(2, " + EncodeSecret(uncompressedKey) + ", " + EncodeSecret(keys[1]) + "))"; @@ -553,7 +554,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig wrapped in P2SH - Legacy { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); @@ -583,7 +584,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // P2WSH multisig wrapped in P2SH - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "sh(wsh(multi(2, " + EncodeSecret(keys[0]) + ", " + EncodeSecret(keys[1]) + ")))"; @@ -598,7 +599,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // Combo - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "combo(" + EncodeSecret(keys[0]) + ")"; @@ -642,7 +643,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // Taproot - Descriptor { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); std::string desc_str = "tr(" + EncodeSecret(keys[0]) + ")"; @@ -660,7 +661,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // OP_RETURN { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -675,7 +676,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // witness unspendable { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -690,7 +691,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // witness unknown { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); @@ -705,7 +706,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) // Nonstandard { - CWallet keystore(chain.get(), "", CreateDummyWalletDatabase()); + CWallet keystore(chain.get(), "", CreateMockableWalletDatabase()); keystore.SetupLegacyScriptPubKeyMan(); LOCK(keystore.GetLegacyScriptPubKeyMan()->cs_KeyStore); BOOST_CHECK(keystore.GetLegacyScriptPubKeyMan()->AddKey(keys[0])); diff --git a/src/wallet/test/scriptpubkeyman_tests.cpp b/src/wallet/test/scriptpubkeyman_tests.cpp index 90042f5252..d4997c418a 100644 --- a/src/wallet/test/scriptpubkeyman_tests.cpp +++ b/src/wallet/test/scriptpubkeyman_tests.cpp @@ -7,6 +7,7 @@ #include <test/util/setup_common.h> #include <wallet/scriptpubkeyman.h> #include <wallet/wallet.h> +#include <wallet/test/util.h> #include <boost/test/unit_test.hpp> @@ -18,7 +19,7 @@ BOOST_FIXTURE_TEST_SUITE(scriptpubkeyman_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(CanProvide) { // Set up wallet and keyman variables. - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", CreateMockableWalletDatabase()); LegacyScriptPubKeyMan& keyman = *wallet.GetOrCreateLegacyScriptPubKeyMan(); // Make a 1 of 2 multisig script diff --git a/src/wallet/test/util.cpp b/src/wallet/test/util.cpp index b7bf312edf..069ab25f26 100644 --- a/src/wallet/test/util.cpp +++ b/src/wallet/test/util.cpp @@ -7,7 +7,9 @@ #include <chain.h> #include <key.h> #include <key_io.h> +#include <streams.h> #include <test/util/setup_common.h> +#include <wallet/context.h> #include <wallet/wallet.h> #include <wallet/walletdb.h> @@ -16,7 +18,7 @@ namespace wallet { std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cchain, const CKey& key) { - auto wallet = std::make_unique<CWallet>(&chain, "", CreateMockWalletDatabase()); + auto wallet = std::make_unique<CWallet>(&chain, "", CreateMockableWalletDatabase()); { LOCK2(wallet->cs_wallet, ::cs_main); wallet->SetLastBlockProcessed(cchain.Height(), cchain.Tip()->GetBlockHash()); @@ -44,28 +46,39 @@ std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cc return wallet; } -std::unique_ptr<WalletDatabase> DuplicateMockDatabase(WalletDatabase& database, DatabaseOptions& options) +std::shared_ptr<CWallet> TestLoadWallet(std::unique_ptr<WalletDatabase> database, WalletContext& context, uint64_t create_flags) { - auto new_database = CreateMockWalletDatabase(options); - - // Get a cursor to the original database - auto batch = database.MakeBatch(); - std::unique_ptr<wallet::DatabaseCursor> cursor = batch->GetNewCursor(); + bilingual_str error; + std::vector<bilingual_str> warnings; + auto wallet = CWallet::Create(context, "", std::move(database), create_flags, error, warnings); + NotifyWalletLoaded(context, wallet); + if (context.chain) { + wallet->postInitProcess(); + } + return wallet; +} - // Get a batch for the new database - auto new_batch = new_database->MakeBatch(); +std::shared_ptr<CWallet> TestLoadWallet(WalletContext& context) +{ + DatabaseOptions options; + options.create_flags = WALLET_FLAG_DESCRIPTORS; + DatabaseStatus status; + bilingual_str error; + std::vector<bilingual_str> warnings; + auto database = MakeWalletDatabase("", options, status, error); + return TestLoadWallet(std::move(database), context, options.create_flags); +} - // Read all records from the original database and write them to the new one - while (true) { - DataStream key{}; - DataStream value{}; - DatabaseCursor::Status status = cursor->Next(key, value); - assert(status != DatabaseCursor::Status::FAIL); - if (status == DatabaseCursor::Status::DONE) break; - new_batch->Write(key, value); - } +void TestUnloadWallet(std::shared_ptr<CWallet>&& wallet) +{ + SyncWithValidationInterfaceQueue(); + wallet->m_chain_notifications_handler.reset(); + UnloadWallet(std::move(wallet)); +} - return new_database; +std::unique_ptr<WalletDatabase> DuplicateMockDatabase(WalletDatabase& database) +{ + return std::make_unique<MockableDatabase>(dynamic_cast<MockableDatabase&>(database).m_records); } std::string getnewaddress(CWallet& w) @@ -79,4 +92,107 @@ CTxDestination getNewDestination(CWallet& w, OutputType output_type) return *Assert(w.GetNewDestination(output_type, "")); } +// BytePrefix compares equality with other byte spans that begin with the same prefix. +struct BytePrefix { Span<const std::byte> prefix; }; +bool operator<(BytePrefix a, Span<const std::byte> b) { return a.prefix < b.subspan(0, std::min(a.prefix.size(), b.size())); } +bool operator<(Span<const std::byte> a, BytePrefix b) { return a.subspan(0, std::min(a.size(), b.prefix.size())) < b.prefix; } + +MockableCursor::MockableCursor(const MockableData& records, bool pass, Span<const std::byte> prefix) +{ + m_pass = pass; + std::tie(m_cursor, m_cursor_end) = records.equal_range(BytePrefix{prefix}); +} + +DatabaseCursor::Status MockableCursor::Next(DataStream& key, DataStream& value) +{ + if (!m_pass) { + return Status::FAIL; + } + if (m_cursor == m_cursor_end) { + return Status::DONE; + } + key.clear(); + value.clear(); + const auto& [key_data, value_data] = *m_cursor; + key.write(key_data); + value.write(value_data); + m_cursor++; + return Status::MORE; +} + +bool MockableBatch::ReadKey(DataStream&& key, DataStream& value) +{ + if (!m_pass) { + return false; + } + SerializeData key_data{key.begin(), key.end()}; + const auto& it = m_records.find(key_data); + if (it == m_records.end()) { + return false; + } + value.clear(); + value.write(it->second); + return true; +} + +bool MockableBatch::WriteKey(DataStream&& key, DataStream&& value, bool overwrite) +{ + if (!m_pass) { + return false; + } + SerializeData key_data{key.begin(), key.end()}; + SerializeData value_data{value.begin(), value.end()}; + auto [it, inserted] = m_records.emplace(key_data, value_data); + if (!inserted && overwrite) { // Overwrite if requested + it->second = value_data; + inserted = true; + } + return inserted; +} + +bool MockableBatch::EraseKey(DataStream&& key) +{ + if (!m_pass) { + return false; + } + SerializeData key_data{key.begin(), key.end()}; + m_records.erase(key_data); + return true; +} + +bool MockableBatch::HasKey(DataStream&& key) +{ + if (!m_pass) { + return false; + } + SerializeData key_data{key.begin(), key.end()}; + return m_records.count(key_data) > 0; +} + +bool MockableBatch::ErasePrefix(Span<const std::byte> prefix) +{ + if (!m_pass) { + return false; + } + auto it = m_records.begin(); + while (it != m_records.end()) { + auto& key = it->first; + if (key.size() < prefix.size() || std::search(key.begin(), key.end(), prefix.begin(), prefix.end()) != key.begin()) { + it++; + continue; + } + it = m_records.erase(it); + } + return true; +} + +std::unique_ptr<WalletDatabase> CreateMockableWalletDatabase(MockableData records) +{ + return std::make_unique<MockableDatabase>(records); +} + +MockableDatabase& GetMockableDatabase(CWallet& wallet) +{ + return dynamic_cast<MockableDatabase&>(wallet.GetDatabase()); +} } // namespace wallet diff --git a/src/wallet/test/util.h b/src/wallet/test/util.h index d726517e21..2a1fe639de 100644 --- a/src/wallet/test/util.h +++ b/src/wallet/test/util.h @@ -6,6 +6,8 @@ #define BITCOIN_WALLET_TEST_UTIL_H #include <script/standard.h> +#include <wallet/db.h> + #include <memory> class ArgsManager; @@ -18,19 +20,112 @@ class Chain; namespace wallet { class CWallet; -struct DatabaseOptions; class WalletDatabase; +struct WalletContext; + +static const DatabaseFormat DATABASE_FORMATS[] = { +#ifdef USE_SQLITE + DatabaseFormat::SQLITE, +#endif +#ifdef USE_BDB + DatabaseFormat::BERKELEY, +#endif +}; + +const std::string ADDRESS_BCRT1_UNSPENDABLE = "bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqq3xueyj"; std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cchain, const CKey& key); +std::shared_ptr<CWallet> TestLoadWallet(WalletContext& context); +std::shared_ptr<CWallet> TestLoadWallet(std::unique_ptr<WalletDatabase> database, WalletContext& context, uint64_t create_flags); +void TestUnloadWallet(std::shared_ptr<CWallet>&& wallet); + // Creates a copy of the provided database -std::unique_ptr<WalletDatabase> DuplicateMockDatabase(WalletDatabase& database, DatabaseOptions& options); +std::unique_ptr<WalletDatabase> DuplicateMockDatabase(WalletDatabase& database); /** Returns a new encoded destination from the wallet (hardcoded to BECH32) */ std::string getnewaddress(CWallet& w); /** Returns a new destination, of an specific type, from the wallet */ CTxDestination getNewDestination(CWallet& w, OutputType output_type); +using MockableData = std::map<SerializeData, SerializeData, std::less<>>; + +class MockableCursor: public DatabaseCursor +{ +public: + MockableData::const_iterator m_cursor; + MockableData::const_iterator m_cursor_end; + bool m_pass; + + explicit MockableCursor(const MockableData& records, bool pass) : m_cursor(records.begin()), m_cursor_end(records.end()), m_pass(pass) {} + MockableCursor(const MockableData& records, bool pass, Span<const std::byte> prefix); + ~MockableCursor() {} + + Status Next(DataStream& key, DataStream& value) override; +}; + +class MockableBatch : public DatabaseBatch +{ +private: + MockableData& m_records; + bool m_pass; + + bool ReadKey(DataStream&& key, DataStream& value) override; + bool WriteKey(DataStream&& key, DataStream&& value, bool overwrite=true) override; + bool EraseKey(DataStream&& key) override; + bool HasKey(DataStream&& key) override; + bool ErasePrefix(Span<const std::byte> prefix) override; + +public: + explicit MockableBatch(MockableData& records, bool pass) : m_records(records), m_pass(pass) {} + ~MockableBatch() {} + + void Flush() override {} + void Close() override {} + + std::unique_ptr<DatabaseCursor> GetNewCursor() override + { + return std::make_unique<MockableCursor>(m_records, m_pass); + } + std::unique_ptr<DatabaseCursor> GetNewPrefixCursor(Span<const std::byte> prefix) override { + return std::make_unique<MockableCursor>(m_records, m_pass, prefix); + } + bool TxnBegin() override { return m_pass; } + bool TxnCommit() override { return m_pass; } + bool TxnAbort() override { return m_pass; } +}; + +/** A WalletDatabase whose contents and return values can be modified as needed for testing + **/ +class MockableDatabase : public WalletDatabase +{ +public: + MockableData m_records; + bool m_pass{true}; + + MockableDatabase(MockableData records = {}) : WalletDatabase(), m_records(records) {} + ~MockableDatabase() {}; + + void Open() override {} + void AddRef() override {} + void RemoveRef() override {} + + bool Rewrite(const char* pszSkip=nullptr) override { return m_pass; } + bool Backup(const std::string& strDest) const override { return m_pass; } + void Flush() override {} + void Close() override {} + bool PeriodicFlush() override { return m_pass; } + void IncrementUpdateCounter() override {} + void ReloadDbEnv() override {} + + std::string Filename() override { return "mockable"; } + std::string Format() override { return "mock"; } + std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) override { return std::make_unique<MockableBatch>(m_records, m_pass); } +}; + +std::unique_ptr<WalletDatabase> CreateMockableWalletDatabase(MockableData records = {}); + +MockableDatabase& GetMockableDatabase(CWallet& wallet); } // namespace wallet #endif // BITCOIN_WALLET_TEST_UTIL_H diff --git a/src/wallet/test/wallet_test_fixture.cpp b/src/wallet/test/wallet_test_fixture.cpp index 2dd8f9ad33..57c538e4ef 100644 --- a/src/wallet/test/wallet_test_fixture.cpp +++ b/src/wallet/test/wallet_test_fixture.cpp @@ -2,15 +2,17 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <wallet/test/util.h> #include <wallet/test/wallet_test_fixture.h> #include <scheduler.h> +#include <util/chaintype.h> namespace wallet { -WalletTestingSetup::WalletTestingSetup(const std::string& chainName) - : TestingSetup(chainName), +WalletTestingSetup::WalletTestingSetup(const ChainType chainType) + : TestingSetup(chainType), m_wallet_loader{interfaces::MakeWalletLoader(*m_node.chain, *Assert(m_node.args))}, - m_wallet(m_node.chain.get(), "", CreateMockWalletDatabase()) + m_wallet(m_node.chain.get(), "", CreateMockableWalletDatabase()) { m_wallet.LoadWallet(); m_chain_notifications_handler = m_node.chain->handleNotifications({ &m_wallet, [](CWallet*) {} }); diff --git a/src/wallet/test/wallet_test_fixture.h b/src/wallet/test/wallet_test_fixture.h index f1ef15a282..4aae02e075 100644 --- a/src/wallet/test/wallet_test_fixture.h +++ b/src/wallet/test/wallet_test_fixture.h @@ -10,6 +10,7 @@ #include <interfaces/chain.h> #include <interfaces/wallet.h> #include <node/context.h> +#include <util/chaintype.h> #include <util/check.h> #include <wallet/wallet.h> @@ -19,7 +20,7 @@ namespace wallet { /** Testing setup and teardown for wallet. */ struct WalletTestingSetup : public TestingSetup { - explicit WalletTestingSetup(const std::string& chainName = CBaseChainParams::MAIN); + explicit WalletTestingSetup(const ChainType chainType = ChainType::MAIN); ~WalletTestingSetup(); std::unique_ptr<interfaces::WalletLoader> m_wallet_loader; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index da4b553394..65b02c267d 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -29,7 +29,6 @@ #include <univalue.h> using node::MAX_BLOCKFILE_SIZE; -using node::UnlinkPrunedFiles; namespace wallet { RPCHelpMan importmulti(); @@ -43,26 +42,6 @@ static_assert(WALLET_INCREMENTAL_RELAY_FEE >= DEFAULT_INCREMENTAL_RELAY_FEE, "wa BOOST_FIXTURE_TEST_SUITE(wallet_tests, WalletTestingSetup) -static std::shared_ptr<CWallet> TestLoadWallet(WalletContext& context) -{ - DatabaseOptions options; - options.create_flags = WALLET_FLAG_DESCRIPTORS; - DatabaseStatus status; - bilingual_str error; - std::vector<bilingual_str> warnings; - auto database = MakeWalletDatabase("", options, status, error); - auto wallet = CWallet::Create(context, "", std::move(database), options.create_flags, error, warnings); - NotifyWalletLoaded(context, wallet); - return wallet; -} - -static void TestUnloadWallet(std::shared_ptr<CWallet>&& wallet) -{ - SyncWithValidationInterfaceQueue(); - wallet->m_chain_notifications_handler.reset(); - UnloadWallet(std::move(wallet)); -} - static CMutableTransaction TestSimpleSpend(const CTransaction& from, uint32_t index, const CKey& key, const CScript& pubkey) { CMutableTransaction mtx; @@ -98,7 +77,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions fails to read an unknown start block. { - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", CreateMockableWalletDatabase()); { LOCK(wallet.cs_wallet); LOCK(Assert(m_node.chainman)->GetMutex()); @@ -119,7 +98,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions picks up transactions in both the old // and new block files. { - CWallet wallet(m_node.chain.get(), "", CreateMockWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", CreateMockableWalletDatabase()); { LOCK(wallet.cs_wallet); LOCK(Assert(m_node.chainman)->GetMutex()); @@ -159,12 +138,12 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) file_number = oldTip->GetBlockPos().nFile; Assert(m_node.chainman)->m_blockman.PruneOneBlockFile(file_number); } - UnlinkPrunedFiles({file_number}); + m_node.chainman->m_blockman.UnlinkPrunedFiles({file_number}); // Verify ScanForWalletTransactions only picks transactions in the new block // file. { - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", CreateMockableWalletDatabase()); { LOCK(wallet.cs_wallet); LOCK(Assert(m_node.chainman)->GetMutex()); @@ -188,11 +167,11 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) file_number = newTip->GetBlockPos().nFile; Assert(m_node.chainman)->m_blockman.PruneOneBlockFile(file_number); } - UnlinkPrunedFiles({file_number}); + m_node.chainman->m_blockman.UnlinkPrunedFiles({file_number}); // Verify ScanForWalletTransactions scans no blocks. { - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", CreateMockableWalletDatabase()); { LOCK(wallet.cs_wallet); LOCK(Assert(m_node.chainman)->GetMutex()); @@ -226,13 +205,13 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) file_number = oldTip->GetBlockPos().nFile; Assert(m_node.chainman)->m_blockman.PruneOneBlockFile(file_number); } - UnlinkPrunedFiles({file_number}); + m_node.chainman->m_blockman.UnlinkPrunedFiles({file_number}); // Verify importmulti RPC returns failure for a key whose creation time is // before the missing block, and success for a key whose creation time is // after. { - const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateMockableWalletDatabase()); wallet->SetupLegacyScriptPubKeyMan(); WITH_LOCK(wallet->cs_wallet, wallet->SetLastBlockProcessed(newTip->nHeight, newTip->GetBlockHash())); WalletContext context; @@ -298,7 +277,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) { WalletContext context; context.args = &m_args; - const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateMockableWalletDatabase()); { auto spk_man = wallet->GetOrCreateLegacyScriptPubKeyMan(); LOCK2(wallet->cs_wallet, spk_man->cs_KeyStore); @@ -321,7 +300,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // Call importwallet RPC and verify all blocks with timestamps >= BLOCK_TIME // were scanned, and no prior blocks were scanned. { - const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateMockableWalletDatabase()); LOCK(wallet->cs_wallet); wallet->SetupLegacyScriptPubKeyMan(); @@ -355,7 +334,7 @@ BOOST_FIXTURE_TEST_CASE(importwallet_rescan, TestChain100Setup) // debit functions. BOOST_FIXTURE_TEST_CASE(coin_mark_dirty_immature_credit, TestChain100Setup) { - CWallet wallet(m_node.chain.get(), "", CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", CreateMockableWalletDatabase()); LOCK(wallet.cs_wallet); LOCK(Assert(m_node.chainman)->GetMutex()); @@ -427,15 +406,6 @@ BOOST_AUTO_TEST_CASE(ComputeTimeSmart) BOOST_CHECK_EQUAL(AddTx(*m_node.chainman, m_wallet, 5, 50, 600), 300); } -static const DatabaseFormat DATABASE_FORMATS[] = { -#ifdef USE_SQLITE - DatabaseFormat::SQLITE, -#endif -#ifdef USE_BDB - DatabaseFormat::BERKELEY, -#endif -}; - void TestLoadWallet(const std::string& name, DatabaseFormat format, std::function<void(std::shared_ptr<CWallet>)> f) { node::NodeContext node; @@ -709,7 +679,7 @@ BOOST_FIXTURE_TEST_CASE(BasicOutputTypesTest, ListCoinsTest) BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) { { - const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateMockableWalletDatabase()); wallet->SetupLegacyScriptPubKeyMan(); wallet->SetMinVersion(FEATURE_LATEST); wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); @@ -717,7 +687,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) BOOST_CHECK(!wallet->GetNewDestination(OutputType::BECH32, "")); } { - const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateDummyWalletDatabase()); + const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", CreateMockableWalletDatabase()); LOCK(wallet->cs_wallet); wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet->SetMinVersion(FEATURE_LATEST); @@ -855,10 +825,11 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) // Reload wallet and make sure new transactions are detected despite events // being blocked + // Loading will also ask for current mempool transactions wallet = TestLoadWallet(context); BOOST_CHECK(rescan_completed); - // AddToWallet events for block_tx and mempool_tx - BOOST_CHECK_EQUAL(addtx_count, 2); + // AddToWallet events for block_tx and mempool_tx (x2) + BOOST_CHECK_EQUAL(addtx_count, 3); { LOCK(wallet->cs_wallet); BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_tx.GetHash()), 1U); @@ -872,7 +843,7 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) SyncWithValidationInterfaceQueue(); // AddToWallet events for block_tx and mempool_tx events are counted a // second time as the notification queue is processed - BOOST_CHECK_EQUAL(addtx_count, 4); + BOOST_CHECK_EQUAL(addtx_count, 5); TestUnloadWallet(std::move(wallet)); @@ -895,7 +866,9 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) SyncWithValidationInterfaceQueue(); }); wallet = TestLoadWallet(context); - BOOST_CHECK_EQUAL(addtx_count, 2); + // Since mempool transactions are requested at the end of loading, there will + // be 2 additional AddToWallet calls, one from the previous test, and a duplicate for mempool_tx + BOOST_CHECK_EQUAL(addtx_count, 2 + 2); { LOCK(wallet->cs_wallet); BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_tx.GetHash()), 1U); @@ -951,62 +924,13 @@ BOOST_FIXTURE_TEST_CASE(ZapSelectTx, TestChain100Setup) TestUnloadWallet(std::move(wallet)); } -class FailCursor : public DatabaseCursor -{ -public: - Status Next(DataStream& key, DataStream& value) override { return Status::FAIL; } -}; - -/** RAII class that provides access to a FailDatabase. Which fails if needed. */ -class FailBatch : public DatabaseBatch -{ -private: - bool m_pass{true}; - bool ReadKey(DataStream&& key, DataStream& value) override { return m_pass; } - bool WriteKey(DataStream&& key, DataStream&& value, bool overwrite = true) override { return m_pass; } - bool EraseKey(DataStream&& key) override { return m_pass; } - bool HasKey(DataStream&& key) override { return m_pass; } - bool ErasePrefix(Span<const std::byte> prefix) override { return m_pass; } - -public: - explicit FailBatch(bool pass) : m_pass(pass) {} - void Flush() override {} - void Close() override {} - - std::unique_ptr<DatabaseCursor> GetNewCursor() override { return std::make_unique<FailCursor>(); } - bool TxnBegin() override { return false; } - bool TxnCommit() override { return false; } - bool TxnAbort() override { return false; } -}; - -/** A dummy WalletDatabase that does nothing, only fails if needed.**/ -class FailDatabase : public WalletDatabase -{ -public: - bool m_pass{true}; // false when this db should fail - - void Open() override {}; - void AddRef() override {} - void RemoveRef() override {} - bool Rewrite(const char* pszSkip=nullptr) override { return true; } - bool Backup(const std::string& strDest) const override { return true; } - void Close() override {} - void Flush() override {} - bool PeriodicFlush() override { return true; } - void IncrementUpdateCounter() override { ++nUpdateCounter; } - void ReloadDbEnv() override {} - std::string Filename() override { return "faildb"; } - std::string Format() override { return "faildb"; } - std::unique_ptr<DatabaseBatch> MakeBatch(bool flush_on_close = true) override { return std::make_unique<FailBatch>(m_pass); } -}; - /** * Checks a wallet invalid state where the inputs (prev-txs) of a new arriving transaction are not marked dirty, * while the transaction that spends them exist inside the in-memory wallet tx map (not stored on db due a db write failure). */ BOOST_FIXTURE_TEST_CASE(wallet_sync_tx_invalid_state_test, TestingSetup) { - CWallet wallet(m_node.chain.get(), "", std::make_unique<FailDatabase>()); + CWallet wallet(m_node.chain.get(), "", CreateMockableWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -1014,11 +938,10 @@ BOOST_FIXTURE_TEST_CASE(wallet_sync_tx_invalid_state_test, TestingSetup) } // Add tx to wallet - const auto& op_dest = wallet.GetNewDestination(OutputType::BECH32M, ""); - BOOST_ASSERT(op_dest); + const auto op_dest{*Assert(wallet.GetNewDestination(OutputType::BECH32M, ""))}; CMutableTransaction mtx; - mtx.vout.push_back({COIN, GetScriptForDestination(*op_dest)}); + mtx.vout.push_back({COIN, GetScriptForDestination(op_dest)}); mtx.vin.push_back(CTxIn(g_insecure_rand_ctx.rand256(), 0)); const auto& tx_id_to_spend = wallet.AddToWallet(MakeTransactionRef(mtx), TxStateInMempool{})->GetHash(); @@ -1053,7 +976,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_sync_tx_invalid_state_test, TestingSetup) // 1) Make db always fail // 2) Try to add a transaction that spends the previously created transaction and // verify that we are not moving forward if the wallet cannot store it - static_cast<FailDatabase&>(wallet.GetDatabase()).m_pass = false; + GetMockableDatabase(wallet).m_pass = false; mtx.vin.clear(); mtx.vin.push_back(CTxIn(good_tx_id, 0)); BOOST_CHECK_EXCEPTION(wallet.transactionAddedToMempool(MakeTransactionRef(mtx)), diff --git a/src/wallet/test/walletdb_tests.cpp b/src/wallet/test/walletdb_tests.cpp index 21842fe780..17b6c4f7ed 100644 --- a/src/wallet/test/walletdb_tests.cpp +++ b/src/wallet/test/walletdb_tests.cpp @@ -6,6 +6,8 @@ #include <clientversion.h> #include <streams.h> #include <uint256.h> +#include <wallet/test/util.h> +#include <wallet/wallet.h> #include <boost/test/unit_test.hpp> @@ -27,5 +29,31 @@ BOOST_AUTO_TEST_CASE(walletdb_readkeyvalue) BOOST_CHECK_THROW(ssValue >> dummy, std::ios_base::failure); } +BOOST_AUTO_TEST_CASE(walletdb_read_write_deadlock) +{ + // Exercises a db read write operation that shouldn't deadlock. + for (const DatabaseFormat& db_format : DATABASE_FORMATS) { + // Context setup + DatabaseOptions options; + options.require_format = db_format; + DatabaseStatus status; + bilingual_str error_string; + std::unique_ptr<WalletDatabase> db = MakeDatabase(m_path_root / strprintf("wallet_%d_.dat", db_format).c_str(), options, status, error_string); + BOOST_CHECK_EQUAL(status, DatabaseStatus::SUCCESS); + + std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", std::move(db))); + wallet->m_keypool_size = 4; + + // Create legacy spkm + LOCK(wallet->cs_wallet); + auto legacy_spkm = wallet->GetOrCreateLegacyScriptPubKeyMan(); + BOOST_CHECK(legacy_spkm->SetupGeneration(true)); + wallet->Flush(); + + // Now delete all records, which performs a read write operation. + BOOST_CHECK(wallet->GetLegacyScriptPubKeyMan()->DeleteRecords()); + } +} + BOOST_AUTO_TEST_SUITE_END() } // namespace wallet diff --git a/src/wallet/test/walletload_tests.cpp b/src/wallet/test/walletload_tests.cpp index 9f5a4b14d3..73a4b77188 100644 --- a/src/wallet/test/walletload_tests.cpp +++ b/src/wallet/test/walletload_tests.cpp @@ -34,7 +34,7 @@ public: BOOST_FIXTURE_TEST_CASE(wallet_load_unknown_descriptor, TestingSetup) { - std::unique_ptr<WalletDatabase> database = CreateMockWalletDatabase(); + std::unique_ptr<WalletDatabase> database = CreateMockableWalletDatabase(); { // Write unknown active descriptor WalletBatch batch(*database, false); @@ -70,38 +70,45 @@ bool HasAnyRecordOfType(WalletDatabase& db, const std::string& key) return false; } -BOOST_FIXTURE_TEST_CASE(wallet_load_verif_crypted_key_checksum, TestingSetup) +template<typename... Args> +SerializeData MakeSerializeData(const Args&... args) { - // The test duplicates the db so each case has its own db instance. - int NUMBER_OF_TESTS = 4; - std::vector<std::unique_ptr<WalletDatabase>> dbs; - CKey first_key; - auto get_db = [](std::vector<std::unique_ptr<WalletDatabase>>& dbs) { - std::unique_ptr<WalletDatabase> db = std::move(dbs.back()); - dbs.pop_back(); - return db; - }; - - { // Context setup. + CDataStream s(0, 0); + SerializeMany(s, args...); + return {s.begin(), s.end()}; +} + + +BOOST_FIXTURE_TEST_CASE(wallet_load_ckey, TestingSetup) +{ + SerializeData ckey_record_key; + SerializeData ckey_record_value; + MockableData records; + + { + // Context setup. // Create and encrypt legacy wallet - std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", CreateMockWalletDatabase())); + std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", CreateMockableWalletDatabase())); LOCK(wallet->cs_wallet); auto legacy_spkm = wallet->GetOrCreateLegacyScriptPubKeyMan(); BOOST_CHECK(legacy_spkm->SetupGeneration(true)); - // Get the first key in the wallet + // Retrieve a key CTxDestination dest = *Assert(legacy_spkm->GetNewDestination(OutputType::LEGACY)); CKeyID key_id = GetKeyForDestination(*legacy_spkm, dest); + CKey first_key; BOOST_CHECK(legacy_spkm->GetKey(key_id, first_key)); - // Encrypt the wallet and duplicate database + // Encrypt the wallet BOOST_CHECK(wallet->EncryptWallet("encrypt")); wallet->Flush(); - DatabaseOptions options; - for (int i=0; i < NUMBER_OF_TESTS; i++) { - dbs.emplace_back(DuplicateMockDatabase(wallet->GetDatabase(), options)); - } + // Store a copy of all the records + records = GetMockableDatabase(*wallet).m_records; + + // Get the record for the retrieved key + ckey_record_key = MakeSerializeData(DBKeys::CRYPTED_KEY, first_key.GetPubKey()); + ckey_record_value = records.at(ckey_record_key); } { @@ -112,7 +119,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_load_verif_crypted_key_checksum, TestingSetup) // the records every time that 'CWallet::Unlock' gets called, which is not good. // Load the wallet and check that is encrypted - std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", get_db(dbs))); + std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", CreateMockableWalletDatabase(records))); BOOST_CHECK_EQUAL(wallet->LoadWallet(), DBErrors::LOAD_OK); BOOST_CHECK(wallet->IsCrypted()); BOOST_CHECK(HasAnyRecordOfType(wallet->GetDatabase(), DBKeys::CRYPTED_KEY)); @@ -127,18 +134,12 @@ BOOST_FIXTURE_TEST_CASE(wallet_load_verif_crypted_key_checksum, TestingSetup) { // Second test case: // Verify that loading up a 'ckey' with no checksum triggers a complete re-write of the crypted keys. - std::unique_ptr<WalletDatabase> db = get_db(dbs); - { - std::unique_ptr<DatabaseBatch> batch = db->MakeBatch(false); - std::pair<std::vector<unsigned char>, uint256> value; - BOOST_CHECK(batch->Read(std::make_pair(DBKeys::CRYPTED_KEY, first_key.GetPubKey()), value)); - const auto key = std::make_pair(DBKeys::CRYPTED_KEY, first_key.GetPubKey()); - BOOST_CHECK(batch->Write(key, value.first, /*fOverwrite=*/true)); - } + // Cut off the 32 byte checksum from a ckey record + records[ckey_record_key].resize(ckey_record_value.size() - 32); // Load the wallet and check that is encrypted - std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", std::move(db))); + std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", CreateMockableWalletDatabase(records))); BOOST_CHECK_EQUAL(wallet->LoadWallet(), DBErrors::LOAD_OK); BOOST_CHECK(wallet->IsCrypted()); BOOST_CHECK(HasAnyRecordOfType(wallet->GetDatabase(), DBKeys::CRYPTED_KEY)); @@ -154,35 +155,25 @@ BOOST_FIXTURE_TEST_CASE(wallet_load_verif_crypted_key_checksum, TestingSetup) { // Third test case: // Verify that loading up a 'ckey' with an invalid checksum throws an error. - std::unique_ptr<WalletDatabase> db = get_db(dbs); - { - std::unique_ptr<DatabaseBatch> batch = db->MakeBatch(false); - std::vector<unsigned char> crypted_data; - BOOST_CHECK(batch->Read(std::make_pair(DBKeys::CRYPTED_KEY, first_key.GetPubKey()), crypted_data)); - - // Write an invalid checksum - std::pair<std::vector<unsigned char>, uint256> value = std::make_pair(crypted_data, uint256::ONE); - const auto key = std::make_pair(DBKeys::CRYPTED_KEY, first_key.GetPubKey()); - BOOST_CHECK(batch->Write(key, value, /*fOverwrite=*/true)); - } - - std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", std::move(db))); + + // Cut off the 32 byte checksum from a ckey record + records[ckey_record_key].resize(ckey_record_value.size() - 32); + // Fill in the checksum space with 0s + records[ckey_record_key].resize(ckey_record_value.size()); + + std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", CreateMockableWalletDatabase(records))); BOOST_CHECK_EQUAL(wallet->LoadWallet(), DBErrors::CORRUPT); } { // Fourth test case: // Verify that loading up a 'ckey' with an invalid pubkey throws an error - std::unique_ptr<WalletDatabase> db = get_db(dbs); - { - CPubKey invalid_key; - BOOST_ASSERT(!invalid_key.IsValid()); - const auto key = std::make_pair(DBKeys::CRYPTED_KEY, invalid_key); - std::pair<std::vector<unsigned char>, uint256> value; - BOOST_CHECK(db->MakeBatch(false)->Write(key, value, /*fOverwrite=*/true)); - } - - std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", std::move(db))); + CPubKey invalid_key; + BOOST_CHECK(!invalid_key.IsValid()); + SerializeData key = MakeSerializeData(DBKeys::CRYPTED_KEY, invalid_key); + records[key] = ckey_record_value; + + std::shared_ptr<CWallet> wallet(new CWallet(m_node.chain.get(), "", CreateMockableWalletDatabase(records))); BOOST_CHECK_EQUAL(wallet->LoadWallet(), DBErrors::CORRUPT); } } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index caf95a3f03..62f0f53b01 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -42,6 +42,7 @@ #include <wallet/context.h> #include <wallet/external_signer_scriptpubkeyman.h> #include <wallet/fees.h> +#include <wallet/scriptpubkeyman.h> #include <univalue.h> @@ -55,9 +56,9 @@ namespace wallet { bool AddWalletSetting(interfaces::Chain& chain, const std::string& wallet_name) { - util::SettingsValue setting_value = chain.getRwSetting("wallet"); + common::SettingsValue setting_value = chain.getRwSetting("wallet"); if (!setting_value.isArray()) setting_value.setArray(); - for (const util::SettingsValue& value : setting_value.getValues()) { + for (const common::SettingsValue& value : setting_value.getValues()) { if (value.isStr() && value.get_str() == wallet_name) return true; } setting_value.push_back(wallet_name); @@ -66,10 +67,10 @@ bool AddWalletSetting(interfaces::Chain& chain, const std::string& wallet_name) bool RemoveWalletSetting(interfaces::Chain& chain, const std::string& wallet_name) { - util::SettingsValue setting_value = chain.getRwSetting("wallet"); + common::SettingsValue setting_value = chain.getRwSetting("wallet"); if (!setting_value.isArray()) return true; - util::SettingsValue new_value(util::SettingsValue::VARR); - for (const util::SettingsValue& value : setting_value.getValues()) { + common::SettingsValue new_value(common::SettingsValue::VARR); + for (const common::SettingsValue& value : setting_value.getValues()) { if (!value.isStr() || value.get_str() != wallet_name) new_value.push_back(value); } if (new_value.size() == setting_value.size()) return true; @@ -559,13 +560,14 @@ bool CWallet::ChangeWalletPassphrase(const SecureString& strOldWalletPassphrase, return false; if (Unlock(_vMasterKey)) { - int64_t nStartTime = GetTimeMillis(); + constexpr MillisecondsDouble target{100}; + auto start{SteadyClock::now()}; crypter.SetKeyFromPassphrase(strNewWalletPassphrase, pMasterKey.second.vchSalt, pMasterKey.second.nDeriveIterations, pMasterKey.second.nDerivationMethod); - pMasterKey.second.nDeriveIterations = static_cast<unsigned int>(pMasterKey.second.nDeriveIterations * (100 / ((double)(GetTimeMillis() - nStartTime)))); + pMasterKey.second.nDeriveIterations = static_cast<unsigned int>(pMasterKey.second.nDeriveIterations * target / (SteadyClock::now() - start)); - nStartTime = GetTimeMillis(); + start = SteadyClock::now(); crypter.SetKeyFromPassphrase(strNewWalletPassphrase, pMasterKey.second.vchSalt, pMasterKey.second.nDeriveIterations, pMasterKey.second.nDerivationMethod); - pMasterKey.second.nDeriveIterations = (pMasterKey.second.nDeriveIterations + static_cast<unsigned int>(pMasterKey.second.nDeriveIterations * 100 / ((double)(GetTimeMillis() - nStartTime)))) / 2; + pMasterKey.second.nDeriveIterations = (pMasterKey.second.nDeriveIterations + static_cast<unsigned int>(pMasterKey.second.nDeriveIterations * target / (SteadyClock::now() - start))) / 2; if (pMasterKey.second.nDeriveIterations < 25000) pMasterKey.second.nDeriveIterations = 25000; @@ -762,13 +764,14 @@ bool CWallet::EncryptWallet(const SecureString& strWalletPassphrase) GetStrongRandBytes(kMasterKey.vchSalt); CCrypter crypter; - int64_t nStartTime = GetTimeMillis(); + constexpr MillisecondsDouble target{100}; + auto start{SteadyClock::now()}; crypter.SetKeyFromPassphrase(strWalletPassphrase, kMasterKey.vchSalt, 25000, kMasterKey.nDerivationMethod); - kMasterKey.nDeriveIterations = static_cast<unsigned int>(2500000 / ((double)(GetTimeMillis() - nStartTime))); + kMasterKey.nDeriveIterations = static_cast<unsigned int>(25000 * target / (SteadyClock::now() - start)); - nStartTime = GetTimeMillis(); + start = SteadyClock::now(); crypter.SetKeyFromPassphrase(strWalletPassphrase, kMasterKey.vchSalt, kMasterKey.nDeriveIterations, kMasterKey.nDerivationMethod); - kMasterKey.nDeriveIterations = (kMasterKey.nDeriveIterations + static_cast<unsigned int>(kMasterKey.nDeriveIterations * 100 / ((double)(GetTimeMillis() - nStartTime)))) / 2; + kMasterKey.nDeriveIterations = (kMasterKey.nDeriveIterations + static_cast<unsigned int>(kMasterKey.nDeriveIterations * target / (SteadyClock::now() - start))) / 2; if (kMasterKey.nDeriveIterations < 25000) kMasterKey.nDeriveIterations = 25000; @@ -1264,11 +1267,6 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) { LOCK(cs_wallet); - WalletBatch batch(GetDatabase()); - - std::set<uint256> todo; - std::set<uint256> done; - // Can't mark abandoned if confirmed or in mempool auto it = mapWallet.find(hashTx); assert(it != mapWallet.end()); @@ -1277,44 +1275,25 @@ bool CWallet::AbandonTransaction(const uint256& hashTx) return false; } - todo.insert(hashTx); - - while (!todo.empty()) { - uint256 now = *todo.begin(); - todo.erase(now); - done.insert(now); - auto it = mapWallet.find(now); - assert(it != mapWallet.end()); - CWalletTx& wtx = it->second; - int currentconfirm = GetTxDepthInMainChain(wtx); - // If the orig tx was not in block, none of its spends can be - assert(currentconfirm <= 0); - // if (currentconfirm < 0) {Tx and spends are already conflicted, no need to abandon} - if (currentconfirm == 0 && !wtx.isAbandoned()) { - // If the orig tx was not in block/mempool, none of its spends can be in mempool - assert(!wtx.InMempool()); + auto try_updating_state = [](CWalletTx& wtx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { + // If the orig tx was not in block/mempool, none of its spends can be. + assert(!wtx.isConfirmed()); + assert(!wtx.InMempool()); + // If already conflicted or abandoned, no need to set abandoned + if (!wtx.isConflicted() && !wtx.isAbandoned()) { wtx.m_state = TxStateInactive{/*abandoned=*/true}; - wtx.MarkDirty(); - batch.WriteTx(wtx); - NotifyTransactionChanged(wtx.GetHash(), CT_UPDATED); - // Iterate over all its outputs, and mark transactions in the wallet that spend them abandoned too. - // States are not permanent, so these transactions can become unabandoned if they are re-added to the - // mempool, or confirmed in a block, or conflicted. - // Note: If the reorged coinbase is re-added to the main chain, the descendants that have not had their - // states change will remain abandoned and will require manual broadcast if the user wants them. - for (unsigned int i = 0; i < wtx.tx->vout.size(); ++i) { - std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(COutPoint(now, i)); - for (TxSpends::const_iterator iter = range.first; iter != range.second; ++iter) { - if (!done.count(iter->second)) { - todo.insert(iter->second); - } - } - } - // If a transaction changes 'conflicted' state, that changes the balance - // available of the outputs it spends. So force those to be recomputed - MarkInputsDirty(wtx.tx); + return TxUpdate::NOTIFY_CHANGED; } - } + return TxUpdate::UNCHANGED; + }; + + // Iterate over all its outputs, and mark transactions in the wallet that spend them abandoned too. + // States are not permanent, so these transactions can become unabandoned if they are re-added to the + // mempool, or confirmed in a block, or conflicted. + // Note: If the reorged coinbase is re-added to the main chain, the descendants that have not had their + // states change will remain abandoned and will require manual broadcast if the user wants them. + + RecursiveUpdateTxState(hashTx, try_updating_state); return true; } @@ -1331,13 +1310,29 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c if (conflictconfirms >= 0) return; + auto try_updating_state = [&](CWalletTx& wtx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { + if (conflictconfirms < GetTxDepthInMainChain(wtx)) { + // Block is 'more conflicted' than current confirm; update. + // Mark transaction as conflicted with this block. + wtx.m_state = TxStateConflicted{hashBlock, conflicting_height}; + return TxUpdate::CHANGED; + } + return TxUpdate::UNCHANGED; + }; + + // Iterate over all its outputs, and mark transactions in the wallet that spend them conflicted too. + RecursiveUpdateTxState(hashTx, try_updating_state); + +} + +void CWallet::RecursiveUpdateTxState(const uint256& tx_hash, const TryUpdatingStateFn& try_updating_state) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) { // Do not flush the wallet here for performance reasons WalletBatch batch(GetDatabase(), false); std::set<uint256> todo; std::set<uint256> done; - todo.insert(hashTx); + todo.insert(tx_hash); while (!todo.empty()) { uint256 now = *todo.begin(); @@ -1346,14 +1341,12 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c auto it = mapWallet.find(now); assert(it != mapWallet.end()); CWalletTx& wtx = it->second; - int currentconfirm = GetTxDepthInMainChain(wtx); - if (conflictconfirms < currentconfirm) { - // Block is 'more conflicted' than current confirm; update. - // Mark transaction as conflicted with this block. - wtx.m_state = TxStateConflicted{hashBlock, conflicting_height}; + + TxUpdate update_state = try_updating_state(wtx); + if (update_state != TxUpdate::UNCHANGED) { wtx.MarkDirty(); batch.WriteTx(wtx); - // Iterate over all its outputs, and mark transactions in the wallet that spend them conflicted too + // Iterate over all its outputs, and update those tx states as well (if applicable) for (unsigned int i = 0; i < wtx.tx->vout.size(); ++i) { std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(COutPoint(now, i)); for (TxSpends::const_iterator iter = range.first; iter != range.second; ++iter) { @@ -1362,7 +1355,12 @@ void CWallet::MarkConflicted(const uint256& hashBlock, int conflicting_height, c } } } - // If a transaction changes 'conflicted' state, that changes the balance + + if (update_state == TxUpdate::NOTIFY_CHANGED) { + NotifyTransactionChanged(wtx.GetHash(), CT_UPDATED); + } + + // If a transaction changes its tx state, that usually changes the balance // available of the outputs it spends. So force those to be recomputed MarkInputsDirty(wtx.tx); } @@ -1434,6 +1432,12 @@ void CWallet::blockConnected(const interfaces::BlockInfo& block) m_last_block_processed_height = block.height; m_last_block_processed = block.hash; + + // No need to scan block if it was created before the wallet birthday. + // Uses chain max time and twice the grace period to adjust time for block time variability. + if (block.chain_time_max < m_birth_time.load() - (TIMESTAMP_WINDOW * 2)) return; + + // Scan block for (size_t index = 0; index < block.data->vtx.size(); index++) { SyncTransaction(block.data->vtx[index], TxStateConfirmed{block.hash, block.height, static_cast<int>(index)}); transactionRemovedFromMempool(block.data->vtx[index], MemPoolRemovalReason::BLOCK); @@ -1451,8 +1455,36 @@ void CWallet::blockDisconnected(const interfaces::BlockInfo& block) // future with a stickier abandoned state or even removing abandontransaction call. m_last_block_processed_height = block.height - 1; m_last_block_processed = *Assert(block.prev_hash); + + int disconnect_height = block.height; + for (const CTransactionRef& ptx : Assert(block.data)->vtx) { SyncTransaction(ptx, TxStateInactive{}); + + for (const CTxIn& tx_in : ptx->vin) { + // No other wallet transactions conflicted with this transaction + if (mapTxSpends.count(tx_in.prevout) < 1) continue; + + std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(tx_in.prevout); + + // For all of the spends that conflict with this transaction + for (TxSpends::const_iterator _it = range.first; _it != range.second; ++_it) { + CWalletTx& wtx = mapWallet.find(_it->second)->second; + + if (!wtx.isConflicted()) continue; + + auto try_updating_state = [&](CWalletTx& tx) { + if (!tx.isConflicted()) return TxUpdate::UNCHANGED; + if (tx.state<TxStateConflicted>()->conflicting_block_height >= disconnect_height) { + tx.m_state = TxStateInactive{}; + return TxUpdate::CHANGED; + } + return TxUpdate::UNCHANGED; + }; + + RecursiveUpdateTxState(wtx.tx->GetHash(), try_updating_state); + } + } } } @@ -1777,6 +1809,14 @@ bool CWallet::ImportScriptPubKeys(const std::string& label, const std::set<CScri return true; } +void CWallet::FirstKeyTimeChanged(const ScriptPubKeyMan* spkm, int64_t new_birth_time) +{ + int64_t birthtime = m_birth_time.load(); + if (new_birth_time < birthtime) { + m_birth_time = new_birth_time; + } +} + /** * Scan active chain for relevant transactions after importing keys. This should * be called whenever new keys are added to the wallet, with the oldest key @@ -2792,7 +2832,7 @@ bool CWallet::SetAddressPreviouslySpent(WalletBatch& batch, const CTxDestination return false; if (!used) { - if (auto* data{util::FindKey(m_address_book, dest)}) data->previously_spent = false; + if (auto* data{common::FindKey(m_address_book, dest)}) data->previously_spent = false; return batch.WriteAddressPreviouslySpent(dest, false); } @@ -2812,7 +2852,7 @@ void CWallet::LoadAddressReceiveRequest(const CTxDestination& dest, const std::s bool CWallet::IsAddressPreviouslySpent(const CTxDestination& dest) const { - if (auto* data{util::FindKey(m_address_book, dest)}) return data->previously_spent; + if (auto* data{common::FindKey(m_address_book, dest)}) return data->previously_spent; return false; } @@ -3105,6 +3145,14 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri // Try to top up keypool. No-op if the wallet is locked. walletInstance->TopUpKeyPool(); + // Cache the first key time + std::optional<int64_t> time_first_key; + for (auto spk_man : walletInstance->GetAllScriptPubKeyMans()) { + int64_t time = spk_man->GetTimeFirstKey(); + if (!time_first_key || time < *time_first_key) time_first_key = time; + } + if (time_first_key) walletInstance->m_birth_time = *time_first_key; + if (chain && !AttachChain(walletInstance, *chain, rescan_required, error, warnings)) { return nullptr; } @@ -3180,11 +3228,7 @@ bool CWallet::AttachChain(const std::shared_ptr<CWallet>& walletInstance, interf { // No need to read and scan block if block was created before // our wallet birthday (as adjusted for block time variability) - std::optional<int64_t> time_first_key; - for (auto spk_man : walletInstance->GetAllScriptPubKeyMans()) { - int64_t time = spk_man->GetTimeFirstKey(); - if (!time_first_key || time < *time_first_key) time_first_key = time; - } + std::optional<int64_t> time_first_key = walletInstance->m_birth_time.load(); if (time_first_key) { FoundBlock found = FoundBlock().height(rescan_height); chain.findFirstBlockWithTimeAndHeight(*time_first_key - TIMESTAMP_WINDOW, rescan_height, found); @@ -3496,6 +3540,14 @@ LegacyScriptPubKeyMan* CWallet::GetOrCreateLegacyScriptPubKeyMan() return GetLegacyScriptPubKeyMan(); } +void CWallet::AddScriptPubKeyMan(const uint256& id, std::unique_ptr<ScriptPubKeyMan> spkm_man) +{ + const auto& spkm = m_spk_managers[id] = std::move(spkm_man); + + // Update birth time if needed + FirstKeyTimeChanged(spkm.get(), spkm->GetTimeFirstKey()); +} + void CWallet::SetupLegacyScriptPubKeyMan() { if (!m_internal_spk_managers.empty() || !m_external_spk_managers.empty() || !m_spk_managers.empty() || IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { @@ -3507,7 +3559,8 @@ void CWallet::SetupLegacyScriptPubKeyMan() m_internal_spk_managers[type] = spk_manager.get(); m_external_spk_managers[type] = spk_manager.get(); } - m_spk_managers[spk_manager->GetID()] = std::move(spk_manager); + uint256 id = spk_manager->GetID(); + AddScriptPubKeyMan(id, std::move(spk_manager)); } const CKeyingMaterial& CWallet::GetEncryptionKey() const @@ -3525,6 +3578,7 @@ void CWallet::ConnectScriptPubKeyManNotifiers() for (const auto& spk_man : GetActiveScriptPubKeyMans()) { spk_man->NotifyWatchonlyChanged.connect(NotifyWatchonlyChanged); spk_man->NotifyCanGetAddressesChanged.connect(NotifyCanGetAddressesChanged); + spk_man->NotifyFirstKeyTimeChanged.connect(std::bind(&CWallet::FirstKeyTimeChanged, this, std::placeholders::_1, std::placeholders::_2)); } } @@ -3532,10 +3586,10 @@ void CWallet::LoadDescriptorScriptPubKeyMan(uint256 id, WalletDescriptor& desc) { if (IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER)) { auto spk_manager = std::unique_ptr<ScriptPubKeyMan>(new ExternalSignerScriptPubKeyMan(*this, desc, m_keypool_size)); - m_spk_managers[id] = std::move(spk_manager); + AddScriptPubKeyMan(id, std::move(spk_manager)); } else { auto spk_manager = std::unique_ptr<ScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, desc, m_keypool_size)); - m_spk_managers[id] = std::move(spk_manager); + AddScriptPubKeyMan(id, std::move(spk_manager)); } } @@ -3556,7 +3610,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans(const CExtKey& master_key) } spk_manager->SetupDescriptorGeneration(master_key, t, internal); uint256 id = spk_manager->GetID(); - m_spk_managers[id] = std::move(spk_manager); + AddScriptPubKeyMan(id, std::move(spk_manager)); AddActiveScriptPubKeyMan(id, t, internal); } } @@ -3587,7 +3641,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans() if (!signer_res.isObject()) throw std::runtime_error(std::string(__func__) + ": Unexpected result"); for (bool internal : {false, true}) { - const UniValue& descriptor_vals = find_value(signer_res, internal ? "internal" : "receive"); + const UniValue& descriptor_vals = signer_res.find_value(internal ? "internal" : "receive"); if (!descriptor_vals.isArray()) throw std::runtime_error(std::string(__func__) + ": Unexpected result"); for (const UniValue& desc_val : descriptor_vals.get_array().getValues()) { const std::string& desc_str = desc_val.getValStr(); @@ -3604,7 +3658,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans() auto spk_manager = std::unique_ptr<ExternalSignerScriptPubKeyMan>(new ExternalSignerScriptPubKeyMan(*this, m_keypool_size)); spk_manager->SetupDescriptor(std::move(desc)); uint256 id = spk_manager->GetID(); - m_spk_managers[id] = std::move(spk_manager); + AddScriptPubKeyMan(id, std::move(spk_manager)); AddActiveScriptPubKeyMan(id, t, internal); } } @@ -3721,7 +3775,8 @@ ScriptPubKeyMan* CWallet::AddWalletDescriptor(WalletDescriptor& desc, const Flat spk_man = new_spk_man.get(); // Save the descriptor to memory - m_spk_managers[new_spk_man->GetID()] = std::move(new_spk_man); + uint256 id = new_spk_man->GetID(); + AddScriptPubKeyMan(id, std::move(new_spk_man)); } // Add the private keys to the descriptor @@ -3820,9 +3875,7 @@ bool CWallet::MigrateToSQLite(bilingual_str& error) bool began = batch->TxnBegin(); assert(began); // This is a critical error, the new db could not be written to. The original db exists as a backup, but we should not continue execution. for (const auto& [key, value] : records) { - DataStream ss_key{key}; - DataStream ss_value{value}; - if (!batch->Write(ss_key, ss_value)) { + if (!batch->Write(MakeUCharSpan(key), MakeUCharSpan(value))) { batch->TxnAbort(); m_database->Close(); fs::remove(m_database->Filename()); @@ -3864,7 +3917,8 @@ bool CWallet::ApplyMigrationData(MigrationData& data, bilingual_str& error) error = _("Error: Duplicate descriptors created during migration. Your wallet may be corrupted."); return false; } - m_spk_managers[desc_spkm->GetID()] = std::move(desc_spkm); + uint256 id = desc_spkm->GetID(); + AddScriptPubKeyMan(id, std::move(desc_spkm)); } // Remove the LegacyScriptPubKeyMan from disk diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 5252b46cdc..cbd5008366 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -284,7 +284,7 @@ private: std::atomic<bool> fScanningWallet{false}; // controlled by WalletRescanReserver std::atomic<bool> m_attaching_chain{false}; std::atomic<bool> m_scanning_with_passphrase{false}; - std::atomic<int64_t> m_scanning_start{0}; + std::atomic<SteadyClock::time_point> m_scanning_start{SteadyClock::time_point{}}; std::atomic<double> m_scanning_progress{0}; friend class WalletRescanReserver; @@ -299,6 +299,10 @@ private: // Local time that the tip block was received. Used to schedule wallet rebroadcasts. std::atomic<int64_t> m_best_block_time {0}; + // First created key time. Used to skip blocks prior to this time. + // 'std::numeric_limits<int64_t>::max()' if wallet is blank. + std::atomic<int64_t> m_birth_time{std::numeric_limits<int64_t>::max()}; + /** * Used to keep track of spent outpoints, and * detect and report conflicts (double-spends or @@ -330,6 +334,13 @@ private: /** Mark a transaction (and its in-wallet descendants) as conflicting with a particular block. */ void MarkConflicted(const uint256& hashBlock, int conflicting_height, const uint256& hashTx); + enum class TxUpdate { UNCHANGED, CHANGED, NOTIFY_CHANGED }; + + using TryUpdatingStateFn = std::function<TxUpdate(CWalletTx& wtx)>; + + /** Mark a transaction (and its in-wallet descendants) as a particular tx state. */ + void RecursiveUpdateTxState(const uint256& tx_hash, const TryUpdatingStateFn& try_updating_state) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /** Mark a transaction's inputs dirty, thus forcing the outputs to be recomputed */ void MarkInputsDirty(const CTransactionRef& tx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -380,6 +391,10 @@ private: // ScriptPubKeyMan::GetID. In many cases it will be the hash of an internal structure std::map<uint256, std::unique_ptr<ScriptPubKeyMan>> m_spk_managers; + // Appends spk managers into the main 'm_spk_managers'. + // Must be the only method adding data to it. + void AddScriptPubKeyMan(const uint256& id, std::unique_ptr<ScriptPubKeyMan> spkm_man); + /** * Catch wallet up to current chain, scanning new blocks, updating the best * block locator and m_last_block_processed, and registering for @@ -505,7 +520,7 @@ public: bool IsAbortingRescan() const { return fAbortRescan; } bool IsScanning() const { return fScanningWallet; } bool IsScanningWithPassphrase() const { return m_scanning_with_passphrase; } - int64_t ScanningDuration() const { return fScanningWallet ? GetTimeMillis() - m_scanning_start : 0; } + SteadyClock::duration ScanningDuration() const { return fScanningWallet ? SteadyClock::now() - m_scanning_start.load() : SteadyClock::duration{}; } double ScanningProgress() const { return fScanningWallet ? (double) m_scanning_progress : 0; } //! Upgrade stored CKeyMetadata objects to store key origin info as KeyOriginInfo @@ -641,6 +656,9 @@ public: bool ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool ImportScriptPubKeys(const std::string& label, const std::set<CScript>& script_pub_keys, const bool have_solving_data, const bool apply_label, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + /** Updates wallet birth time if 'new_birth_time' is below it */ + void FirstKeyTimeChanged(const ScriptPubKeyMan* spkm, int64_t new_birth_time); + CFeeRate m_pay_tx_fee{DEFAULT_PAY_TX_FEE}; unsigned int m_confirm_target{DEFAULT_TX_CONFIRM_TARGET}; /** Allow Coin Selection to pick unconfirmed UTXOs that were sent from our own wallet if it @@ -1014,7 +1032,7 @@ public: return false; } m_wallet.m_scanning_with_passphrase.exchange(with_passphrase); - m_wallet.m_scanning_start = GetTimeMillis(); + m_wallet.m_scanning_start = SteadyClock::now(); m_wallet.m_scanning_progress = 0; m_could_reserve = true; return true; @@ -1056,7 +1074,7 @@ struct MigrationResult { }; //! Do all steps to migrate a legacy wallet to a descriptor wallet -util::Result<MigrationResult> MigrateLegacyToDescriptor(const std::string& wallet_name, const SecureString& passphrase, WalletContext& context); +[[nodiscard]] util::Result<MigrationResult> MigrateLegacyToDescriptor(const std::string& wallet_name, const SecureString& passphrase, WalletContext& context); } // namespace wallet #endif // BITCOIN_WALLET_WALLET_H diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 005592d720..34fe8ab17f 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -5,13 +5,13 @@ #include <wallet/walletdb.h> +#include <common/system.h> #include <key_io.h> #include <protocol.h> #include <serialize.h> #include <sync.h> #include <util/bip32.h> #include <util/fs.h> -#include <util/system.h> #include <util/time.h> #include <util/translation.h> #ifdef USE_BDB @@ -1136,6 +1136,9 @@ bool WalletBatch::WriteWalletFlags(const uint64_t flags) bool WalletBatch::EraseRecords(const std::unordered_set<std::string>& types) { + // Begin db txn + if (!m_batch->TxnBegin()) return false; + // Get cursor std::unique_ptr<DatabaseCursor> cursor = m_batch->GetNewCursor(); if (!cursor) @@ -1144,8 +1147,7 @@ bool WalletBatch::EraseRecords(const std::unordered_set<std::string>& types) } // Iterate the DB and look for any records that have the type prefixes - while (true) - { + while (true) { // Read next record DataStream key{}; DataStream value{}; @@ -1153,6 +1155,8 @@ bool WalletBatch::EraseRecords(const std::unordered_set<std::string>& types) if (status == DatabaseCursor::Status::DONE) { break; } else if (status == DatabaseCursor::Status::FAIL) { + cursor.reset(nullptr); + m_batch->TxnAbort(); // abort db txn return false; } @@ -1163,10 +1167,16 @@ bool WalletBatch::EraseRecords(const std::unordered_set<std::string>& types) key >> type; if (types.count(type) > 0) { - m_batch->Erase(key_data); + if (!m_batch->Erase(key_data)) { + cursor.reset(nullptr); + m_batch->TxnAbort(); + return false; // erase failed + } } } - return true; + // Finish db txn + cursor.reset(nullptr); + return m_batch->TxnCommit(); } bool WalletBatch::TxnBegin() @@ -1262,44 +1272,4 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas status = DatabaseStatus::FAILED_BAD_FORMAT; return nullptr; } - -/** Return object for accessing dummy database with no read/write capabilities. */ -std::unique_ptr<WalletDatabase> CreateDummyWalletDatabase() -{ - return std::make_unique<DummyDatabase>(); -} - -/** Return object for accessing temporary in-memory database. */ -std::unique_ptr<WalletDatabase> CreateMockWalletDatabase(DatabaseOptions& options) -{ - - std::optional<DatabaseFormat> format; - if (options.require_format) format = options.require_format; - if (!format) { -#ifdef USE_BDB - format = DatabaseFormat::BERKELEY; -#endif -#ifdef USE_SQLITE - format = DatabaseFormat::SQLITE; -#endif - } - - if (format == DatabaseFormat::SQLITE) { -#ifdef USE_SQLITE - return std::make_unique<SQLiteDatabase>(":memory:", "", options, true); -#endif - assert(false); - } - -#ifdef USE_BDB - return std::make_unique<BerkeleyDatabase>(std::make_shared<BerkeleyEnvironment>(), "", options); -#endif - assert(false); -} - -std::unique_ptr<WalletDatabase> CreateMockWalletDatabase() -{ - DatabaseOptions options; - return CreateMockWalletDatabase(options); -} } // namespace wallet diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 72086e950a..f84a89b23f 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -305,13 +305,6 @@ using KeyFilterFn = std::function<bool(const std::string&)>; //! Unserialize a given Key-Value pair and load it into the wallet bool ReadKeyValue(CWallet* pwallet, DataStream& ssKey, CDataStream& ssValue, std::string& strType, std::string& strErr, const KeyFilterFn& filter_fn = nullptr); - -/** Return object for accessing dummy database with no read/write capabilities. */ -std::unique_ptr<WalletDatabase> CreateDummyWalletDatabase(); - -/** Return object for accessing temporary in-memory database. */ -std::unique_ptr<WalletDatabase> CreateMockWalletDatabase(DatabaseOptions& options); -std::unique_ptr<WalletDatabase> CreateMockWalletDatabase(); } // namespace wallet #endif // BITCOIN_WALLET_WALLETDB_H diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index 37847b64ef..c5975144c1 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -112,20 +112,6 @@ public: WalletDescriptor() {} WalletDescriptor(std::shared_ptr<Descriptor> descriptor, uint64_t creation_time, int32_t range_start, int32_t range_end, int32_t next_index) : descriptor(descriptor), creation_time(creation_time), range_start(range_start), range_end(range_end), next_index(next_index) {} }; - -class CWallet; -class DescriptorScriptPubKeyMan; - -/** struct containing information needed for migrating legacy wallets to descriptor wallets */ -struct MigrationData -{ - CExtKey master_key; - std::vector<std::pair<std::string, int64_t>> watch_descs; - std::vector<std::pair<std::string, int64_t>> solvable_descs; - std::vector<std::unique_ptr<DescriptorScriptPubKeyMan>> desc_spkms; - std::shared_ptr<CWallet> watchonly_wallet{nullptr}; - std::shared_ptr<CWallet> solvable_wallet{nullptr}; -}; } // namespace wallet #endif // BITCOIN_WALLET_WALLETUTIL_H diff --git a/src/warnings.cpp b/src/warnings.cpp index d0de706953..cb73c7aea2 100644 --- a/src/warnings.cpp +++ b/src/warnings.cpp @@ -5,9 +5,9 @@ #include <warnings.h> +#include <common/system.h> #include <sync.h> #include <util/string.h> -#include <util/system.h> #include <util/translation.h> #include <vector> diff --git a/src/zmq/zmqabstractnotifier.h b/src/zmq/zmqabstractnotifier.h index cf0ee48f47..17fa7bbaa9 100644 --- a/src/zmq/zmqabstractnotifier.h +++ b/src/zmq/zmqabstractnotifier.h @@ -6,6 +6,7 @@ #define BITCOIN_ZMQ_ZMQABSTRACTNOTIFIER_H #include <cstdint> +#include <functional> #include <memory> #include <string> @@ -13,7 +14,7 @@ class CBlockIndex; class CTransaction; class CZMQAbstractNotifier; -using CZMQNotifierFactory = std::unique_ptr<CZMQAbstractNotifier> (*)(); +using CZMQNotifierFactory = std::function<std::unique_ptr<CZMQAbstractNotifier>()>; class CZMQAbstractNotifier { diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index 9920d80a69..6755368249 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -39,12 +39,14 @@ std::list<const CZMQAbstractNotifier*> CZMQNotificationInterface::GetActiveNotif return result; } -CZMQNotificationInterface* CZMQNotificationInterface::Create() +std::unique_ptr<CZMQNotificationInterface> CZMQNotificationInterface::Create(std::function<bool(CBlock&, const CBlockIndex&)> get_block_by_index) { std::map<std::string, CZMQNotifierFactory> factories; factories["pubhashblock"] = CZMQAbstractNotifier::Create<CZMQPublishHashBlockNotifier>; factories["pubhashtx"] = CZMQAbstractNotifier::Create<CZMQPublishHashTransactionNotifier>; - factories["pubrawblock"] = CZMQAbstractNotifier::Create<CZMQPublishRawBlockNotifier>; + factories["pubrawblock"] = [&get_block_by_index]() -> std::unique_ptr<CZMQAbstractNotifier> { + return std::make_unique<CZMQPublishRawBlockNotifier>(get_block_by_index); + }; factories["pubrawtx"] = CZMQAbstractNotifier::Create<CZMQPublishRawTransactionNotifier>; factories["pubsequence"] = CZMQAbstractNotifier::Create<CZMQPublishSequenceNotifier>; @@ -68,7 +70,7 @@ CZMQNotificationInterface* CZMQNotificationInterface::Create() notificationInterface->notifiers = std::move(notifiers); if (notificationInterface->Initialize()) { - return notificationInterface.release(); + return notificationInterface; } } @@ -198,4 +200,4 @@ void CZMQNotificationInterface::BlockDisconnected(const std::shared_ptr<const CB }); } -CZMQNotificationInterface* g_zmq_notification_interface = nullptr; +std::unique_ptr<CZMQNotificationInterface> g_zmq_notification_interface; diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h index a43f9bfef3..ce67633b30 100644 --- a/src/zmq/zmqnotificationinterface.h +++ b/src/zmq/zmqnotificationinterface.h @@ -9,6 +9,7 @@ #include <validationinterface.h> #include <cstdint> +#include <functional> #include <list> #include <memory> @@ -23,7 +24,7 @@ public: std::list<const CZMQAbstractNotifier*> GetActiveNotifiers() const; - static CZMQNotificationInterface* Create(); + static std::unique_ptr<CZMQNotificationInterface> Create(std::function<bool(CBlock&, const CBlockIndex&)> get_block_by_index); protected: bool Initialize(); @@ -43,6 +44,6 @@ private: std::list<std::unique_ptr<CZMQAbstractNotifier>> notifiers; }; -extern CZMQNotificationInterface* g_zmq_notification_interface; +extern std::unique_ptr<CZMQNotificationInterface> g_zmq_notification_interface; #endif // BITCOIN_ZMQ_ZMQNOTIFICATIONINTERFACE_H diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index 55f3d4e934..1241431523 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -39,8 +39,6 @@ namespace Consensus { struct Params; } -using node::ReadBlockFromDisk; - static std::multimap<std::string, CZMQAbstractPublishNotifier*> mapPublishNotifiers; static const char *MSG_HASHBLOCK = "hashblock"; @@ -99,9 +97,8 @@ static bool IsZMQAddressIPV6(const std::string &zmq_address) const size_t colon_index = zmq_address.rfind(':'); if (tcp_index == 0 && colon_index != std::string::npos) { const std::string ip = zmq_address.substr(tcp_prefix.length(), colon_index - tcp_prefix.length()); - CNetAddr addr; - LookupHost(ip, addr, false); - if (addr.IsIPv6()) return true; + const std::optional<CNetAddr> addr{LookupHost(ip, false)}; + if (addr.has_value() && addr.value().IsIPv6()) return true; } return false; } @@ -247,10 +244,9 @@ bool CZMQPublishRawBlockNotifier::NotifyBlock(const CBlockIndex *pindex) { LogPrint(BCLog::ZMQ, "Publish rawblock %s to %s\n", pindex->GetBlockHash().GetHex(), this->address); - const Consensus::Params& consensusParams = Params().GetConsensus(); CDataStream ss(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); CBlock block; - if (!ReadBlockFromDisk(block, pindex, consensusParams)) { + if (!m_get_block_by_index(block, *pindex)) { zmqError("Can't read block from disk"); return false; } diff --git a/src/zmq/zmqpublishnotifier.h b/src/zmq/zmqpublishnotifier.h index 18336a5eb0..a5cd433761 100644 --- a/src/zmq/zmqpublishnotifier.h +++ b/src/zmq/zmqpublishnotifier.h @@ -9,7 +9,9 @@ #include <cstddef> #include <cstdint> +#include <functional> +class CBlock; class CBlockIndex; class CTransaction; @@ -46,7 +48,12 @@ public: class CZMQPublishRawBlockNotifier : public CZMQAbstractPublishNotifier { +private: + const std::function<bool(CBlock&, const CBlockIndex&)> m_get_block_by_index; + public: + CZMQPublishRawBlockNotifier(std::function<bool(CBlock&, const CBlockIndex&)> get_block_by_index) + : m_get_block_by_index{std::move(get_block_by_index)} {} bool NotifyBlock(const CBlockIndex *pindex) override; }; |