diff options
Diffstat (limited to 'src')
162 files changed, 3616 insertions, 2290 deletions
diff --git a/src/.clang-tidy b/src/.clang-tidy index e9807d4cb7..df2a080075 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -1,12 +1,14 @@ Checks: ' -*, bugprone-argument-comment, +misc-unused-using-decls, modernize-use-default-member-init, modernize-use-nullptr, readability-redundant-declaration, ' WarningsAsErrors: ' bugprone-argument-comment, +misc-unused-using-decls, modernize-use-default-member-init, modernize-use-nullptr, readability-redundant-declaration, diff --git a/src/Makefile.am b/src/Makefile.am index 488ff0e273..7745f4d7c8 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -139,6 +139,7 @@ BITCOIN_CORE_H = \ compat/cpuid.h \ compat/endian.h \ compressor.h \ + node/connection_types.h \ consensus/consensus.h \ consensus/tx_check.h \ consensus/tx_verify.h \ @@ -148,6 +149,7 @@ BITCOIN_CORE_H = \ dbwrapper.h \ deploymentinfo.h \ deploymentstatus.h \ + node/eviction.h \ external_signer.h \ flatfile.h \ fs.h \ @@ -169,12 +171,14 @@ BITCOIN_CORE_H = \ interfaces/ipc.h \ interfaces/node.h \ interfaces/wallet.h \ + kernel/chain.h \ kernel/chainstatemanager_opts.h \ kernel/checks.h \ kernel/coinstats.h \ kernel/context.h \ kernel/mempool_limits.h \ kernel/mempool_options.h \ + kernel/mempool_persist.h \ key.h \ key_io.h \ logging.h \ @@ -196,6 +200,7 @@ BITCOIN_CORE_H = \ node/chainstate.h \ node/coin.h \ node/context.h \ + node/mempool_persist_args.h \ node/miner.h \ node/minisketchwrapper.h \ node/psbt.h \ @@ -259,7 +264,6 @@ BITCOIN_CORE_H = \ util/bip32.h \ util/bytevectorhash.h \ util/check.h \ - util/designator.h \ util/epochguard.h \ util/error.h \ util/fastrange.h \ @@ -275,6 +279,7 @@ BITCOIN_CORE_H = \ util/overloaded.h \ util/rbf.h \ util/readwritefile.h \ + util/result.h \ util/serfloat.h \ util/settings.h \ util/sock.h \ @@ -361,9 +366,11 @@ libbitcoin_node_a_SOURCES = \ index/coinstatsindex.cpp \ index/txindex.cpp \ init.cpp \ + kernel/chain.cpp \ kernel/checks.cpp \ kernel/coinstats.cpp \ kernel/context.cpp \ + kernel/mempool_persist.cpp \ mapport.cpp \ mempool_args.cpp \ net.cpp \ @@ -373,8 +380,11 @@ libbitcoin_node_a_SOURCES = \ node/caches.cpp \ node/chainstate.cpp \ node/coin.cpp \ + node/connection_types.cpp \ node/context.cpp \ + node/eviction.cpp \ node/interfaces.cpp \ + node/mempool_persist_args.cpp \ node/miner.cpp \ node/minisketchwrapper.cpp \ node/psbt.cpp \ @@ -854,8 +864,7 @@ endif # TODO: libbitcoinkernel is a work in progress consensus engine library, as more # and more modules are decoupled from the consensus engine, this list will -# shrink to only those which are absolutely necessary. For example, things -# like index/*.cpp will be removed. +# shrink to only those which are absolutely necessary. libbitcoinkernel_la_SOURCES = \ kernel/bitcoinkernel.cpp \ arith_uint256.cpp \ @@ -875,9 +884,11 @@ libbitcoinkernel_la_SOURCES = \ flatfile.cpp \ fs.cpp \ hash.cpp \ + kernel/chain.cpp \ kernel/checks.cpp \ kernel/coinstats.cpp \ kernel/context.cpp \ + kernel/mempool_persist.cpp \ key.cpp \ logging.cpp \ node/blockstorage.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 098feacb3d..d9195ad32e 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -325,6 +325,7 @@ test_fuzz_fuzz_SOURCES = \ test/fuzz/tx_in.cpp \ test/fuzz/tx_out.cpp \ test/fuzz/tx_pool.cpp \ + test/fuzz/txorphan.cpp \ test/fuzz/txrequest.cpp \ test/fuzz/utxo_snapshot.cpp \ test/fuzz/validation_load_mempool.cpp \ @@ -372,7 +373,7 @@ endif $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check if ENABLE_TESTS -UNIVALUE_TESTS = univalue/test/object univalue/test/unitester univalue/test/no_nul +UNIVALUE_TESTS = univalue/test/object univalue/test/unitester noinst_PROGRAMS += $(UNIVALUE_TESTS) TESTS += $(UNIVALUE_TESTS) @@ -381,11 +382,6 @@ univalue_test_unitester_LDADD = $(LIBUNIVALUE) univalue_test_unitester_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) -DJSON_TEST_SRC=\"$(srcdir)/$(UNIVALUE_TEST_DATA_DIR_INT)\" univalue_test_unitester_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) -univalue_test_no_nul_SOURCES = $(UNIVALUE_TEST_NO_NUL_INT) -univalue_test_no_nul_LDADD = $(LIBUNIVALUE) -univalue_test_no_nul_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) -univalue_test_no_nul_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) - univalue_test_object_SOURCES = $(UNIVALUE_TEST_OBJECT_INT) univalue_test_object_LDADD = $(LIBUNIVALUE) univalue_test_object_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) diff --git a/src/Makefile.test_fuzz.include b/src/Makefile.test_fuzz.include index 8922dda3ad..b43816636f 100644 --- a/src/Makefile.test_fuzz.include +++ b/src/Makefile.test_fuzz.include @@ -10,6 +10,7 @@ EXTRA_LIBRARIES += \ TEST_FUZZ_H = \ test/fuzz/fuzz.h \ test/fuzz/FuzzedDataProvider.h \ + test/fuzz/mempool_utils.h \ test/fuzz/util.h libtest_fuzz_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 52e5cb743f..53aa470042 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -8,6 +8,7 @@ #include <chainparams.h> #include <consensus/validation.h> #include <streams.h> +#include <util/system.h> #include <validation.h> // These are the two major time-sinks which happen after we have fully received diff --git a/src/bench/coin_selection.cpp b/src/bench/coin_selection.cpp index b2958bcc9f..eaefb9b63a 100644 --- a/src/bench/coin_selection.cpp +++ b/src/bench/coin_selection.cpp @@ -58,7 +58,8 @@ static void CoinSelection(benchmark::Bench& bench) // Create coins std::vector<COutput> coins; for (const auto& wtx : wtxs) { - coins.emplace_back(COutPoint(wtx->GetHash(), 0), wtx->tx->vout.at(0), /*depth=*/6 * 24, GetTxSpendSize(wallet, *wtx, 0), /*spendable=*/true, /*solvable=*/true, /*safe=*/true, wtx->GetTxTime(), /*from_me=*/true, /*fees=*/ 0); + const auto txout = wtx->tx->vout.at(0); + coins.emplace_back(COutPoint(wtx->GetHash(), 0), txout, /*depth=*/6 * 24, CalculateMaximumSignedInputSize(txout, &wallet, /*coin_control=*/nullptr), /*spendable=*/true, /*solvable=*/true, /*safe=*/true, wtx->GetTxTime(), /*from_me=*/true, /*fees=*/ 0); } const CoinEligibilityFilter filter_standard(1, 6, 0); diff --git a/src/bench/gcs_filter.cpp b/src/bench/gcs_filter.cpp index 607e4392b7..80babb213b 100644 --- a/src/bench/gcs_filter.cpp +++ b/src/bench/gcs_filter.cpp @@ -5,39 +5,84 @@ #include <bench/bench.h> #include <blockfilter.h> -static void ConstructGCSFilter(benchmark::Bench& bench) +static const GCSFilter::ElementSet GenerateGCSTestElements() { GCSFilter::ElementSet elements; - for (int i = 0; i < 10000; ++i) { + + // Testing the benchmarks with different number of elements show that a filter + // with at least 100,000 elements results in benchmarks that have the same + // ns/op. This makes it easy to reason about how long (in nanoseconds) a single + // filter element takes to process. + for (int i = 0; i < 100000; ++i) { GCSFilter::Element element(32); element[0] = static_cast<unsigned char>(i); element[1] = static_cast<unsigned char>(i >> 8); elements.insert(std::move(element)); } + return elements; +} + +static void GCSBlockFilterGetHash(benchmark::Bench& bench) +{ + auto elements = GenerateGCSTestElements(); + + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); + BlockFilter block_filter(BlockFilterType::BASIC, {}, filter.GetEncoded(), /*skip_decode_check=*/false); + + bench.run([&] { + block_filter.GetHash(); + }); +} + +static void GCSFilterConstruct(benchmark::Bench& bench) +{ + auto elements = GenerateGCSTestElements(); + uint64_t siphash_k0 = 0; - bench.batch(elements.size()).unit("elem").run([&] { - GCSFilter filter({siphash_k0, 0, 20, 1 << 20}, elements); + bench.run([&]{ + GCSFilter filter({siphash_k0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); siphash_k0++; }); } -static void MatchGCSFilter(benchmark::Bench& bench) +static void GCSFilterDecode(benchmark::Bench& bench) { - GCSFilter::ElementSet elements; - for (int i = 0; i < 10000; ++i) { - GCSFilter::Element element(32); - element[0] = static_cast<unsigned char>(i); - element[1] = static_cast<unsigned char>(i >> 8); - elements.insert(std::move(element)); - } - GCSFilter filter({0, 0, 20, 1 << 20}, elements); + auto elements = GenerateGCSTestElements(); - bench.unit("elem").run([&] { - filter.Match(GCSFilter::Element()); + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); + auto encoded = filter.GetEncoded(); + + bench.run([&] { + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, encoded, /*skip_decode_check=*/false); }); } -BENCHMARK(ConstructGCSFilter); -BENCHMARK(MatchGCSFilter); +static void GCSFilterDecodeSkipCheck(benchmark::Bench& bench) +{ + auto elements = GenerateGCSTestElements(); + + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); + auto encoded = filter.GetEncoded(); + + bench.run([&] { + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, encoded, /*skip_decode_check=*/true); + }); +} + +static void GCSFilterMatch(benchmark::Bench& bench) +{ + auto elements = GenerateGCSTestElements(); + + GCSFilter filter({0, 0, BASIC_FILTER_P, BASIC_FILTER_M}, elements); + + bench.run([&] { + filter.Match(GCSFilter::Element()); + }); +} +BENCHMARK(GCSBlockFilterGetHash); +BENCHMARK(GCSFilterConstruct); +BENCHMARK(GCSFilterDecode); +BENCHMARK(GCSFilterDecodeSkipCheck); +BENCHMARK(GCSFilterMatch); diff --git a/src/bench/wallet_loading.cpp b/src/bench/wallet_loading.cpp index f611383788..a10f7ff7d1 100644 --- a/src/bench/wallet_loading.cpp +++ b/src/bench/wallet_loading.cpp @@ -19,8 +19,6 @@ using wallet::CWallet; using wallet::DatabaseFormat; using wallet::DatabaseOptions; -using wallet::ISMINE_SPENDABLE; -using wallet::MakeWalletDatabase; using wallet::TxStateInactive; using wallet::WALLET_FLAG_DESCRIPTORS; using wallet::WalletContext; @@ -47,12 +45,11 @@ static void BenchUnloadWallet(std::shared_ptr<CWallet>&& wallet) static void AddTx(CWallet& wallet) { - bilingual_str error; - CTxDestination dest; - wallet.GetNewDestination(OutputType::BECH32, "", dest, error); + const auto& dest = wallet.GetNewDestination(OutputType::BECH32, ""); + assert(dest.HasRes()); CMutableTransaction mtx; - mtx.vout.push_back({COIN, GetScriptForDestination(dest)}); + mtx.vout.push_back({COIN, GetScriptForDestination(dest.GetObj())}); mtx.vin.push_back(CTxIn()); wallet.AddToWallet(MakeTransactionRef(mtx), TxStateInactive{}); diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index 1817aa1a53..4656cb23e7 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -18,6 +18,7 @@ #include <consensus/validation.h> #include <core_io.h> #include <node/blockstorage.h> +#include <node/caches.h> #include <node/chainstate.h> #include <scheduler.h> #include <script/sigcache.h> @@ -78,32 +79,24 @@ int main(int argc, char* argv[]) // SETUP: Chainstate const ChainstateManager::Options chainman_opts{ - chainparams, - static_cast<int64_t(*)()>(GetTime), + .chainparams = chainparams, + .adjusted_time_callback = static_cast<int64_t (*)()>(GetTime), }; ChainstateManager chainman{chainman_opts}; - auto rv = node::LoadChainstate(false, - std::ref(chainman), - nullptr, - false, - false, - 2 << 20, - 2 << 22, - (450 << 20) - (2 << 20) - (2 << 22), - false, - false, - []() { return false; }); - if (rv.has_value()) { + node::CacheSizes cache_sizes; + cache_sizes.block_tree_db = 2 << 20; + cache_sizes.coins_db = 2 << 22; + cache_sizes.coins = (450 << 20) - (2 << 20) - (2 << 22); + node::ChainstateLoadOptions options; + options.check_interrupt = [] { return false; }; + auto [status, error] = node::LoadChainstate(chainman, cache_sizes, options); + if (status != node::ChainstateLoadStatus::SUCCESS) { std::cerr << "Failed to load Chain state from your datadir." << std::endl; goto epilogue; } else { - auto maybe_verify_error = node::VerifyLoadedChainstate(std::ref(chainman), - false, - false, - DEFAULT_CHECKBLOCKS, - DEFAULT_CHECKLEVEL); - if (maybe_verify_error.has_value()) { + std::tie(status, error) = node::VerifyLoadedChainstate(chainman, options); + if (status != node::ChainstateLoadStatus::SUCCESS) { std::cerr << "Failed to verify loaded Chain state from your datadir." << std::endl; goto epilogue; } diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index 63a9ba498f..1ad6872143 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -47,7 +47,7 @@ GCSFilter::GCSFilter(const Params& params) : m_params(params), m_N(0), m_F(0), m_encoded{0} {} -GCSFilter::GCSFilter(const Params& params, std::vector<unsigned char> encoded_filter) +GCSFilter::GCSFilter(const Params& params, std::vector<unsigned char> encoded_filter, bool skip_decode_check) : m_params(params), m_encoded(std::move(encoded_filter)) { SpanReader stream{GCS_SER_TYPE, GCS_SER_VERSION, m_encoded}; @@ -59,6 +59,8 @@ GCSFilter::GCSFilter(const Params& params, std::vector<unsigned char> encoded_fi } m_F = static_cast<uint64_t>(m_N) * static_cast<uint64_t>(m_params.m_M); + if (skip_decode_check) return; + // Verify that the encoded filter contains exactly N elements. If it has too much or too little // data, a std::ios_base::failure exception will be raised. BitStreamReader<SpanReader> bitreader{stream}; @@ -219,14 +221,14 @@ static GCSFilter::ElementSet BasicFilterElements(const CBlock& block, } BlockFilter::BlockFilter(BlockFilterType filter_type, const uint256& block_hash, - std::vector<unsigned char> filter) + std::vector<unsigned char> filter, bool skip_decode_check) : m_filter_type(filter_type), m_block_hash(block_hash) { GCSFilter::Params params; if (!BuildParams(params)) { throw std::invalid_argument("unknown filter_type"); } - m_filter = GCSFilter(params, std::move(filter)); + m_filter = GCSFilter(params, std::move(filter), skip_decode_check); } BlockFilter::BlockFilter(BlockFilterType filter_type, const CBlock& block, const CBlockUndo& block_undo) diff --git a/src/blockfilter.h b/src/blockfilter.h index 96cefbf3b2..d6a51e95c2 100644 --- a/src/blockfilter.h +++ b/src/blockfilter.h @@ -59,7 +59,7 @@ public: explicit GCSFilter(const Params& params = Params()); /** Reconstructs an already-created filter from an encoding. */ - GCSFilter(const Params& params, std::vector<unsigned char> encoded_filter); + GCSFilter(const Params& params, std::vector<unsigned char> encoded_filter, bool skip_decode_check); /** Builds a new filter from the params and set of elements. */ GCSFilter(const Params& params, const ElementSet& elements); @@ -122,7 +122,7 @@ public: //! Reconstruct a BlockFilter from parts. BlockFilter(BlockFilterType filter_type, const uint256& block_hash, - std::vector<unsigned char> filter); + std::vector<unsigned char> filter, bool skip_decode_check); //! Construct a new BlockFilter of the specified type from a block. BlockFilter(BlockFilterType filter_type, const CBlock& block, const CBlockUndo& block_undo); @@ -164,7 +164,7 @@ public: if (!BuildParams(params)) { throw std::ios_base::failure("unknown filter_type"); } - m_filter = GCSFilter(params, std::move(encoded_filter)); + m_filter = GCSFilter(params, std::move(encoded_filter), /*skip_decode_check=*/false); } }; diff --git a/src/dbwrapper.cpp b/src/dbwrapper.cpp index d4a8e4f35a..4dbc839941 100644 --- a/src/dbwrapper.cpp +++ b/src/dbwrapper.cpp @@ -4,15 +4,28 @@ #include <dbwrapper.h> -#include <memory> +#include <fs.h> +#include <logging.h> #include <random.h> +#include <tinyformat.h> +#include <util/strencodings.h> +#include <util/system.h> +#include <algorithm> +#include <cassert> +#include <cstdarg> +#include <cstdint> +#include <cstdio> #include <leveldb/cache.h> +#include <leveldb/db.h> #include <leveldb/env.h> #include <leveldb/filter_policy.h> #include <leveldb/helpers/memenv/memenv.h> -#include <stdint.h> -#include <algorithm> +#include <leveldb/iterator.h> +#include <leveldb/options.h> +#include <leveldb/status.h> +#include <memory> +#include <optional> class CBitcoinLevelDBLogger : public leveldb::Logger { public: diff --git a/src/dbwrapper.h b/src/dbwrapper.h index cef8426d61..665eaa0e98 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -7,14 +7,26 @@ #include <clientversion.h> #include <fs.h> +#include <logging.h> #include <serialize.h> #include <span.h> #include <streams.h> -#include <util/strencodings.h> -#include <util/system.h> +#include <cstddef> +#include <cstdint> +#include <exception> #include <leveldb/db.h> +#include <leveldb/iterator.h> +#include <leveldb/options.h> +#include <leveldb/slice.h> +#include <leveldb/status.h> #include <leveldb/write_batch.h> +#include <stdexcept> +#include <string> +#include <vector> +namespace leveldb { +class Env; +} static const size_t DBWRAPPER_PREALLOC_KEY_SIZE = 64; static const size_t DBWRAPPER_PREALLOC_VALUE_SIZE = 1024; @@ -9,6 +9,7 @@ #include <cstdio> #include <filesystem> +#include <functional> #include <iomanip> #include <ios> #include <ostream> @@ -199,6 +200,7 @@ bool create_directories(const std::filesystem::path& p, std::error_code& ec) = d /** Bridge operations to C stdio */ namespace fsbridge { + using FopenFn = std::function<FILE*(const fs::path&, const char*)>; FILE *fopen(const fs::path& p, const char *mode); /** diff --git a/src/index/base.cpp b/src/index/base.cpp index 323547900d..2368189da2 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -4,11 +4,15 @@ #include <chainparams.h> #include <index/base.h> +#include <interfaces/chain.h> +#include <kernel/chain.h> #include <node/blockstorage.h> +#include <node/context.h> #include <node/interface_ui.h> #include <shutdown.h> #include <tinyformat.h> #include <util/syscall_sandbox.h> +#include <util/system.h> #include <util/thread.h> #include <util/translation.h> #include <validation.h> // For g_chainman @@ -31,6 +35,15 @@ static void FatalError(const char* fmt, const Args&... args) StartShutdown(); } +CBlockLocator GetLocator(interfaces::Chain& chain, const uint256& block_hash) +{ + CBlockLocator locator; + bool found = chain.findBlock(block_hash, interfaces::FoundBlock().locator(locator)); + assert(found); + assert(!locator.IsNull()); + return locator; +} + BaseIndex::DB::DB(const fs::path& path, size_t n_cache_size, bool f_memory, bool f_wipe, bool f_obfuscate) : CDBWrapper(path, n_cache_size, f_memory, f_wipe, f_obfuscate) {} @@ -49,6 +62,9 @@ void BaseIndex::DB::WriteBestBlock(CDBBatch& batch, const CBlockLocator& locator batch.Write(DB_BEST_BLOCK, locator); } +BaseIndex::BaseIndex(std::unique_ptr<interfaces::Chain> chain) + : m_chain{std::move(chain)} {} + BaseIndex::~BaseIndex() { Interrupt(); @@ -175,12 +191,15 @@ void BaseIndex::ThreadSync() } CBlock block; + interfaces::BlockInfo block_info = kernel::MakeBlockInfo(pindex); if (!ReadBlockFromDisk(block, pindex, consensus_params)) { FatalError("%s: Failed to read block %s from disk", __func__, pindex->GetBlockHash().ToString()); return; + } else { + block_info.data = █ } - if (!WriteBlock(block, pindex)) { + if (!CustomAppend(block_info)) { FatalError("%s: Failed to write block %s to index database", __func__, pindex->GetBlockHash().ToString()); return; @@ -197,22 +216,20 @@ void BaseIndex::ThreadSync() bool BaseIndex::Commit() { - CDBBatch batch(GetDB()); - if (!CommitInternal(batch) || !GetDB().WriteBatch(batch)) { - return error("%s: Failed to commit latest %s state", __func__, GetName()); - } - return true; -} - -bool BaseIndex::CommitInternal(CDBBatch& batch) -{ - LOCK(cs_main); // Don't commit anything if we haven't indexed any block yet // (this could happen if init is interrupted). - if (m_best_block_index == nullptr) { - return false; + bool ok = m_best_block_index != nullptr; + if (ok) { + CDBBatch batch(GetDB()); + ok = CustomCommit(batch); + if (ok) { + GetDB().WriteBestBlock(batch, GetLocator(*m_chain, m_best_block_index.load()->GetBlockHash())); + ok = GetDB().WriteBatch(batch); + } + } + if (!ok) { + return error("%s: Failed to commit latest %s state", __func__, GetName()); } - GetDB().WriteBestBlock(batch, m_chainstate->m_chain.GetLocator(m_best_block_index)); return true; } @@ -221,6 +238,10 @@ bool BaseIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_ti assert(current_tip == m_best_block_index); assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); + if (!CustomRewind({current_tip->GetBlockHash(), current_tip->nHeight}, {new_tip->GetBlockHash(), new_tip->nHeight})) { + return false; + } + // In the case of a reorg, ensure persisted block locator is not stale. // Pruning has a minimum of 288 blocks-to-keep and getting the index // out of sync may be possible but a users fault. @@ -268,8 +289,8 @@ void BaseIndex::BlockConnected(const std::shared_ptr<const CBlock>& block, const return; } } - - if (WriteBlock(*block, pindex)) { + interfaces::BlockInfo block_info = kernel::MakeBlockInfo(pindex, block.get()); + if (CustomAppend(block_info)) { SetBestBlockIndex(pindex); } else { FatalError("%s: Failed to write block %s to index", @@ -346,13 +367,18 @@ void BaseIndex::Interrupt() m_interrupt(); } -bool BaseIndex::Start(CChainState& active_chainstate) +bool BaseIndex::Start() { - m_chainstate = &active_chainstate; + // m_chainstate member gives indexing code access to node internals. It is + // removed in followup https://github.com/bitcoin/bitcoin/pull/24230 + m_chainstate = &m_chain->context()->chainman->ActiveChainstate(); // Need to register this ValidationInterface before running Init(), so that // callbacks are not missed if Init sets m_synced to true. RegisterValidationInterface(this); - if (!Init()) { + if (!Init()) return false; + + const CBlockIndex* index = m_best_block_index.load(); + if (!CustomInit(index ? std::make_optional(interfaces::BlockKey{index->GetBlockHash(), index->nHeight}) : std::nullopt)) { return false; } diff --git a/src/index/base.h b/src/index/base.h index a8f6a18c8d..77c46dd428 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -6,12 +6,16 @@ #define BITCOIN_INDEX_BASE_H #include <dbwrapper.h> +#include <interfaces/chain.h> #include <threadinterrupt.h> #include <validationinterface.h> class CBlock; class CBlockIndex; class CChainState; +namespace interfaces { +class Chain; +} // namespace interfaces struct IndexSummary { std::string name; @@ -59,6 +63,9 @@ private: std::thread m_thread_sync; CThreadInterrupt m_interrupt; + /// Read best block locator and check that data needed to sync has not been pruned. + bool Init(); + /// Sync the index with the block index starting from the current best block. /// Intended to be run in its own thread, m_thread_sync, and can be /// interrupted with m_interrupt. Once the index gets in sync, the m_synced @@ -76,30 +83,32 @@ private: /// getting corrupted. bool Commit(); + /// Loop over disconnected blocks and call CustomRewind. + bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip); + virtual bool AllowPrune() const = 0; protected: + std::unique_ptr<interfaces::Chain> m_chain; CChainState* m_chainstate{nullptr}; void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* pindex) override; void ChainStateFlushed(const CBlockLocator& locator) override; - const CBlockIndex* CurrentIndex() { return m_best_block_index.load(); }; - /// Initialize internal state from the database and block index. - [[nodiscard]] virtual bool Init(); + [[nodiscard]] virtual bool CustomInit(const std::optional<interfaces::BlockKey>& block) { return true; } /// Write update index entries for a newly connected block. - virtual bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) { return true; } + [[nodiscard]] virtual bool CustomAppend(const interfaces::BlockInfo& block) { return true; } /// Virtual method called internally by Commit that can be overridden to atomically /// commit more index state. - virtual bool CommitInternal(CDBBatch& batch); + virtual bool CustomCommit(CDBBatch& batch) { return true; } /// Rewind index to an earlier chain tip during a chain reorg. The tip must /// be an ancestor of the current best block. - virtual bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip); + [[nodiscard]] virtual bool CustomRewind(const interfaces::BlockKey& current_tip, const interfaces::BlockKey& new_tip) { return true; } virtual DB& GetDB() const = 0; @@ -110,6 +119,7 @@ protected: void SetBestBlockIndex(const CBlockIndex* block); public: + BaseIndex(std::unique_ptr<interfaces::Chain> chain); /// Destructor interrupts sync thread if running and blocks until it exits. virtual ~BaseIndex(); @@ -124,7 +134,7 @@ public: /// Start initializes the sync state and registers the instance as a /// ValidationInterface so that it stays in sync with blockchain updates. - [[nodiscard]] bool Start(CChainState& active_chainstate); + [[nodiscard]] bool Start(); /// Stops the instance from staying in sync with blockchain updates. void Stop(); diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index bc44a90c0e..f4837f3456 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -5,9 +5,11 @@ #include <map> #include <dbwrapper.h> +#include <hash.h> #include <index/blockfilterindex.h> #include <node/blockstorage.h> #include <util/system.h> +#include <validation.h> using node::UndoReadFromDisk; @@ -93,9 +95,9 @@ struct DBHashKey { static std::map<BlockFilterType, BlockFilterIndex> g_filter_indexes; -BlockFilterIndex::BlockFilterIndex(BlockFilterType filter_type, +BlockFilterIndex::BlockFilterIndex(std::unique_ptr<interfaces::Chain> chain, BlockFilterType filter_type, size_t n_cache_size, bool f_memory, bool f_wipe) - : m_filter_type(filter_type) + : BaseIndex(std::move(chain)), m_filter_type(filter_type) { const std::string& filter_name = BlockFilterTypeName(filter_type); if (filter_name.empty()) throw std::invalid_argument("unknown filter_type"); @@ -108,7 +110,7 @@ BlockFilterIndex::BlockFilterIndex(BlockFilterType filter_type, m_filter_fileseq = std::make_unique<FlatFileSeq>(std::move(path), "fltr", FLTR_FILE_CHUNK_SIZE); } -bool BlockFilterIndex::Init() +bool BlockFilterIndex::CustomInit(const std::optional<interfaces::BlockKey>& block) { if (!m_db->Read(DB_FILTER_POS, m_next_filter_pos)) { // Check that the cause of the read failure is that the key does not exist. Any other errors @@ -123,10 +125,10 @@ bool BlockFilterIndex::Init() m_next_filter_pos.nFile = 0; m_next_filter_pos.nPos = 0; } - return BaseIndex::Init(); + return true; } -bool BlockFilterIndex::CommitInternal(CDBBatch& batch) +bool BlockFilterIndex::CustomCommit(CDBBatch& batch) { const FlatFilePos& pos = m_next_filter_pos; @@ -140,21 +142,25 @@ bool BlockFilterIndex::CommitInternal(CDBBatch& batch) } batch.Write(DB_FILTER_POS, pos); - return BaseIndex::CommitInternal(batch); + return true; } -bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const +bool BlockFilterIndex::ReadFilterFromDisk(const FlatFilePos& pos, const uint256& hash, BlockFilter& filter) const { AutoFile filein{m_filter_fileseq->Open(pos, true)}; if (filein.IsNull()) { return false; } + // Check that the hash of the encoded_filter matches the one stored in the db. uint256 block_hash; std::vector<uint8_t> encoded_filter; try { filein >> block_hash >> encoded_filter; - filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter)); + uint256 result; + CHash256().Write(encoded_filter).Finalize(result); + if (result != hash) return error("Checksum mismatch in filter decode."); + filter = BlockFilter(GetFilterType(), block_hash, std::move(encoded_filter), /*skip_decode_check=*/true); } catch (const std::exception& e) { return error("%s: Failed to deserialize block filter from disk: %s", __func__, e.what()); @@ -209,22 +215,25 @@ size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& return data_size; } -bool BlockFilterIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) +bool BlockFilterIndex::CustomAppend(const interfaces::BlockInfo& block) { CBlockUndo block_undo; uint256 prev_header; - if (pindex->nHeight > 0) { + if (block.height > 0) { + // pindex variable gives indexing code access to node internals. It + // will be removed in upcoming commit + const CBlockIndex* pindex = WITH_LOCK(cs_main, return m_chainstate->m_blockman.LookupBlockIndex(block.hash)); if (!UndoReadFromDisk(block_undo, pindex)) { return false; } std::pair<uint256, DBVal> read_out; - if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { + if (!m_db->Read(DBHeightKey(block.height - 1), read_out)) { return false; } - uint256 expected_block_hash = pindex->pprev->GetBlockHash(); + uint256 expected_block_hash = *Assert(block.prev_hash); if (read_out.first != expected_block_hash) { return error("%s: previous block header belongs to unexpected block %s; expected %s", __func__, read_out.first.ToString(), expected_block_hash.ToString()); @@ -233,18 +242,18 @@ bool BlockFilterIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex prev_header = read_out.second.header; } - BlockFilter filter(m_filter_type, block, block_undo); + BlockFilter filter(m_filter_type, *Assert(block.data), block_undo); size_t bytes_written = WriteFilterToDisk(m_next_filter_pos, filter); if (bytes_written == 0) return false; std::pair<uint256, DBVal> value; - value.first = pindex->GetBlockHash(); + value.first = block.hash; value.second.hash = filter.GetHash(); value.second.header = filter.ComputeHeader(prev_header); value.second.pos = m_next_filter_pos; - if (!m_db->Write(DBHeightKey(pindex->nHeight), value)) { + if (!m_db->Write(DBHeightKey(block.height), value)) { return false; } @@ -278,17 +287,15 @@ static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch, return true; } -bool BlockFilterIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) +bool BlockFilterIndex::CustomRewind(const interfaces::BlockKey& current_tip, const interfaces::BlockKey& new_tip) { - assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); - CDBBatch batch(*m_db); std::unique_ptr<CDBIterator> db_it(m_db->NewIterator()); // During a reorg, we need to copy all filters for blocks that are getting disconnected from the // height index to the hash index so we can still find them when the height index entries are // overwritten. - if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight, current_tip->nHeight)) { + if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip.height, current_tip.height)) { return false; } @@ -298,7 +305,7 @@ bool BlockFilterIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* batch.Write(DB_FILTER_POS, m_next_filter_pos); if (!m_db->WriteBatch(batch)) return false; - return BaseIndex::Rewind(current_tip, new_tip); + return true; } static bool LookupOne(const CDBWrapper& db, const CBlockIndex* block_index, DBVal& result) @@ -381,7 +388,7 @@ bool BlockFilterIndex::LookupFilter(const CBlockIndex* block_index, BlockFilter& return false; } - return ReadFilterFromDisk(entry.pos, filter_out); + return ReadFilterFromDisk(entry.pos, entry.hash, filter_out); } bool BlockFilterIndex::LookupFilterHeader(const CBlockIndex* block_index, uint256& header_out) @@ -425,7 +432,7 @@ bool BlockFilterIndex::LookupFilterRange(int start_height, const CBlockIndex* st filters_out.resize(entries.size()); auto filter_pos_it = filters_out.begin(); for (const auto& entry : entries) { - if (!ReadFilterFromDisk(entry.pos, *filter_pos_it)) { + if (!ReadFilterFromDisk(entry.pos, entry.hash, *filter_pos_it)) { return false; } ++filter_pos_it; @@ -462,12 +469,12 @@ void ForEachBlockFilterIndex(std::function<void (BlockFilterIndex&)> fn) for (auto& entry : g_filter_indexes) fn(entry.second); } -bool InitBlockFilterIndex(BlockFilterType filter_type, +bool InitBlockFilterIndex(std::function<std::unique_ptr<interfaces::Chain>()> make_chain, BlockFilterType filter_type, size_t n_cache_size, bool f_memory, bool f_wipe) { auto result = g_filter_indexes.emplace(std::piecewise_construct, std::forward_as_tuple(filter_type), - std::forward_as_tuple(filter_type, + std::forward_as_tuple(make_chain(), filter_type, n_cache_size, f_memory, f_wipe)); return result.second; } diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h index 6deff59000..968eccb6b3 100644 --- a/src/index/blockfilterindex.h +++ b/src/index/blockfilterindex.h @@ -31,7 +31,7 @@ private: FlatFilePos m_next_filter_pos; std::unique_ptr<FlatFileSeq> m_filter_fileseq; - bool ReadFilterFromDisk(const FlatFilePos& pos, BlockFilter& filter) const; + bool ReadFilterFromDisk(const FlatFilePos& pos, const uint256& hash, BlockFilter& filter) const; size_t WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& filter); Mutex m_cs_headers_cache; @@ -41,13 +41,13 @@ private: bool AllowPrune() const override { return true; } protected: - bool Init() override; + bool CustomInit(const std::optional<interfaces::BlockKey>& block) override; - bool CommitInternal(CDBBatch& batch) override; + bool CustomCommit(CDBBatch& batch) override; - bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; + bool CustomAppend(const interfaces::BlockInfo& block) override; - bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) override; + bool CustomRewind(const interfaces::BlockKey& current_tip, const interfaces::BlockKey& new_tip) override; BaseIndex::DB& GetDB() const override { return *m_db; } @@ -55,7 +55,7 @@ protected: public: /** Constructs the index, which becomes available to be queried. */ - explicit BlockFilterIndex(BlockFilterType filter_type, + explicit BlockFilterIndex(std::unique_ptr<interfaces::Chain> chain, BlockFilterType filter_type, size_t n_cache_size, bool f_memory = false, bool f_wipe = false); BlockFilterType GetFilterType() const { return m_filter_type; } @@ -88,7 +88,7 @@ void ForEachBlockFilterIndex(std::function<void (BlockFilterIndex&)> fn); * Initialize a block filter index for the given type if one does not already exist. Returns true if * a new index is created and false if one has already been initialized. */ -bool InitBlockFilterIndex(BlockFilterType filter_type, +bool InitBlockFilterIndex(std::function<std::unique_ptr<interfaces::Chain>()> make_chain, BlockFilterType filter_type, size_t n_cache_size, bool f_memory = false, bool f_wipe = false); /** diff --git a/src/index/coinstatsindex.cpp b/src/index/coinstatsindex.cpp index 687e330fe0..b9029e946a 100644 --- a/src/index/coinstatsindex.cpp +++ b/src/index/coinstatsindex.cpp @@ -10,6 +10,7 @@ #include <serialize.h> #include <txdb.h> #include <undo.h> +#include <util/system.h> #include <validation.h> using kernel::CCoinsStats; @@ -102,7 +103,8 @@ struct DBHashKey { std::unique_ptr<CoinStatsIndex> g_coin_stats_index; -CoinStatsIndex::CoinStatsIndex(size_t n_cache_size, bool f_memory, bool f_wipe) +CoinStatsIndex::CoinStatsIndex(std::unique_ptr<interfaces::Chain> chain, size_t n_cache_size, bool f_memory, bool f_wipe) + : BaseIndex(std::move(chain)) { fs::path path{gArgs.GetDataDirNet() / "indexes" / "coinstats"}; fs::create_directories(path); @@ -110,24 +112,27 @@ CoinStatsIndex::CoinStatsIndex(size_t n_cache_size, bool f_memory, bool f_wipe) m_db = std::make_unique<CoinStatsIndex::DB>(path / "db", n_cache_size, f_memory, f_wipe); } -bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) +bool CoinStatsIndex::CustomAppend(const interfaces::BlockInfo& block) { CBlockUndo block_undo; - const CAmount block_subsidy{GetBlockSubsidy(pindex->nHeight, Params().GetConsensus())}; + const CAmount block_subsidy{GetBlockSubsidy(block.height, Params().GetConsensus())}; m_total_subsidy += block_subsidy; // Ignore genesis block - if (pindex->nHeight > 0) { + if (block.height > 0) { + // pindex variable gives indexing code access to node internals. It + // will be removed in upcoming commit + const CBlockIndex* pindex = WITH_LOCK(cs_main, return m_chainstate->m_blockman.LookupBlockIndex(block.hash)); if (!UndoReadFromDisk(block_undo, pindex)) { return false; } std::pair<uint256, DBVal> read_out; - if (!m_db->Read(DBHeightKey(pindex->nHeight - 1), read_out)) { + if (!m_db->Read(DBHeightKey(block.height - 1), read_out)) { return false; } - uint256 expected_block_hash{pindex->pprev->GetBlockHash()}; + uint256 expected_block_hash{*Assert(block.prev_hash)}; if (read_out.first != expected_block_hash) { LogPrintf("WARNING: previous block header belongs to unexpected block %s; expected %s\n", read_out.first.ToString(), expected_block_hash.ToString()); @@ -139,12 +144,13 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) } // TODO: Deduplicate BIP30 related code - bool is_bip30_block{(pindex->nHeight == 91722 && pindex->GetBlockHash() == uint256S("0x00000000000271a2dc26e7667f8419f2e15416dc6955e5a6c6cdf3f2574dd08e")) || - (pindex->nHeight == 91812 && pindex->GetBlockHash() == uint256S("0x00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f"))}; + bool is_bip30_block{(block.height == 91722 && block.hash == uint256S("0x00000000000271a2dc26e7667f8419f2e15416dc6955e5a6c6cdf3f2574dd08e")) || + (block.height == 91812 && block.hash == uint256S("0x00000000000af0aed4792b1acee3d966af36cf5def14935db8de83d6f9306f2f"))}; // Add the new utxos created from the block - for (size_t i = 0; i < block.vtx.size(); ++i) { - const auto& tx{block.vtx.at(i)}; + assert(block.data); + for (size_t i = 0; i < block.data->vtx.size(); ++i) { + const auto& tx{block.data->vtx.at(i)}; // Skip duplicate txid coinbase transactions (BIP30). if (is_bip30_block && tx->IsCoinBase()) { @@ -155,7 +161,7 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) for (uint32_t j = 0; j < tx->vout.size(); ++j) { const CTxOut& out{tx->vout[j]}; - Coin coin{out, pindex->nHeight, tx->IsCoinBase()}; + Coin coin{out, block.height, tx->IsCoinBase()}; COutPoint outpoint{tx->GetHash(), j}; // Skip unspendable coins @@ -211,7 +217,7 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) m_total_unspendables_unclaimed_rewards += unclaimed_rewards; std::pair<uint256, DBVal> value; - value.first = pindex->GetBlockHash(); + value.first = block.hash; value.second.transaction_output_count = m_transaction_output_count; value.second.bogo_size = m_bogo_size; value.second.total_amount = m_total_amount; @@ -231,7 +237,7 @@ bool CoinStatsIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) // Intentionally do not update DB_MUHASH here so it stays in sync with // DB_BEST_BLOCK, and the index is not corrupted if there is an unclean shutdown. - return m_db->Write(DBHeightKey(pindex->nHeight), value); + return m_db->Write(DBHeightKey(block.height), value); } static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch, @@ -260,17 +266,15 @@ static bool CopyHeightIndexToHashIndex(CDBIterator& db_it, CDBBatch& batch, return true; } -bool CoinStatsIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) +bool CoinStatsIndex::CustomRewind(const interfaces::BlockKey& current_tip, const interfaces::BlockKey& new_tip) { - assert(current_tip->GetAncestor(new_tip->nHeight) == new_tip); - CDBBatch batch(*m_db); std::unique_ptr<CDBIterator> db_it(m_db->NewIterator()); // During a reorg, we need to copy all hash digests for blocks that are // getting disconnected from the height index to the hash index so we can // still find them when the height index entries are overwritten. - if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip->nHeight, current_tip->nHeight)) { + if (!CopyHeightIndexToHashIndex(*db_it, batch, m_name, new_tip.height, current_tip.height)) { return false; } @@ -278,7 +282,8 @@ bool CoinStatsIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* n { LOCK(cs_main); - const CBlockIndex* iter_tip{m_chainstate->m_blockman.LookupBlockIndex(current_tip->GetBlockHash())}; + const CBlockIndex* iter_tip{m_chainstate->m_blockman.LookupBlockIndex(current_tip.hash)}; + const CBlockIndex* new_tip_index{m_chainstate->m_blockman.LookupBlockIndex(new_tip.hash)}; const auto& consensus_params{Params().GetConsensus()}; do { @@ -292,29 +297,29 @@ bool CoinStatsIndex::Rewind(const CBlockIndex* current_tip, const CBlockIndex* n ReverseBlock(block, iter_tip); iter_tip = iter_tip->GetAncestor(iter_tip->nHeight - 1); - } while (new_tip != iter_tip); + } while (new_tip_index != iter_tip); } - return BaseIndex::Rewind(current_tip, new_tip); + return true; } -static bool LookUpOne(const CDBWrapper& db, const CBlockIndex* block_index, DBVal& result) +static bool LookUpOne(const CDBWrapper& db, const interfaces::BlockKey& block, DBVal& result) { // First check if the result is stored under the height index and the value // there matches the block hash. This should be the case if the block is on // the active chain. std::pair<uint256, DBVal> read_out; - if (!db.Read(DBHeightKey(block_index->nHeight), read_out)) { + if (!db.Read(DBHeightKey(block.height), read_out)) { return false; } - if (read_out.first == block_index->GetBlockHash()) { + if (read_out.first == block.hash) { result = std::move(read_out.second); return true; } // If value at the height index corresponds to an different block, the // result will be stored in the hash index. - return db.Read(DBHashKey(block_index->GetBlockHash()), result); + return db.Read(DBHashKey(block.hash), result); } std::optional<CCoinsStats> CoinStatsIndex::LookUpStats(const CBlockIndex* block_index) const @@ -323,7 +328,7 @@ std::optional<CCoinsStats> CoinStatsIndex::LookUpStats(const CBlockIndex* block_ stats.index_used = true; DBVal entry; - if (!LookUpOne(*m_db, block_index, entry)) { + if (!LookUpOne(*m_db, {block_index->GetBlockHash(), block_index->nHeight}, entry)) { return std::nullopt; } @@ -344,7 +349,7 @@ std::optional<CCoinsStats> CoinStatsIndex::LookUpStats(const CBlockIndex* block_ return stats; } -bool CoinStatsIndex::Init() +bool CoinStatsIndex::CustomInit(const std::optional<interfaces::BlockKey>& block) { if (!m_db->Read(DB_MUHASH, m_muhash)) { // Check that the cause of the read failure is that the key does not @@ -356,13 +361,9 @@ bool CoinStatsIndex::Init() } } - if (!BaseIndex::Init()) return false; - - const CBlockIndex* pindex{CurrentIndex()}; - - if (pindex) { + if (block) { DBVal entry; - if (!LookUpOne(*m_db, pindex, entry)) { + if (!LookUpOne(*m_db, *block, entry)) { return error("%s: Cannot read current %s state; index may be corrupted", __func__, GetName()); } @@ -391,12 +392,12 @@ bool CoinStatsIndex::Init() return true; } -bool CoinStatsIndex::CommitInternal(CDBBatch& batch) +bool CoinStatsIndex::CustomCommit(CDBBatch& batch) { // DB_MUHASH should always be committed in a batch together with DB_BEST_BLOCK // to prevent an inconsistent state of the DB. batch.Write(DB_MUHASH, m_muhash); - return BaseIndex::CommitInternal(batch); + return true; } // Reverse a single block as part of a reorg diff --git a/src/index/coinstatsindex.h b/src/index/coinstatsindex.h index cae052d913..c4af223388 100644 --- a/src/index/coinstatsindex.h +++ b/src/index/coinstatsindex.h @@ -39,13 +39,13 @@ private: bool AllowPrune() const override { return true; } protected: - bool Init() override; + bool CustomInit(const std::optional<interfaces::BlockKey>& block) override; - bool CommitInternal(CDBBatch& batch) override; + bool CustomCommit(CDBBatch& batch) override; - bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; + bool CustomAppend(const interfaces::BlockInfo& block) override; - bool Rewind(const CBlockIndex* current_tip, const CBlockIndex* new_tip) override; + bool CustomRewind(const interfaces::BlockKey& current_tip, const interfaces::BlockKey& new_tip) override; BaseIndex::DB& GetDB() const override { return *m_db; } @@ -53,7 +53,7 @@ protected: public: // Constructs the index, which becomes available to be queried. - explicit CoinStatsIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + explicit CoinStatsIndex(std::unique_ptr<interfaces::Chain> chain, size_t n_cache_size, bool f_memory = false, bool f_wipe = false); // Look up stats for a specific block using CBlockIndex std::optional<kernel::CCoinsStats> LookUpStats(const CBlockIndex* block_index) const; diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 97c11c4383..b719aface8 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -48,23 +48,22 @@ bool TxIndex::DB::WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_ return WriteBatch(batch); } -TxIndex::TxIndex(size_t n_cache_size, bool f_memory, bool f_wipe) - : m_db(std::make_unique<TxIndex::DB>(n_cache_size, f_memory, f_wipe)) +TxIndex::TxIndex(std::unique_ptr<interfaces::Chain> chain, size_t n_cache_size, bool f_memory, bool f_wipe) + : BaseIndex(std::move(chain)), m_db(std::make_unique<TxIndex::DB>(n_cache_size, f_memory, f_wipe)) {} TxIndex::~TxIndex() = default; -bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) +bool TxIndex::CustomAppend(const interfaces::BlockInfo& block) { // Exclude genesis block transaction because outputs are not spendable. - if (pindex->nHeight == 0) return true; + if (block.height == 0) return true; - CDiskTxPos pos{ - WITH_LOCK(::cs_main, return pindex->GetBlockPos()), - GetSizeOfCompactSize(block.vtx.size())}; + assert(block.data); + CDiskTxPos pos({block.file_number, block.data_pos}, GetSizeOfCompactSize(block.data->vtx.size())); std::vector<std::pair<uint256, CDiskTxPos>> vPos; - vPos.reserve(block.vtx.size()); - for (const auto& tx : block.vtx) { + vPos.reserve(block.data->vtx.size()); + for (const auto& tx : block.data->vtx) { vPos.emplace_back(tx->GetHash(), pos); pos.nTxOffset += ::GetSerializeSize(*tx, CLIENT_VERSION); } diff --git a/src/index/txindex.h b/src/index/txindex.h index ec339abaa1..be240c4582 100644 --- a/src/index/txindex.h +++ b/src/index/txindex.h @@ -23,7 +23,7 @@ private: bool AllowPrune() const override { return false; } protected: - bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; + bool CustomAppend(const interfaces::BlockInfo& block) override; BaseIndex::DB& GetDB() const override; @@ -31,7 +31,7 @@ protected: public: /// Constructs the index, which becomes available to be queried. - explicit TxIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + explicit TxIndex(std::unique_ptr<interfaces::Chain> chain, size_t n_cache_size, bool f_memory = false, bool f_wipe = false); // Destructor is declared because this class contains a unique_ptr to an incomplete type. virtual ~TxIndex() override; diff --git a/src/init.cpp b/src/init.cpp index ee90abcc93..a94bbe6460 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -10,6 +10,7 @@ #include <init.h> #include <kernel/checks.h> +#include <kernel/mempool_persist.h> #include <addrman.h> #include <banman.h> @@ -41,6 +42,7 @@ #include <node/chainstate.h> #include <node/context.h> #include <node/interface_ui.h> +#include <node/mempool_persist_args.h> #include <node/miner.h> #include <policy/feerate.h> #include <policy/fees.h> @@ -64,7 +66,6 @@ #include <txorphanage.h> #include <util/asmap.h> #include <util/check.h> -#include <util/designator.h> #include <util/moneystr.h> #include <util/strencodings.h> #include <util/string.h> @@ -103,14 +104,16 @@ #include <zmq/zmqrpc.h> #endif +using kernel::DumpMempool; + using node::CacheSizes; using node::CalculateCacheSizes; -using node::ChainstateLoadVerifyError; -using node::ChainstateLoadingError; -using node::CleanupBlockRevFiles; +using node::DEFAULT_PERSIST_MEMPOOL; using node::DEFAULT_PRINTPRIORITY; using node::DEFAULT_STOPAFTERBLOCKIMPORT; using node::LoadChainstate; +using node::MempoolPath; +using node::ShouldPersistMempool; using node::NodeContext; using node::ThreadImport; using node::VerifyLoadedChainstate; @@ -246,8 +249,8 @@ void Shutdown(NodeContext& node) node.addrman.reset(); node.netgroupman.reset(); - if (node.mempool && node.mempool->IsLoaded() && node.args->GetBoolArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { - DumpMempool(*node.mempool); + if (node.mempool && node.mempool->GetLoadTried() && ShouldPersistMempool(*node.args)) { + DumpMempool(*node.mempool, MempoolPath(*node.args)); } // Drop transactions we were still watching, and record fee estimations. @@ -558,6 +561,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-bytespersigop", strprintf("Equivalent bytes per sigop in transactions for relay and mining (default: %u)", DEFAULT_BYTES_PER_SIGOP), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); argsman.AddArg("-datacarrier", strprintf("Relay and mine data carrier transactions (default: %u)", DEFAULT_ACCEPT_DATACARRIER), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); argsman.AddArg("-datacarriersize", strprintf("Maximum size of data in data carrier transactions we relay and mine (default: %u)", MAX_OP_RETURN_RELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); + argsman.AddArg("-mempoolfullrbf", strprintf("Accept transaction replace-by-fee without requiring replaceability signaling (default: %u)", DEFAULT_MEMPOOL_FULL_RBF), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); argsman.AddArg("-minrelaytxfee=<amt>", strprintf("Fees (in %s/kvB) smaller than this are considered zero fee for relaying, mining and transaction creation (default: %s)", CURRENCY_UNIT, FormatMoney(DEFAULT_MIN_RELAY_TX_FEE)), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); argsman.AddArg("-whitelistforcerelay", strprintf("Add 'forcerelay' permission to whitelisted inbound peers with default permissions. This will relay transactions even if the transactions were already in the mempool. (default: %d)", DEFAULT_WHITELISTFORCERELAY), ArgsManager::ALLOW_ANY, OptionsCategory::NODE_RELAY); @@ -966,7 +970,7 @@ bool AppInitParameterInteraction(const ArgsManager& args, bool use_syscall_sandb peer_connect_timeout = args.GetIntArg("-peertimeout", DEFAULT_PEER_CONNECT_TIMEOUT); if (peer_connect_timeout <= 0) { - return InitError(Untranslated("peertimeout cannot be configured with a negative value.")); + return InitError(Untranslated("peertimeout must be a positive integer.")); } if (args.IsArgSet("-minrelaytxfee")) { @@ -1092,21 +1096,8 @@ static bool LockDataDirectory(bool probeOnly) bool AppInitSanityChecks(const kernel::Context& kernel) { // ********************************************************* Step 4: sanity checks - auto maybe_error = kernel::SanityChecks(kernel); - - if (maybe_error.has_value()) { - switch (maybe_error.value()) { - case kernel::SanityCheckError::ERROR_ECC: - InitError(Untranslated("Elliptic curve cryptography sanity check failure. Aborting.")); - break; - case kernel::SanityCheckError::ERROR_RANDOM: - InitError(Untranslated("OS cryptographic RNG sanity check failure. Aborting.")); - break; - case kernel::SanityCheckError::ERROR_CHRONO: - InitError(Untranslated("Clock epoch mismatch. Aborting.")); - break; - } // no default case, so the compiler can warn about missing cases - + if (auto error = kernel::SanityChecks(kernel)) { + InitError(*error); return InitError(strprintf(_("Initialization sanity check failed. %s is shutting down."), PACKAGE_NAME)); } @@ -1424,8 +1415,8 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) assert(!node.chainman); CTxMemPool::Options mempool_opts{ - Desig(estimator) node.fee_estimator.get(), - Desig(check_ratio) chainparams.DefaultConsistencyChecks() ? 1 : 0, + .estimator = node.fee_estimator.get(), + .check_ratio = chainparams.DefaultConsistencyChecks() ? 1 : 0, }; ApplyArgsManOptions(args, mempool_opts); mempool_opts.check_ratio = std::clamp<int>(mempool_opts.check_ratio, 0, 1'000'000); @@ -1440,118 +1431,60 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) node.mempool = std::make_unique<CTxMemPool>(mempool_opts); const ChainstateManager::Options chainman_opts{ - chainparams, - GetAdjustedTime, + .chainparams = chainparams, + .adjusted_time_callback = GetAdjustedTime, }; node.chainman = std::make_unique<ChainstateManager>(chainman_opts); ChainstateManager& chainman = *node.chainman; - const bool fReset = fReindex; - bilingual_str strLoadError; + node::ChainstateLoadOptions options; + options.mempool = Assert(node.mempool.get()); + options.reindex = node::fReindex; + options.reindex_chainstate = fReindexChainState; + options.prune = node::fPruneMode; + options.check_blocks = args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); + options.check_level = args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL); + options.check_interrupt = ShutdownRequested; + options.coins_error_cb = [] { + uiInterface.ThreadSafeMessageBox( + _("Error reading from database, shutting down."), + "", CClientUIInterface::MSG_ERROR); + }; uiInterface.InitMessage(_("Loading block index…").translated); const int64_t load_block_index_start_time = GetTimeMillis(); - std::optional<ChainstateLoadingError> maybe_load_error; - try { - maybe_load_error = LoadChainstate(fReset, - chainman, - Assert(node.mempool.get()), - fPruneMode, - fReindexChainState, - cache_sizes.block_tree_db, - cache_sizes.coins_db, - cache_sizes.coins, - /*block_tree_db_in_memory=*/false, - /*coins_db_in_memory=*/false, - /*shutdown_requested=*/ShutdownRequested, - /*coins_error_cb=*/[]() { - uiInterface.ThreadSafeMessageBox( - _("Error reading from database, shutting down."), - "", CClientUIInterface::MSG_ERROR); - }); - } catch (const std::exception& e) { - LogPrintf("%s\n", e.what()); - maybe_load_error = ChainstateLoadingError::ERROR_GENERIC_BLOCKDB_OPEN_FAILED; - } - if (maybe_load_error.has_value()) { - switch (maybe_load_error.value()) { - case ChainstateLoadingError::ERROR_LOADING_BLOCK_DB: - strLoadError = _("Error loading block database"); - break; - case ChainstateLoadingError::ERROR_BAD_GENESIS_BLOCK: - // If the loaded chain has a wrong genesis, bail out immediately - // (we're likely using a testnet datadir, or the other way around). - return InitError(_("Incorrect or no genesis block found. Wrong datadir for network?")); - case ChainstateLoadingError::ERROR_PRUNED_NEEDS_REINDEX: - strLoadError = _("You need to rebuild the database using -reindex to go back to unpruned mode. This will redownload the entire blockchain"); - break; - case ChainstateLoadingError::ERROR_LOAD_GENESIS_BLOCK_FAILED: - strLoadError = _("Error initializing block database"); - break; - case ChainstateLoadingError::ERROR_CHAINSTATE_UPGRADE_FAILED: - return InitError(_("Unsupported chainstate database format found. " - "Please restart with -reindex-chainstate. This will " - "rebuild the chainstate database.")); - case ChainstateLoadingError::ERROR_REPLAYBLOCKS_FAILED: - strLoadError = _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate."); - break; - case ChainstateLoadingError::ERROR_LOADCHAINTIP_FAILED: - strLoadError = _("Error initializing block database"); - break; - case ChainstateLoadingError::ERROR_GENERIC_BLOCKDB_OPEN_FAILED: - strLoadError = _("Error opening block database"); - break; - case ChainstateLoadingError::ERROR_BLOCKS_WITNESS_INSUFFICIENTLY_VALIDATED: - strLoadError = strprintf(_("Witness data for blocks after height %d requires validation. Please restart with -reindex."), - chainman.GetConsensus().SegwitHeight); - break; - case ChainstateLoadingError::SHUTDOWN_PROBED: - break; - } - } else { - std::optional<ChainstateLoadVerifyError> maybe_verify_error; + auto catch_exceptions = [](auto&& f) { try { - uiInterface.InitMessage(_("Verifying blocks…").translated); - auto check_blocks = args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); - if (chainman.m_blockman.m_have_pruned && check_blocks > MIN_BLOCKS_TO_KEEP) { - LogPrintfCategory(BCLog::PRUNE, "pruned datadir may not have more than %d blocks; only checking available blocks\n", - MIN_BLOCKS_TO_KEEP); - } - maybe_verify_error = VerifyLoadedChainstate(chainman, - fReset, - fReindexChainState, - check_blocks, - args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL)); + return f(); } catch (const std::exception& e) { LogPrintf("%s\n", e.what()); - maybe_verify_error = ChainstateLoadVerifyError::ERROR_GENERIC_FAILURE; + return std::make_tuple(node::ChainstateLoadStatus::FAILURE, _("Error opening block database")); } - if (maybe_verify_error.has_value()) { - switch (maybe_verify_error.value()) { - case ChainstateLoadVerifyError::ERROR_BLOCK_FROM_FUTURE: - strLoadError = _("The block database contains a block which appears to be from the future. " - "This may be due to your computer's date and time being set incorrectly. " - "Only rebuild the block database if you are sure that your computer's date and time are correct"); - break; - case ChainstateLoadVerifyError::ERROR_CORRUPTED_BLOCK_DB: - strLoadError = _("Corrupted block database detected"); - break; - case ChainstateLoadVerifyError::ERROR_GENERIC_FAILURE: - strLoadError = _("Error opening block database"); - break; - } - } else { + }; + auto [status, error] = catch_exceptions([&]{ return LoadChainstate(chainman, cache_sizes, options); }); + if (status == node::ChainstateLoadStatus::SUCCESS) { + uiInterface.InitMessage(_("Verifying blocks…").translated); + if (chainman.m_blockman.m_have_pruned && options.check_blocks > MIN_BLOCKS_TO_KEEP) { + LogPrintfCategory(BCLog::PRUNE, "pruned datadir may not have more than %d blocks; only checking available blocks\n", + MIN_BLOCKS_TO_KEEP); + } + std::tie(status, error) = catch_exceptions([&]{ return VerifyLoadedChainstate(chainman, options);}); + if (status == node::ChainstateLoadStatus::SUCCESS) { fLoaded = true; LogPrintf(" block index %15dms\n", GetTimeMillis() - load_block_index_start_time); } } + if (status == node::ChainstateLoadStatus::FAILURE_INCOMPATIBLE_DB) { + return InitError(error); + } + if (!fLoaded && !ShutdownRequested()) { // first suggest a reindex - if (!fReset) { + if (!options.reindex) { bool fRet = uiInterface.ThreadSafeQuestion( - strLoadError + Untranslated(".\n\n") + _("Do you want to rebuild the block database now?"), - strLoadError.original + ".\nPlease restart with -reindex or -reindex-chainstate to recover.", + error + Untranslated(".\n\n") + _("Do you want to rebuild the block database now?"), + error.original + ".\nPlease restart with -reindex or -reindex-chainstate to recover.", "", CClientUIInterface::MSG_ERROR | CClientUIInterface::BTN_ABORT); if (fRet) { fReindex = true; @@ -1561,7 +1494,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) return false; } } else { - return InitError(strLoadError); + return InitError(error); } } } @@ -1587,22 +1520,22 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) return InitError(*error); } - g_txindex = std::make_unique<TxIndex>(cache_sizes.tx_index, false, fReindex); - if (!g_txindex->Start(chainman.ActiveChainstate())) { + g_txindex = std::make_unique<TxIndex>(interfaces::MakeChain(node), cache_sizes.tx_index, false, fReindex); + if (!g_txindex->Start()) { return false; } } for (const auto& filter_type : g_enabled_filter_types) { - InitBlockFilterIndex(filter_type, cache_sizes.filter_index, false, fReindex); - if (!GetBlockFilterIndex(filter_type)->Start(chainman.ActiveChainstate())) { + InitBlockFilterIndex([&]{ return interfaces::MakeChain(node); }, filter_type, cache_sizes.filter_index, false, fReindex); + if (!GetBlockFilterIndex(filter_type)->Start()) { return false; } } if (args.GetBoolArg("-coinstatsindex", DEFAULT_COINSTATSINDEX)) { - g_coin_stats_index = std::make_unique<CoinStatsIndex>(/* cache size */ 0, false, fReindex); - if (!g_coin_stats_index->Start(chainman.ActiveChainstate())) { + g_coin_stats_index = std::make_unique<CoinStatsIndex>(interfaces::MakeChain(node), /* cache size */ 0, false, fReindex); + if (!g_coin_stats_index->Start()) { return false; } } @@ -1669,7 +1602,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } chainman.m_load_block = std::thread(&util::TraceThread, "loadblk", [=, &chainman, &args] { - ThreadImport(chainman, vImportFiles, args); + ThreadImport(chainman, vImportFiles, args, ShouldPersistMempool(args) ? MempoolPath(args) : fs::path{}); }); // Wait for genesis block to be processed diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index ddfb4bda95..5fc0e540a9 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -18,6 +18,7 @@ class ArgsManager; class CBlock; +class CBlockUndo; class CFeeRate; class CRPCCommand; class CScheduler; @@ -37,6 +38,12 @@ namespace interfaces { class Handler; class Wallet; +//! Hash/height pair to help track and identify blocks. +struct BlockKey { + uint256 hash; + int height = -1; +}; + //! Helper for findBlock to selectively return pieces of block data. If block is //! found, data will be returned by setting specified output variables. If block //! is not found, output variables will keep their previous values. @@ -50,6 +57,8 @@ public: FoundBlock& mtpTime(int64_t& mtp_time) { m_mtp_time = &mtp_time; return *this; } //! Return whether block is in the active (most-work) chain. FoundBlock& inActiveChain(bool& in_active_chain) { m_in_active_chain = &in_active_chain; return *this; } + //! Return locator if block is in the active chain. + FoundBlock& locator(CBlockLocator& locator) { m_locator = &locator; return *this; } //! Return next block in the active chain if current block is in the active chain. FoundBlock& nextBlock(const FoundBlock& next_block) { m_next_block = &next_block; return *this; } //! Read block data from disk. If the block exists but doesn't have data @@ -62,11 +71,25 @@ public: int64_t* m_max_time = nullptr; int64_t* m_mtp_time = nullptr; bool* m_in_active_chain = nullptr; + CBlockLocator* m_locator = nullptr; const FoundBlock* m_next_block = nullptr; CBlock* m_data = nullptr; mutable bool found = false; }; +//! Block data sent with blockConnected, blockDisconnected notifications. +struct BlockInfo { + const uint256& hash; + const uint256* prev_hash = nullptr; + int height = -1; + int file_number = -1; + unsigned data_pos = 0; + const CBlock* data = nullptr; + const CBlockUndo* undo_data = nullptr; + + BlockInfo(const uint256& hash LIFETIMEBOUND) : hash(hash) {} +}; + //! Interface giving clients (wallet processes, maybe other analysis tools in //! the future) ability to access to the chain state, receive notifications, //! estimate fees, and submit transactions. @@ -111,6 +134,10 @@ public: //! Get locator for the current chain tip. virtual CBlockLocator getTipLocator() = 0; + //! Return a locator that refers to a block in the active chain. + //! If specified block is not in the active chain, return locator for the latest ancestor that is in the chain. + virtual CBlockLocator getActiveChainLocator(const uint256& block_hash) = 0; + //! Return height of the highest block on chain in common with the locator, //! which will either be the original block used to create the locator, //! or one of its ancestors. @@ -235,8 +262,8 @@ public: virtual ~Notifications() {} virtual void transactionAddedToMempool(const CTransactionRef& tx, uint64_t mempool_sequence) {} virtual void transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason, uint64_t mempool_sequence) {} - virtual void blockConnected(const CBlock& block, int height) {} - virtual void blockDisconnected(const CBlock& block, int height) {} + virtual void blockConnected(const BlockInfo& block) {} + virtual void blockDisconnected(const BlockInfo& block) {} virtual void updatedBlockTip() {} virtual void chainStateFlushed(const CBlockLocator& locator) {} }; @@ -283,6 +310,13 @@ public: //! to be prepared to handle this by ignoring notifications about unknown //! removed transactions and already added new transactions. virtual void requestMempoolTransactions(Notifications& notifications) = 0; + + //! Return true if an assumed-valid chain is in use. + virtual bool hasAssumedValidChain() = 0; + + //! Get internal node context. Useful for testing, but not + //! accessible across processes. + virtual node::NodeContext* context() { return nullptr; } }; //! Interface to let node manage chain clients (wallets, or maybe tools for diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index f26ac866dc..e29fefae56 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -12,6 +12,7 @@ #include <script/standard.h> // For CTxDestination #include <support/allocators/secure.h> // For SecureString #include <util/message.h> +#include <util/result.h> #include <util/ui_change_type.h> #include <cstdint> @@ -87,7 +88,7 @@ public: virtual std::string getWalletName() = 0; // Get a new address. - virtual bool getNewDestination(const OutputType type, const std::string label, CTxDestination& dest) = 0; + virtual BResult<CTxDestination> getNewDestination(const OutputType type, const std::string label) = 0; //! Get public key. virtual bool getPubKey(const CScript& script, const CKeyID& address, CPubKey& pub_key) = 0; @@ -114,7 +115,7 @@ public: std::string* purpose) = 0; //! Get wallet address list. - virtual std::vector<WalletAddress> getAddresses() = 0; + virtual std::vector<WalletAddress> getAddresses() const = 0; //! Get receive requests. virtual std::vector<std::string> getAddressReceiveRequests() = 0; @@ -138,12 +139,11 @@ public: virtual void listLockedCoins(std::vector<COutPoint>& outputs) = 0; //! Create transaction. - virtual CTransactionRef createTransaction(const std::vector<wallet::CRecipient>& recipients, + virtual BResult<CTransactionRef> createTransaction(const std::vector<wallet::CRecipient>& recipients, const wallet::CCoinControl& coin_control, bool sign, int& change_pos, - CAmount& fee, - bilingual_str& fail_reason) = 0; + CAmount& fee) = 0; //! Commit transaction. virtual void commitTransaction(CTransactionRef tx, @@ -329,7 +329,7 @@ public: virtual std::string getWalletDir() = 0; //! Restore backup wallet - virtual std::unique_ptr<Wallet> restoreWallet(const fs::path& backup_file, const std::string& wallet_name, bilingual_str& error, std::vector<bilingual_str>& warnings) = 0; + virtual BResult<std::unique_ptr<Wallet>> restoreWallet(const fs::path& backup_file, const std::string& wallet_name, std::vector<bilingual_str>& warnings) = 0; //! Return available wallets in wallet directory. virtual std::vector<std::string> listWalletDir() = 0; diff --git a/src/kernel/chain.cpp b/src/kernel/chain.cpp new file mode 100644 index 0000000000..82e77125d7 --- /dev/null +++ b/src/kernel/chain.cpp @@ -0,0 +1,26 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chain.h> +#include <interfaces/chain.h> +#include <sync.h> +#include <uint256.h> + +class CBlock; + +namespace kernel { +interfaces::BlockInfo MakeBlockInfo(const CBlockIndex* index, const CBlock* data) +{ + interfaces::BlockInfo info{index ? *index->phashBlock : uint256::ZERO}; + if (index) { + info.prev_hash = index->pprev ? index->pprev->phashBlock : nullptr; + info.height = index->nHeight; + LOCK(::cs_main); + info.file_number = index->nFile; + info.data_pos = index->nDataPos; + } + info.data = data; + return info; +} +} // namespace kernel diff --git a/src/kernel/chain.h b/src/kernel/chain.h new file mode 100644 index 0000000000..f0750f8266 --- /dev/null +++ b/src/kernel/chain.h @@ -0,0 +1,19 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_KERNEL_CHAIN_H +#define BITCOIN_KERNEL_CHAIN_H + +class CBlock; +class CBlockIndex; +namespace interfaces { +struct BlockInfo; +} // namespace interfaces + +namespace kernel { +//! Return data from block index. +interfaces::BlockInfo MakeBlockInfo(const CBlockIndex* block_index, const CBlock* data = nullptr); +} // namespace kernel + +#endif // BITCOIN_KERNEL_CHAIN_H diff --git a/src/kernel/chainstatemanager_opts.h b/src/kernel/chainstatemanager_opts.h index 575d94e2e9..510a1f9edc 100644 --- a/src/kernel/chainstatemanager_opts.h +++ b/src/kernel/chainstatemanager_opts.h @@ -10,6 +10,8 @@ class CChainParams; +namespace kernel { + /** * An options struct for `ChainstateManager`, more ergonomically referred to as * `ChainstateManager::Options` due to the using-declaration in @@ -20,4 +22,6 @@ struct ChainstateManagerOpts { const std::function<int64_t()> adjusted_time_callback{nullptr}; }; +} // namespace kernel + #endif // BITCOIN_KERNEL_CHAINSTATEMANAGER_OPTS_H diff --git a/src/kernel/checks.cpp b/src/kernel/checks.cpp index 2a1dd3bfa2..4c303c172c 100644 --- a/src/kernel/checks.cpp +++ b/src/kernel/checks.cpp @@ -7,21 +7,24 @@ #include <key.h> #include <random.h> #include <util/time.h> +#include <util/translation.h> + +#include <memory> namespace kernel { -std::optional<SanityCheckError> SanityChecks(const Context&) +std::optional<bilingual_str> SanityChecks(const Context&) { if (!ECC_InitSanityCheck()) { - return SanityCheckError::ERROR_ECC; + return Untranslated("Elliptic curve cryptography sanity check failure. Aborting."); } if (!Random_SanityCheck()) { - return SanityCheckError::ERROR_RANDOM; + return Untranslated("OS cryptographic RNG sanity check failure. Aborting."); } if (!ChronoSanityCheck()) { - return SanityCheckError::ERROR_CHRONO; + return Untranslated("Clock epoch mismatch. Aborting."); } return std::nullopt; diff --git a/src/kernel/checks.h b/src/kernel/checks.h index 80b207f607..3eb14824fb 100644 --- a/src/kernel/checks.h +++ b/src/kernel/checks.h @@ -7,20 +7,16 @@ #include <optional> +struct bilingual_str; + namespace kernel { struct Context; -enum class SanityCheckError { - ERROR_ECC, - ERROR_RANDOM, - ERROR_CHRONO, -}; - /** * Ensure a usable environment with all necessary library support. */ -std::optional<SanityCheckError> SanityChecks(const Context&); +std::optional<bilingual_str> SanityChecks(const Context&); } diff --git a/src/kernel/coinstats.cpp b/src/kernel/coinstats.cpp index f380871627..8d948bfc1d 100644 --- a/src/kernel/coinstats.cpp +++ b/src/kernel/coinstats.cpp @@ -4,16 +4,32 @@ #include <kernel/coinstats.h> +#include <chain.h> #include <coins.h> #include <crypto/muhash.h> #include <hash.h> +#include <node/blockstorage.h> +#include <primitives/transaction.h> +#include <script/script.h> #include <serialize.h> +#include <span.h> +#include <streams.h> +#include <sync.h> +#include <tinyformat.h> #include <uint256.h> +#include <util/check.h> #include <util/overflow.h> #include <util/system.h> #include <validation.h> +#include <version.h> +#include <cassert> +#include <iosfwd> +#include <iterator> #include <map> +#include <memory> +#include <string> +#include <utility> namespace kernel { diff --git a/src/kernel/coinstats.h b/src/kernel/coinstats.h index a15957233f..b7c1328e93 100644 --- a/src/kernel/coinstats.h +++ b/src/kernel/coinstats.h @@ -5,16 +5,18 @@ #ifndef BITCOIN_KERNEL_COINSTATS_H #define BITCOIN_KERNEL_COINSTATS_H -#include <chain.h> -#include <coins.h> #include <consensus/amount.h> #include <streams.h> #include <uint256.h> #include <cstdint> #include <functional> +#include <optional> class CCoinsView; +class Coin; +class COutPoint; +class CScript; namespace node { class BlockManager; } // namespace node diff --git a/src/kernel/mempool_options.h b/src/kernel/mempool_options.h index a14abb6628..07953b443b 100644 --- a/src/kernel/mempool_options.h +++ b/src/kernel/mempool_options.h @@ -15,6 +15,8 @@ class CBlockPolicyEstimator; static constexpr unsigned int DEFAULT_MAX_MEMPOOL_SIZE_MB{300}; /** Default for -mempoolexpiry, expiration time for mempool transactions in hours */ static constexpr unsigned int DEFAULT_MEMPOOL_EXPIRY_HOURS{336}; +/** Default for -mempoolfullrbf, if the transaction replaceability signaling is ignored */ +static constexpr bool DEFAULT_MEMPOOL_FULL_RBF{false}; namespace kernel { /** @@ -31,6 +33,7 @@ struct MemPoolOptions { int check_ratio{0}; int64_t max_size_bytes{DEFAULT_MAX_MEMPOOL_SIZE_MB * 1'000'000}; std::chrono::seconds expiry{std::chrono::hours{DEFAULT_MEMPOOL_EXPIRY_HOURS}}; + bool full_rbf{DEFAULT_MEMPOOL_FULL_RBF}; MemPoolLimits limits{}; }; } // namespace kernel diff --git a/src/kernel/mempool_persist.cpp b/src/kernel/mempool_persist.cpp new file mode 100644 index 0000000000..1a1cf2bbdc --- /dev/null +++ b/src/kernel/mempool_persist.cpp @@ -0,0 +1,189 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <kernel/mempool_persist.h> + +#include <clientversion.h> +#include <consensus/amount.h> +#include <fs.h> +#include <logging.h> +#include <primitives/transaction.h> +#include <serialize.h> +#include <shutdown.h> +#include <streams.h> +#include <sync.h> +#include <txmempool.h> +#include <uint256.h> +#include <util/system.h> +#include <util/time.h> +#include <validation.h> + +#include <chrono> +#include <cstdint> +#include <cstdio> +#include <exception> +#include <functional> +#include <map> +#include <memory> +#include <set> +#include <stdexcept> +#include <utility> +#include <vector> + +using fsbridge::FopenFn; + +namespace kernel { + +static const uint64_t MEMPOOL_DUMP_VERSION = 1; + +bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, CChainState& active_chainstate, FopenFn mockable_fopen_function) +{ + if (load_path.empty()) return false; + + FILE* filestr{mockable_fopen_function(load_path, "rb")}; + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); + return false; + } + + int64_t count = 0; + int64_t expired = 0; + int64_t failed = 0; + int64_t already_there = 0; + int64_t unbroadcast = 0; + auto now = NodeClock::now(); + + try { + uint64_t version; + file >> version; + if (version != MEMPOOL_DUMP_VERSION) { + return false; + } + uint64_t num; + file >> num; + while (num) { + --num; + CTransactionRef tx; + int64_t nTime; + int64_t nFeeDelta; + file >> tx; + file >> nTime; + file >> nFeeDelta; + + CAmount amountdelta = nFeeDelta; + if (amountdelta) { + pool.PrioritiseTransaction(tx->GetHash(), amountdelta); + } + if (nTime > TicksSinceEpoch<std::chrono::seconds>(now - pool.m_expiry)) { + LOCK(cs_main); + const auto& accepted = AcceptToMemoryPool(active_chainstate, tx, nTime, /*bypass_limits=*/false, /*test_accept=*/false); + if (accepted.m_result_type == MempoolAcceptResult::ResultType::VALID) { + ++count; + } else { + // mempool may contain the transaction already, e.g. from + // wallet(s) having loaded it while we were processing + // mempool transactions; consider these as valid, instead of + // failed, but mark them as 'already there' + if (pool.exists(GenTxid::Txid(tx->GetHash()))) { + ++already_there; + } else { + ++failed; + } + } + } else { + ++expired; + } + if (ShutdownRequested()) + return false; + } + std::map<uint256, CAmount> mapDeltas; + file >> mapDeltas; + + for (const auto& i : mapDeltas) { + pool.PrioritiseTransaction(i.first, i.second); + } + + std::set<uint256> unbroadcast_txids; + file >> unbroadcast_txids; + unbroadcast = unbroadcast_txids.size(); + for (const auto& txid : unbroadcast_txids) { + // Ensure transactions were accepted to mempool then add to + // unbroadcast set. + if (pool.get(txid) != nullptr) pool.AddUnbroadcastTx(txid); + } + } catch (const std::exception& e) { + LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); + return false; + } + + LogPrintf("Imported mempool transactions from disk: %i succeeded, %i failed, %i expired, %i already there, %i waiting for initial broadcast\n", count, failed, expired, already_there, unbroadcast); + return true; +} + +bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mockable_fopen_function, bool skip_file_commit) +{ + auto start = SteadyClock::now(); + + std::map<uint256, CAmount> mapDeltas; + std::vector<TxMempoolInfo> vinfo; + std::set<uint256> unbroadcast_txids; + + static Mutex dump_mutex; + LOCK(dump_mutex); + + { + LOCK(pool.cs); + for (const auto &i : pool.mapDeltas) { + mapDeltas[i.first] = i.second; + } + vinfo = pool.infoAll(); + unbroadcast_txids = pool.GetUnbroadcastTxs(); + } + + auto mid = SteadyClock::now(); + + try { + FILE* filestr{mockable_fopen_function(dump_path + ".new", "wb")}; + if (!filestr) { + return false; + } + + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + + uint64_t version = MEMPOOL_DUMP_VERSION; + file << version; + + file << (uint64_t)vinfo.size(); + for (const auto& i : vinfo) { + file << *(i.tx); + file << int64_t{count_seconds(i.m_time)}; + file << int64_t{i.nFeeDelta}; + mapDeltas.erase(i.tx->GetHash()); + } + + file << mapDeltas; + + LogPrintf("Writing %d unbroadcast transactions to disk.\n", unbroadcast_txids.size()); + file << unbroadcast_txids; + + if (!skip_file_commit && !FileCommit(file.Get())) + throw std::runtime_error("FileCommit failed"); + file.fclose(); + if (!RenameOver(dump_path + ".new", dump_path)) { + throw std::runtime_error("Rename failed"); + } + auto last = SteadyClock::now(); + + LogPrintf("Dumped mempool: %gs to copy, %gs to dump\n", + Ticks<SecondsDouble>(mid - start), + Ticks<SecondsDouble>(last - mid)); + } catch (const std::exception& e) { + LogPrintf("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); + return false; + } + return true; +} + +} // namespace kernel diff --git a/src/kernel/mempool_persist.h b/src/kernel/mempool_persist.h new file mode 100644 index 0000000000..9a15ec6dca --- /dev/null +++ b/src/kernel/mempool_persist.h @@ -0,0 +1,28 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_KERNEL_MEMPOOL_PERSIST_H +#define BITCOIN_KERNEL_MEMPOOL_PERSIST_H + +#include <fs.h> + +class CChainState; +class CTxMemPool; + +namespace kernel { + +/** Dump the mempool to disk. */ +bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, + fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen, + bool skip_file_commit = false); + +/** Load the mempool from disk. */ +bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, + CChainState& active_chainstate, + fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen); + +} // namespace kernel + + +#endif // BITCOIN_KERNEL_MEMPOOL_PERSIST_H diff --git a/src/mempool_args.cpp b/src/mempool_args.cpp index e26cbe0275..77caa127e9 100644 --- a/src/mempool_args.cpp +++ b/src/mempool_args.cpp @@ -33,5 +33,7 @@ void ApplyArgsManOptions(const ArgsManager& argsman, MemPoolOptions& mempool_opt if (auto hours = argsman.GetIntArg("-mempoolexpiry")) mempool_opts.expiry = std::chrono::hours{*hours}; + mempool_opts.full_rbf = argsman.GetBoolArg("-mempoolfullrbf", mempool_opts.full_rbf); + ApplyArgsManOptions(argsman, mempool_opts.limits); } diff --git a/src/minisketch/README.md b/src/minisketch/README.md index c0cfdc1623..f8b89ff33e 100644 --- a/src/minisketch/README.md +++ b/src/minisketch/README.md @@ -203,8 +203,8 @@ Some improvements that are still TODO: * <a name="myfootnote4">[4]</a> Bhaskar Biswas, Vincent Herbert. *Efficient Root Finding of Polynomials over Fields of Characteristic 2.* 2009. hal-00626997. [[URL]](https://hal.archives-ouvertes.fr/hal-00626997) [[PDF]](https://hal.archives-ouvertes.fr/hal-00626997/document) * <a name="myfootnote6">[6]</a> Eppstein, David, Michael T. Goodrich, Frank Uyeda, and George Varghese. *What's the difference?: efficient set reconciliation without prior context.* ACM SIGCOMM Computer Communication Review, vol. 41, no. 4, pp. 218-229. ACM, 2011. [[PDF]](https://www.ics.uci.edu/~eppstein/pubs/EppGooUye-SIGCOMM-11.pdf) * <a name="myfootnote7">[7]</a> Goodrich, Michael T. and Michael Mitzenmacher. *Invertible bloom lookup tables.* 2011 49th Annual Allerton Conference on Communication, Control, and Computing (Allerton) (2011): 792-799. [[PDF]](https://arxiv.org/pdf/1101.2245.pdf) -* <a name="myfootnote8">[8]</a> Maxwell, Gregory F. *[Blocksonly mode BW savings, the limits of efficient block xfer, and better relay](https://bitcointalk.org/index.php?topic=1377345.0)* Bitcointalk 2016, *[Technical notes on mempool synchronizing relay](https://people.xiph.org/~greg/mempool_sync_relay.txt)* #bitcoin-wizards 2016. -* <a name="myfootnote9">[9]</a> Maxwell, Gregory F. *[Block network coding](https://en.bitcoin.it/wiki/User:Gmaxwell/block_network_coding)* Bitcoin Wiki 2014, *[Technical notes on efficient block xfer](https://people.xiph.org/~greg/efficient.block.xfer.txt)* #bitcoin-wizards 2015. +* <a name="myfootnote8">[8]</a> Maxwell, Gregory F. *[Blocksonly mode BW savings, the limits of efficient block xfer, and better relay](https://bitcointalk.org/index.php?topic=1377345.0)* Bitcointalk 2016, *[Technical notes on mempool synchronizing relay](https://nt4tn.net/tech-notes/2016.mempool_sync_relay.txt)* #bitcoin-wizards 2016. +* <a name="myfootnote9">[9]</a> Maxwell, Gregory F. *[Block network coding](https://en.bitcoin.it/wiki/User:Gmaxwell/block_network_coding)* Bitcoin Wiki 2014, *[Technical notes on efficient block xfer](https://nt4tn.net/tech-notes/201512.efficient.block.xfer.txt)* #bitcoin-wizards 2015. * <a name="myfootnote10">[10]</a> Ruffing, Tim, Moreno-Sanchez, Pedro, Aniket, Kate, *P2P Mixing and Unlinkable Bitcoin Transactions* NDSS Symposium 2017 [[URL]](https://eprint.iacr.org/2016/824) [[PDF]](https://eprint.iacr.org/2016/824.pdf) * <a name="myfootnote11">[11]</a> Y. Misky, A. Trachtenberg, R. Zippel. *Set Reconciliation with Nearly Optimal Communication Complexity.* Cornell University, 2000. [[URL]](https://ecommons.cornell.edu/handle/1813/5803) [[PDF]](https://ecommons.cornell.edu/bitstream/handle/1813/5803/2000-1813.pdf) * <a name="myfootnote12">[12]</a> Itoh, Toshiya, and Shigeo Tsujii. "A fast algorithm for computing multiplicative inverses in GF (2m) using normal bases." Information and computation 78, no. 3 (1988): 171-177. [[URL]](https://www.sciencedirect.com/science/article/pii/0890540188900247) diff --git a/src/minisketch/include/minisketch.h b/src/minisketch/include/minisketch.h index 0b5d8372e8..24d6b4e1c0 100644 --- a/src/minisketch/include/minisketch.h +++ b/src/minisketch/include/minisketch.h @@ -5,7 +5,8 @@ #include <stdlib.h> #ifdef _MSC_VER -# include <compat.h> +# include <BaseTsd.h> + typedef SSIZE_T ssize_t; #else # include <unistd.h> #endif diff --git a/src/net.cpp b/src/net.cpp index a91abae944..6151ab9e08 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -16,6 +16,7 @@ #include <compat.h> #include <consensus/consensus.h> #include <crypto/sha256.h> +#include <node/eviction.h> #include <fs.h> #include <i2p.h> #include <net_permissions.h> @@ -25,7 +26,6 @@ #include <protocol.h> #include <random.h> #include <scheduler.h> -#include <util/designator.h> #include <util/sock.h> #include <util/strencodings.h> #include <util/syscall_sandbox.h> @@ -206,15 +206,13 @@ static std::vector<CAddress> ConvertSeeds(const std::vector<uint8_t> &vSeedsIn) // Otherwise, return the unroutable 0.0.0.0 but filled in with // the normal parameters, since the IP may be changed to a useful // one by discovery. -CAddress GetLocalAddress(const CNetAddr *paddrPeer, ServiceFlags nLocalServices) +CService GetLocalAddress(const CNetAddr& addrPeer) { - CAddress ret(CService(CNetAddr(),GetListenPort()), nLocalServices); + CService ret{CNetAddr(), GetListenPort()}; CService addr; - if (GetLocal(addr, paddrPeer)) - { - ret = CAddress(addr, nLocalServices); + if (GetLocal(addr, &addrPeer)) { + ret = CService{addr}; } - ret.nTime = GetAdjustedTime(); return ret; } @@ -233,35 +231,35 @@ bool IsPeerAddrLocalGood(CNode *pnode) IsReachable(addrLocal.GetNetwork()); } -std::optional<CAddress> GetLocalAddrForPeer(CNode *pnode) +std::optional<CService> GetLocalAddrForPeer(CNode& node) { - CAddress addrLocal = GetLocalAddress(&pnode->addr, pnode->GetLocalServices()); + CService addrLocal{GetLocalAddress(node.addr)}; if (gArgs.GetBoolArg("-addrmantest", false)) { // use IPv4 loopback during addrmantest - addrLocal = CAddress(CService(LookupNumeric("127.0.0.1", GetListenPort())), pnode->GetLocalServices()); + addrLocal = CService(LookupNumeric("127.0.0.1", GetListenPort())); } // If discovery is enabled, sometimes give our peer the address it // tells us that it sees us as in case it has a better idea of our // address than we do. FastRandomContext rng; - if (IsPeerAddrLocalGood(pnode) && (!addrLocal.IsRoutable() || + if (IsPeerAddrLocalGood(&node) && (!addrLocal.IsRoutable() || rng.randbits((GetnScore(addrLocal) > LOCAL_MANUAL) ? 3 : 1) == 0)) { - if (pnode->IsInboundConn()) { + if (node.IsInboundConn()) { // For inbound connections, assume both the address and the port // as seen from the peer. - addrLocal = CAddress{pnode->GetAddrLocal(), addrLocal.nServices, addrLocal.nTime}; + addrLocal = CService{node.GetAddrLocal()}; } else { // For outbound connections, assume just the address as seen from // the peer and leave the port in `addrLocal` as returned by // `GetLocalAddress()` above. The peer has no way to observe our // listening port when we have initiated the connection. - addrLocal.SetIP(pnode->GetAddrLocal()); + addrLocal.SetIP(node.GetAddrLocal()); } } if (addrLocal.IsRoutable() || gArgs.GetBoolArg("-addrmantest", false)) { - LogPrint(BCLog::NET, "Advertising address %s to peer=%d\n", addrLocal.ToString(), pnode->GetId()); + LogPrint(BCLog::NET, "Advertising address %s to peer=%d\n", addrLocal.ToString(), node.GetId()); return addrLocal; } // Address is unroutable. Don't advertise. @@ -543,7 +541,6 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo addr_bind = GetBindAddress(*sock); } CNode* pnode = new CNode(id, - nLocalServices, std::move(sock), addrConnect, CalculateKeyedNetGroup(addrConnect), @@ -576,26 +573,6 @@ void CConnman::AddWhitelistPermissionFlags(NetPermissionFlags& flags, const CNet } } -std::string ConnectionTypeAsString(ConnectionType conn_type) -{ - switch (conn_type) { - case ConnectionType::INBOUND: - return "inbound"; - case ConnectionType::MANUAL: - return "manual"; - case ConnectionType::FEELER: - return "feeler"; - case ConnectionType::OUTBOUND_FULL_RELAY: - return "outbound-full-relay"; - case ConnectionType::BLOCK_RELAY: - return "block-relay-only"; - case ConnectionType::ADDR_FETCH: - return "addr-fetch"; - } // no default case, so the compiler can warn about missing cases - - assert(false); -} - CService CNode::GetAddrLocal() const { AssertLockNotHeld(m_addr_local_mutex); @@ -623,7 +600,6 @@ Network CNode::ConnectedThroughNetwork() const void CNode::CopyStats(CNodeStats& stats) { stats.nodeid = this->GetId(); - X(nServices); X(addr); X(addrBind); stats.m_network = ConnectedThroughNetwork(); @@ -877,210 +853,6 @@ size_t CConnman::SocketSendData(CNode& node) const return nSentSize; } -static bool ReverseCompareNodeMinPingTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) -{ - return a.m_min_ping_time > b.m_min_ping_time; -} - -static bool ReverseCompareNodeTimeConnected(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) -{ - return a.m_connected > b.m_connected; -} - -static bool CompareNetGroupKeyed(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) { - return a.nKeyedNetGroup < b.nKeyedNetGroup; -} - -static bool CompareNodeBlockTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) -{ - // There is a fall-through here because it is common for a node to have many peers which have not yet relayed a block. - if (a.m_last_block_time != b.m_last_block_time) return a.m_last_block_time < b.m_last_block_time; - if (a.fRelevantServices != b.fRelevantServices) return b.fRelevantServices; - return a.m_connected > b.m_connected; -} - -static bool CompareNodeTXTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) -{ - // There is a fall-through here because it is common for a node to have more than a few peers that have not yet relayed txn. - if (a.m_last_tx_time != b.m_last_tx_time) return a.m_last_tx_time < b.m_last_tx_time; - if (a.m_relay_txs != b.m_relay_txs) return b.m_relay_txs; - if (a.fBloomFilter != b.fBloomFilter) return a.fBloomFilter; - return a.m_connected > b.m_connected; -} - -// Pick out the potential block-relay only peers, and sort them by last block time. -static bool CompareNodeBlockRelayOnlyTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) -{ - if (a.m_relay_txs != b.m_relay_txs) return a.m_relay_txs; - if (a.m_last_block_time != b.m_last_block_time) return a.m_last_block_time < b.m_last_block_time; - if (a.fRelevantServices != b.fRelevantServices) return b.fRelevantServices; - return a.m_connected > b.m_connected; -} - -/** - * Sort eviction candidates by network/localhost and connection uptime. - * Candidates near the beginning are more likely to be evicted, and those - * near the end are more likely to be protected, e.g. less likely to be evicted. - * - First, nodes that are not `is_local` and that do not belong to `network`, - * sorted by increasing uptime (from most recently connected to connected longer). - * - Then, nodes that are `is_local` or belong to `network`, sorted by increasing uptime. - */ -struct CompareNodeNetworkTime { - const bool m_is_local; - const Network m_network; - CompareNodeNetworkTime(bool is_local, Network network) : m_is_local(is_local), m_network(network) {} - bool operator()(const NodeEvictionCandidate& a, const NodeEvictionCandidate& b) const - { - if (m_is_local && a.m_is_local != b.m_is_local) return b.m_is_local; - if ((a.m_network == m_network) != (b.m_network == m_network)) return b.m_network == m_network; - return a.m_connected > b.m_connected; - }; -}; - -//! Sort an array by the specified comparator, then erase the last K elements where predicate is true. -template <typename T, typename Comparator> -static void EraseLastKElements( - std::vector<T>& elements, Comparator comparator, size_t k, - std::function<bool(const NodeEvictionCandidate&)> predicate = [](const NodeEvictionCandidate& n) { return true; }) -{ - std::sort(elements.begin(), elements.end(), comparator); - size_t eraseSize = std::min(k, elements.size()); - elements.erase(std::remove_if(elements.end() - eraseSize, elements.end(), predicate), elements.end()); -} - -void ProtectEvictionCandidatesByRatio(std::vector<NodeEvictionCandidate>& eviction_candidates) -{ - // Protect the half of the remaining nodes which have been connected the longest. - // This replicates the non-eviction implicit behavior, and precludes attacks that start later. - // To favorise the diversity of our peer connections, reserve up to half of these protected - // spots for Tor/onion, localhost, I2P, and CJDNS peers, even if they're not longest uptime - // overall. This helps protect these higher-latency peers that tend to be otherwise - // disadvantaged under our eviction criteria. - const size_t initial_size = eviction_candidates.size(); - const size_t total_protect_size{initial_size / 2}; - - // Disadvantaged networks to protect. In the case of equal counts, earlier array members - // have the first opportunity to recover unused slots from the previous iteration. - struct Net { bool is_local; Network id; size_t count; }; - std::array<Net, 4> networks{ - {{false, NET_CJDNS, 0}, {false, NET_I2P, 0}, {/*localhost=*/true, NET_MAX, 0}, {false, NET_ONION, 0}}}; - - // Count and store the number of eviction candidates per network. - for (Net& n : networks) { - n.count = std::count_if(eviction_candidates.cbegin(), eviction_candidates.cend(), - [&n](const NodeEvictionCandidate& c) { - return n.is_local ? c.m_is_local : c.m_network == n.id; - }); - } - // Sort `networks` by ascending candidate count, to give networks having fewer candidates - // the first opportunity to recover unused protected slots from the previous iteration. - std::stable_sort(networks.begin(), networks.end(), [](Net a, Net b) { return a.count < b.count; }); - - // Protect up to 25% of the eviction candidates by disadvantaged network. - const size_t max_protect_by_network{total_protect_size / 2}; - size_t num_protected{0}; - - while (num_protected < max_protect_by_network) { - // Count the number of disadvantaged networks from which we have peers to protect. - auto num_networks = std::count_if(networks.begin(), networks.end(), [](const Net& n) { return n.count; }); - if (num_networks == 0) { - break; - } - const size_t disadvantaged_to_protect{max_protect_by_network - num_protected}; - const size_t protect_per_network{std::max(disadvantaged_to_protect / num_networks, static_cast<size_t>(1))}; - // Early exit flag if there are no remaining candidates by disadvantaged network. - bool protected_at_least_one{false}; - - for (Net& n : networks) { - if (n.count == 0) continue; - const size_t before = eviction_candidates.size(); - EraseLastKElements(eviction_candidates, CompareNodeNetworkTime(n.is_local, n.id), - protect_per_network, [&n](const NodeEvictionCandidate& c) { - return n.is_local ? c.m_is_local : c.m_network == n.id; - }); - const size_t after = eviction_candidates.size(); - if (before > after) { - protected_at_least_one = true; - const size_t delta{before - after}; - num_protected += delta; - if (num_protected >= max_protect_by_network) { - break; - } - n.count -= delta; - } - } - if (!protected_at_least_one) { - break; - } - } - - // Calculate how many we removed, and update our total number of peers that - // we want to protect based on uptime accordingly. - assert(num_protected == initial_size - eviction_candidates.size()); - const size_t remaining_to_protect{total_protect_size - num_protected}; - EraseLastKElements(eviction_candidates, ReverseCompareNodeTimeConnected, remaining_to_protect); -} - -[[nodiscard]] std::optional<NodeId> SelectNodeToEvict(std::vector<NodeEvictionCandidate>&& vEvictionCandidates) -{ - // Protect connections with certain characteristics - - // Deterministically select 4 peers to protect by netgroup. - // An attacker cannot predict which netgroups will be protected - EraseLastKElements(vEvictionCandidates, CompareNetGroupKeyed, 4); - // Protect the 8 nodes with the lowest minimum ping time. - // An attacker cannot manipulate this metric without physically moving nodes closer to the target. - EraseLastKElements(vEvictionCandidates, ReverseCompareNodeMinPingTime, 8); - // Protect 4 nodes that most recently sent us novel transactions accepted into our mempool. - // An attacker cannot manipulate this metric without performing useful work. - EraseLastKElements(vEvictionCandidates, CompareNodeTXTime, 4); - // Protect up to 8 non-tx-relay peers that have sent us novel blocks. - EraseLastKElements(vEvictionCandidates, CompareNodeBlockRelayOnlyTime, 8, - [](const NodeEvictionCandidate& n) { return !n.m_relay_txs && n.fRelevantServices; }); - - // Protect 4 nodes that most recently sent us novel blocks. - // An attacker cannot manipulate this metric without performing useful work. - EraseLastKElements(vEvictionCandidates, CompareNodeBlockTime, 4); - - // Protect some of the remaining eviction candidates by ratios of desirable - // or disadvantaged characteristics. - ProtectEvictionCandidatesByRatio(vEvictionCandidates); - - if (vEvictionCandidates.empty()) return std::nullopt; - - // If any remaining peers are preferred for eviction consider only them. - // This happens after the other preferences since if a peer is really the best by other criteria (esp relaying blocks) - // then we probably don't want to evict it no matter what. - if (std::any_of(vEvictionCandidates.begin(),vEvictionCandidates.end(),[](NodeEvictionCandidate const &n){return n.prefer_evict;})) { - vEvictionCandidates.erase(std::remove_if(vEvictionCandidates.begin(),vEvictionCandidates.end(), - [](NodeEvictionCandidate const &n){return !n.prefer_evict;}),vEvictionCandidates.end()); - } - - // Identify the network group with the most connections and youngest member. - // (vEvictionCandidates is already sorted by reverse connect time) - uint64_t naMostConnections; - unsigned int nMostConnections = 0; - std::chrono::seconds nMostConnectionsTime{0}; - std::map<uint64_t, std::vector<NodeEvictionCandidate> > mapNetGroupNodes; - for (const NodeEvictionCandidate &node : vEvictionCandidates) { - std::vector<NodeEvictionCandidate> &group = mapNetGroupNodes[node.nKeyedNetGroup]; - group.push_back(node); - const auto grouptime{group[0].m_connected}; - - if (group.size() > nMostConnections || (group.size() == nMostConnections && grouptime > nMostConnectionsTime)) { - nMostConnections = group.size(); - nMostConnectionsTime = grouptime; - naMostConnections = node.nKeyedNetGroup; - } - } - - // Reduce to the network group with the most connections - vEvictionCandidates = std::move(mapNetGroupNodes[naMostConnections]); - - // Disconnect from the network group with the most connections - return vEvictionCandidates.front().id; -} - /** Try to find a connection to evict when the node is full. * Extreme care must be taken to avoid opening the node to attacker * triggered network partitioning. @@ -1096,25 +868,23 @@ bool CConnman::AttemptToEvictConnection() LOCK(m_nodes_mutex); for (const CNode* node : m_nodes) { - if (node->HasPermission(NetPermissionFlags::NoBan)) - continue; - if (!node->IsInboundConn()) - continue; if (node->fDisconnect) continue; NodeEvictionCandidate candidate{ - Desig(id) node->GetId(), - Desig(m_connected) node->m_connected, - Desig(m_min_ping_time) node->m_min_ping_time, - Desig(m_last_block_time) node->m_last_block_time, - Desig(m_last_tx_time) node->m_last_tx_time, - Desig(fRelevantServices) HasAllDesirableServiceFlags(node->nServices), - Desig(m_relay_txs) node->m_relays_txs.load(), - Desig(fBloomFilter) node->m_bloom_filter_loaded.load(), - Desig(nKeyedNetGroup) node->nKeyedNetGroup, - Desig(prefer_evict) node->m_prefer_evict, - Desig(m_is_local) node->addr.IsLocal(), - Desig(m_network) node->ConnectedThroughNetwork(), + .id = node->GetId(), + .m_connected = node->m_connected, + .m_min_ping_time = node->m_min_ping_time, + .m_last_block_time = node->m_last_block_time, + .m_last_tx_time = node->m_last_tx_time, + .fRelevantServices = node->m_has_all_wanted_services, + .m_relay_txs = node->m_relays_txs.load(), + .fBloomFilter = node->m_bloom_filter_loaded.load(), + .nKeyedNetGroup = node->nKeyedNetGroup, + .prefer_evict = node->m_prefer_evict, + .m_is_local = node->addr.IsLocal(), + .m_network = node->ConnectedThroughNetwork(), + .m_noban = node->HasPermission(NetPermissionFlags::NoBan), + .m_conn_type = node->m_conn_type, }; vEvictionCandidates.push_back(candidate); } @@ -1240,7 +1010,6 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, const bool inbound_onion = std::find(m_onion_binds.begin(), m_onion_binds.end(), addr_bind) != m_onion_binds.end(); CNode* pnode = new CNode(id, - nodeServices, std::move(sock), addr, CalculateKeyedNetGroup(addr), @@ -1252,7 +1021,7 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, pnode->AddRef(); pnode->m_permissionFlags = permissionFlags; pnode->m_prefer_evict = discouraged; - m_msgproc->InitializeNode(pnode); + m_msgproc->InitializeNode(*pnode, nodeServices); LogPrint(BCLog::NET, "connection from %s accepted\n", addr.ToString()); @@ -2190,7 +1959,7 @@ void CConnman::OpenNetworkConnection(const CAddress& addrConnect, bool fCountFai if (grantOutbound) grantOutbound->MoveTo(pnode->grantOutbound); - m_msgproc->InitializeNode(pnode); + m_msgproc->InitializeNode(*pnode, nLocalServices); { LOCK(m_nodes_mutex); m_nodes.push_back(pnode); @@ -2934,7 +2703,10 @@ ServiceFlags CConnman::GetLocalServices() const unsigned int CConnman::GetReceiveFloodSize() const { return nReceiveFloodSize; } -CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, std::shared_ptr<Sock> sock, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress& addrBindIn, const std::string& addrNameIn, ConnectionType conn_type_in, bool inbound_onion) +CNode::CNode(NodeId idIn, std::shared_ptr<Sock> sock, const CAddress& addrIn, + uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, + const CAddress& addrBindIn, const std::string& addrNameIn, + ConnectionType conn_type_in, bool inbound_onion) : m_sock{sock}, m_connected{GetTime<std::chrono::seconds>()}, addr(addrIn), @@ -2944,8 +2716,7 @@ CNode::CNode(NodeId idIn, ServiceFlags nLocalServicesIn, std::shared_ptr<Sock> s nKeyedNetGroup(nKeyedNetGroupIn), id(idIn), nLocalHostNonce(nLocalHostNonceIn), - m_conn_type(conn_type_in), - nLocalServices(nLocalServicesIn) + m_conn_type(conn_type_in) { if (inbound_onion) assert(conn_type_in == ConnectionType::INBOUND); @@ -9,6 +9,7 @@ #include <chainparams.h> #include <common/bloom.h> #include <compat.h> +#include <node/connection_types.h> #include <consensus/amount.h> #include <crypto/siphash.h> #include <hash.h> @@ -121,78 +122,6 @@ struct CSerializedNetMsg { std::string m_type; }; -/** Different types of connections to a peer. This enum encapsulates the - * information we have available at the time of opening or accepting the - * connection. Aside from INBOUND, all types are initiated by us. - * - * If adding or removing types, please update CONNECTION_TYPE_DOC in - * src/rpc/net.cpp and src/qt/rpcconsole.cpp, as well as the descriptions in - * src/qt/guiutil.cpp and src/bitcoin-cli.cpp::NetinfoRequestHandler. */ -enum class ConnectionType { - /** - * Inbound connections are those initiated by a peer. This is the only - * property we know at the time of connection, until P2P messages are - * exchanged. - */ - INBOUND, - - /** - * These are the default connections that we use to connect with the - * network. There is no restriction on what is relayed; by default we relay - * blocks, addresses & transactions. We automatically attempt to open - * MAX_OUTBOUND_FULL_RELAY_CONNECTIONS using addresses from our AddrMan. - */ - OUTBOUND_FULL_RELAY, - - - /** - * We open manual connections to addresses that users explicitly requested - * via the addnode RPC or the -addnode/-connect configuration options. Even if a - * manual connection is misbehaving, we do not automatically disconnect or - * add it to our discouragement filter. - */ - MANUAL, - - /** - * Feeler connections are short-lived connections made to check that a node - * is alive. They can be useful for: - * - test-before-evict: if one of the peers is considered for eviction from - * our AddrMan because another peer is mapped to the same slot in the tried table, - * evict only if this longer-known peer is offline. - * - move node addresses from New to Tried table, so that we have more - * connectable addresses in our AddrMan. - * Note that in the literature ("Eclipse Attacks on Bitcoin’s Peer-to-Peer Network") - * only the latter feature is referred to as "feeler connections", - * although in our codebase feeler connections encompass test-before-evict as well. - * We make these connections approximately every FEELER_INTERVAL: - * first we resolve previously found collisions if they exist (test-before-evict), - * otherwise we connect to a node from the new table. - */ - FEELER, - - /** - * We use block-relay-only connections to help prevent against partition - * attacks. By not relaying transactions or addresses, these connections - * are harder to detect by a third party, thus helping obfuscate the - * network topology. We automatically attempt to open - * MAX_BLOCK_RELAY_ONLY_ANCHORS using addresses from our anchors.dat. Then - * addresses from our AddrMan if MAX_BLOCK_RELAY_ONLY_CONNECTIONS - * isn't reached yet. - */ - BLOCK_RELAY, - - /** - * AddrFetch connections are short lived connections used to solicit - * addresses from peers. These are initiated to addresses submitted via the - * -seednode command line argument, or under certain conditions when the - * AddrMan is empty. - */ - ADDR_FETCH, -}; - -/** Convert ConnectionType enum to a string value */ -std::string ConnectionTypeAsString(ConnectionType conn_type); - /** * Look up IP addresses from all interfaces on the machine and add them to the * list of local addresses to self-advertise. @@ -215,8 +144,8 @@ enum }; bool IsPeerAddrLocalGood(CNode *pnode); -/** Returns a local address that we should advertise to this peer */ -std::optional<CAddress> GetLocalAddrForPeer(CNode *pnode); +/** Returns a local address that we should advertise to this peer. */ +std::optional<CService> GetLocalAddrForPeer(CNode& node); /** * Mark a network as reachable or unreachable (no automatic connects to it) @@ -234,7 +163,7 @@ void RemoveLocal(const CService& addr); bool SeenLocal(const CService& addr); bool IsLocal(const CService& addr); bool GetLocal(CService &addr, const CNetAddr *paddrPeer = nullptr); -CAddress GetLocalAddress(const CNetAddr *paddrPeer, ServiceFlags nLocalServices); +CService GetLocalAddress(const CNetAddr& addrPeer); extern bool fDiscover; @@ -258,7 +187,6 @@ class CNodeStats { public: NodeId nodeid; - ServiceFlags nServices; std::chrono::seconds m_last_send; std::chrono::seconds m_last_recv; std::chrono::seconds m_last_tx_time; @@ -417,7 +345,6 @@ public: std::unique_ptr<TransportSerializer> m_serializer; NetPermissionFlags m_permissionFlags{NetPermissionFlags::None}; - std::atomic<ServiceFlags> nServices{NODE_NONE}; /** * Socket used for communication with the node. @@ -470,8 +397,6 @@ public: bool HasPermission(NetPermissionFlags permission) const { return NetPermissions::HasFlag(m_permissionFlags, permission); } - bool fClient{false}; // set by version message - bool m_limited_node{false}; //after BIP159, set by version message /** fSuccessfullyConnected is set to true on receiving VERACK from the peer. */ std::atomic_bool fSuccessfullyConnected{false}; // Setting fDisconnect to true will cause the node to be disconnected the @@ -555,6 +480,9 @@ public: // Peer selected us as (compact blocks) high-bandwidth peer (BIP152) std::atomic<bool> m_bip152_highbandwidth_from{false}; + /** Whether this peer provides all services that we want. Used for eviction decisions */ + std::atomic_bool m_has_all_wanted_services{false}; + /** Whether we should relay transactions to this peer (their version * message did not include fRelay=false and this is not a block-relay-only * connection). This only changes from false to true. It will never change @@ -585,7 +513,10 @@ public: * criterium in CConnman::AttemptToEvictConnection. */ std::atomic<std::chrono::microseconds> m_min_ping_time{std::chrono::microseconds::max()}; - CNode(NodeId id, ServiceFlags nLocalServicesIn, std::shared_ptr<Sock> sock, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress& addrBindIn, const std::string& addrNameIn, ConnectionType conn_type_in, bool inbound_onion); + CNode(NodeId id, std::shared_ptr<Sock> sock, const CAddress& addrIn, + uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, + const CAddress& addrBindIn, const std::string& addrNameIn, + ConnectionType conn_type_in, bool inbound_onion); CNode(const CNode&) = delete; CNode& operator=(const CNode&) = delete; @@ -643,11 +574,6 @@ public: void CopyStats(CNodeStats& stats) EXCLUSIVE_LOCKS_REQUIRED(!m_subver_mutex, !m_addr_local_mutex, !cs_vSend, !cs_vRecv); - ServiceFlags GetLocalServices() const - { - return nLocalServices; - } - std::string ConnectionTypeAsString() const { return ::ConnectionTypeAsString(m_conn_type); } /** A ping-pong round trip has completed successfully. Update latest and minimum ping times. */ @@ -662,23 +588,6 @@ private: const ConnectionType m_conn_type; std::atomic<int> m_greatest_common_version{INIT_PROTO_VERSION}; - //! Services offered to this peer. - //! - //! This is supplied by the parent CConnman during peer connection - //! (CConnman::ConnectNode()) from its attribute of the same name. - //! - //! This is const because there is no protocol defined for renegotiating - //! services initially offered to a peer. The set of local services we - //! offer should not change after initialization. - //! - //! An interesting example of this is NODE_NETWORK and initial block - //! download: a node which starts up from scratch doesn't have any blocks - //! to serve, but still advertises NODE_NETWORK because it will eventually - //! fulfill this role after IBD completes. P2P code is written in such a - //! way that it can gracefully handle peers who don't make good on their - //! service advertisements. - const ServiceFlags nLocalServices; - std::list<CNetMessage> vRecvMsg; // Used only by SocketHandler thread // Our address, as reported by the peer @@ -696,7 +605,7 @@ class NetEventsInterface { public: /** Initialize a peer (setup state, queue any initial messages) */ - virtual void InitializeNode(CNode* pnode) = 0; + virtual void InitializeNode(CNode& node, ServiceFlags our_services) = 0; /** Handle removal of a peer (clear state) */ virtual void FinalizeNode(const CNode& node) = 0; @@ -1106,16 +1015,14 @@ private: std::map<uint64_t, CachedAddrResponse> m_addr_response_caches; /** - * Services this instance offers. + * Services this node offers. * - * This data is replicated in each CNode instance we create during peer - * connection (in ConnectNode()) under a member also called - * nLocalServices. + * This data is replicated in each Peer instance we create. * * This data is not marked const, but after being set it should not - * change. See the note in CNode::nLocalServices documentation. + * change. * - * \sa CNode::nLocalServices + * \sa Peer::our_services */ ServiceFlags nLocalServices; @@ -1247,54 +1154,4 @@ extern std::function<void(const CAddress& addr, bool is_incoming)> CaptureMessage; -struct NodeEvictionCandidate -{ - NodeId id; - std::chrono::seconds m_connected; - std::chrono::microseconds m_min_ping_time; - std::chrono::seconds m_last_block_time; - std::chrono::seconds m_last_tx_time; - bool fRelevantServices; - bool m_relay_txs; - bool fBloomFilter; - uint64_t nKeyedNetGroup; - bool prefer_evict; - bool m_is_local; - Network m_network; -}; - -/** - * Select an inbound peer to evict after filtering out (protecting) peers having - * distinct, difficult-to-forge characteristics. The protection logic picks out - * fixed numbers of desirable peers per various criteria, followed by (mostly) - * ratios of desirable or disadvantaged peers. If any eviction candidates - * remain, the selection logic chooses a peer to evict. - */ -[[nodiscard]] std::optional<NodeId> SelectNodeToEvict(std::vector<NodeEvictionCandidate>&& vEvictionCandidates); - -/** Protect desirable or disadvantaged inbound peers from eviction by ratio. - * - * This function protects half of the peers which have been connected the - * longest, to replicate the non-eviction implicit behavior and preclude attacks - * that start later. - * - * Half of these protected spots (1/4 of the total) are reserved for the - * following categories of peers, sorted by longest uptime, even if they're not - * longest uptime overall: - * - * - onion peers connected via our tor control service - * - * - localhost peers, as manually configured hidden services not using - * `-bind=addr[:port]=onion` will not be detected as inbound onion connections - * - * - I2P peers - * - * - CJDNS peers - * - * This helps protect these privacy network peers, which tend to be otherwise - * disadvantaged under our eviction criteria for their higher min ping times - * relative to IPv4/IPv6 peers, and favorise the diversity of peer connections. - */ -void ProtectEvictionCandidatesByRatio(std::vector<NodeEvictionCandidate>& vEvictionCandidates); - #endif // BITCOIN_NET_H diff --git a/src/net_processing.cpp b/src/net_processing.cpp index db711be130..74d1bf44d2 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -61,6 +61,8 @@ static constexpr auto UNCONDITIONAL_RELAY_DELAY = 2min; * Timeout = base + per_header * (expected number of headers) */ static constexpr auto HEADERS_DOWNLOAD_TIMEOUT_BASE = 15min; static constexpr auto HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER = 1ms; +/** How long to wait for a peer to respond to a getheaders request */ +static constexpr auto HEADERS_RESPONSE_TIME{2min}; /** Protect at least this many outbound peers from disconnection due to slow/ * behind headers chain. */ @@ -205,6 +207,23 @@ struct Peer { /** Same id as the CNode object for this peer */ const NodeId m_id{0}; + /** Services we offered to this peer. + * + * This is supplied by CConnman during peer initialization. It's const + * because there is no protocol defined for renegotiating services + * initially offered to a peer. The set of local services we offer should + * not change after initialization. + * + * An interesting example of this is NODE_NETWORK and initial block + * download: a node which starts up from scratch doesn't have any blocks + * to serve, but still advertises NODE_NETWORK because it will eventually + * fulfill this role after IBD completes. P2P code is written in such a + * way that it can gracefully handle peers who don't make good on their + * service advertisements. */ + const ServiceFlags m_our_services; + /** Services this peer offered to us. */ + std::atomic<ServiceFlags> m_their_services{NODE_NONE}; + /** Protects misbehavior data members */ Mutex m_misbehavior_mutex; /** Accumulated misbehavior score for this peer */ @@ -355,8 +374,12 @@ struct Peer { /** Work queue of items requested by this peer **/ std::deque<CInv> m_getdata_requests GUARDED_BY(m_getdata_requests_mutex); - Peer(NodeId id) + /** Time of the last getheaders message to this peer */ + NodeClock::time_point m_last_getheaders_timestamp{}; + + explicit Peer(NodeId id, ServiceFlags our_services) : m_id{id} + , m_our_services{our_services} {} private: @@ -405,8 +428,6 @@ struct CNodeState { bool m_requested_hb_cmpctblocks{false}; /** Whether this peer will send us cmpctblocks if we request them. */ bool m_provides_cmpctblocks{false}; - //! Whether this peer can give us witnesses - bool fHaveWitness{false}; /** State used to enforce CHAIN_SYNC_TIMEOUT and EXTRA_PEER_CHECK_INTERVAL logic. * @@ -477,7 +498,7 @@ public: EXCLUSIVE_LOCKS_REQUIRED(!m_most_recent_block_mutex); /** Implement NetEventsInterface */ - void InitializeNode(CNode* pnode) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); + void InitializeNode(CNode& node, ServiceFlags our_services) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); void FinalizeNode(const CNode& node) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); bool ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, !m_recent_confirmed_transactions_mutex, !m_most_recent_block_mutex); @@ -487,7 +508,8 @@ public: /** Implement PeerManager */ void StartScheduledTasks(CScheduler& scheduler) override; void CheckForStaleTipAndEvictPeers() override; - std::optional<std::string> FetchBlock(NodeId peer_id, const CBlockIndex& block_index) override; + std::optional<std::string> FetchBlock(NodeId peer_id, const CBlockIndex& block_index) override + EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) const override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); bool IgnoresIncomingTxs() override { return m_ignore_incoming_txs; } void SendPings() override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); @@ -501,7 +523,7 @@ public: private: /** Consider evicting an outbound peer based on the amount of time they've been behind our tip */ - void ConsiderEviction(CNode& pto, std::chrono::seconds time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void ConsiderEviction(CNode& pto, Peer& peer, std::chrono::seconds time_in_seconds) EXCLUSIVE_LOCKS_REQUIRED(cs_main); /** If we have extra outbound peers, try to disconnect the one with the oldest block announcement */ void EvictExtraOutboundPeers(std::chrono::seconds now) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -560,6 +582,22 @@ private: const std::vector<CBlockHeader>& headers, bool via_compact_block) EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); + /** Various helpers for headers processing, invoked by ProcessHeadersMessage() */ + /** Deal with state tracking and headers sync for peers that send the + * occasional non-connecting header (this can happen due to BIP 130 headers + * announcements for blocks interacting with the 2hr (MAX_FUTURE_BLOCK_TIME) rule). */ + void HandleFewUnconnectingHeaders(CNode& pfrom, Peer& peer, const std::vector<CBlockHeader>& headers); + /** Return true if the headers connect to each other, false otherwise */ + bool CheckHeadersAreContinuous(const std::vector<CBlockHeader>& headers) const; + /** Request further headers from this peer with a given locator. + * We don't issue a getheaders message if we have a recent one outstanding. + * This returns true if a getheaders is actually sent, and false otherwise. + */ + bool MaybeSendGetHeaders(CNode& pfrom, const CBlockLocator& locator, Peer& peer); + /** Potentially fetch blocks from this peer upon receipt of a new headers tip */ + void HeadersDirectFetchBlocks(CNode& pfrom, const Peer& peer, const CBlockIndex* pindexLast); + /** Update peer state based on received headers message */ + void UpdatePeerStateForReceivedHeaders(CNode& pfrom, const CBlockIndex *pindexLast, bool received_new_header, bool may_have_more_headers); void SendBlockTransactions(CNode& pfrom, Peer& peer, const CBlock& block, const BlockTransactionsRequest& req); @@ -636,7 +674,7 @@ private: /** Get a pointer to a mutable CNodeState. */ CNodeState* State(NodeId pnode) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - uint32_t GetFetchFlags(const CNode& pfrom) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); + uint32_t GetFetchFlags(const Peer& peer) const; std::atomic<std::chrono::microseconds> m_next_inv_to_inbounds{0us}; @@ -757,7 +795,7 @@ private: /** Update pindexLastCommonBlock and add not-in-flight missing successors to vBlocks, until it has * at most count entries. */ - void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vector<const CBlockIndex*>& vBlocks, NodeId& nodeStaller) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void FindNextBlocksToDownload(const Peer& peer, unsigned int count, std::vector<const CBlockIndex*>& vBlocks, NodeId& nodeStaller) EXCLUSIVE_LOCKS_REQUIRED(cs_main); std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> > mapBlocksInFlight GUARDED_BY(cs_main); @@ -827,6 +865,7 @@ private: * * May disconnect from the peer in the case of a bad request. * + * @param[in] node The node that we received the request from * @param[in] peer The peer that we received the request from * @param[in] filter_type The filter type the request is for. Must be basic filters. * @param[in] start_height The start height for the request @@ -836,7 +875,7 @@ private: * @param[out] filter_index The filter index, if the request can be serviced. * @return True if the request can be serviced. */ - bool PrepareBlockFilterRequest(CNode& peer, + bool PrepareBlockFilterRequest(CNode& node, Peer& peer, BlockFilterType filter_type, uint32_t start_height, const uint256& stop_hash, uint32_t max_height_diff, const CBlockIndex*& stop_index, @@ -847,30 +886,33 @@ private: * * May disconnect from the peer in the case of a bad request. * + * @param[in] node The node that we received the request from * @param[in] peer The peer that we received the request from * @param[in] vRecv The raw message received */ - void ProcessGetCFilters(CNode& peer, CDataStream& vRecv); + void ProcessGetCFilters(CNode& node, Peer& peer, CDataStream& vRecv); /** * Handle a cfheaders request. * * May disconnect from the peer in the case of a bad request. * + * @param[in] node The node that we received the request from * @param[in] peer The peer that we received the request from * @param[in] vRecv The raw message received */ - void ProcessGetCFHeaders(CNode& peer, CDataStream& vRecv); + void ProcessGetCFHeaders(CNode& node, Peer& peer, CDataStream& vRecv); /** * Handle a getcfcheckpt request. * * May disconnect from the peer in the case of a bad request. * + * @param[in] node The node that we received the request from * @param[in] peer The peer that we received the request from * @param[in] vRecv The raw message received */ - void ProcessGetCFCheckPt(CNode& peer, CDataStream& vRecv); + void ProcessGetCFCheckPt(CNode& node, Peer& peer, CDataStream& vRecv); /** Checks if address relay is permitted with peer. If needed, initializes * the m_addr_known bloom filter and sets m_addr_relay_enabled to true. @@ -934,6 +976,26 @@ static void AddKnownTx(Peer& peer, const uint256& hash) tx_relay->m_tx_inventory_known_filter.insert(hash); } +/** Whether this peer can serve us blocks. */ +static bool CanServeBlocks(const Peer& peer) +{ + return peer.m_their_services & (NODE_NETWORK|NODE_NETWORK_LIMITED); +} + +/** Whether this peer can only serve limited recent blocks (e.g. because + * it prunes old blocks) */ +static bool IsLimitedPeer(const Peer& peer) +{ + return (!(peer.m_their_services & NODE_NETWORK) && + (peer.m_their_services & NODE_NETWORK_LIMITED)); +} + +/** Whether this peer can serve us witness data */ +static bool CanServeWitnesses(const Peer& peer) +{ + return peer.m_their_services & NODE_WITNESS; +} + std::chrono::microseconds PeerManagerImpl::NextInvToInbounds(std::chrono::microseconds now, std::chrono::seconds average_interval) { @@ -1127,17 +1189,17 @@ void PeerManagerImpl::UpdateBlockAvailability(NodeId nodeid, const uint256 &hash } } -void PeerManagerImpl::FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vector<const CBlockIndex*>& vBlocks, NodeId& nodeStaller) +void PeerManagerImpl::FindNextBlocksToDownload(const Peer& peer, unsigned int count, std::vector<const CBlockIndex*>& vBlocks, NodeId& nodeStaller) { if (count == 0) return; vBlocks.reserve(vBlocks.size() + count); - CNodeState *state = State(nodeid); + CNodeState *state = State(peer.m_id); assert(state != nullptr); // Make sure pindexBestKnownBlock is up to date, we'll need it. - ProcessBlockAvailability(nodeid); + ProcessBlockAvailability(peer.m_id); if (state->pindexBestKnownBlock == nullptr || state->pindexBestKnownBlock->nChainWork < m_chainman.ActiveChain().Tip()->nChainWork || state->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { // This peer has nothing interesting. @@ -1185,7 +1247,7 @@ void PeerManagerImpl::FindNextBlocksToDownload(NodeId nodeid, unsigned int count // We consider the chain that this peer is on invalid. return; } - if (!State(nodeid)->fHaveWitness && DeploymentActiveAt(*pindex, m_chainman, Consensus::DEPLOYMENT_SEGWIT)) { + if (!CanServeWitnesses(peer) && DeploymentActiveAt(*pindex, m_chainman, Consensus::DEPLOYMENT_SEGWIT)) { // We wouldn't download this block or its descendants from this peer. return; } @@ -1196,7 +1258,7 @@ void PeerManagerImpl::FindNextBlocksToDownload(NodeId nodeid, unsigned int count // The block is not already downloaded, and not yet in flight. if (pindex->nHeight > nWindowEnd) { // We reached the end of the window. - if (vBlocks.size() == 0 && waitingfor != nodeid) { + if (vBlocks.size() == 0 && waitingfor != peer.m_id) { // We aren't able to fetch anything, but we would be if the download window was one larger. nodeStaller = waitingfor; } @@ -1218,10 +1280,7 @@ void PeerManagerImpl::FindNextBlocksToDownload(NodeId nodeid, unsigned int count void PeerManagerImpl::PushNodeVersion(CNode& pnode, const Peer& peer) { - // Note that pnode->GetLocalServices() is a reflection of the local - // services we were offering when the CNode object was created for this - // peer. - uint64_t my_services{pnode.GetLocalServices()}; + uint64_t my_services{peer.m_our_services}; const int64_t nTime{count_seconds(GetTime<std::chrono::seconds>())}; uint64_t nonce = pnode.GetLocalNonce(); const int nNodeStartingHeight{m_best_height}; @@ -1278,21 +1337,21 @@ void PeerManagerImpl::UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_s if (state) state->m_last_block_announcement = time_in_seconds; } -void PeerManagerImpl::InitializeNode(CNode *pnode) +void PeerManagerImpl::InitializeNode(CNode& node, ServiceFlags our_services) { - NodeId nodeid = pnode->GetId(); + NodeId nodeid = node.GetId(); { LOCK(cs_main); - m_node_states.emplace_hint(m_node_states.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(pnode->IsInboundConn())); + m_node_states.emplace_hint(m_node_states.end(), std::piecewise_construct, std::forward_as_tuple(nodeid), std::forward_as_tuple(node.IsInboundConn())); assert(m_txrequest.Count(nodeid) == 0); } - PeerRef peer = std::make_shared<Peer>(nodeid); + PeerRef peer = std::make_shared<Peer>(nodeid, our_services); { LOCK(m_peer_mutex); m_peer_map.emplace_hint(m_peer_map.end(), nodeid, peer); } - if (!pnode->IsInboundConn()) { - PushNodeVersion(*pnode, *peer); + if (!node.IsInboundConn()) { + PushNodeVersion(node, *peer); } } @@ -1410,6 +1469,7 @@ bool PeerManagerImpl::GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) c PeerRef peer = GetPeerRef(nodeid); if (peer == nullptr) return false; + stats.their_services = peer->m_their_services; stats.m_starting_height = peer->m_starting_height; // It is common for nodes with good ping times to suddenly become lagged, // due to a new block arriving or other large transfer. @@ -1564,12 +1624,14 @@ std::optional<std::string> PeerManagerImpl::FetchBlock(NodeId peer_id, const CBl if (fImporting) return "Importing..."; if (fReindex) return "Reindexing..."; - LOCK(cs_main); // Ensure this peer exists and hasn't been disconnected - CNodeState* state = State(peer_id); - if (state == nullptr) return "Peer does not exist"; + PeerRef peer = GetPeerRef(peer_id); + if (peer == nullptr) return "Peer does not exist"; + // Ignore pre-segwit peers - if (!state->fHaveWitness) return "Pre-SegWit peer"; + if (!CanServeWitnesses(*peer)) return "Pre-SegWit peer"; + + LOCK(cs_main); // Mark block as in-flight unless it already is (for this peer). // If a block was already in-flight for a different peer, its BLOCKTXN @@ -1953,7 +2015,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& } // Avoid leaking prune-height by never sending blocks below the NODE_NETWORK_LIMITED threshold if (!pfrom.HasPermission(NetPermissionFlags::NoBan) && ( - (((pfrom.GetLocalServices() & NODE_NETWORK_LIMITED) == NODE_NETWORK_LIMITED) && ((pfrom.GetLocalServices() & NODE_NETWORK) != NODE_NETWORK) && (m_chainman.ActiveChain().Tip()->nHeight - pindex->nHeight > (int)NODE_NETWORK_LIMITED_MIN_BLOCKS + 2 /* add two blocks buffer extension for possible races */) ) + (((peer.m_our_services & NODE_NETWORK_LIMITED) == NODE_NETWORK_LIMITED) && ((peer.m_our_services & NODE_NETWORK) != NODE_NETWORK) && (m_chainman.ActiveChain().Tip()->nHeight - pindex->nHeight > (int)NODE_NETWORK_LIMITED_MIN_BLOCKS + 2 /* add two blocks buffer extension for possible races */) ) )) { LogPrint(BCLog::NET, "Ignore block request below NODE_NETWORK_LIMITED threshold, disconnect peer=%d\n", pfrom.GetId()); //disconnect node and prevent it from stalling (would otherwise wait for the missing block) @@ -2170,10 +2232,10 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic } } -uint32_t PeerManagerImpl::GetFetchFlags(const CNode& pfrom) const EXCLUSIVE_LOCKS_REQUIRED(cs_main) +uint32_t PeerManagerImpl::GetFetchFlags(const Peer& peer) const { uint32_t nFetchFlags = 0; - if (State(pfrom.GetId())->fHaveWitness) { + if (CanServeWitnesses(peer)) { nFetchFlags |= MSG_WITNESS_FLAG; } return nFetchFlags; @@ -2194,6 +2256,203 @@ void PeerManagerImpl::SendBlockTransactions(CNode& pfrom, Peer& peer, const CBlo m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCKTXN, resp)); } +/** + * Special handling for unconnecting headers that might be part of a block + * announcement. + * + * We'll send a getheaders message in response to try to connect the chain. + * + * The peer can send up to MAX_UNCONNECTING_HEADERS in a row that + * don't connect before given DoS points. + * + * Once a headers message is received that is valid and does connect, + * nUnconnectingHeaders gets reset back to 0. + */ +void PeerManagerImpl::HandleFewUnconnectingHeaders(CNode& pfrom, Peer& peer, + const std::vector<CBlockHeader>& headers) +{ + const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); + + LOCK(cs_main); + CNodeState *nodestate = State(pfrom.GetId()); + + nodestate->nUnconnectingHeaders++; + // Try to fill in the missing headers. + if (MaybeSendGetHeaders(pfrom, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), peer)) { + LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", + headers[0].GetHash().ToString(), + headers[0].hashPrevBlock.ToString(), + m_chainman.m_best_header->nHeight, + pfrom.GetId(), nodestate->nUnconnectingHeaders); + } + // Set hashLastUnknownBlock for this peer, so that if we + // eventually get the headers - even from a different peer - + // we can use this peer to download. + UpdateBlockAvailability(pfrom.GetId(), headers.back().GetHash()); + + // The peer may just be broken, so periodically assign DoS points if this + // condition persists. + if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { + Misbehaving(peer, 20, strprintf("%d non-connecting headers", nodestate->nUnconnectingHeaders)); + } +} + +bool PeerManagerImpl::CheckHeadersAreContinuous(const std::vector<CBlockHeader>& headers) const +{ + uint256 hashLastBlock; + for (const CBlockHeader& header : headers) { + if (!hashLastBlock.IsNull() && header.hashPrevBlock != hashLastBlock) { + return false; + } + hashLastBlock = header.GetHash(); + } + return true; +} + +bool PeerManagerImpl::MaybeSendGetHeaders(CNode& pfrom, const CBlockLocator& locator, Peer& peer) +{ + const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); + + const auto current_time = NodeClock::now(); + + // Only allow a new getheaders message to go out if we don't have a recent + // one already in-flight + if (current_time - peer.m_last_getheaders_timestamp > HEADERS_RESPONSE_TIME) { + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, locator, uint256())); + peer.m_last_getheaders_timestamp = current_time; + return true; + } + return false; +} + +/* + * Given a new headers tip ending in pindexLast, potentially request blocks towards that tip. + * We require that the given tip have at least as much work as our tip, and for + * our current tip to be "close to synced" (see CanDirectFetch()). + */ +void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const Peer& peer, const CBlockIndex* pindexLast) +{ + const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); + + LOCK(cs_main); + CNodeState *nodestate = State(pfrom.GetId()); + + if (CanDirectFetch() && pindexLast->IsValid(BLOCK_VALID_TREE) && m_chainman.ActiveChain().Tip()->nChainWork <= pindexLast->nChainWork) { + + std::vector<const CBlockIndex*> vToFetch; + const CBlockIndex *pindexWalk = pindexLast; + // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. + while (pindexWalk && !m_chainman.ActiveChain().Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && + !IsBlockRequested(pindexWalk->GetBlockHash()) && + (!DeploymentActiveAt(*pindexWalk, m_chainman, Consensus::DEPLOYMENT_SEGWIT) || CanServeWitnesses(peer))) { + // We don't have this block, and it's not yet in flight. + vToFetch.push_back(pindexWalk); + } + pindexWalk = pindexWalk->pprev; + } + // If pindexWalk still isn't on our main chain, we're looking at a + // very large reorg at a time we think we're close to caught up to + // the main chain -- this shouldn't really happen. Bail out on the + // direct fetch and rely on parallel download instead. + if (!m_chainman.ActiveChain().Contains(pindexWalk)) { + LogPrint(BCLog::NET, "Large reorg, won't direct fetch to %s (%d)\n", + pindexLast->GetBlockHash().ToString(), + pindexLast->nHeight); + } else { + std::vector<CInv> vGetData; + // Download as much as possible, from earliest to latest. + for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { + if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + // Can't download any more from this peer + break; + } + uint32_t nFetchFlags = GetFetchFlags(peer); + vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); + BlockRequested(pfrom.GetId(), *pindex); + LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", + pindex->GetBlockHash().ToString(), pfrom.GetId()); + } + if (vGetData.size() > 1) { + LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", + pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); + } + if (vGetData.size() > 0) { + if (!m_ignore_incoming_txs && + nodestate->m_provides_cmpctblocks && + vGetData.size() == 1 && + mapBlocksInFlight.size() == 1 && + pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { + // In any case, we want to download using a compact block, not a regular one + vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); + } + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + } + } + } +} + +/** + * Given receipt of headers from a peer ending in pindexLast, along with + * whether that header was new and whether the headers message was full, + * update the state we keep for the peer. + */ +void PeerManagerImpl::UpdatePeerStateForReceivedHeaders(CNode& pfrom, + const CBlockIndex *pindexLast, bool received_new_header, bool may_have_more_headers) +{ + LOCK(cs_main); + CNodeState *nodestate = State(pfrom.GetId()); + if (nodestate->nUnconnectingHeaders > 0) { + LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom.GetId(), nodestate->nUnconnectingHeaders); + } + nodestate->nUnconnectingHeaders = 0; + + assert(pindexLast); + UpdateBlockAvailability(pfrom.GetId(), pindexLast->GetBlockHash()); + + // From here, pindexBestKnownBlock should be guaranteed to be non-null, + // because it is set in UpdateBlockAvailability. Some nullptr checks + // are still present, however, as belt-and-suspenders. + + if (received_new_header && pindexLast->nChainWork > m_chainman.ActiveChain().Tip()->nChainWork) { + nodestate->m_last_block_announcement = GetTime(); + } + + // If we're in IBD, we want outbound peers that will serve us a useful + // chain. Disconnect peers that are on chains with insufficient work. + if (m_chainman.ActiveChainstate().IsInitialBlockDownload() && !may_have_more_headers) { + // If the peer has no more headers to give us, then we know we have + // their tip. + if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { + // This peer has too little work on their headers chain to help + // us sync -- disconnect if it is an outbound disconnection + // candidate. + // Note: We compare their tip to nMinimumChainWork (rather than + // m_chainman.ActiveChain().Tip()) because we won't start block download + // until we have a headers chain that has at least + // nMinimumChainWork, even if a peer has a chain past our tip, + // as an anti-DoS measure. + if (pfrom.IsOutboundOrBlockRelayConn()) { + LogPrintf("Disconnecting outbound peer %d -- headers chain has insufficient work\n", pfrom.GetId()); + pfrom.fDisconnect = true; + } + } + } + + // If this is an outbound full-relay peer, check to see if we should protect + // it from the bad/lagging chain logic. + // Note that outbound block-relay peers are excluded from this protection, and + // thus always subject to eviction under the bad/lagging chain logic. + // See ChainSyncTimeoutState. + if (!pfrom.fDisconnect && pfrom.IsFullOutboundConn() && nodestate->pindexBestKnownBlock != nullptr) { + if (m_outbound_peers_with_protect_from_disconnect < MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT && nodestate->pindexBestKnownBlock->nChainWork >= m_chainman.ActiveChain().Tip()->nChainWork && !nodestate->m_chain_sync.m_protect) { + LogPrint(BCLog::NET, "Protecting outbound peer=%d from eviction\n", pfrom.GetId()); + nodestate->m_chain_sync.m_protect = true; + ++m_outbound_peers_with_protect_from_disconnect; + } + } +} + void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, const std::vector<CBlockHeader>& headers, bool via_compact_block) @@ -2206,55 +2465,33 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, return; } - bool received_new_header = false; const CBlockIndex *pindexLast = nullptr; - { - LOCK(cs_main); - CNodeState *nodestate = State(pfrom.GetId()); - // If this looks like it could be a block announcement (nCount <= - // MAX_BLOCKS_TO_ANNOUNCE), use special logic for handling headers that - // don't connect: - // - Send a getheaders message in response to try to connect the chain. - // - The peer can send up to MAX_UNCONNECTING_HEADERS in a row that - // don't connect before giving DoS points - // - Once a headers message is received that is valid and does connect, - // nUnconnectingHeaders gets reset back to 0. - if (!m_chainman.m_blockman.LookupBlockIndex(headers[0].hashPrevBlock) && nCount <= MAX_BLOCKS_TO_ANNOUNCE) { - nodestate->nUnconnectingHeaders++; - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), uint256())); - LogPrint(BCLog::NET, "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", - headers[0].GetHash().ToString(), - headers[0].hashPrevBlock.ToString(), - m_chainman.m_best_header->nHeight, - pfrom.GetId(), nodestate->nUnconnectingHeaders); - // Set hashLastUnknownBlock for this peer, so that if we - // eventually get the headers - even from a different peer - - // we can use this peer to download. - UpdateBlockAvailability(pfrom.GetId(), headers.back().GetHash()); - - if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { - Misbehaving(peer, 20, strprintf("%d non-connecting headers", nodestate->nUnconnectingHeaders)); - } - return; - } + // Do these headers connect to something in our block index? + bool headers_connect_blockindex{WITH_LOCK(::cs_main, return m_chainman.m_blockman.LookupBlockIndex(headers[0].hashPrevBlock) != nullptr)}; - uint256 hashLastBlock; - for (const CBlockHeader& header : headers) { - if (!hashLastBlock.IsNull() && header.hashPrevBlock != hashLastBlock) { - Misbehaving(peer, 20, "non-continuous headers sequence"); - return; - } - hashLastBlock = header.GetHash(); + if (!headers_connect_blockindex) { + if (nCount <= MAX_BLOCKS_TO_ANNOUNCE) { + // If this looks like it could be a BIP 130 block announcement, use + // special logic for handling headers that don't connect, as this + // could be benign. + HandleFewUnconnectingHeaders(pfrom, peer, headers); + } else { + Misbehaving(peer, 10, "invalid header received"); } + return; + } - // If we don't have the last header, then they'll have given us - // something new (if these headers are valid). - if (!m_chainman.m_blockman.LookupBlockIndex(hashLastBlock)) { - received_new_header = true; - } + // At this point, the headers connect to something in our block index. + if (!CheckHeadersAreContinuous(headers)) { + Misbehaving(peer, 20, "non-continuous headers sequence"); + return; } + // If we don't have the last header, then this peer will have given us + // something new (if these headers are valid). + bool received_new_header{WITH_LOCK(::cs_main, return m_chainman.m_blockman.LookupBlockIndex(headers.back().GetHash()) == nullptr)}; + BlockValidationState state; if (!m_chainman.ProcessNewBlockHeaders(headers, state, &pindexLast)) { if (state.IsInvalid()) { @@ -2263,123 +2500,20 @@ void PeerManagerImpl::ProcessHeadersMessage(CNode& pfrom, Peer& peer, } } - { - LOCK(cs_main); - CNodeState *nodestate = State(pfrom.GetId()); - if (nodestate->nUnconnectingHeaders > 0) { - LogPrint(BCLog::NET, "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom.GetId(), nodestate->nUnconnectingHeaders); - } - nodestate->nUnconnectingHeaders = 0; - - assert(pindexLast); - UpdateBlockAvailability(pfrom.GetId(), pindexLast->GetBlockHash()); - - // From here, pindexBestKnownBlock should be guaranteed to be non-null, - // because it is set in UpdateBlockAvailability. Some nullptr checks - // are still present, however, as belt-and-suspenders. - - if (received_new_header && pindexLast->nChainWork > m_chainman.ActiveChain().Tip()->nChainWork) { - nodestate->m_last_block_announcement = GetTime(); - } - - if (nCount == MAX_HEADERS_RESULTS) { - // Headers message had its maximum size; the peer may have more headers. - // TODO: optimize: if pindexLast is an ancestor of m_chainman.ActiveChain().Tip or m_chainman.m_best_header, continue - // from there instead. + // Consider fetching more headers. + if (nCount == MAX_HEADERS_RESULTS) { + // Headers message had its maximum size; the peer may have more headers. + if (MaybeSendGetHeaders(pfrom, m_chainman.ActiveChain().GetLocator(pindexLast), peer)) { LogPrint(BCLog::NET, "more getheaders (%d) to end to peer=%d (startheight:%d)\n", - pindexLast->nHeight, pfrom.GetId(), peer.m_starting_height); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(pindexLast), uint256())); - } - - // If this set of headers is valid and ends in a block with at least as - // much work as our tip, download as much as possible. - if (CanDirectFetch() && pindexLast->IsValid(BLOCK_VALID_TREE) && m_chainman.ActiveChain().Tip()->nChainWork <= pindexLast->nChainWork) { - std::vector<const CBlockIndex*> vToFetch; - const CBlockIndex *pindexWalk = pindexLast; - // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. - while (pindexWalk && !m_chainman.ActiveChain().Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && - !IsBlockRequested(pindexWalk->GetBlockHash()) && - (!DeploymentActiveAt(*pindexWalk, m_chainman, Consensus::DEPLOYMENT_SEGWIT) || State(pfrom.GetId())->fHaveWitness)) { - // We don't have this block, and it's not yet in flight. - vToFetch.push_back(pindexWalk); - } - pindexWalk = pindexWalk->pprev; - } - // If pindexWalk still isn't on our main chain, we're looking at a - // very large reorg at a time we think we're close to caught up to - // the main chain -- this shouldn't really happen. Bail out on the - // direct fetch and rely on parallel download instead. - if (!m_chainman.ActiveChain().Contains(pindexWalk)) { - LogPrint(BCLog::NET, "Large reorg, won't direct fetch to %s (%d)\n", - pindexLast->GetBlockHash().ToString(), - pindexLast->nHeight); - } else { - std::vector<CInv> vGetData; - // Download as much as possible, from earliest to latest. - for (const CBlockIndex *pindex : reverse_iterate(vToFetch)) { - if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { - // Can't download any more from this peer - break; - } - uint32_t nFetchFlags = GetFetchFlags(pfrom); - vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); - BlockRequested(pfrom.GetId(), *pindex); - LogPrint(BCLog::NET, "Requesting block %s from peer=%d\n", - pindex->GetBlockHash().ToString(), pfrom.GetId()); - } - if (vGetData.size() > 1) { - LogPrint(BCLog::NET, "Downloading blocks toward %s (%d) via headers direct fetch\n", - pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); - } - if (vGetData.size() > 0) { - if (!m_ignore_incoming_txs && - nodestate->m_provides_cmpctblocks && - vGetData.size() == 1 && - mapBlocksInFlight.size() == 1 && - pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN)) { - // In any case, we want to download using a compact block, not a regular one - vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); - } - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); - } - } - } - // If we're in IBD, we want outbound peers that will serve us a useful - // chain. Disconnect peers that are on chains with insufficient work. - if (m_chainman.ActiveChainstate().IsInitialBlockDownload() && nCount != MAX_HEADERS_RESULTS) { - // When nCount < MAX_HEADERS_RESULTS, we know we have no more - // headers to fetch from this peer. - if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < nMinimumChainWork) { - // This peer has too little work on their headers chain to help - // us sync -- disconnect if it is an outbound disconnection - // candidate. - // Note: We compare their tip to nMinimumChainWork (rather than - // m_chainman.ActiveChain().Tip()) because we won't start block download - // until we have a headers chain that has at least - // nMinimumChainWork, even if a peer has a chain past our tip, - // as an anti-DoS measure. - if (pfrom.IsOutboundOrBlockRelayConn()) { - LogPrintf("Disconnecting outbound peer %d -- headers chain has insufficient work\n", pfrom.GetId()); - pfrom.fDisconnect = true; - } - } - } - - // If this is an outbound full-relay peer, check to see if we should protect - // it from the bad/lagging chain logic. - // Note that outbound block-relay peers are excluded from this protection, and - // thus always subject to eviction under the bad/lagging chain logic. - // See ChainSyncTimeoutState. - if (!pfrom.fDisconnect && pfrom.IsFullOutboundConn() && nodestate->pindexBestKnownBlock != nullptr) { - if (m_outbound_peers_with_protect_from_disconnect < MAX_OUTBOUND_PEERS_TO_PROTECT_FROM_DISCONNECT && nodestate->pindexBestKnownBlock->nChainWork >= m_chainman.ActiveChain().Tip()->nChainWork && !nodestate->m_chain_sync.m_protect) { - LogPrint(BCLog::NET, "Protecting outbound peer=%d from eviction\n", pfrom.GetId()); - nodestate->m_chain_sync.m_protect = true; - ++m_outbound_peers_with_protect_from_disconnect; - } + pindexLast->nHeight, pfrom.GetId(), peer.m_starting_height); } } + UpdatePeerStateForReceivedHeaders(pfrom, pindexLast, received_new_header, nCount == MAX_HEADERS_RESULTS); + + // Consider immediately downloading blocks. + HeadersDirectFetchBlocks(pfrom, peer, pindexLast); + return; } @@ -2462,7 +2596,7 @@ void PeerManagerImpl::ProcessOrphanTx(std::set<uint256>& orphan_work_set) } } -bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& peer, +bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& node, Peer& peer, BlockFilterType filter_type, uint32_t start_height, const uint256& stop_hash, uint32_t max_height_diff, const CBlockIndex*& stop_index, @@ -2470,11 +2604,11 @@ bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& peer, { const bool supported_filter_type = (filter_type == BlockFilterType::BASIC && - (peer.GetLocalServices() & NODE_COMPACT_FILTERS)); + (peer.m_our_services & NODE_COMPACT_FILTERS)); if (!supported_filter_type) { LogPrint(BCLog::NET, "peer %d requested unsupported block filter type: %d\n", - peer.GetId(), static_cast<uint8_t>(filter_type)); - peer.fDisconnect = true; + node.GetId(), static_cast<uint8_t>(filter_type)); + node.fDisconnect = true; return false; } @@ -2485,8 +2619,8 @@ bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& peer, // Check that the stop block exists and the peer would be allowed to fetch it. if (!stop_index || !BlockRequestAllowed(stop_index)) { LogPrint(BCLog::NET, "peer %d requested invalid block hash: %s\n", - peer.GetId(), stop_hash.ToString()); - peer.fDisconnect = true; + node.GetId(), stop_hash.ToString()); + node.fDisconnect = true; return false; } } @@ -2495,14 +2629,14 @@ bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& peer, if (start_height > stop_height) { LogPrint(BCLog::NET, "peer %d sent invalid getcfilters/getcfheaders with " /* Continued */ "start height %d and stop height %d\n", - peer.GetId(), start_height, stop_height); - peer.fDisconnect = true; + node.GetId(), start_height, stop_height); + node.fDisconnect = true; return false; } if (stop_height - start_height >= max_height_diff) { LogPrint(BCLog::NET, "peer %d requested too many cfilters/cfheaders: %d / %d\n", - peer.GetId(), stop_height - start_height + 1, max_height_diff); - peer.fDisconnect = true; + node.GetId(), stop_height - start_height + 1, max_height_diff); + node.fDisconnect = true; return false; } @@ -2515,7 +2649,7 @@ bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& peer, return true; } -void PeerManagerImpl::ProcessGetCFilters(CNode& peer, CDataStream& vRecv) +void PeerManagerImpl::ProcessGetCFilters(CNode& node,Peer& peer, CDataStream& vRecv) { uint8_t filter_type_ser; uint32_t start_height; @@ -2527,7 +2661,7 @@ void PeerManagerImpl::ProcessGetCFilters(CNode& peer, CDataStream& vRecv) const CBlockIndex* stop_index; BlockFilterIndex* filter_index; - if (!PrepareBlockFilterRequest(peer, filter_type, start_height, stop_hash, + if (!PrepareBlockFilterRequest(node, peer, filter_type, start_height, stop_hash, MAX_GETCFILTERS_SIZE, stop_index, filter_index)) { return; } @@ -2540,13 +2674,13 @@ void PeerManagerImpl::ProcessGetCFilters(CNode& peer, CDataStream& vRecv) } for (const auto& filter : filters) { - CSerializedNetMsg msg = CNetMsgMaker(peer.GetCommonVersion()) + CSerializedNetMsg msg = CNetMsgMaker(node.GetCommonVersion()) .Make(NetMsgType::CFILTER, filter); - m_connman.PushMessage(&peer, std::move(msg)); + m_connman.PushMessage(&node, std::move(msg)); } } -void PeerManagerImpl::ProcessGetCFHeaders(CNode& peer, CDataStream& vRecv) +void PeerManagerImpl::ProcessGetCFHeaders(CNode& node, Peer& peer, CDataStream& vRecv) { uint8_t filter_type_ser; uint32_t start_height; @@ -2558,7 +2692,7 @@ void PeerManagerImpl::ProcessGetCFHeaders(CNode& peer, CDataStream& vRecv) const CBlockIndex* stop_index; BlockFilterIndex* filter_index; - if (!PrepareBlockFilterRequest(peer, filter_type, start_height, stop_hash, + if (!PrepareBlockFilterRequest(node, peer, filter_type, start_height, stop_hash, MAX_GETCFHEADERS_SIZE, stop_index, filter_index)) { return; } @@ -2581,16 +2715,16 @@ void PeerManagerImpl::ProcessGetCFHeaders(CNode& peer, CDataStream& vRecv) return; } - CSerializedNetMsg msg = CNetMsgMaker(peer.GetCommonVersion()) + CSerializedNetMsg msg = CNetMsgMaker(node.GetCommonVersion()) .Make(NetMsgType::CFHEADERS, filter_type_ser, stop_index->GetBlockHash(), prev_header, filter_hashes); - m_connman.PushMessage(&peer, std::move(msg)); + m_connman.PushMessage(&node, std::move(msg)); } -void PeerManagerImpl::ProcessGetCFCheckPt(CNode& peer, CDataStream& vRecv) +void PeerManagerImpl::ProcessGetCFCheckPt(CNode& node, Peer& peer, CDataStream& vRecv) { uint8_t filter_type_ser; uint256 stop_hash; @@ -2601,7 +2735,7 @@ void PeerManagerImpl::ProcessGetCFCheckPt(CNode& peer, CDataStream& vRecv) const CBlockIndex* stop_index; BlockFilterIndex* filter_index; - if (!PrepareBlockFilterRequest(peer, filter_type, /*start_height=*/0, stop_hash, + if (!PrepareBlockFilterRequest(node, peer, filter_type, /*start_height=*/0, stop_hash, /*max_height_diff=*/std::numeric_limits<uint32_t>::max(), stop_index, filter_index)) { return; @@ -2622,12 +2756,12 @@ void PeerManagerImpl::ProcessGetCFCheckPt(CNode& peer, CDataStream& vRecv) } } - CSerializedNetMsg msg = CNetMsgMaker(peer.GetCommonVersion()) + CSerializedNetMsg msg = CNetMsgMaker(node.GetCommonVersion()) .Make(NetMsgType::CFCHECKPT, filter_type_ser, stop_index->GetBlockHash(), headers); - m_connman.PushMessage(&peer, std::move(msg)); + m_connman.PushMessage(&node, std::move(msg)); } void PeerManagerImpl::ProcessBlock(CNode& node, const std::shared_ptr<const CBlock>& block, bool force_processing) @@ -2749,7 +2883,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::VERACK)); - pfrom.nServices = nServices; + pfrom.m_has_all_wanted_services = HasAllDesirableServiceFlags(nServices); + peer->m_their_services = nServices; pfrom.SetAddrLocal(addrMe); { LOCK(pfrom.m_subver_mutex); @@ -2757,18 +2892,12 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } peer->m_starting_height = starting_height; - // set nodes not relaying blocks and tx and not serving (parts) of the historical blockchain as "clients" - pfrom.fClient = (!(nServices & NODE_NETWORK) && !(nServices & NODE_NETWORK_LIMITED)); - - // set nodes not capable of serving the complete blockchain history as "limited nodes" - pfrom.m_limited_node = (!(nServices & NODE_NETWORK) && (nServices & NODE_NETWORK_LIMITED)); - // We only initialize the m_tx_relay data structure if: // - this isn't an outbound block-relay-only connection; and // - fRelay=true or we're offering NODE_BLOOM to this peer // (NODE_BLOOM means that the peer may turn on tx relay later) if (!pfrom.IsBlockOnlyConn() && - (fRelay || (pfrom.GetLocalServices() & NODE_BLOOM))) { + (fRelay || (peer->m_our_services & NODE_BLOOM))) { auto* const tx_relay = peer->SetTxRelay(); { LOCK(tx_relay->m_bloom_filter_mutex); @@ -2777,17 +2906,11 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (fRelay) pfrom.m_relays_txs = true; } - if((nServices & NODE_WITNESS)) - { - LOCK(cs_main); - State(pfrom.GetId())->fHaveWitness = true; - } - // Potentially mark this peer as a preferred download peer. { LOCK(cs_main); CNodeState* state = State(pfrom.GetId()); - state->fPreferredDownload = (!pfrom.IsInboundConn() || pfrom.HasPermission(NetPermissionFlags::NoBan)) && !pfrom.IsAddrFetchConn() && !pfrom.fClient; + state->fPreferredDownload = (!pfrom.IsInboundConn() || pfrom.HasPermission(NetPermissionFlags::NoBan)) && !pfrom.IsAddrFetchConn() && CanServeBlocks(*peer); m_num_preferred_download_peers += state->fPreferredDownload; } @@ -2806,7 +2929,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // indicate to the peer that we will participate in addr relay. if (fListen && !m_chainman.ActiveChainstate().IsInitialBlockDownload()) { - CAddress addr = GetLocalAddress(&pfrom.addr, pfrom.GetLocalServices()); + CAddress addr{GetLocalAddress(pfrom.addr), peer->m_our_services, (uint32_t)GetAdjustedTime()}; FastRandomContext insecure_rand; if (addr.IsRoutable()) { @@ -3142,8 +3265,11 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } if (best_block != nullptr) { - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), *best_block)); - LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", m_chainman.m_best_header->nHeight, best_block->ToString(), pfrom.GetId()); + if (MaybeSendGetHeaders(pfrom, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), *peer)) { + LogPrint(BCLog::NET, "getheaders (%d) %s to peer=%d\n", + m_chainman.m_best_header->nHeight, best_block->ToString(), + pfrom.GetId()); + } } return; @@ -3316,7 +3442,10 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // others. if (m_chainman.ActiveTip() == nullptr || (m_chainman.ActiveTip()->nChainWork < nMinimumChainWork && !pfrom.HasPermission(NetPermissionFlags::Download))) { - LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because active chain has too little work\n", pfrom.GetId()); + LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because active chain has too little work; sending empty response\n", pfrom.GetId()); + // Just respond with an empty headers message, to tell the peer to + // go away but not treat us as unresponsive. + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, std::vector<CBlock>())); return; } @@ -3597,8 +3726,9 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (!m_chainman.m_blockman.LookupBlockIndex(cmpctblock.header.hashPrevBlock)) { // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers - if (!m_chainman.ActiveChainstate().IsInitialBlockDownload()) - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), uint256())); + if (!m_chainman.ActiveChainstate().IsInitialBlockDownload()) { + MaybeSendGetHeaders(pfrom, m_chainman.ActiveChain().GetLocator(m_chainman.m_best_header), *peer); + } return; } @@ -3658,7 +3788,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // We requested this block for some reason, but our mempool will probably be useless // so we just grab the block via normal getdata std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(*peer), cmpctblock.header.GetHash()); m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); } return; @@ -3694,7 +3824,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } else if (status == READ_STATUS_FAILED) { // Duplicate txindexes, the block is now in-flight, so just request it std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(*peer), cmpctblock.header.GetHash()); m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return; } @@ -3737,7 +3867,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // We requested this block, but its far into the future, so our // mempool will probably be useless - request the block normally std::vector<CInv> vInv(1); - vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(pfrom), cmpctblock.header.GetHash()); + vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(*peer), cmpctblock.header.GetHash()); m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); return; } else { @@ -3821,7 +3951,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } else if (status == READ_STATUS_FAILED) { // Might have collided, fall back to getdata now :( std::vector<CInv> invs; - invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(pfrom), resp.blockhash)); + invs.push_back(CInv(MSG_BLOCK | GetFetchFlags(*peer), resp.blockhash)); m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); } else { // Block is either okay, or possibly we received @@ -3872,6 +4002,10 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } + // Assume that this is in response to any outstanding getheaders + // request we may have sent, and clear out the time of our last request + peer->m_last_getheaders_timestamp = {}; + std::vector<CBlockHeader> headers; // Bypass the normal CBlock deserialization, as we don't want to risk deserializing 2000 full blocks. @@ -3957,7 +4091,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } if (msg_type == NetMsgType::MEMPOOL) { - if (!(pfrom.GetLocalServices() & NODE_BLOOM) && !pfrom.HasPermission(NetPermissionFlags::Mempool)) + if (!(peer->m_our_services & NODE_BLOOM) && !pfrom.HasPermission(NetPermissionFlags::Mempool)) { if (!pfrom.HasPermission(NetPermissionFlags::NoBan)) { @@ -4060,7 +4194,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } if (msg_type == NetMsgType::FILTERLOAD) { - if (!(pfrom.GetLocalServices() & NODE_BLOOM)) { + if (!(peer->m_our_services & NODE_BLOOM)) { LogPrint(BCLog::NET, "filterload received despite not offering bloom services from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; @@ -4085,7 +4219,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } if (msg_type == NetMsgType::FILTERADD) { - if (!(pfrom.GetLocalServices() & NODE_BLOOM)) { + if (!(peer->m_our_services & NODE_BLOOM)) { LogPrint(BCLog::NET, "filteradd received despite not offering bloom services from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; @@ -4113,7 +4247,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } if (msg_type == NetMsgType::FILTERCLEAR) { - if (!(pfrom.GetLocalServices() & NODE_BLOOM)) { + if (!(peer->m_our_services & NODE_BLOOM)) { LogPrint(BCLog::NET, "filterclear received despite not offering bloom services from peer=%d; disconnecting\n", pfrom.GetId()); pfrom.fDisconnect = true; return; @@ -4144,17 +4278,17 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } if (msg_type == NetMsgType::GETCFILTERS) { - ProcessGetCFilters(pfrom, vRecv); + ProcessGetCFilters(pfrom, *peer, vRecv); return; } if (msg_type == NetMsgType::GETCFHEADERS) { - ProcessGetCFHeaders(pfrom, vRecv); + ProcessGetCFHeaders(pfrom, *peer, vRecv); return; } if (msg_type == NetMsgType::GETCFCHECKPT) { - ProcessGetCFCheckPt(pfrom, vRecv); + ProcessGetCFCheckPt(pfrom, *peer, vRecv); return; } @@ -4300,7 +4434,7 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt return fMoreWork; } -void PeerManagerImpl::ConsiderEviction(CNode& pto, std::chrono::seconds time_in_seconds) +void PeerManagerImpl::ConsiderEviction(CNode& pto, Peer& peer, std::chrono::seconds time_in_seconds) { AssertLockHeld(cs_main); @@ -4338,10 +4472,15 @@ void PeerManagerImpl::ConsiderEviction(CNode& pto, std::chrono::seconds time_in_ pto.fDisconnect = true; } else { assert(state.m_chain_sync.m_work_header); + // Here, we assume that the getheaders message goes out, + // because it'll either go out or be skipped because of a + // getheaders in-flight already, in which case the peer should + // still respond to us with a sufficiently high work chain tip. + MaybeSendGetHeaders(pto, + m_chainman.ActiveChain().GetLocator(state.m_chain_sync.m_work_header->pprev), + peer); LogPrint(BCLog::NET, "sending getheaders to outbound peer=%d to verify chain work (current best known block:%s, benchmark blockhash: %s)\n", pto.GetId(), state.pindexBestKnownBlock != nullptr ? state.pindexBestKnownBlock->GetBlockHash().ToString() : "<none>", state.m_chain_sync.m_work_header->GetBlockHash().ToString()); - m_connman.PushMessage(&pto, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(state.m_chain_sync.m_work_header->pprev), uint256())); state.m_chain_sync.m_sent_getheaders = true; - constexpr auto HEADERS_RESPONSE_TIME{2min}; // Bump the timeout to allow a response, which could clear the timeout // (if the response shows the peer has synced), reset the timeout (if // the peer syncs to the required work but not to our tip), or result @@ -4545,9 +4684,10 @@ void PeerManagerImpl::MaybeSendAddr(CNode& node, Peer& peer, std::chrono::micros if (peer.m_next_local_addr_send != 0us) { peer.m_addr_known->reset(); } - if (std::optional<CAddress> local_addr = GetLocalAddrForPeer(&node)) { + if (std::optional<CService> local_service = GetLocalAddrForPeer(node)) { + CAddress local_addr{*local_service, peer.m_our_services, (uint32_t)GetAdjustedTime()}; FastRandomContext insecure_rand; - PushAddress(peer, *local_addr, insecure_rand); + PushAddress(peer, local_addr, insecure_rand); } peer.m_next_local_addr_send = GetExponentialRand(current_time, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); } @@ -4731,7 +4871,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) bool sync_blocks_and_headers_from_peer = false; if (state.fPreferredDownload) { sync_blocks_and_headers_from_peer = true; - } else if (!pto->fClient && !pto->IsAddrFetchConn()) { + } else if (CanServeBlocks(*peer) && !pto->IsAddrFetchConn()) { // Typically this is an inbound peer. If we don't have any outbound // peers, or if we aren't downloading any blocks from such peers, // then allow block downloads from this peer, too. @@ -4746,18 +4886,9 @@ bool PeerManagerImpl::SendMessages(CNode* pto) } } - if (!state.fSyncStarted && !pto->fClient && !fImporting && !fReindex) { + if (!state.fSyncStarted && CanServeBlocks(*peer) && !fImporting && !fReindex) { // Only actively request headers from a single peer, unless we're close to today. if ((nSyncStarted == 0 && sync_blocks_and_headers_from_peer) || m_chainman.m_best_header->GetBlockTime() > GetAdjustedTime() - 24 * 60 * 60) { - state.fSyncStarted = true; - state.m_headers_sync_timeout = current_time + HEADERS_DOWNLOAD_TIMEOUT_BASE + - ( - // Convert HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER to microseconds before scaling - // to maintain precision - std::chrono::microseconds{HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER} * - (GetAdjustedTime() - m_chainman.m_best_header->GetBlockTime()) / consensusParams.nPowTargetSpacing - ); - nSyncStarted++; const CBlockIndex* pindexStart = m_chainman.m_best_header; /* If possible, start at the block preceding the currently best known header. This ensures that we always get a @@ -4768,8 +4899,19 @@ bool PeerManagerImpl::SendMessages(CNode* pto) got back an empty response. */ if (pindexStart->pprev) pindexStart = pindexStart->pprev; - LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), peer->m_starting_height); - m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETHEADERS, m_chainman.ActiveChain().GetLocator(pindexStart), uint256())); + if (MaybeSendGetHeaders(*pto, m_chainman.ActiveChain().GetLocator(pindexStart), *peer)) { + LogPrint(BCLog::NET, "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->GetId(), peer->m_starting_height); + + state.fSyncStarted = true; + state.m_headers_sync_timeout = current_time + HEADERS_DOWNLOAD_TIMEOUT_BASE + + ( + // Convert HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER to microseconds before scaling + // to maintain precision + std::chrono::microseconds{HEADERS_DOWNLOAD_TIMEOUT_PER_HEADER} * + (GetAdjustedTime() - m_chainman.m_best_header->GetBlockTime()) / consensusParams.nPowTargetSpacing + ); + nSyncStarted++; + } } } @@ -5115,18 +5257,18 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // Check that outbound peers have reasonable chains // GetTime() is used by this anti-DoS logic so we can test this using mocktime - ConsiderEviction(*pto, GetTime<std::chrono::seconds>()); + ConsiderEviction(*pto, *peer, GetTime<std::chrono::seconds>()); // // Message: getdata (blocks) // std::vector<CInv> vGetData; - if (!pto->fClient && ((sync_blocks_and_headers_from_peer && !pto->m_limited_node) || !m_chainman.ActiveChainstate().IsInitialBlockDownload()) && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (CanServeBlocks(*peer) && ((sync_blocks_and_headers_from_peer && !IsLimitedPeer(*peer)) || !m_chainman.ActiveChainstate().IsInitialBlockDownload()) && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { std::vector<const CBlockIndex*> vToDownload; NodeId staller = -1; - FindNextBlocksToDownload(pto->GetId(), MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller); + FindNextBlocksToDownload(*peer, MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller); for (const CBlockIndex *pindex : vToDownload) { - uint32_t nFetchFlags = GetFetchFlags(*pto); + uint32_t nFetchFlags = GetFetchFlags(*peer); vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); BlockRequested(pto->GetId(), *pindex); LogPrint(BCLog::NET, "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), @@ -5153,7 +5295,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) if (!AlreadyHaveTx(gtxid)) { LogPrint(BCLog::NET, "Requesting %s %s peer=%d\n", gtxid.IsWtxid() ? "wtx" : "tx", gtxid.GetHash().ToString(), pto->GetId()); - vGetData.emplace_back(gtxid.IsWtxid() ? MSG_WTX : (MSG_TX | GetFetchFlags(*pto)), gtxid.GetHash()); + vGetData.emplace_back(gtxid.IsWtxid() ? MSG_WTX : (MSG_TX | GetFetchFlags(*peer)), gtxid.GetHash()); if (vGetData.size() >= MAX_GETDATA_SZ) { m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); vGetData.clear(); diff --git a/src/net_processing.h b/src/net_processing.h index 5fbae98c27..bcda9614d4 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -34,6 +34,7 @@ struct CNodeStateStats { uint64_t m_addr_processed = 0; uint64_t m_addr_rate_limited = 0; bool m_addr_relay_enabled{false}; + ServiceFlags their_services; }; class PeerManager : public CValidationInterface, public NetEventsInterface diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index cadafcaa8d..103f4f0d7f 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -823,7 +823,7 @@ struct CImportingNow { } }; -void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args) +void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args, const fs::path& mempool_path) { SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION_LOAD_BLOCKS); ScheduleBatchPriority(); @@ -893,6 +893,6 @@ void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFile return; } } // End scope of CImportingNow - chainman.ActiveChainstate().LoadMempool(args); + chainman.ActiveChainstate().LoadMempool(mempool_path); } } // namespace node diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index e017f3f427..9b76371aae 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -211,7 +211,7 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, c bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); -void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args); +void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args, const fs::path& mempool_path); } // namespace node #endif // BITCOIN_NODE_BLOCKSTORAGE_H diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index 54ba5b7966..bb6b2df1b7 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -4,66 +4,73 @@ #include <node/chainstate.h> +#include <chain.h> +#include <coins.h> #include <consensus/params.h> #include <node/blockstorage.h> +#include <node/caches.h> +#include <sync.h> +#include <threadsafety.h> +#include <txdb.h> +#include <uint256.h> +#include <util/time.h> #include <validation.h> +#include <algorithm> +#include <atomic> +#include <cassert> +#include <memory> +#include <vector> + namespace node { -std::optional<ChainstateLoadingError> LoadChainstate(bool fReset, - ChainstateManager& chainman, - CTxMemPool* mempool, - bool fPruneMode, - bool fReindexChainState, - int64_t nBlockTreeDBCache, - int64_t nCoinDBCache, - int64_t nCoinCacheUsage, - bool block_tree_db_in_memory, - bool coins_db_in_memory, - std::function<bool()> shutdown_requested, - std::function<void()> coins_error_cb) +ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSizes& cache_sizes, + const ChainstateLoadOptions& options) { auto is_coinsview_empty = [&](CChainState* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { - return fReset || fReindexChainState || chainstate->CoinsTip().GetBestBlock().IsNull(); + return options.reindex || options.reindex_chainstate || chainstate->CoinsTip().GetBestBlock().IsNull(); }; LOCK(cs_main); - chainman.InitializeChainstate(mempool); - chainman.m_total_coinstip_cache = nCoinCacheUsage; - chainman.m_total_coinsdb_cache = nCoinDBCache; + chainman.InitializeChainstate(options.mempool); + chainman.m_total_coinstip_cache = cache_sizes.coins; + chainman.m_total_coinsdb_cache = cache_sizes.coins_db; auto& pblocktree{chainman.m_blockman.m_block_tree_db}; // new CBlockTreeDB tries to delete the existing file, which // fails if it's still open from the previous loop. Close it first: pblocktree.reset(); - pblocktree.reset(new CBlockTreeDB(nBlockTreeDBCache, block_tree_db_in_memory, fReset)); + pblocktree.reset(new CBlockTreeDB(cache_sizes.block_tree_db, options.block_tree_db_in_memory, options.reindex)); - if (fReset) { + if (options.reindex) { pblocktree->WriteReindexing(true); //If we're reindexing in prune mode, wipe away unusable block files and all undo data files - if (fPruneMode) + if (options.prune) { CleanupBlockRevFiles(); + } } - if (shutdown_requested && shutdown_requested()) return ChainstateLoadingError::SHUTDOWN_PROBED; + if (options.check_interrupt && options.check_interrupt()) return {ChainstateLoadStatus::INTERRUPTED, {}}; // LoadBlockIndex will load m_have_pruned if we've ever removed a // block file from disk. - // Note that it also sets fReindex based on the disk flag! - // From here on out fReindex and fReset mean something different! + // Note that it also sets fReindex global based on the disk flag! + // From here on, fReindex and options.reindex values may be different! if (!chainman.LoadBlockIndex()) { - if (shutdown_requested && shutdown_requested()) return ChainstateLoadingError::SHUTDOWN_PROBED; - return ChainstateLoadingError::ERROR_LOADING_BLOCK_DB; + if (options.check_interrupt && options.check_interrupt()) return {ChainstateLoadStatus::INTERRUPTED, {}}; + return {ChainstateLoadStatus::FAILURE, _("Error loading block database")}; } if (!chainman.BlockIndex().empty() && !chainman.m_blockman.LookupBlockIndex(chainman.GetConsensus().hashGenesisBlock)) { - return ChainstateLoadingError::ERROR_BAD_GENESIS_BLOCK; + // If the loaded chain has a wrong genesis, bail out immediately + // (we're likely using a testnet datadir, or the other way around). + return {ChainstateLoadStatus::FAILURE_INCOMPATIBLE_DB, _("Incorrect or no genesis block found. Wrong datadir for network?")}; } // Check for changed -prune state. What we are concerned about is a user who has pruned blocks // in the past, but is now trying to run unpruned. - if (chainman.m_blockman.m_have_pruned && !fPruneMode) { - return ChainstateLoadingError::ERROR_PRUNED_NEEDS_REINDEX; + if (chainman.m_blockman.m_have_pruned && !options.prune) { + return {ChainstateLoadStatus::FAILURE, _("You need to rebuild the database using -reindex to go back to unpruned mode. This will redownload the entire blockchain")}; } // At this point blocktree args are consistent with what's on disk. @@ -71,7 +78,7 @@ std::optional<ChainstateLoadingError> LoadChainstate(bool fReset, // (otherwise we use the one already on disk). // This is called again in ThreadImport after the reindex completes. if (!fReindex && !chainman.ActiveChainstate().LoadGenesisBlock()) { - return ChainstateLoadingError::ERROR_LOAD_GENESIS_BLOCK_FAILED; + return {ChainstateLoadStatus::FAILURE, _("Error initializing block database")}; } // At this point we're either in reindex or we've loaded a useful @@ -79,57 +86,56 @@ std::optional<ChainstateLoadingError> LoadChainstate(bool fReset, for (CChainState* chainstate : chainman.GetAll()) { chainstate->InitCoinsDB( - /*cache_size_bytes=*/nCoinDBCache, - /*in_memory=*/coins_db_in_memory, - /*should_wipe=*/fReset || fReindexChainState); + /*cache_size_bytes=*/cache_sizes.coins_db, + /*in_memory=*/options.coins_db_in_memory, + /*should_wipe=*/options.reindex || options.reindex_chainstate); - if (coins_error_cb) { - chainstate->CoinsErrorCatcher().AddReadErrCallback(coins_error_cb); + if (options.coins_error_cb) { + chainstate->CoinsErrorCatcher().AddReadErrCallback(options.coins_error_cb); } // Refuse to load unsupported database format. // This is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate if (chainstate->CoinsDB().NeedsUpgrade()) { - return ChainstateLoadingError::ERROR_CHAINSTATE_UPGRADE_FAILED; + return {ChainstateLoadStatus::FAILURE_INCOMPATIBLE_DB, _("Unsupported chainstate database format found. " + "Please restart with -reindex-chainstate. This will " + "rebuild the chainstate database.")}; } // ReplayBlocks is a no-op if we cleared the coinsviewdb with -reindex or -reindex-chainstate if (!chainstate->ReplayBlocks()) { - return ChainstateLoadingError::ERROR_REPLAYBLOCKS_FAILED; + return {ChainstateLoadStatus::FAILURE, _("Unable to replay blocks. You will need to rebuild the database using -reindex-chainstate.")}; } // The on-disk coinsdb is now in a good state, create the cache - chainstate->InitCoinsCache(nCoinCacheUsage); + chainstate->InitCoinsCache(cache_sizes.coins); assert(chainstate->CanFlushToDisk()); if (!is_coinsview_empty(chainstate)) { // LoadChainTip initializes the chain based on CoinsTip()'s best block if (!chainstate->LoadChainTip()) { - return ChainstateLoadingError::ERROR_LOADCHAINTIP_FAILED; + return {ChainstateLoadStatus::FAILURE, _("Error initializing block database")}; } assert(chainstate->m_chain.Tip() != nullptr); } } - if (!fReset) { + if (!options.reindex) { auto chainstates{chainman.GetAll()}; if (std::any_of(chainstates.begin(), chainstates.end(), [](const CChainState* cs) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return cs->NeedsRedownload(); })) { - return ChainstateLoadingError::ERROR_BLOCKS_WITNESS_INSUFFICIENTLY_VALIDATED; - } + return {ChainstateLoadStatus::FAILURE, strprintf(_("Witness data for blocks after height %d requires validation. Please restart with -reindex."), + chainman.GetConsensus().SegwitHeight)}; + }; } - return std::nullopt; + return {ChainstateLoadStatus::SUCCESS, {}}; } -std::optional<ChainstateLoadVerifyError> VerifyLoadedChainstate(ChainstateManager& chainman, - bool fReset, - bool fReindexChainState, - int check_blocks, - int check_level) +ChainstateLoadResult VerifyLoadedChainstate(ChainstateManager& chainman, const ChainstateLoadOptions& options) { auto is_coinsview_empty = [&](CChainState* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { - return fReset || fReindexChainState || chainstate->CoinsTip().GetBestBlock().IsNull(); + return options.reindex || options.reindex_chainstate || chainstate->CoinsTip().GetBestBlock().IsNull(); }; LOCK(cs_main); @@ -138,18 +144,20 @@ std::optional<ChainstateLoadVerifyError> VerifyLoadedChainstate(ChainstateManage if (!is_coinsview_empty(chainstate)) { const CBlockIndex* tip = chainstate->m_chain.Tip(); if (tip && tip->nTime > GetTime() + MAX_FUTURE_BLOCK_TIME) { - return ChainstateLoadVerifyError::ERROR_BLOCK_FROM_FUTURE; + return {ChainstateLoadStatus::FAILURE, _("The block database contains a block which appears to be from the future. " + "This may be due to your computer's date and time being set incorrectly. " + "Only rebuild the block database if you are sure that your computer's date and time are correct")}; } if (!CVerifyDB().VerifyDB( *chainstate, chainman.GetConsensus(), chainstate->CoinsDB(), - check_level, - check_blocks)) { - return ChainstateLoadVerifyError::ERROR_CORRUPTED_BLOCK_DB; + options.check_level, + options.check_blocks)) { + return {ChainstateLoadStatus::FAILURE, _("Corrupted block database detected")}; } } } - return std::nullopt; + return {ChainstateLoadStatus::SUCCESS, {}}; } } // namespace node diff --git a/src/node/chainstate.h b/src/node/chainstate.h index ff7935e8e0..649cfb2526 100644 --- a/src/node/chainstate.h +++ b/src/node/chainstate.h @@ -5,30 +5,41 @@ #ifndef BITCOIN_NODE_CHAINSTATE_H #define BITCOIN_NODE_CHAINSTATE_H +#include <validation.h> + #include <cstdint> #include <functional> #include <optional> class ChainstateManager; class CTxMemPool; -namespace Consensus { -struct Params; -} // namespace Consensus namespace node { -enum class ChainstateLoadingError { - ERROR_LOADING_BLOCK_DB, - ERROR_BAD_GENESIS_BLOCK, - ERROR_PRUNED_NEEDS_REINDEX, - ERROR_LOAD_GENESIS_BLOCK_FAILED, - ERROR_CHAINSTATE_UPGRADE_FAILED, - ERROR_REPLAYBLOCKS_FAILED, - ERROR_LOADCHAINTIP_FAILED, - ERROR_GENERIC_BLOCKDB_OPEN_FAILED, - ERROR_BLOCKS_WITNESS_INSUFFICIENTLY_VALIDATED, - SHUTDOWN_PROBED, + +struct CacheSizes; + +struct ChainstateLoadOptions { + CTxMemPool* mempool{nullptr}; + bool block_tree_db_in_memory{false}; + bool coins_db_in_memory{false}; + bool reindex{false}; + bool reindex_chainstate{false}; + bool prune{false}; + int64_t check_blocks{DEFAULT_CHECKBLOCKS}; + int64_t check_level{DEFAULT_CHECKLEVEL}; + std::function<bool()> check_interrupt; + std::function<void()> coins_error_cb; }; +//! Chainstate load status. Simple applications can just check for the success +//! case, and treat other cases as errors. More complex applications may want to +//! try reindexing in the generic failure case, and pass an interrupt callback +//! and exit cleanly in the interrupted case. +enum class ChainstateLoadStatus { SUCCESS, FAILURE, FAILURE_INCOMPATIBLE_DB, INTERRUPTED }; + +//! Chainstate load status code and optional error string. +using ChainstateLoadResult = std::tuple<ChainstateLoadStatus, bilingual_str>; + /** This sequence can have 4 types of outcomes: * * 1. Success @@ -40,45 +51,11 @@ enum class ChainstateLoadingError { * 4. Hard failure * - a failure that definitively cannot be recovered from with a reindex * - * Currently, LoadChainstate returns a std::optional<ChainstateLoadingError> - * which: - * - * - if has_value() - * - Either "Soft failure", "Hard failure", or "Shutdown requested", - * differentiable by the specific enumerator. - * - * Note that a return value of SHUTDOWN_PROBED means ONLY that "during - * this sequence, when we explicitly checked shutdown_requested() at - * arbitrary points, one of those calls returned true". Therefore, a - * return value other than SHUTDOWN_PROBED does not guarantee that - * shutdown hasn't been called indirectly. - * - else - * - Success! + * LoadChainstate returns a (status code, error string) tuple. */ -std::optional<ChainstateLoadingError> LoadChainstate(bool fReset, - ChainstateManager& chainman, - CTxMemPool* mempool, - bool fPruneMode, - bool fReindexChainState, - int64_t nBlockTreeDBCache, - int64_t nCoinDBCache, - int64_t nCoinCacheUsage, - bool block_tree_db_in_memory, - bool coins_db_in_memory, - std::function<bool()> shutdown_requested = nullptr, - std::function<void()> coins_error_cb = nullptr); - -enum class ChainstateLoadVerifyError { - ERROR_BLOCK_FROM_FUTURE, - ERROR_CORRUPTED_BLOCK_DB, - ERROR_GENERIC_FAILURE, -}; - -std::optional<ChainstateLoadVerifyError> VerifyLoadedChainstate(ChainstateManager& chainman, - bool fReset, - bool fReindexChainState, - int check_blocks, - int check_level); +ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSizes& cache_sizes, + const ChainstateLoadOptions& options); +ChainstateLoadResult VerifyLoadedChainstate(ChainstateManager& chainman, const ChainstateLoadOptions& options); } // namespace node #endif // BITCOIN_NODE_CHAINSTATE_H diff --git a/src/node/connection_types.cpp b/src/node/connection_types.cpp new file mode 100644 index 0000000000..904f4371aa --- /dev/null +++ b/src/node/connection_types.cpp @@ -0,0 +1,26 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/connection_types.h> +#include <cassert> + +std::string ConnectionTypeAsString(ConnectionType conn_type) +{ + switch (conn_type) { + case ConnectionType::INBOUND: + return "inbound"; + case ConnectionType::MANUAL: + return "manual"; + case ConnectionType::FEELER: + return "feeler"; + case ConnectionType::OUTBOUND_FULL_RELAY: + return "outbound-full-relay"; + case ConnectionType::BLOCK_RELAY: + return "block-relay-only"; + case ConnectionType::ADDR_FETCH: + return "addr-fetch"; + } // no default case, so the compiler can warn about missing cases + + assert(false); +} diff --git a/src/node/connection_types.h b/src/node/connection_types.h new file mode 100644 index 0000000000..5e1abcace6 --- /dev/null +++ b/src/node/connection_types.h @@ -0,0 +1,82 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_CONNECTION_TYPES_H +#define BITCOIN_NODE_CONNECTION_TYPES_H + +#include <string> + +/** Different types of connections to a peer. This enum encapsulates the + * information we have available at the time of opening or accepting the + * connection. Aside from INBOUND, all types are initiated by us. + * + * If adding or removing types, please update CONNECTION_TYPE_DOC in + * src/rpc/net.cpp and src/qt/rpcconsole.cpp, as well as the descriptions in + * src/qt/guiutil.cpp and src/bitcoin-cli.cpp::NetinfoRequestHandler. */ +enum class ConnectionType { + /** + * Inbound connections are those initiated by a peer. This is the only + * property we know at the time of connection, until P2P messages are + * exchanged. + */ + INBOUND, + + /** + * These are the default connections that we use to connect with the + * network. There is no restriction on what is relayed; by default we relay + * blocks, addresses & transactions. We automatically attempt to open + * MAX_OUTBOUND_FULL_RELAY_CONNECTIONS using addresses from our AddrMan. + */ + OUTBOUND_FULL_RELAY, + + + /** + * We open manual connections to addresses that users explicitly requested + * via the addnode RPC or the -addnode/-connect configuration options. Even if a + * manual connection is misbehaving, we do not automatically disconnect or + * add it to our discouragement filter. + */ + MANUAL, + + /** + * Feeler connections are short-lived connections made to check that a node + * is alive. They can be useful for: + * - test-before-evict: if one of the peers is considered for eviction from + * our AddrMan because another peer is mapped to the same slot in the tried table, + * evict only if this longer-known peer is offline. + * - move node addresses from New to Tried table, so that we have more + * connectable addresses in our AddrMan. + * Note that in the literature ("Eclipse Attacks on Bitcoin’s Peer-to-Peer Network") + * only the latter feature is referred to as "feeler connections", + * although in our codebase feeler connections encompass test-before-evict as well. + * We make these connections approximately every FEELER_INTERVAL: + * first we resolve previously found collisions if they exist (test-before-evict), + * otherwise we connect to a node from the new table. + */ + FEELER, + + /** + * We use block-relay-only connections to help prevent against partition + * attacks. By not relaying transactions or addresses, these connections + * are harder to detect by a third party, thus helping obfuscate the + * network topology. We automatically attempt to open + * MAX_BLOCK_RELAY_ONLY_ANCHORS using addresses from our anchors.dat. Then + * addresses from our AddrMan if MAX_BLOCK_RELAY_ONLY_CONNECTIONS + * isn't reached yet. + */ + BLOCK_RELAY, + + /** + * AddrFetch connections are short lived connections used to solicit + * addresses from peers. These are initiated to addresses submitted via the + * -seednode command line argument, or under certain conditions when the + * AddrMan is empty. + */ + ADDR_FETCH, +}; + +/** Convert ConnectionType enum to a string value */ +std::string ConnectionTypeAsString(ConnectionType conn_type); + +#endif // BITCOIN_NODE_CONNECTION_TYPES_H diff --git a/src/node/eviction.cpp b/src/node/eviction.cpp new file mode 100644 index 0000000000..33406931d4 --- /dev/null +++ b/src/node/eviction.cpp @@ -0,0 +1,240 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/eviction.h> + +#include <algorithm> +#include <array> +#include <chrono> +#include <cstdint> +#include <functional> +#include <map> +#include <vector> + + +static bool ReverseCompareNodeMinPingTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) +{ + return a.m_min_ping_time > b.m_min_ping_time; +} + +static bool ReverseCompareNodeTimeConnected(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) +{ + return a.m_connected > b.m_connected; +} + +static bool CompareNetGroupKeyed(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) { + return a.nKeyedNetGroup < b.nKeyedNetGroup; +} + +static bool CompareNodeBlockTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) +{ + // There is a fall-through here because it is common for a node to have many peers which have not yet relayed a block. + if (a.m_last_block_time != b.m_last_block_time) return a.m_last_block_time < b.m_last_block_time; + if (a.fRelevantServices != b.fRelevantServices) return b.fRelevantServices; + return a.m_connected > b.m_connected; +} + +static bool CompareNodeTXTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) +{ + // There is a fall-through here because it is common for a node to have more than a few peers that have not yet relayed txn. + if (a.m_last_tx_time != b.m_last_tx_time) return a.m_last_tx_time < b.m_last_tx_time; + if (a.m_relay_txs != b.m_relay_txs) return b.m_relay_txs; + if (a.fBloomFilter != b.fBloomFilter) return a.fBloomFilter; + return a.m_connected > b.m_connected; +} + +// Pick out the potential block-relay only peers, and sort them by last block time. +static bool CompareNodeBlockRelayOnlyTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) +{ + if (a.m_relay_txs != b.m_relay_txs) return a.m_relay_txs; + if (a.m_last_block_time != b.m_last_block_time) return a.m_last_block_time < b.m_last_block_time; + if (a.fRelevantServices != b.fRelevantServices) return b.fRelevantServices; + return a.m_connected > b.m_connected; +} + +/** + * Sort eviction candidates by network/localhost and connection uptime. + * Candidates near the beginning are more likely to be evicted, and those + * near the end are more likely to be protected, e.g. less likely to be evicted. + * - First, nodes that are not `is_local` and that do not belong to `network`, + * sorted by increasing uptime (from most recently connected to connected longer). + * - Then, nodes that are `is_local` or belong to `network`, sorted by increasing uptime. + */ +struct CompareNodeNetworkTime { + const bool m_is_local; + const Network m_network; + CompareNodeNetworkTime(bool is_local, Network network) : m_is_local(is_local), m_network(network) {} + bool operator()(const NodeEvictionCandidate& a, const NodeEvictionCandidate& b) const + { + if (m_is_local && a.m_is_local != b.m_is_local) return b.m_is_local; + if ((a.m_network == m_network) != (b.m_network == m_network)) return b.m_network == m_network; + return a.m_connected > b.m_connected; + }; +}; + +//! Sort an array by the specified comparator, then erase the last K elements where predicate is true. +template <typename T, typename Comparator> +static void EraseLastKElements( + std::vector<T>& elements, Comparator comparator, size_t k, + std::function<bool(const NodeEvictionCandidate&)> predicate = [](const NodeEvictionCandidate& n) { return true; }) +{ + std::sort(elements.begin(), elements.end(), comparator); + size_t eraseSize = std::min(k, elements.size()); + elements.erase(std::remove_if(elements.end() - eraseSize, elements.end(), predicate), elements.end()); +} + +void ProtectNoBanConnections(std::vector<NodeEvictionCandidate>& eviction_candidates) +{ + eviction_candidates.erase(std::remove_if(eviction_candidates.begin(), eviction_candidates.end(), + [](NodeEvictionCandidate const& n) { + return n.m_noban; + }), + eviction_candidates.end()); +} + +void ProtectOutboundConnections(std::vector<NodeEvictionCandidate>& eviction_candidates) +{ + eviction_candidates.erase(std::remove_if(eviction_candidates.begin(), eviction_candidates.end(), + [](NodeEvictionCandidate const& n) { + return n.m_conn_type != ConnectionType::INBOUND; + }), + eviction_candidates.end()); +} + +void ProtectEvictionCandidatesByRatio(std::vector<NodeEvictionCandidate>& eviction_candidates) +{ + // Protect the half of the remaining nodes which have been connected the longest. + // This replicates the non-eviction implicit behavior, and precludes attacks that start later. + // To favorise the diversity of our peer connections, reserve up to half of these protected + // spots for Tor/onion, localhost, I2P, and CJDNS peers, even if they're not longest uptime + // overall. This helps protect these higher-latency peers that tend to be otherwise + // disadvantaged under our eviction criteria. + const size_t initial_size = eviction_candidates.size(); + const size_t total_protect_size{initial_size / 2}; + + // Disadvantaged networks to protect. In the case of equal counts, earlier array members + // have the first opportunity to recover unused slots from the previous iteration. + struct Net { bool is_local; Network id; size_t count; }; + std::array<Net, 4> networks{ + {{false, NET_CJDNS, 0}, {false, NET_I2P, 0}, {/*localhost=*/true, NET_MAX, 0}, {false, NET_ONION, 0}}}; + + // Count and store the number of eviction candidates per network. + for (Net& n : networks) { + n.count = std::count_if(eviction_candidates.cbegin(), eviction_candidates.cend(), + [&n](const NodeEvictionCandidate& c) { + return n.is_local ? c.m_is_local : c.m_network == n.id; + }); + } + // Sort `networks` by ascending candidate count, to give networks having fewer candidates + // the first opportunity to recover unused protected slots from the previous iteration. + std::stable_sort(networks.begin(), networks.end(), [](Net a, Net b) { return a.count < b.count; }); + + // Protect up to 25% of the eviction candidates by disadvantaged network. + const size_t max_protect_by_network{total_protect_size / 2}; + size_t num_protected{0}; + + while (num_protected < max_protect_by_network) { + // Count the number of disadvantaged networks from which we have peers to protect. + auto num_networks = std::count_if(networks.begin(), networks.end(), [](const Net& n) { return n.count; }); + if (num_networks == 0) { + break; + } + const size_t disadvantaged_to_protect{max_protect_by_network - num_protected}; + const size_t protect_per_network{std::max(disadvantaged_to_protect / num_networks, static_cast<size_t>(1))}; + // Early exit flag if there are no remaining candidates by disadvantaged network. + bool protected_at_least_one{false}; + + for (Net& n : networks) { + if (n.count == 0) continue; + const size_t before = eviction_candidates.size(); + EraseLastKElements(eviction_candidates, CompareNodeNetworkTime(n.is_local, n.id), + protect_per_network, [&n](const NodeEvictionCandidate& c) { + return n.is_local ? c.m_is_local : c.m_network == n.id; + }); + const size_t after = eviction_candidates.size(); + if (before > after) { + protected_at_least_one = true; + const size_t delta{before - after}; + num_protected += delta; + if (num_protected >= max_protect_by_network) { + break; + } + n.count -= delta; + } + } + if (!protected_at_least_one) { + break; + } + } + + // Calculate how many we removed, and update our total number of peers that + // we want to protect based on uptime accordingly. + assert(num_protected == initial_size - eviction_candidates.size()); + const size_t remaining_to_protect{total_protect_size - num_protected}; + EraseLastKElements(eviction_candidates, ReverseCompareNodeTimeConnected, remaining_to_protect); +} + +[[nodiscard]] std::optional<NodeId> SelectNodeToEvict(std::vector<NodeEvictionCandidate>&& vEvictionCandidates) +{ + // Protect connections with certain characteristics + + ProtectNoBanConnections(vEvictionCandidates); + + ProtectOutboundConnections(vEvictionCandidates); + + // Deterministically select 4 peers to protect by netgroup. + // An attacker cannot predict which netgroups will be protected + EraseLastKElements(vEvictionCandidates, CompareNetGroupKeyed, 4); + // Protect the 8 nodes with the lowest minimum ping time. + // An attacker cannot manipulate this metric without physically moving nodes closer to the target. + EraseLastKElements(vEvictionCandidates, ReverseCompareNodeMinPingTime, 8); + // Protect 4 nodes that most recently sent us novel transactions accepted into our mempool. + // An attacker cannot manipulate this metric without performing useful work. + EraseLastKElements(vEvictionCandidates, CompareNodeTXTime, 4); + // Protect up to 8 non-tx-relay peers that have sent us novel blocks. + EraseLastKElements(vEvictionCandidates, CompareNodeBlockRelayOnlyTime, 8, + [](const NodeEvictionCandidate& n) { return !n.m_relay_txs && n.fRelevantServices; }); + + // Protect 4 nodes that most recently sent us novel blocks. + // An attacker cannot manipulate this metric without performing useful work. + EraseLastKElements(vEvictionCandidates, CompareNodeBlockTime, 4); + + // Protect some of the remaining eviction candidates by ratios of desirable + // or disadvantaged characteristics. + ProtectEvictionCandidatesByRatio(vEvictionCandidates); + + if (vEvictionCandidates.empty()) return std::nullopt; + + // If any remaining peers are preferred for eviction consider only them. + // This happens after the other preferences since if a peer is really the best by other criteria (esp relaying blocks) + // then we probably don't want to evict it no matter what. + if (std::any_of(vEvictionCandidates.begin(),vEvictionCandidates.end(),[](NodeEvictionCandidate const &n){return n.prefer_evict;})) { + vEvictionCandidates.erase(std::remove_if(vEvictionCandidates.begin(),vEvictionCandidates.end(), + [](NodeEvictionCandidate const &n){return !n.prefer_evict;}),vEvictionCandidates.end()); + } + + // Identify the network group with the most connections and youngest member. + // (vEvictionCandidates is already sorted by reverse connect time) + uint64_t naMostConnections; + unsigned int nMostConnections = 0; + std::chrono::seconds nMostConnectionsTime{0}; + std::map<uint64_t, std::vector<NodeEvictionCandidate> > mapNetGroupNodes; + for (const NodeEvictionCandidate &node : vEvictionCandidates) { + std::vector<NodeEvictionCandidate> &group = mapNetGroupNodes[node.nKeyedNetGroup]; + group.push_back(node); + const auto grouptime{group[0].m_connected}; + + if (group.size() > nMostConnections || (group.size() == nMostConnections && grouptime > nMostConnectionsTime)) { + nMostConnections = group.size(); + nMostConnectionsTime = grouptime; + naMostConnections = node.nKeyedNetGroup; + } + } + + // Reduce to the network group with the most connections + vEvictionCandidates = std::move(mapNetGroupNodes[naMostConnections]); + + // Disconnect from the network group with the most connections + return vEvictionCandidates.front().id; +} diff --git a/src/node/eviction.h b/src/node/eviction.h new file mode 100644 index 0000000000..1bb32e5327 --- /dev/null +++ b/src/node/eviction.h @@ -0,0 +1,69 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_EVICTION_H +#define BITCOIN_NODE_EVICTION_H + +#include <node/connection_types.h> +#include <net_permissions.h> + +#include <chrono> +#include <cstdint> +#include <optional> +#include <vector> + +typedef int64_t NodeId; + +struct NodeEvictionCandidate { + NodeId id; + std::chrono::seconds m_connected; + std::chrono::microseconds m_min_ping_time; + std::chrono::seconds m_last_block_time; + std::chrono::seconds m_last_tx_time; + bool fRelevantServices; + bool m_relay_txs; + bool fBloomFilter; + uint64_t nKeyedNetGroup; + bool prefer_evict; + bool m_is_local; + Network m_network; + bool m_noban; + ConnectionType m_conn_type; +}; + +/** + * Select an inbound peer to evict after filtering out (protecting) peers having + * distinct, difficult-to-forge characteristics. The protection logic picks out + * fixed numbers of desirable peers per various criteria, followed by (mostly) + * ratios of desirable or disadvantaged peers. If any eviction candidates + * remain, the selection logic chooses a peer to evict. + */ +[[nodiscard]] std::optional<NodeId> SelectNodeToEvict(std::vector<NodeEvictionCandidate>&& vEvictionCandidates); + +/** Protect desirable or disadvantaged inbound peers from eviction by ratio. + * + * This function protects half of the peers which have been connected the + * longest, to replicate the non-eviction implicit behavior and preclude attacks + * that start later. + * + * Half of these protected spots (1/4 of the total) are reserved for the + * following categories of peers, sorted by longest uptime, even if they're not + * longest uptime overall: + * + * - onion peers connected via our tor control service + * + * - localhost peers, as manually configured hidden services not using + * `-bind=addr[:port]=onion` will not be detected as inbound onion connections + * + * - I2P peers + * + * - CJDNS peers + * + * This helps protect these privacy network peers, which tend to be otherwise + * disadvantaged under our eviction criteria for their higher min ping times + * relative to IPv4/IPv6 peers, and favorise the diversity of peer connections. + */ +void ProtectEvictionCandidatesByRatio(std::vector<NodeEvictionCandidate>& vEvictionCandidates); + +#endif // BITCOIN_NODE_EVICTION_H diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 3c085ae6fb..56b5a928ad 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -19,6 +19,7 @@ #include <netaddress.h> #include <netbase.h> #include <node/blockstorage.h> +#include <kernel/chain.h> #include <node/coin.h> #include <node/context.h> #include <node/transaction.h> @@ -401,6 +402,7 @@ bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<Rec if (block.m_max_time) *block.m_max_time = index->GetBlockTimeMax(); if (block.m_mtp_time) *block.m_mtp_time = index->GetMedianTimePast(); if (block.m_in_active_chain) *block.m_in_active_chain = active[index->nHeight] == index; + if (block.m_locator) { *block.m_locator = active.GetLocator(index); } if (block.m_next_block) FillBlock(active[index->nHeight] == index ? active[index->nHeight + 1] : nullptr, *block.m_next_block, lock, active); if (block.m_data) { REVERSE_LOCK(lock); @@ -426,11 +428,11 @@ public: } void BlockConnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* index) override { - m_notifications->blockConnected(*block, index->nHeight); + m_notifications->blockConnected(kernel::MakeBlockInfo(index, block.get())); } void BlockDisconnected(const std::shared_ptr<const CBlock>& block, const CBlockIndex* index) override { - m_notifications->blockDisconnected(*block, index->nHeight); + m_notifications->blockDisconnected(kernel::MakeBlockInfo(index, block.get())); } void UpdatedBlockTip(const CBlockIndex* index, const CBlockIndex* fork_index, bool is_ibd) override { @@ -524,20 +526,27 @@ public: } bool haveBlockOnDisk(int height) override { - LOCK(cs_main); + LOCK(::cs_main); const CChain& active = Assert(m_node.chainman)->ActiveChain(); CBlockIndex* block = active[height]; return block && ((block->nStatus & BLOCK_HAVE_DATA) != 0) && block->nTx > 0; } CBlockLocator getTipLocator() override { - LOCK(cs_main); + LOCK(::cs_main); const CChain& active = Assert(m_node.chainman)->ActiveChain(); return active.GetLocator(); } + CBlockLocator getActiveChainLocator(const uint256& block_hash) override + { + LOCK(::cs_main); + const CBlockIndex* index = chainman().m_blockman.LookupBlockIndex(block_hash); + if (!index) return {}; + return chainman().ActiveChain().GetLocator(index); + } std::optional<int> findLocatorFork(const CBlockLocator& locator) override { - LOCK(cs_main); + LOCK(::cs_main); const CChainState& active = Assert(m_node.chainman)->ActiveChainstate(); if (const CBlockIndex* fork = active.FindForkInGlobalIndex(locator)) { return fork->nHeight; @@ -593,7 +602,7 @@ public: void findCoins(std::map<COutPoint, Coin>& coins) override { return FindCoins(m_node, coins); } double guessVerificationProgress(const uint256& block_hash) override { - LOCK(cs_main); + LOCK(::cs_main); return GuessVerificationProgress(chainman().GetParams().TxData(), chainman().m_blockman.LookupBlockIndex(block_hash)); } bool hasBlocks(const uint256& block_hash, int min_height, std::optional<int> max_height) override @@ -693,7 +702,7 @@ public: CFeeRate relayDustFee() override { return ::dustRelayFee; } bool havePruned() override { - LOCK(cs_main); + LOCK(::cs_main); return m_node.chainman->m_blockman.m_have_pruned; } bool isReadyToBroadcast() override { return !node::fImporting && !node::fReindex && !isInitialBlockDownload(); } @@ -768,6 +777,12 @@ public: notifications.transactionAddedToMempool(entry.GetSharedTx(), 0 /* mempool_sequence */); } } + bool hasAssumedValidChain() override + { + return Assert(m_node.chainman)->IsSnapshotActive(); + } + + NodeContext* context() override { return &m_node; } NodeContext& m_node; }; } // namespace diff --git a/src/node/mempool_persist_args.cpp b/src/node/mempool_persist_args.cpp new file mode 100644 index 0000000000..4e775869c6 --- /dev/null +++ b/src/node/mempool_persist_args.cpp @@ -0,0 +1,23 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/mempool_persist_args.h> + +#include <fs.h> +#include <util/system.h> +#include <validation.h> + +namespace node { + +bool ShouldPersistMempool(const ArgsManager& argsman) +{ + return argsman.GetBoolArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL); +} + +fs::path MempoolPath(const ArgsManager& argsman) +{ + return argsman.GetDataDirNet() / "mempool.dat"; +} + +} // namespace node diff --git a/src/node/mempool_persist_args.h b/src/node/mempool_persist_args.h new file mode 100644 index 0000000000..f719ec62ab --- /dev/null +++ b/src/node/mempool_persist_args.h @@ -0,0 +1,25 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_MEMPOOL_PERSIST_ARGS_H +#define BITCOIN_NODE_MEMPOOL_PERSIST_ARGS_H + +#include <fs.h> + +class ArgsManager; + +namespace node { + +/** + * Default for -persistmempool, indicating whether the node should attempt to + * automatically load the mempool on start and save to disk on shutdown + */ +static constexpr bool DEFAULT_PERSIST_MEMPOOL{true}; + +bool ShouldPersistMempool(const ArgsManager& argsman); +fs::path MempoolPath(const ArgsManager& argsman); + +} // namespace node + +#endif // BITCOIN_NODE_MEMPOOL_PERSIST_ARGS_H diff --git a/src/psbt.h b/src/psbt.h index a143a99988..c390bb67d3 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -866,6 +866,12 @@ struct PSBTOutput s_tree >> depth; s_tree >> leaf_ver; s_tree >> script; + if (depth > TAPROOT_CONTROL_MAX_NODE_COUNT) { + throw std::ios_base::failure("Output Taproot tree has as leaf greater than Taproot maximum depth"); + } + if ((leaf_ver & ~TAPROOT_LEAF_MASK) != 0) { + throw std::ios_base::failure("Output Taproot tree has a leaf with an invalid leaf version"); + } m_tap_tree->Add((int)depth, script, (int)leaf_ver, true /* track */); } if (!m_tap_tree->IsComplete()) { diff --git a/src/qt/addresstablemodel.cpp b/src/qt/addresstablemodel.cpp index 27ee9509e6..bb50b5384a 100644 --- a/src/qt/addresstablemodel.cpp +++ b/src/qt/addresstablemodel.cpp @@ -370,23 +370,21 @@ QString AddressTableModel::addRow(const QString &type, const QString &label, con else if(type == Receive) { // Generate a new address to associate with given label - CTxDestination dest; - if(!walletModel->wallet().getNewDestination(address_type, strLabel, dest)) - { + auto op_dest = walletModel->wallet().getNewDestination(address_type, strLabel); + if (!op_dest) { WalletModel::UnlockContext ctx(walletModel->requestUnlock()); - if(!ctx.isValid()) - { + if (!ctx.isValid()) { // Unlock wallet failed or was cancelled editStatus = WALLET_UNLOCK_FAILURE; return QString(); } - if(!walletModel->wallet().getNewDestination(address_type, strLabel, dest)) - { + op_dest = walletModel->wallet().getNewDestination(address_type, strLabel); + if (!op_dest) { editStatus = KEY_GENERATION_FAILURE; return QString(); } } - strAddress = EncodeDestination(dest); + strAddress = EncodeDestination(op_dest.GetObj()); } else { diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index 27d3a1b9e2..33c60deafb 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -77,8 +77,6 @@ Q_DECLARE_METATYPE(CAmount) Q_DECLARE_METATYPE(SynchronizationState) Q_DECLARE_METATYPE(uint256) -using node::NodeContext; - static void RegisterMetaTypes() { // Register meta types used for QMetaObject::invokeMethod and Qt::QueuedConnection diff --git a/src/qt/bitcoingui.cpp b/src/qt/bitcoingui.cpp index d65fc58865..baad0fdd91 100644 --- a/src/qt/bitcoingui.cpp +++ b/src/qt/bitcoingui.cpp @@ -47,6 +47,7 @@ #include <QCursor> #include <QDateTime> #include <QDragEnterEvent> +#include <QInputDialog> #include <QKeySequence> #include <QListWidget> #include <QMenu> @@ -348,6 +349,12 @@ void BitcoinGUI::createActions() m_create_wallet_action->setEnabled(false); m_create_wallet_action->setStatusTip(tr("Create a new wallet")); + //: Name of the menu item that restores wallet from a backup file. + m_restore_wallet_action = new QAction(tr("Restore Wallet…"), this); + m_restore_wallet_action->setEnabled(false); + //: Status tip for Restore Wallet menu item + m_restore_wallet_action->setStatusTip(tr("Restore a wallet from a backup file")); + m_close_all_wallets_action = new QAction(tr("Close All Wallets…"), this); m_close_all_wallets_action->setStatusTip(tr("Close all wallets")); @@ -412,6 +419,27 @@ void BitcoinGUI::createActions() action->setEnabled(false); } }); + connect(m_restore_wallet_action, &QAction::triggered, [this] { + //: Name of the wallet data file format. + QString name_data_file = tr("Wallet Data"); + + //: The title for Restore Wallet File Windows + QString title_windows = tr("Load Wallet Backup"); + + QString backup_file = GUIUtil::getOpenFileName(this, title_windows, QString(), name_data_file + QLatin1String(" (*.dat)"), nullptr); + if (backup_file.isEmpty()) return; + + bool wallet_name_ok; + //: Title of the Restore Wallet input dialog (where the wallet name is entered) + QString wallet_name = QInputDialog::getText(this, tr("Restore Name"), tr("Wallet Name:"), QLineEdit::Normal, "", &wallet_name_ok); + if (!wallet_name_ok || wallet_name.isEmpty()) return; + + auto activity = new RestoreWalletActivity(m_wallet_controller, this); + connect(activity, &RestoreWalletActivity::restored, this, &BitcoinGUI::setCurrentWallet, Qt::QueuedConnection); + + auto backup_file_path = fs::PathFromString(backup_file.toStdString()); + activity->restore(backup_file_path, wallet_name.toStdString()); + }); connect(m_close_wallet_action, &QAction::triggered, [this] { m_wallet_controller->closeWallet(walletFrame->currentWalletModel(), this); }); @@ -450,8 +478,10 @@ void BitcoinGUI::createMenuBar() file->addAction(m_close_wallet_action); file->addAction(m_close_all_wallets_action); file->addSeparator(); - file->addAction(openAction); file->addAction(backupWalletAction); + file->addAction(m_restore_wallet_action); + file->addSeparator(); + file->addAction(openAction); file->addAction(signMessageAction); file->addAction(verifyMessageAction); file->addAction(m_load_psbt_action); @@ -642,6 +672,7 @@ void BitcoinGUI::setWalletController(WalletController* wallet_controller) m_create_wallet_action->setEnabled(true); m_open_wallet_action->setEnabled(true); m_open_wallet_action->setMenu(m_open_wallet_menu); + m_restore_wallet_action->setEnabled(true); GUIUtil::ExceptionSafeConnect(wallet_controller, &WalletController::walletAdded, this, &BitcoinGUI::addWallet); connect(wallet_controller, &WalletController::walletRemoved, this, &BitcoinGUI::removeWallet); @@ -1319,6 +1350,12 @@ void BitcoinGUI::setEncryptionStatus(int status) { switch(status) { + case WalletModel::NoKeys: + labelWalletEncryptionIcon->hide(); + encryptWalletAction->setChecked(false); + changePassphraseAction->setEnabled(false); + encryptWalletAction->setEnabled(false); + break; case WalletModel::Unencrypted: labelWalletEncryptionIcon->hide(); encryptWalletAction->setChecked(false); diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 6272d5d947..b2e13245e1 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -157,6 +157,7 @@ private: QAction* m_create_wallet_action{nullptr}; QAction* m_open_wallet_action{nullptr}; QMenu* m_open_wallet_menu{nullptr}; + QAction* m_restore_wallet_action{nullptr}; QAction* m_close_wallet_action{nullptr}; QAction* m_close_all_wallets_action{nullptr}; QAction* m_wallet_selector_label_action = nullptr; diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 462b923d61..2b6711ca40 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -19,6 +19,7 @@ #include <validation.h> // for DEFAULT_SCRIPTCHECK_THREADS and MAX_SCRIPTCHECK_THREADS #include <netbase.h> #include <txdb.h> // for -dbcache defaults +#include <util/system.h> #include <chrono> @@ -286,11 +287,19 @@ void OptionsDialog::on_resetButton_clicked() { if (model) { // confirmation dialog + /*: Text explaining that the settings changed will not come into effect + until the client is restarted. */ + QString reset_dialog_text = tr("Client restart required to activate changes.") + "<br><br>"; + /*: Text explaining to the user that the client's current settings + will be backed up at a specific location. %1 is a stand-in + argument for the backup location's path. */ + reset_dialog_text.append(tr("Current settings will be backed up at \"%1\".").arg(m_client_model->dataDir()) + "<br><br>"); + /*: Text asking the user to confirm if they would like to proceed + with a client shutdown. */ + reset_dialog_text.append(tr("Client will be shut down. Do you want to proceed?")); + //: Window title text of pop-up window shown when the user has chosen to reset options. QMessageBox::StandardButton btnRetVal = QMessageBox::question(this, tr("Confirm options reset"), - tr("Client restart required to activate changes.") + "<br><br>" + - tr("Current settings will be backed up at \"%1\".").arg(m_client_model->dataDir()) + "<br><br>" + - tr("Client will be shut down. Do you want to proceed?"), - QMessageBox::Yes | QMessageBox::Cancel, QMessageBox::Cancel); + reset_dialog_text, QMessageBox::Yes | QMessageBox::Cancel, QMessageBox::Cancel); if (btnRetVal == QMessageBox::Cancel) return; diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index b791fd30c4..70fccdef1c 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -1162,7 +1162,6 @@ void RPCConsole::updateDetailWidget() if (!stats->nodeStats.addrLocal.empty()) peerAddrDetails += "<br />" + tr("via %1").arg(QString::fromStdString(stats->nodeStats.addrLocal)); ui->peerHeading->setText(peerAddrDetails); - ui->peerServices->setText(GUIUtil::formatServicesStr(stats->nodeStats.nServices)); QString bip152_hb_settings; if (stats->nodeStats.m_bip152_highbandwidth_to) bip152_hb_settings = ts.to; if (stats->nodeStats.m_bip152_highbandwidth_from) bip152_hb_settings += (bip152_hb_settings.isEmpty() ? ts.from : QLatin1Char('/') + ts.from); @@ -1197,6 +1196,7 @@ void RPCConsole::updateDetailWidget() // This check fails for example if the lock was busy and // nodeStateStats couldn't be fetched. if (stats->fNodeStateStatsAvailable) { + ui->peerServices->setText(GUIUtil::formatServicesStr(stats->nodeStateStats.their_services)); // Sync height is init to -1 if (stats->nodeStateStats.nSyncHeight > -1) { ui->peerSyncHeight->setText(QString("%1").arg(stats->nodeStateStats.nSyncHeight)); diff --git a/src/qt/test/test_main.cpp b/src/qt/test/test_main.cpp index de23f51c92..846fa519ee 100644 --- a/src/qt/test/test_main.cpp +++ b/src/qt/test/test_main.cpp @@ -43,8 +43,6 @@ Q_IMPORT_PLUGIN(QAndroidPlatformIntegrationPlugin) #endif #endif -using node::NodeContext; - const std::function<void(const std::string&)> G_TEST_LOG_FUN{}; const std::function<std::vector<const char*>()> G_TEST_COMMAND_LINE_ARGUMENTS{}; diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 7671bfc739..b8ae62ecab 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -173,7 +173,7 @@ void TestGUI(interfaces::Node& node) { WalletRescanReserver reserver(*wallet); reserver.reserve(); - CWallet::ScanResult result = wallet->ScanForWalletTransactions(Params().GetConsensus().hashGenesisBlock, 0 /* block height */, {} /* max height */, reserver, true /* fUpdate */); + CWallet::ScanResult result = wallet->ScanForWalletTransactions(Params().GetConsensus().hashGenesisBlock, /*start_height=*/0, /*max_height=*/{}, reserver, /*fUpdate=*/true, /*save_progress=*/false); QCOMPARE(result.status, CWallet::ScanResult::SUCCESS); QCOMPARE(result.last_scanned_block, node.context()->chainman->ActiveChain().Tip()->GetBlockHash()); QVERIFY(result.last_failed_block.IsNull()); diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index d27ddf1aba..01d84624e8 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -373,3 +373,47 @@ void LoadWalletsActivity::load() QTimer::singleShot(0, this, [this] { Q_EMIT finished(); }); }); } + +RestoreWalletActivity::RestoreWalletActivity(WalletController* wallet_controller, QWidget* parent_widget) + : WalletControllerActivity(wallet_controller, parent_widget) +{ +} + +void RestoreWalletActivity::restore(const fs::path& backup_file, const std::string& wallet_name) +{ + QString name = QString::fromStdString(wallet_name); + + showProgressDialog( + //: Title of progress window which is displayed when wallets are being restored. + tr("Restore Wallet"), + /*: Descriptive text of the restore wallets progress window which indicates to + the user that wallets are currently being restored.*/ + tr("Restoring Wallet <b>%1</b>…").arg(name.toHtmlEscaped())); + + QTimer::singleShot(0, worker(), [this, backup_file, wallet_name] { + auto wallet{node().walletLoader().restoreWallet(backup_file, wallet_name, m_warning_message)}; + + m_error_message = wallet ? bilingual_str{} : wallet.GetError(); + if (wallet) m_wallet_model = m_wallet_controller->getOrCreateWallet(wallet.ReleaseObj()); + + QTimer::singleShot(0, this, &RestoreWalletActivity::finish); + }); +} + +void RestoreWalletActivity::finish() +{ + if (!m_error_message.empty()) { + //: Title of message box which is displayed when the wallet could not be restored. + QMessageBox::critical(m_parent_widget, tr("Restore wallet failed"), QString::fromStdString(m_error_message.translated)); + } else if (!m_warning_message.empty()) { + //: Title of message box which is displayed when the wallet is restored with some warning. + QMessageBox::warning(m_parent_widget, tr("Restore wallet warning"), QString::fromStdString(Join(m_warning_message, Untranslated("\n")).translated)); + } else { + //: Title of message box which is displayed when the wallet is successfully restored. + QMessageBox::information(m_parent_widget, tr("Restore wallet message"), QString::fromStdString(Untranslated("Wallet restored successfully \n").translated)); + } + + if (m_wallet_model) Q_EMIT restored(m_wallet_model); + + Q_EMIT finished(); +} diff --git a/src/qt/walletcontroller.h b/src/qt/walletcontroller.h index 24f85c258c..fcd65756c6 100644 --- a/src/qt/walletcontroller.h +++ b/src/qt/walletcontroller.h @@ -33,6 +33,10 @@ class Node; class Wallet; } // namespace interfaces +namespace fs { +class path; +} + class AskPassphraseDialog; class CreateWalletActivity; class CreateWalletDialog; @@ -155,4 +159,20 @@ public: void load(); }; +class RestoreWalletActivity : public WalletControllerActivity +{ + Q_OBJECT + +public: + RestoreWalletActivity(WalletController* wallet_controller, QWidget* parent_widget); + + void restore(const fs::path& backup_file, const std::string& wallet_name); + +Q_SIGNALS: + void restored(WalletModel* wallet_model); + +private: + void finish(); +}; + #endif // BITCOIN_QT_WALLETCONTROLLER_H diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index 6b38e207d3..8dc97e66a2 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -213,6 +213,14 @@ void WalletFrame::gotoLoadPSBT(bool from_clipboard) } std::ifstream in{filename.toLocal8Bit().data(), std::ios::binary}; data.assign(std::istream_iterator<unsigned char>{in}, {}); + + // Some psbt files may be base64 strings in the file rather than binary data + std::string b64_str{data.begin(), data.end()}; + b64_str.erase(b64_str.find_last_not_of(" \t\n\r\f\v") + 1); // Trim trailing whitespace + auto b64_dec = DecodeBase64(b64_str); + if (b64_dec.has_value()) { + data = b64_dec.value(); + } } std::string error; diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index ab4d1cae3f..fde136b727 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -204,10 +204,10 @@ WalletModel::SendCoinsReturn WalletModel::prepareTransaction(WalletModelTransact { CAmount nFeeRequired = 0; int nChangePosRet = -1; - bilingual_str error; auto& newTx = transaction.getWtx(); - newTx = m_wallet->createTransaction(vecSend, coinControl, !wallet().privateKeysDisabled() /* sign */, nChangePosRet, nFeeRequired, error); + const auto& res = m_wallet->createTransaction(vecSend, coinControl, !wallet().privateKeysDisabled() /* sign */, nChangePosRet, nFeeRequired); + newTx = res ? res.GetObj() : nullptr; transaction.setTransactionFee(nFeeRequired); if (fSubtractFeeFromAmount && newTx) transaction.reassignAmounts(nChangePosRet); @@ -218,7 +218,7 @@ WalletModel::SendCoinsReturn WalletModel::prepareTransaction(WalletModelTransact { return SendCoinsReturn(AmountWithFeeExceedsBalance); } - Q_EMIT message(tr("Send Coins"), QString::fromStdString(error.translated), + Q_EMIT message(tr("Send Coins"), QString::fromStdString(res.GetError().translated), CClientUIInterface::MSG_ERROR); return TransactionCreationFailed; } @@ -306,6 +306,11 @@ WalletModel::EncryptionStatus WalletModel::getEncryptionStatus() const { if(!m_wallet->isCrypted()) { + // A previous bug allowed for watchonly wallets to be encrypted (encryption keys set, but nothing is actually encrypted). + // To avoid misrepresenting the encryption status of such wallets, we only return NoKeys for watchonly wallets that are unencrypted. + if (m_wallet->privateKeysDisabled()) { + return NoKeys; + } return Unencrypted; } else if(m_wallet->isLocked()) diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index a52290dee8..0184fb8ec2 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -71,6 +71,7 @@ public: enum EncryptionStatus { + NoKeys, // wallet->IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS) Unencrypted, // !wallet->IsCrypted() Locked, // wallet->IsCrypted() && wallet->IsLocked() Unlocked // wallet->IsCrypted() && !wallet->IsLocked() diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index c1b4e863b5..1635851afb 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -39,6 +39,7 @@ #include <univalue.h> #include <util/check.h> #include <util/strencodings.h> +#include <util/system.h> #include <util/translation.h> #include <validation.h> #include <validationinterface.h> @@ -440,6 +441,11 @@ static RPCHelpMan getblockfrompeer() }, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { + RPCTypeCheck(request.params, { + UniValue::VSTR, // blockhash + UniValue::VNUM, // peer_id + }); + const NodeContext& node = EnsureAnyNodeContext(request.context); ChainstateManager& chainman = EnsureChainman(node); PeerManager& peerman = EnsurePeerman(node); @@ -598,6 +604,29 @@ static CBlockUndo GetUndoChecked(BlockManager& blockman, const CBlockIndex* pblo return blockUndo; } +const RPCResult getblock_vin{ + RPCResult::Type::ARR, "vin", "", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::ELISION, "", "The same output as verbosity = 2"}, + {RPCResult::Type::OBJ, "prevout", "(Only if undo information is available)", + { + {RPCResult::Type::BOOL, "generated", "Coinbase or not"}, + {RPCResult::Type::NUM, "height", "The height of the prevout"}, + {RPCResult::Type::NUM, "value", "The value in " + CURRENCY_UNIT}, + {RPCResult::Type::OBJ, "scriptPubKey", "", + { + {RPCResult::Type::STR, "asm", "The asm"}, + {RPCResult::Type::STR_HEX, "hex", "The hex"}, + {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, + {RPCResult::Type::STR, "type", "The type (one of: " + GetAllOutputTypes() + ")"}, + }}, + }}, + }}, + } +}; + static RPCHelpMan getblock() { return RPCHelpMan{"getblock", @@ -657,26 +686,7 @@ static RPCHelpMan getblock() { {RPCResult::Type::OBJ, "", "", { - {RPCResult::Type::ARR, "vin", "", - { - {RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::ELISION, "", "The same output as verbosity = 2"}, - {RPCResult::Type::OBJ, "prevout", "(Only if undo information is available)", - { - {RPCResult::Type::BOOL, "generated", "Coinbase or not"}, - {RPCResult::Type::NUM, "height", "The height of the prevout"}, - {RPCResult::Type::NUM, "value", "The value in " + CURRENCY_UNIT}, - {RPCResult::Type::OBJ, "scriptPubKey", "", - { - {RPCResult::Type::STR, "asm", "The asm"}, - {RPCResult::Type::STR, "hex", "The hex"}, - {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, - {RPCResult::Type::STR, "type", "The type (one of: " + GetAllOutputTypes() + ")"}, - }}, - }}, - }}, - }}, + getblock_vin, }}, }}, }}, @@ -845,7 +855,7 @@ static RPCHelpMan gettxoutsetinfo() "Note this call may take some time if you are not using coinstatsindex.\n", { {"hash_type", RPCArg::Type::STR, RPCArg::Default{"hash_serialized_2"}, "Which UTXO set hash should be calculated. Options: 'hash_serialized_2' (the legacy algorithm), 'muhash', 'none'."}, - {"hash_or_height", RPCArg::Type::NUM, RPCArg::Optional::OMITTED_NAMED_ARG, "The block hash or height of the target height (only available with coinstatsindex).", "", {"", "string or numeric"}}, + {"hash_or_height", RPCArg::Type::NUM, RPCArg::DefaultHint{"the current best block"}, "The block hash or height of the target height (only available with coinstatsindex).", "", {"", "string or numeric"}}, {"use_index", RPCArg::Type::BOOL, RPCArg::Default{true}, "Use coinstatsindex, if available."}, }, RPCResult{ @@ -881,6 +891,7 @@ static RPCHelpMan gettxoutsetinfo() HelpExampleCli("gettxoutsetinfo", R"("none")") + HelpExampleCli("gettxoutsetinfo", R"("none" 1000)") + HelpExampleCli("gettxoutsetinfo", R"("none" '"00000000c937983704a73af28acdec37b049d214adbda81d7e2a3dd146f6ed09"')") + + HelpExampleCli("-named gettxoutsetinfo", R"(hash_type='muhash' use_index='false')") + HelpExampleRpc("gettxoutsetinfo", "") + HelpExampleRpc("gettxoutsetinfo", R"("none")") + HelpExampleRpc("gettxoutsetinfo", R"("none", 1000)") + @@ -917,6 +928,9 @@ static RPCHelpMan gettxoutsetinfo() throw JSONRPCError(RPC_INVALID_PARAMETER, "hash_serialized_2 hash type cannot be queried for a specific block"); } + if (!index_requested) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Cannot set use_index to false when querying for a specific block"); + } pindex = ParseHashOrHeight(request.params[1], chainman); } @@ -1001,9 +1015,9 @@ static RPCHelpMan gettxout() {RPCResult::Type::NUM, "confirmations", "The number of confirmations"}, {RPCResult::Type::STR_AMOUNT, "value", "The transaction value in " + CURRENCY_UNIT}, {RPCResult::Type::OBJ, "scriptPubKey", "", { - {RPCResult::Type::STR, "asm", ""}, + {RPCResult::Type::STR, "asm", "The asm"}, {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, - {RPCResult::Type::STR_HEX, "hex", ""}, + {RPCResult::Type::STR_HEX, "hex", "The hex"}, {RPCResult::Type::STR, "type", "The type, eg pubkeyhash"}, {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, }}, diff --git a/src/rpc/client.cpp b/src/rpc/client.cpp index ae0d0112ba..9be3ab7df0 100644 --- a/src/rpc/client.cpp +++ b/src/rpc/client.cpp @@ -110,6 +110,7 @@ static const CRPCConvertParam vRPCConvertParams[] = { "sendrawtransaction", 1, "maxfeerate" }, { "testmempoolaccept", 0, "rawtxs" }, { "testmempoolaccept", 1, "maxfeerate" }, + { "submitpackage", 0, "package" }, { "combinerawtransaction", 0, "txs" }, { "fundrawtransaction", 1, "options" }, { "fundrawtransaction", 2, "iswitness" }, diff --git a/src/rpc/fees.cpp b/src/rpc/fees.cpp index dd1a6441a0..41f386d443 100644 --- a/src/rpc/fees.cpp +++ b/src/rpc/fees.cpp @@ -6,7 +6,6 @@ #include <core_io.h> #include <policy/feerate.h> #include <policy/fees.h> -#include <policy/policy.h> #include <policy/settings.h> #include <rpc/protocol.h> #include <rpc/request.h> @@ -16,7 +15,6 @@ #include <txmempool.h> #include <univalue.h> #include <util/fees.h> -#include <util/system.h> #include <algorithm> #include <array> diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 0d614d5ec0..0ae10b6c39 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -5,8 +5,12 @@ #include <rpc/blockchain.h> +#include <kernel/mempool_persist.h> + +#include <chainparams.h> #include <core_io.h> #include <fs.h> +#include <node/mempool_persist_args.h> #include <policy/rbf.h> #include <policy/settings.h> #include <primitives/transaction.h> @@ -16,8 +20,12 @@ #include <txmempool.h> #include <univalue.h> #include <util/moneystr.h> +#include <util/time.h> + +using kernel::DumpMempool; using node::DEFAULT_MAX_RAW_TX_FEE_RATE; +using node::MempoolPath; using node::NodeContext; static RPCHelpMan sendrawtransaction() @@ -652,7 +660,7 @@ UniValue MempoolInfoToJSON(const CTxMemPool& pool) // Make sure this call is atomic in the pool. LOCK(pool.cs); UniValue ret(UniValue::VOBJ); - ret.pushKV("loaded", pool.IsLoaded()); + ret.pushKV("loaded", pool.GetLoadTried()); ret.pushKV("size", (int64_t)pool.size()); ret.pushKV("bytes", (int64_t)pool.GetTotalTxSize()); ret.pushKV("usage", (int64_t)pool.DynamicMemoryUsage()); @@ -662,6 +670,7 @@ UniValue MempoolInfoToJSON(const CTxMemPool& pool) ret.pushKV("minrelaytxfee", ValueFromAmount(::minRelayTxFee.GetFeePerK())); ret.pushKV("incrementalrelayfee", ValueFromAmount(::incrementalRelayFee.GetFeePerK())); ret.pushKV("unbroadcastcount", uint64_t{pool.GetUnbroadcastTxs().size()}); + ret.pushKV("fullrbf", pool.m_full_rbf); return ret; } @@ -683,6 +692,7 @@ static RPCHelpMan getmempoolinfo() {RPCResult::Type::STR_AMOUNT, "minrelaytxfee", "Current minimum relay fee for transactions"}, {RPCResult::Type::NUM, "incrementalrelayfee", "minimum fee rate increment for mempool limiting or BIP 125 replacement in " + CURRENCY_UNIT + "/kvB"}, {RPCResult::Type::NUM, "unbroadcastcount", "Current number of transactions that haven't passed initial broadcast yet"}, + {RPCResult::Type::BOOL, "fullrbf", "True if the mempool accepts RBF without replaceability signaling inspection"}, }}, RPCExamples{ HelpExampleCli("getmempoolinfo", "") @@ -714,22 +724,168 @@ static RPCHelpMan savemempool() const ArgsManager& args{EnsureAnyArgsman(request.context)}; const CTxMemPool& mempool = EnsureAnyMemPool(request.context); - if (!mempool.IsLoaded()) { + if (!mempool.GetLoadTried()) { throw JSONRPCError(RPC_MISC_ERROR, "The mempool was not loaded yet"); } - if (!DumpMempool(mempool)) { + const fs::path& dump_path = MempoolPath(args); + + if (!DumpMempool(mempool, dump_path)) { throw JSONRPCError(RPC_MISC_ERROR, "Unable to dump mempool to disk"); } UniValue ret(UniValue::VOBJ); - ret.pushKV("filename", fs::path((args.GetDataDirNet() / "mempool.dat")).u8string()); + ret.pushKV("filename", dump_path.u8string()); return ret; }, }; } +static RPCHelpMan submitpackage() +{ + return RPCHelpMan{"submitpackage", + "Submit a package of raw transactions (serialized, hex-encoded) to local node (-regtest only).\n" + "The package will be validated according to consensus and mempool policy rules. If all transactions pass, they will be accepted to mempool.\n" + "This RPC is experimental and the interface may be unstable. Refer to doc/policy/packages.md for documentation on package policies.\n" + "Warning: until package relay is in use, successful submission does not mean the transaction will propagate to other nodes on the network.\n" + "Currently, each transaction is broadcasted individually after submission, which means they must meet other nodes' feerate requirements alone.\n" + , + { + {"package", RPCArg::Type::ARR, RPCArg::Optional::NO, "An array of raw transactions.", + { + {"rawtx", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, ""}, + }, + }, + }, + RPCResult{ + RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::OBJ_DYN, "tx-results", "transaction results keyed by wtxid", + { + {RPCResult::Type::OBJ, "wtxid", "transaction wtxid", { + {RPCResult::Type::STR_HEX, "txid", "The transaction hash in hex"}, + {RPCResult::Type::STR_HEX, "other-wtxid", /*optional=*/true, "The wtxid of a different transaction with the same txid but different witness found in the mempool. This means the submitted transaction was ignored."}, + {RPCResult::Type::NUM, "vsize", "Virtual transaction size as defined in BIP 141."}, + {RPCResult::Type::OBJ, "fees", "Transaction fees", { + {RPCResult::Type::STR_AMOUNT, "base", "transaction fee in " + CURRENCY_UNIT}, + }}, + }} + }}, + {RPCResult::Type::STR_AMOUNT, "package-feerate", /*optional=*/true, "package feerate used for feerate checks in " + CURRENCY_UNIT + " per KvB. Excludes transactions which were deduplicated or accepted individually."}, + {RPCResult::Type::ARR, "replaced-transactions", /*optional=*/true, "List of txids of replaced transactions", + { + {RPCResult::Type::STR_HEX, "", "The transaction id"}, + }}, + }, + }, + RPCExamples{ + HelpExampleCli("testmempoolaccept", "[rawtx1, rawtx2]") + + HelpExampleCli("submitpackage", "[rawtx1, rawtx2]") + }, + [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue + { + if (!Params().IsMockableChain()) { + throw std::runtime_error("submitpackage is for regression testing (-regtest mode) only"); + } + RPCTypeCheck(request.params, { + UniValue::VARR, + }); + const UniValue raw_transactions = request.params[0].get_array(); + if (raw_transactions.size() < 1 || raw_transactions.size() > MAX_PACKAGE_COUNT) { + throw JSONRPCError(RPC_INVALID_PARAMETER, + "Array must contain between 1 and " + ToString(MAX_PACKAGE_COUNT) + " transactions."); + } + + std::vector<CTransactionRef> txns; + txns.reserve(raw_transactions.size()); + for (const auto& rawtx : raw_transactions.getValues()) { + CMutableTransaction mtx; + if (!DecodeHexTx(mtx, rawtx.get_str())) { + throw JSONRPCError(RPC_DESERIALIZATION_ERROR, + "TX decode failed: " + rawtx.get_str() + " Make sure the tx has at least one input."); + } + txns.emplace_back(MakeTransactionRef(std::move(mtx))); + } + + NodeContext& node = EnsureAnyNodeContext(request.context); + CTxMemPool& mempool = EnsureMemPool(node); + CChainState& chainstate = EnsureChainman(node).ActiveChainstate(); + const auto package_result = WITH_LOCK(::cs_main, return ProcessNewPackage(chainstate, mempool, txns, /*test_accept=*/ false)); + + // First catch any errors. + switch(package_result.m_state.GetResult()) { + case PackageValidationResult::PCKG_RESULT_UNSET: break; + case PackageValidationResult::PCKG_POLICY: + { + throw JSONRPCTransactionError(TransactionError::INVALID_PACKAGE, + package_result.m_state.GetRejectReason()); + } + case PackageValidationResult::PCKG_MEMPOOL_ERROR: + { + throw JSONRPCTransactionError(TransactionError::MEMPOOL_ERROR, + package_result.m_state.GetRejectReason()); + } + case PackageValidationResult::PCKG_TX: + { + for (const auto& tx : txns) { + auto it = package_result.m_tx_results.find(tx->GetWitnessHash()); + if (it != package_result.m_tx_results.end() && it->second.m_state.IsInvalid()) { + throw JSONRPCTransactionError(TransactionError::MEMPOOL_REJECTED, + strprintf("%s failed: %s", tx->GetHash().ToString(), it->second.m_state.GetRejectReason())); + } + } + // If a PCKG_TX error was returned, there must have been an invalid transaction. + NONFATAL_UNREACHABLE(); + } + } + for (const auto& tx : txns) { + size_t num_submitted{0}; + std::string err_string; + const auto err = BroadcastTransaction(node, tx, err_string, 0, true, true); + if (err != TransactionError::OK) { + throw JSONRPCTransactionError(err, + strprintf("transaction broadcast failed: %s (all transactions were submitted, %d transactions were broadcast successfully)", + err_string, num_submitted)); + } + } + UniValue rpc_result{UniValue::VOBJ}; + UniValue tx_result_map{UniValue::VOBJ}; + std::set<uint256> replaced_txids; + for (const auto& tx : txns) { + auto it = package_result.m_tx_results.find(tx->GetWitnessHash()); + CHECK_NONFATAL(it != package_result.m_tx_results.end()); + UniValue result_inner{UniValue::VOBJ}; + result_inner.pushKV("txid", tx->GetHash().GetHex()); + if (it->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS) { + result_inner.pushKV("other-wtxid", it->second.m_other_wtxid.value().GetHex()); + } + if (it->second.m_result_type == MempoolAcceptResult::ResultType::VALID || + it->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY) { + result_inner.pushKV("vsize", int64_t{it->second.m_vsize.value()}); + UniValue fees(UniValue::VOBJ); + fees.pushKV("base", ValueFromAmount(it->second.m_base_fees.value())); + result_inner.pushKV("fees", fees); + if (it->second.m_replaced_transactions.has_value()) { + for (const auto& ptx : it->second.m_replaced_transactions.value()) { + replaced_txids.insert(ptx->GetHash()); + } + } + } + tx_result_map.pushKV(tx->GetWitnessHash().GetHex(), result_inner); + } + rpc_result.pushKV("tx-results", tx_result_map); + if (package_result.m_package_feerate.has_value()) { + rpc_result.pushKV("package-feerate", ValueFromAmount(package_result.m_package_feerate.value().GetFeePerK())); + } + UniValue replaced_list(UniValue::VARR); + for (const uint256& hash : replaced_txids) replaced_list.push_back(hash.ToString()); + rpc_result.pushKV("replaced-transactions", replaced_list); + return rpc_result; + }, + }; +} + void RegisterMempoolRPCCommands(CRPCTable& t) { static const CRPCCommand commands[]{ @@ -742,6 +898,7 @@ void RegisterMempoolRPCCommands(CRPCTable& t) {"blockchain", &getmempoolinfo}, {"blockchain", &getrawmempool}, {"blockchain", &savemempool}, + {"hidden", &submitpackage}, }; for (const auto& c : commands) { t.appendCommand(c.name, &c); diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index fad92629c5..27eea824bc 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -195,8 +195,9 @@ static RPCHelpMan getpeerinfo() if (stats.m_mapped_as != 0) { obj.pushKV("mapped_as", uint64_t(stats.m_mapped_as)); } - obj.pushKV("services", strprintf("%016x", stats.nServices)); - obj.pushKV("servicesnames", GetServicesNames(stats.nServices)); + ServiceFlags services{fStateStats ? statestats.their_services : ServiceFlags::NODE_NONE}; + obj.pushKV("services", strprintf("%016x", services)); + obj.pushKV("servicesnames", GetServicesNames(services)); obj.pushKV("lastsend", count_seconds(stats.m_last_send)); obj.pushKV("lastrecv", count_seconds(stats.m_last_recv)); obj.pushKV("last_transaction", count_seconds(stats.m_last_tx_time)); diff --git a/src/rpc/output_script.cpp b/src/rpc/output_script.cpp index f4bb76f50f..744f809814 100644 --- a/src/rpc/output_script.cpp +++ b/src/rpc/output_script.cpp @@ -26,11 +26,6 @@ #include <tuple> #include <vector> -namespace node { -struct NodeContext; -} -using node::NodeContext; - static RPCHelpMan validateaddress() { return RPCHelpMan{ diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 792a1e13b0..16105a85d5 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -46,12 +46,10 @@ #include <univalue.h> using node::AnalyzePSBT; -using node::BroadcastTransaction; using node::FindCoins; using node::GetTransaction; using node::NodeContext; using node::PSBTAnalysis; -using node::ReadBlockFromDisk; static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& entry, CChainState& active_chainstate) { @@ -680,6 +678,200 @@ static RPCHelpMan signrawtransactionwithkey() }; } +const RPCResult decodepsbt_inputs{ + RPCResult::Type::ARR, "inputs", "", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::OBJ, "non_witness_utxo", /*optional=*/true, "Decoded network transaction for non-witness UTXOs", + { + {RPCResult::Type::ELISION, "",""}, + }}, + {RPCResult::Type::OBJ, "witness_utxo", /*optional=*/true, "Transaction output for witness UTXOs", + { + {RPCResult::Type::NUM, "amount", "The value in " + CURRENCY_UNIT}, + {RPCResult::Type::OBJ, "scriptPubKey", "", + { + {RPCResult::Type::STR, "asm", "The asm"}, + {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, + {RPCResult::Type::STR_HEX, "hex", "The hex"}, + {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, + {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, + }}, + }}, + {RPCResult::Type::OBJ_DYN, "partial_signatures", /*optional=*/true, "", + { + {RPCResult::Type::STR, "pubkey", "The public key and signature that corresponds to it."}, + }}, + {RPCResult::Type::STR, "sighash", /*optional=*/true, "The sighash type to be used"}, + {RPCResult::Type::OBJ, "redeem_script", /*optional=*/true, "", + { + {RPCResult::Type::STR, "asm", "The asm"}, + {RPCResult::Type::STR_HEX, "hex", "The hex"}, + {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, + }}, + {RPCResult::Type::OBJ, "witness_script", /*optional=*/true, "", + { + {RPCResult::Type::STR, "asm", "The asm"}, + {RPCResult::Type::STR_HEX, "hex", "The hex"}, + {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, + }}, + {RPCResult::Type::ARR, "bip32_derivs", /*optional=*/true, "", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR, "pubkey", "The public key with the derivation path as the value."}, + {RPCResult::Type::STR, "master_fingerprint", "The fingerprint of the master key"}, + {RPCResult::Type::STR, "path", "The path"}, + }}, + }}, + {RPCResult::Type::OBJ, "final_scriptSig", /*optional=*/true, "", + { + {RPCResult::Type::STR, "asm", "The asm"}, + {RPCResult::Type::STR_HEX, "hex", "The hex"}, + }}, + {RPCResult::Type::ARR, "final_scriptwitness", /*optional=*/true, "", + { + {RPCResult::Type::STR_HEX, "", "hex-encoded witness data (if any)"}, + }}, + {RPCResult::Type::OBJ_DYN, "ripemd160_preimages", /*optional=*/ true, "", + { + {RPCResult::Type::STR, "hash", "The hash and preimage that corresponds to it."}, + }}, + {RPCResult::Type::OBJ_DYN, "sha256_preimages", /*optional=*/ true, "", + { + {RPCResult::Type::STR, "hash", "The hash and preimage that corresponds to it."}, + }}, + {RPCResult::Type::OBJ_DYN, "hash160_preimages", /*optional=*/ true, "", + { + {RPCResult::Type::STR, "hash", "The hash and preimage that corresponds to it."}, + }}, + {RPCResult::Type::OBJ_DYN, "hash256_preimages", /*optional=*/ true, "", + { + {RPCResult::Type::STR, "hash", "The hash and preimage that corresponds to it."}, + }}, + {RPCResult::Type::STR_HEX, "taproot_key_path_sig", /*optional=*/ true, "hex-encoded signature for the Taproot key path spend"}, + {RPCResult::Type::ARR, "taproot_script_path_sigs", /*optional=*/ true, "", + { + {RPCResult::Type::OBJ, "signature", /*optional=*/ true, "The signature for the pubkey and leaf hash combination", + { + {RPCResult::Type::STR, "pubkey", "The x-only pubkey for this signature"}, + {RPCResult::Type::STR, "leaf_hash", "The leaf hash for this signature"}, + {RPCResult::Type::STR, "sig", "The signature itself"}, + }}, + }}, + {RPCResult::Type::ARR, "taproot_scripts", /*optional=*/ true, "", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR_HEX, "script", "A leaf script"}, + {RPCResult::Type::NUM, "leaf_ver", "The version number for the leaf script"}, + {RPCResult::Type::ARR, "control_blocks", "The control blocks for this script", + { + {RPCResult::Type::STR_HEX, "control_block", "A hex-encoded control block for this script"}, + }}, + }}, + }}, + {RPCResult::Type::ARR, "taproot_bip32_derivs", /*optional=*/ true, "", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR, "pubkey", "The x-only public key this path corresponds to"}, + {RPCResult::Type::STR, "master_fingerprint", "The fingerprint of the master key"}, + {RPCResult::Type::STR, "path", "The path"}, + {RPCResult::Type::ARR, "leaf_hashes", "The hashes of the leaves this pubkey appears in", + { + {RPCResult::Type::STR_HEX, "hash", "The hash of a leaf this pubkey appears in"}, + }}, + }}, + }}, + {RPCResult::Type::STR_HEX, "taproot_internal_key", /*optional=*/ true, "The hex-encoded Taproot x-only internal key"}, + {RPCResult::Type::STR_HEX, "taproot_merkle_root", /*optional=*/ true, "The hex-encoded Taproot merkle root"}, + {RPCResult::Type::OBJ_DYN, "unknown", /*optional=*/ true, "The unknown input fields", + { + {RPCResult::Type::STR_HEX, "key", "(key-value pair) An unknown key-value pair"}, + }}, + {RPCResult::Type::ARR, "proprietary", /*optional=*/true, "The input proprietary map", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR_HEX, "identifier", "The hex string for the proprietary identifier"}, + {RPCResult::Type::NUM, "subtype", "The number for the subtype"}, + {RPCResult::Type::STR_HEX, "key", "The hex for the key"}, + {RPCResult::Type::STR_HEX, "value", "The hex for the value"}, + }}, + }}, + }}, + } +}; + +const RPCResult decodepsbt_outputs{ + RPCResult::Type::ARR, "outputs", "", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::OBJ, "redeem_script", /*optional=*/true, "", + { + {RPCResult::Type::STR, "asm", "The asm"}, + {RPCResult::Type::STR_HEX, "hex", "The hex"}, + {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, + }}, + {RPCResult::Type::OBJ, "witness_script", /*optional=*/true, "", + { + {RPCResult::Type::STR, "asm", "The asm"}, + {RPCResult::Type::STR_HEX, "hex", "The hex"}, + {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, + }}, + {RPCResult::Type::ARR, "bip32_derivs", /*optional=*/true, "", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR, "pubkey", "The public key this path corresponds to"}, + {RPCResult::Type::STR, "master_fingerprint", "The fingerprint of the master key"}, + {RPCResult::Type::STR, "path", "The path"}, + }}, + }}, + {RPCResult::Type::STR_HEX, "taproot_internal_key", /*optional=*/ true, "The hex-encoded Taproot x-only internal key"}, + {RPCResult::Type::ARR, "taproot_tree", /*optional=*/ true, "The tuples that make up the Taproot tree, in depth first search order", + { + {RPCResult::Type::OBJ, "tuple", /*optional=*/ true, "A single leaf script in the taproot tree", + { + {RPCResult::Type::NUM, "depth", "The depth of this element in the tree"}, + {RPCResult::Type::NUM, "leaf_ver", "The version of this leaf"}, + {RPCResult::Type::STR, "script", "The hex-encoded script itself"}, + }}, + }}, + {RPCResult::Type::ARR, "taproot_bip32_derivs", /*optional=*/ true, "", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR, "pubkey", "The x-only public key this path corresponds to"}, + {RPCResult::Type::STR, "master_fingerprint", "The fingerprint of the master key"}, + {RPCResult::Type::STR, "path", "The path"}, + {RPCResult::Type::ARR, "leaf_hashes", "The hashes of the leaves this pubkey appears in", + { + {RPCResult::Type::STR_HEX, "hash", "The hash of a leaf this pubkey appears in"}, + }}, + }}, + }}, + {RPCResult::Type::OBJ_DYN, "unknown", /*optional=*/true, "The unknown output fields", + { + {RPCResult::Type::STR_HEX, "key", "(key-value pair) An unknown key-value pair"}, + }}, + {RPCResult::Type::ARR, "proprietary", /*optional=*/true, "The output proprietary map", + { + {RPCResult::Type::OBJ, "", "", + { + {RPCResult::Type::STR_HEX, "identifier", "The hex string for the proprietary identifier"}, + {RPCResult::Type::NUM, "subtype", "The number for the subtype"}, + {RPCResult::Type::STR_HEX, "key", "The hex for the key"}, + {RPCResult::Type::STR_HEX, "value", "The hex for the value"}, + }}, + }}, + }}, + } +}; + static RPCHelpMan decodepsbt() { return RPCHelpMan{ @@ -719,194 +911,8 @@ static RPCHelpMan decodepsbt() { {RPCResult::Type::STR_HEX, "key", "(key-value pair) An unknown key-value pair"}, }}, - {RPCResult::Type::ARR, "inputs", "", - { - {RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::OBJ, "non_witness_utxo", /*optional=*/true, "Decoded network transaction for non-witness UTXOs", - { - {RPCResult::Type::ELISION, "",""}, - }}, - {RPCResult::Type::OBJ, "witness_utxo", /*optional=*/true, "Transaction output for witness UTXOs", - { - {RPCResult::Type::NUM, "amount", "The value in " + CURRENCY_UNIT}, - {RPCResult::Type::OBJ, "scriptPubKey", "", - { - {RPCResult::Type::STR, "asm", "The asm"}, - {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, - {RPCResult::Type::STR_HEX, "hex", "The hex"}, - {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, - {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, - }}, - }}, - {RPCResult::Type::OBJ_DYN, "partial_signatures", /*optional=*/true, "", - { - {RPCResult::Type::STR, "pubkey", "The public key and signature that corresponds to it."}, - }}, - {RPCResult::Type::STR, "sighash", /*optional=*/true, "The sighash type to be used"}, - {RPCResult::Type::OBJ, "redeem_script", /*optional=*/true, "", - { - {RPCResult::Type::STR, "asm", "The asm"}, - {RPCResult::Type::STR_HEX, "hex", "The hex"}, - {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, - }}, - {RPCResult::Type::OBJ, "witness_script", /*optional=*/true, "", - { - {RPCResult::Type::STR, "asm", "The asm"}, - {RPCResult::Type::STR_HEX, "hex", "The hex"}, - {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, - }}, - {RPCResult::Type::ARR, "bip32_derivs", /*optional=*/true, "", - { - {RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR, "pubkey", "The public key with the derivation path as the value."}, - {RPCResult::Type::STR, "master_fingerprint", "The fingerprint of the master key"}, - {RPCResult::Type::STR, "path", "The path"}, - }}, - }}, - {RPCResult::Type::OBJ, "final_scriptSig", /*optional=*/true, "", - { - {RPCResult::Type::STR, "asm", "The asm"}, - {RPCResult::Type::STR, "hex", "The hex"}, - }}, - {RPCResult::Type::ARR, "final_scriptwitness", /*optional=*/true, "", - { - {RPCResult::Type::STR_HEX, "", "hex-encoded witness data (if any)"}, - }}, - {RPCResult::Type::OBJ_DYN, "ripemd160_preimages", /*optional=*/ true, "", - { - {RPCResult::Type::STR, "hash", "The hash and preimage that corresponds to it."}, - }}, - {RPCResult::Type::OBJ_DYN, "sha256_preimages", /*optional=*/ true, "", - { - {RPCResult::Type::STR, "hash", "The hash and preimage that corresponds to it."}, - }}, - {RPCResult::Type::OBJ_DYN, "hash160_preimages", /*optional=*/ true, "", - { - {RPCResult::Type::STR, "hash", "The hash and preimage that corresponds to it."}, - }}, - {RPCResult::Type::OBJ_DYN, "hash256_preimages", /*optional=*/ true, "", - { - {RPCResult::Type::STR, "hash", "The hash and preimage that corresponds to it."}, - }}, - {RPCResult::Type::STR_HEX, "taproot_key_path_sig", /*optional=*/ true, "hex-encoded signature for the Taproot key path spend"}, - {RPCResult::Type::ARR, "taproot_script_path_sigs", /*optional=*/ true, "", - { - {RPCResult::Type::OBJ, "signature", /*optional=*/ true, "The signature for the pubkey and leaf hash combination", - { - {RPCResult::Type::STR, "pubkey", "The x-only pubkey for this signature"}, - {RPCResult::Type::STR, "leaf_hash", "The leaf hash for this signature"}, - {RPCResult::Type::STR, "sig", "The signature itself"}, - }}, - }}, - {RPCResult::Type::ARR, "taproot_scripts", /*optional=*/ true, "", - { - {RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR_HEX, "script", "A leaf script"}, - {RPCResult::Type::NUM, "leaf_ver", "The version number for the leaf script"}, - {RPCResult::Type::ARR, "control_blocks", "The control blocks for this script", - { - {RPCResult::Type::STR_HEX, "control_block", "A hex-encoded control block for this script"}, - }}, - }}, - }}, - {RPCResult::Type::ARR, "taproot_bip32_derivs", /*optional=*/ true, "", - { - {RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR, "pubkey", "The x-only public key this path corresponds to"}, - {RPCResult::Type::STR, "master_fingerprint", "The fingerprint of the master key"}, - {RPCResult::Type::STR, "path", "The path"}, - {RPCResult::Type::ARR, "leaf_hashes", "The hashes of the leaves this pubkey appears in", - { - {RPCResult::Type::STR_HEX, "hash", "The hash of a leaf this pubkey appears in"}, - }}, - }}, - }}, - {RPCResult::Type::STR_HEX, "taproot_internal_key", /*optional=*/ true, "The hex-encoded Taproot x-only internal key"}, - {RPCResult::Type::STR_HEX, "taproot_merkle_root", /*optional=*/ true, "The hex-encoded Taproot merkle root"}, - {RPCResult::Type::OBJ_DYN, "unknown", /*optional=*/ true, "The unknown input fields", - { - {RPCResult::Type::STR_HEX, "key", "(key-value pair) An unknown key-value pair"}, - }}, - {RPCResult::Type::ARR, "proprietary", /*optional=*/true, "The input proprietary map", - { - {RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR_HEX, "identifier", "The hex string for the proprietary identifier"}, - {RPCResult::Type::NUM, "subtype", "The number for the subtype"}, - {RPCResult::Type::STR_HEX, "key", "The hex for the key"}, - {RPCResult::Type::STR_HEX, "value", "The hex for the value"}, - }}, - }}, - }}, - }}, - {RPCResult::Type::ARR, "outputs", "", - { - {RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::OBJ, "redeem_script", /*optional=*/true, "", - { - {RPCResult::Type::STR, "asm", "The asm"}, - {RPCResult::Type::STR_HEX, "hex", "The hex"}, - {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, - }}, - {RPCResult::Type::OBJ, "witness_script", /*optional=*/true, "", - { - {RPCResult::Type::STR, "asm", "The asm"}, - {RPCResult::Type::STR_HEX, "hex", "The hex"}, - {RPCResult::Type::STR, "type", "The type, eg 'pubkeyhash'"}, - }}, - {RPCResult::Type::ARR, "bip32_derivs", /*optional=*/true, "", - { - {RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR, "pubkey", "The public key this path corresponds to"}, - {RPCResult::Type::STR, "master_fingerprint", "The fingerprint of the master key"}, - {RPCResult::Type::STR, "path", "The path"}, - }}, - }}, - {RPCResult::Type::STR_HEX, "taproot_internal_key", /*optional=*/ true, "The hex-encoded Taproot x-only internal key"}, - {RPCResult::Type::ARR, "taproot_tree", /*optional=*/ true, "The tuples that make up the Taproot tree, in depth first search order", - { - {RPCResult::Type::OBJ, "tuple", /*optional=*/ true, "A single leaf script in the taproot tree", - { - {RPCResult::Type::NUM, "depth", "The depth of this element in the tree"}, - {RPCResult::Type::NUM, "leaf_ver", "The version of this leaf"}, - {RPCResult::Type::STR, "script", "The hex-encoded script itself"}, - }}, - }}, - {RPCResult::Type::ARR, "taproot_bip32_derivs", /*optional=*/ true, "", - { - {RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR, "pubkey", "The x-only public key this path corresponds to"}, - {RPCResult::Type::STR, "master_fingerprint", "The fingerprint of the master key"}, - {RPCResult::Type::STR, "path", "The path"}, - {RPCResult::Type::ARR, "leaf_hashes", "The hashes of the leaves this pubkey appears in", - { - {RPCResult::Type::STR_HEX, "hash", "The hash of a leaf this pubkey appears in"}, - }}, - }}, - }}, - {RPCResult::Type::OBJ_DYN, "unknown", /*optional=*/true, "The unknown output fields", - { - {RPCResult::Type::STR_HEX, "key", "(key-value pair) An unknown key-value pair"}, - }}, - {RPCResult::Type::ARR, "proprietary", /*optional=*/true, "The output proprietary map", - { - {RPCResult::Type::OBJ, "", "", - { - {RPCResult::Type::STR_HEX, "identifier", "The hex string for the proprietary identifier"}, - {RPCResult::Type::NUM, "subtype", "The number for the subtype"}, - {RPCResult::Type::STR_HEX, "key", "The hex for the key"}, - {RPCResult::Type::STR_HEX, "value", "The hex for the value"}, - }}, - }}, - }}, - }}, + decodepsbt_inputs, + decodepsbt_outputs, {RPCResult::Type::STR_AMOUNT, "fee", /*optional=*/true, "The transaction fee paid if all UTXOs slots in the PSBT have been filled."}, } }, diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index ca0170c84b..34a4da74f8 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -6,6 +6,7 @@ #include <key_io.h> #include <pubkey.h> +#include <script/miniscript.h> #include <script/script.h> #include <script/standard.h> @@ -161,6 +162,20 @@ public: virtual ~PubkeyProvider() = default; + /** Compare two public keys represented by this provider. + * Used by the Miniscript descriptors to check for duplicate keys in the script. + */ + bool operator<(PubkeyProvider& other) const { + CPubKey a, b; + SigningProvider dummy; + KeyOriginInfo dummy_info; + + GetPubKey(0, dummy, a, dummy_info); + other.GetPubKey(0, dummy, b, dummy_info); + + return a < b; + } + /** Derive a public key. * read_cache is the cache to read keys from (if not nullptr) * write_cache is the cache to write keys to (if not nullptr) @@ -493,12 +508,12 @@ public: /** Base class for all Descriptor implementations. */ class DescriptorImpl : public Descriptor { - //! Public key arguments for this descriptor (size 1 for PK, PKH, WPKH; any size for Multisig). +protected: + //! Public key arguments for this descriptor (size 1 for PK, PKH, WPKH; any size for WSH and Multisig). const std::vector<std::unique_ptr<PubkeyProvider>> m_pubkey_args; //! The string name of the descriptor function. const std::string m_name; -protected: //! The sub-descriptor arguments (empty for everything but SH and WSH). //! In doc/descriptors.m this is referred to as SCRIPT expressions sh(SCRIPT) //! and wsh(SCRIPT), and distinct from KEY expressions and ADDR expressions. @@ -563,7 +578,7 @@ public: return true; } - bool ToStringHelper(const SigningProvider* arg, std::string& out, const StringType type, const DescriptorCache* cache = nullptr) const + virtual bool ToStringHelper(const SigningProvider* arg, std::string& out, const StringType type, const DescriptorCache* cache = nullptr) const { std::string extra = ToStringExtra(); size_t pos = extra.size() > 0 ? 1 : 0; @@ -917,6 +932,89 @@ public: bool IsSingleType() const final { return true; } }; +/* We instantiate Miniscript here with a simple integer as key type. + * The value of these key integers are an index in the + * DescriptorImpl::m_pubkey_args vector. + */ + +/** + * The context for converting a Miniscript descriptor into a Script. + */ +class ScriptMaker { + //! Keys contained in the Miniscript (the evaluation of DescriptorImpl::m_pubkey_args). + const std::vector<CPubKey>& m_keys; + +public: + ScriptMaker(const std::vector<CPubKey>& keys LIFETIMEBOUND) : m_keys(keys) {} + + std::vector<unsigned char> ToPKBytes(uint32_t key) const { + return {m_keys[key].begin(), m_keys[key].end()}; + } + + std::vector<unsigned char> ToPKHBytes(uint32_t key) const { + auto id = m_keys[key].GetID(); + return {id.begin(), id.end()}; + } +}; + +/** + * The context for converting a Miniscript descriptor to its textual form. + */ +class StringMaker { + //! To convert private keys for private descriptors. + const SigningProvider* m_arg; + //! Keys contained in the Miniscript (a reference to DescriptorImpl::m_pubkey_args). + const std::vector<std::unique_ptr<PubkeyProvider>>& m_pubkeys; + //! Whether to serialize keys as private or public. + bool m_private; + +public: + StringMaker(const SigningProvider* arg LIFETIMEBOUND, const std::vector<std::unique_ptr<PubkeyProvider>>& pubkeys LIFETIMEBOUND, bool priv) + : m_arg(arg), m_pubkeys(pubkeys), m_private(priv) {} + + std::optional<std::string> ToString(uint32_t key) const + { + std::string ret; + if (m_private) { + if (!m_pubkeys[key]->ToPrivateString(*m_arg, ret)) return {}; + } else { + ret = m_pubkeys[key]->ToString(); + } + return ret; + } +}; + +class MiniscriptDescriptor final : public DescriptorImpl +{ +private: + miniscript::NodeRef<uint32_t> m_node; + +protected: + std::vector<CScript> MakeScripts(const std::vector<CPubKey>& keys, Span<const CScript> scripts, + FlatSigningProvider& provider) const override + { + for (const auto& key : keys) provider.pubkeys.emplace(key.GetID(), key); + return Vector(m_node->ToScript(ScriptMaker(keys))); + } + +public: + MiniscriptDescriptor(std::vector<std::unique_ptr<PubkeyProvider>> providers, miniscript::NodeRef<uint32_t> node) + : DescriptorImpl(std::move(providers), "?"), m_node(std::move(node)) {} + + bool ToStringHelper(const SigningProvider* arg, std::string& out, const StringType type, + const DescriptorCache* cache = nullptr) const override + { + if (const auto res = m_node->ToString(StringMaker(arg, m_pubkey_args, type == StringType::PRIVATE))) { + out = *res; + return true; + } + return false; + } + + bool IsSolvable() const override { return false; } // For now, mark these descriptors as non-solvable (as we don't have signing logic for them). + bool IsSingleType() const final { return true; } +}; + //////////////////////////////////////////////////////////////////////////// // Parser // //////////////////////////////////////////////////////////////////////////// @@ -1058,6 +1156,94 @@ std::unique_ptr<PubkeyProvider> ParsePubkey(uint32_t key_exp_index, const Span<c return std::make_unique<OriginPubkeyProvider>(key_exp_index, std::move(info), std::move(provider)); } +std::unique_ptr<PubkeyProvider> InferPubkey(const CPubKey& pubkey, ParseScriptContext, const SigningProvider& provider) +{ + std::unique_ptr<PubkeyProvider> key_provider = std::make_unique<ConstPubkeyProvider>(0, pubkey, false); + KeyOriginInfo info; + if (provider.GetKeyOrigin(pubkey.GetID(), info)) { + return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider)); + } + return key_provider; +} + +std::unique_ptr<PubkeyProvider> InferXOnlyPubkey(const XOnlyPubKey& xkey, ParseScriptContext ctx, const SigningProvider& provider) +{ + unsigned char full_key[CPubKey::COMPRESSED_SIZE] = {0x02}; + std::copy(xkey.begin(), xkey.end(), full_key + 1); + CPubKey pubkey(full_key); + std::unique_ptr<PubkeyProvider> key_provider = std::make_unique<ConstPubkeyProvider>(0, pubkey, true); + KeyOriginInfo info; + if (provider.GetKeyOriginByXOnly(xkey, info)) { + return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider)); + } + return key_provider; +} + +/** + * The context for parsing a Miniscript descriptor (either from Script or from its textual representation). + */ +struct KeyParser { + //! The Key type is an index in DescriptorImpl::m_pubkey_args + using Key = uint32_t; + //! Must not be nullptr if parsing from string. + FlatSigningProvider* m_out; + //! Must not be nullptr if parsing from Script. + const SigningProvider* m_in; + //! List of keys contained in the Miniscript. + mutable std::vector<std::unique_ptr<PubkeyProvider>> m_keys; + //! Used to detect key parsing errors within a Miniscript. + mutable std::string m_key_parsing_error; + + KeyParser(FlatSigningProvider* out LIFETIMEBOUND, const SigningProvider* in LIFETIMEBOUND) : m_out(out), m_in(in) {} + + bool KeyCompare(const Key& a, const Key& b) const { + return *m_keys.at(a) < *m_keys.at(b); + } + + template<typename I> std::optional<Key> FromString(I begin, I end) const + { + assert(m_out); + Key key = m_keys.size(); + auto pk = ParsePubkey(key, {&*begin, &*end}, ParseScriptContext::P2WSH, *m_out, m_key_parsing_error); + if (!pk) return {}; + m_keys.push_back(std::move(pk)); + return key; + } + + std::optional<std::string> ToString(const Key& key) const + { + return m_keys.at(key)->ToString(); + } + + template<typename I> std::optional<Key> FromPKBytes(I begin, I end) const + { + assert(m_in); + CPubKey pubkey(begin, end); + if (pubkey.IsValid()) { + Key key = m_keys.size(); + m_keys.push_back(InferPubkey(pubkey, ParseScriptContext::P2WSH, *m_in)); + return key; + } + return {}; + } + + template<typename I> std::optional<Key> FromPKHBytes(I begin, I end) const + { + assert(end - begin == 20); + assert(m_in); + uint160 hash; + std::copy(begin, end, hash.begin()); + CKeyID keyid(hash); + CPubKey pubkey; + if (m_in->GetPubKey(keyid, pubkey)) { + Key key = m_keys.size(); + m_keys.push_back(InferPubkey(pubkey, ParseScriptContext::P2WSH, *m_in)); + return key; + } + return {}; + } +}; + /** Parse a script in a particular context. */ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const char>& sp, ParseScriptContext ctx, FlatSigningProvider& out, std::string& error) { @@ -1279,6 +1465,45 @@ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const error = "Can only have raw() at top level"; return nullptr; } + // Process miniscript expressions. + { + KeyParser parser(&out, nullptr); + auto node = miniscript::FromString(std::string(expr.begin(), expr.end()), parser); + if (node) { + if (ctx != ParseScriptContext::P2WSH) { + error = "Miniscript expressions can only be used in wsh"; + return nullptr; + } + if (parser.m_key_parsing_error != "") { + error = std::move(parser.m_key_parsing_error); + return nullptr; + } + if (!node->IsSane()) { + // Try to find the first insane sub for better error reporting. + auto insane_node = node.get(); + if (const auto sub = node->FindInsaneSub()) insane_node = sub; + if (const auto str = insane_node->ToString(parser)) error = *str; + if (!insane_node->IsValid()) { + error += " is invalid"; + } else { + error += " is not sane"; + if (!insane_node->IsNonMalleable()) { + error += ": malleable witnesses exist"; + } else if (insane_node == node.get() && !insane_node->NeedsSignature()) { + error += ": witnesses without signature exist"; + } else if (!insane_node->CheckTimeLocksMix()) { + error += ": contains mixes of timelocks expressed in blocks and seconds"; + } else if (!insane_node->CheckDuplicateKey()) { + error += ": contains duplicate public keys"; + } else if (!insane_node->ValidSatisfactions()) { + error += ": needs witnesses that may exceed resource limits"; + } + } + return nullptr; + } + return std::make_unique<MiniscriptDescriptor>(std::move(parser.m_keys), std::move(node)); + } + } if (ctx == ParseScriptContext::P2SH) { error = "A function is needed within P2SH"; return nullptr; @@ -1290,29 +1515,6 @@ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const return nullptr; } -std::unique_ptr<PubkeyProvider> InferPubkey(const CPubKey& pubkey, ParseScriptContext, const SigningProvider& provider) -{ - std::unique_ptr<PubkeyProvider> key_provider = std::make_unique<ConstPubkeyProvider>(0, pubkey, false); - KeyOriginInfo info; - if (provider.GetKeyOrigin(pubkey.GetID(), info)) { - return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider)); - } - return key_provider; -} - -std::unique_ptr<PubkeyProvider> InferXOnlyPubkey(const XOnlyPubKey& xkey, ParseScriptContext ctx, const SigningProvider& provider) -{ - unsigned char full_key[CPubKey::COMPRESSED_SIZE] = {0x02}; - std::copy(xkey.begin(), xkey.end(), full_key + 1); - CPubKey pubkey(full_key); - std::unique_ptr<PubkeyProvider> key_provider = std::make_unique<ConstPubkeyProvider>(0, pubkey, true); - KeyOriginInfo info; - if (provider.GetKeyOriginByXOnly(xkey, info)) { - return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider)); - } - return key_provider; -} - std::unique_ptr<DescriptorImpl> InferMultiA(const CScript& script, ParseScriptContext ctx, const SigningProvider& provider) { auto match = MatchMultiA(script); @@ -1426,6 +1628,14 @@ std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptCo } } + if (ctx == ParseScriptContext::P2WSH) { + KeyParser parser(nullptr, &provider); + auto node = miniscript::FromScript(script, parser); + if (node && node->IsSane()) { + return std::make_unique<MiniscriptDescriptor>(std::move(parser.m_keys), std::move(node)); + } + } + CTxDestination dest; if (ExtractDestination(script, dest)) { if (GetScriptForDestination(dest) == script) { diff --git a/src/script/miniscript.h b/src/script/miniscript.h index 2c239c2678..f783d1dafc 100644 --- a/src/script/miniscript.h +++ b/src/script/miniscript.h @@ -429,6 +429,21 @@ private: )); } + /** Like TreeEval, but without downfn or State type. + * upfn takes (const Node&, Span<Result>) and returns Result. */ + template<typename Result, typename UpFn> + Result TreeEval(UpFn upfn) const + { + struct DummyState {}; + return std::move(*TreeEvalMaybe<Result>(DummyState{}, + [](DummyState, const Node&, size_t) { return DummyState{}; }, + [&upfn](DummyState, const Node& node, Span<Result> subs) { + Result res{upfn(node, subs)}; + return std::optional<Result>(std::move(res)); + } + )); + } + /** Compare two miniscript subtrees, using a non-recursive algorithm. */ friend int Compare(const Node<Key>& node1, const Node<Key>& node2) { @@ -818,6 +833,15 @@ public: //! Return the expression type. Type GetType() const { return typ; } + //! Find an insane subnode which has no insane children. Nullptr if there is none. + const Node* FindInsaneSub() const { + return TreeEval<const Node*>([](const Node& node, Span<const Node*> subs) -> const Node* { + for (auto& sub: subs) if (sub) return sub; + if (!node.IsSaneSubexpression()) return &node; + return nullptr; + }); + } + //! Check whether this node is valid at all. bool IsValid() const { return !(GetType() == ""_mst) && ScriptSize() <= MAX_STANDARD_P2WSH_SCRIPT_SIZE; } @@ -953,7 +977,11 @@ void BuildBack(const Ctx& ctx, Fragment nt, std::vector<NodeRef<Key>>& construct } } -//! Parse a miniscript from its textual descriptor form. +/** + * Parse a miniscript from its textual descriptor form. + * This does not check whether the script is valid, let alone sane. The caller is expected to use + * the `IsValidTopLevel()` and `IsSaneTopLevel()` to check for these properties on the node. + */ template<typename Key, typename Ctx> inline NodeRef<Key> Parse(Span<const char> in, const Ctx& ctx) { @@ -1255,9 +1283,7 @@ inline NodeRef<Key> Parse(Span<const char> in, const Ctx& ctx) // Sanity checks on the produced miniscript assert(constructed.size() == 1); if (in.size() > 0) return {}; - const NodeRef<Key> tl_node = std::move(constructed.front()); - if (!tl_node->IsValidTopLevel()) return {}; - return tl_node; + return std::move(constructed.front()); } /** Decode a script into opcode/push pairs. diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index c31e4e51f7..1a182209b8 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -7,6 +7,7 @@ #include <consensus/merkle.h> #include <consensus/validation.h> #include <index/blockfilterindex.h> +#include <interfaces/chain.h> #include <node/miner.h> #include <pow.h> #include <script/standard.h> @@ -110,7 +111,7 @@ bool BuildChainTestingSetup::BuildChain(const CBlockIndex* pindex, BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) { - BlockFilterIndex filter_index(BlockFilterType::BASIC, 1 << 20, true); + BlockFilterIndex filter_index(interfaces::MakeChain(m_node), BlockFilterType::BASIC, 1 << 20, true); uint256 last_header; @@ -137,7 +138,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) // BlockUntilSyncedToCurrentChain should return false before index is started. BOOST_CHECK(!filter_index.BlockUntilSyncedToCurrentChain()); - BOOST_REQUIRE(filter_index.Start(m_node.chainman->ActiveChainstate())); + BOOST_REQUIRE(filter_index.Start()); // Allow filter index to catch up with the block index. constexpr int64_t timeout_ms = 10 * 1000; @@ -279,14 +280,14 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_init_destroy, BasicTestingSetup) filter_index = GetBlockFilterIndex(BlockFilterType::BASIC); BOOST_CHECK(filter_index == nullptr); - BOOST_CHECK(InitBlockFilterIndex(BlockFilterType::BASIC, 1 << 20, true, false)); + BOOST_CHECK(InitBlockFilterIndex([&]{ return interfaces::MakeChain(m_node); }, BlockFilterType::BASIC, 1 << 20, true, false)); filter_index = GetBlockFilterIndex(BlockFilterType::BASIC); BOOST_CHECK(filter_index != nullptr); BOOST_CHECK(filter_index->GetFilterType() == BlockFilterType::BASIC); // Initialize returns false if index already exists. - BOOST_CHECK(!InitBlockFilterIndex(BlockFilterType::BASIC, 1 << 20, true, false)); + BOOST_CHECK(!InitBlockFilterIndex([&]{ return interfaces::MakeChain(m_node); }, BlockFilterType::BASIC, 1 << 20, true, false)); int iter_count = 0; ForEachBlockFilterIndex([&iter_count](BlockFilterIndex& _index) { iter_count++; }); @@ -301,7 +302,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_init_destroy, BasicTestingSetup) BOOST_CHECK(filter_index == nullptr); // Reinitialize index. - BOOST_CHECK(InitBlockFilterIndex(BlockFilterType::BASIC, 1 << 20, true, false)); + BOOST_CHECK(InitBlockFilterIndex([&]{ return interfaces::MakeChain(m_node); }, BlockFilterType::BASIC, 1 << 20, true, false)); DestroyAllBlockFilterIndexes(); diff --git a/src/test/coinstatsindex_tests.cpp b/src/test/coinstatsindex_tests.cpp index 50eb479035..c93d05a93b 100644 --- a/src/test/coinstatsindex_tests.cpp +++ b/src/test/coinstatsindex_tests.cpp @@ -4,6 +4,7 @@ #include <chainparams.h> #include <index/coinstatsindex.h> +#include <interfaces/chain.h> #include <test/util/setup_common.h> #include <test/util/validation.h> #include <util/time.h> @@ -13,9 +14,6 @@ #include <chrono> -using kernel::CCoinsStats; -using kernel::CoinStatsHashType; - BOOST_AUTO_TEST_SUITE(coinstatsindex_tests) static void IndexWaitSynced(BaseIndex& index) @@ -31,7 +29,7 @@ static void IndexWaitSynced(BaseIndex& index) BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup) { - CoinStatsIndex coin_stats_index{1 << 20, true}; + CoinStatsIndex coin_stats_index{interfaces::MakeChain(m_node), 1 << 20, true}; const CBlockIndex* block_index; { @@ -46,7 +44,7 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup) // is started. BOOST_CHECK(!coin_stats_index.BlockUntilSyncedToCurrentChain()); - BOOST_REQUIRE(coin_stats_index.Start(m_node.chainman->ActiveChainstate())); + BOOST_REQUIRE(coin_stats_index.Start()); IndexWaitSynced(coin_stats_index); @@ -90,8 +88,8 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_unclean_shutdown, TestChain100Setup) CChainState& chainstate = Assert(m_node.chainman)->ActiveChainstate(); const CChainParams& params = Params(); { - CoinStatsIndex index{1 << 20}; - BOOST_REQUIRE(index.Start(chainstate)); + CoinStatsIndex index{interfaces::MakeChain(m_node), 1 << 20}; + BOOST_REQUIRE(index.Start()); IndexWaitSynced(index); std::shared_ptr<const CBlock> new_block; CBlockIndex* new_block_index = nullptr; @@ -116,9 +114,9 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_unclean_shutdown, TestChain100Setup) } { - CoinStatsIndex index{1 << 20}; + CoinStatsIndex index{interfaces::MakeChain(m_node), 1 << 20}; // Make sure the index can be loaded. - BOOST_REQUIRE(index.Start(chainstate)); + BOOST_REQUIRE(index.Start()); index.Stop(); } } diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index c87ed82c88..66c4605afa 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -15,6 +15,7 @@ #include <serialize.h> #include <test/util/net.h> #include <test/util/setup_common.h> +#include <timedata.h> #include <util/string.h> #include <util/system.h> #include <util/time.h> @@ -44,17 +45,15 @@ BOOST_FIXTURE_TEST_SUITE(denialofservice_tests, TestingSetup) // work. BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) { - auto connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman); + ConnmanTestMsg& connman = static_cast<ConnmanTestMsg&>(*m_node.connman); // Disable inactivity checks for this test to avoid interference - static_cast<ConnmanTestMsg*>(connman.get())->SetPeerConnectTimeout(99999s); - auto peerLogic = PeerManager::make(*connman, *m_node.addrman, nullptr, - *m_node.chainman, *m_node.mempool, false); + connman.SetPeerConnectTimeout(99999s); + PeerManager& peerman = *m_node.peerman; // Mock an outbound peer CAddress addr1(ip(0xa0b0c001), NODE_NONE); NodeId id{0}; CNode dummyNode1{id++, - ServiceFlags(NODE_NETWORK | NODE_WITNESS), /*sock=*/nullptr, addr1, /*nKeyedNetGroupIn=*/0, @@ -63,10 +62,16 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) /*addrNameIn=*/"", ConnectionType::OUTBOUND_FULL_RELAY, /*inbound_onion=*/false}; - dummyNode1.SetCommonVersion(PROTOCOL_VERSION); - peerLogic->InitializeNode(&dummyNode1); - dummyNode1.fSuccessfullyConnected = true; + connman.Handshake( + /*node=*/dummyNode1, + /*successfully_connected=*/true, + /*remote_services=*/ServiceFlags(NODE_NETWORK | NODE_WITNESS), + /*local_services=*/ServiceFlags(NODE_NETWORK | NODE_WITNESS), + /*permission_flags=*/NetPermissionFlags::None, + /*version=*/PROTOCOL_VERSION, + /*relay_txs=*/true); + TestOnlyResetTimeData(); // This test requires that we have a chain with non-zero work. { @@ -78,7 +83,7 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) // Test starts here { LOCK(dummyNode1.cs_sendProcessing); - BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); // should result in getheaders + BOOST_CHECK(peerman.SendMessages(&dummyNode1)); // should result in getheaders } { LOCK(dummyNode1.cs_vSend); @@ -91,7 +96,7 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) SetMockTime(nStartTime+21*60); { LOCK(dummyNode1.cs_sendProcessing); - BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); // should result in getheaders + BOOST_CHECK(peerman.SendMessages(&dummyNode1)); // should result in getheaders } { LOCK(dummyNode1.cs_vSend); @@ -101,18 +106,17 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) SetMockTime(nStartTime+24*60); { LOCK(dummyNode1.cs_sendProcessing); - BOOST_CHECK(peerLogic->SendMessages(&dummyNode1)); // should result in disconnect + BOOST_CHECK(peerman.SendMessages(&dummyNode1)); // should result in disconnect } BOOST_CHECK(dummyNode1.fDisconnect == true); - peerLogic->FinalizeNode(dummyNode1); + peerman.FinalizeNode(dummyNode1); } static void AddRandomOutboundPeer(NodeId& id, std::vector<CNode*>& vNodes, PeerManager& peerLogic, ConnmanTestMsg& connman, ConnectionType connType) { CAddress addr(ip(g_insecure_rand_ctx.randbits(32)), NODE_NONE); vNodes.emplace_back(new CNode{id++, - ServiceFlags(NODE_NETWORK | NODE_WITNESS), /*sock=*/nullptr, addr, /*nKeyedNetGroupIn=*/0, @@ -124,7 +128,7 @@ static void AddRandomOutboundPeer(NodeId& id, std::vector<CNode*>& vNodes, PeerM CNode &node = *vNodes.back(); node.SetCommonVersion(PROTOCOL_VERSION); - peerLogic.InitializeNode(&node); + peerLogic.InitializeNode(node, ServiceFlags(NODE_NETWORK | NODE_WITNESS)); node.fSuccessfullyConnected = true; connman.AddTestNode(node); @@ -292,7 +296,6 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) banman->ClearBanned(); NodeId id{0}; nodes[0] = new CNode{id++, - NODE_NETWORK, /*sock=*/nullptr, addr[0], /*nKeyedNetGroupIn=*/0, @@ -302,7 +305,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) ConnectionType::INBOUND, /*inbound_onion=*/false}; nodes[0]->SetCommonVersion(PROTOCOL_VERSION); - peerLogic->InitializeNode(nodes[0]); + peerLogic->InitializeNode(*nodes[0], NODE_NETWORK); nodes[0]->fSuccessfullyConnected = true; connman->AddTestNode(*nodes[0]); peerLogic->UnitTestMisbehaving(nodes[0]->GetId(), DISCOURAGEMENT_THRESHOLD); // Should be discouraged @@ -315,7 +318,6 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) BOOST_CHECK(!banman->IsDiscouraged(other_addr)); // Different address, not discouraged nodes[1] = new CNode{id++, - NODE_NETWORK, /*sock=*/nullptr, addr[1], /*nKeyedNetGroupIn=*/1, @@ -325,7 +327,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) ConnectionType::INBOUND, /*inbound_onion=*/false}; nodes[1]->SetCommonVersion(PROTOCOL_VERSION); - peerLogic->InitializeNode(nodes[1]); + peerLogic->InitializeNode(*nodes[1], NODE_NETWORK); nodes[1]->fSuccessfullyConnected = true; connman->AddTestNode(*nodes[1]); peerLogic->UnitTestMisbehaving(nodes[1]->GetId(), DISCOURAGEMENT_THRESHOLD - 1); @@ -353,7 +355,6 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) // Make sure non-IP peers are discouraged and disconnected properly. nodes[2] = new CNode{id++, - NODE_NETWORK, /*sock=*/nullptr, addr[2], /*nKeyedNetGroupIn=*/1, @@ -363,7 +364,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) ConnectionType::OUTBOUND_FULL_RELAY, /*inbound_onion=*/false}; nodes[2]->SetCommonVersion(PROTOCOL_VERSION); - peerLogic->InitializeNode(nodes[2]); + peerLogic->InitializeNode(*nodes[2], NODE_NETWORK); nodes[2]->fSuccessfullyConnected = true; connman->AddTestNode(*nodes[2]); peerLogic->UnitTestMisbehaving(nodes[2]->GetId(), DISCOURAGEMENT_THRESHOLD); @@ -398,7 +399,6 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) CAddress addr(ip(0xa0b0c001), NODE_NONE); NodeId id{0}; CNode dummyNode{id++, - NODE_NETWORK, /*sock=*/nullptr, addr, /*nKeyedNetGroupIn=*/4, @@ -408,7 +408,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) ConnectionType::INBOUND, /*inbound_onion=*/false}; dummyNode.SetCommonVersion(PROTOCOL_VERSION); - peerLogic->InitializeNode(&dummyNode); + peerLogic->InitializeNode(dummyNode, NODE_NETWORK); dummyNode.fSuccessfullyConnected = true; peerLogic->UnitTestMisbehaving(dummyNode.GetId(), DISCOURAGEMENT_THRESHOLD); diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 63c86a896d..a8c666079d 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -141,14 +141,13 @@ void DoCheck(const std::string& prv, const std::string& pub, const std::string& } else { parse_priv = Parse(prv, keys_priv, error); } + BOOST_CHECK_MESSAGE(parse_priv, error); if (replace_apostrophe_with_h_in_pub) { parse_pub = Parse(UseHInsteadOfApostrophe(pub), keys_pub, error); } else { parse_pub = Parse(pub, keys_pub, error); } - - BOOST_CHECK(parse_priv); - BOOST_CHECK(parse_pub); + BOOST_CHECK_MESSAGE(parse_pub, error); // Check that the correct OutputType is inferred BOOST_CHECK(parse_priv->GetOutputType() == type); @@ -161,8 +160,8 @@ void DoCheck(const std::string& prv, const std::string& pub, const std::string& // Check that both versions serialize back to the public version. std::string pub1 = parse_priv->ToString(); std::string pub2 = parse_pub->ToString(); - BOOST_CHECK(EqualDescriptor(pub, pub1)); - BOOST_CHECK(EqualDescriptor(pub, pub2)); + BOOST_CHECK_MESSAGE(EqualDescriptor(pub, pub1), "Private ser: " + pub1 + " Public desc: " + pub); + BOOST_CHECK_MESSAGE(EqualDescriptor(pub, pub2), "Public ser: " + pub2 + " Public desc: " + pub); // Check that both can be serialized with private key back to the private version, but not without private key. if (!(flags & MISSING_PRIVKEYS)) { @@ -486,6 +485,29 @@ Check("sh(wsh(multi(20,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGN } nonminimalmultisig << std::vector<unsigned char>{4} << OP_CHECKMULTISIG; CheckInferRaw(nonminimalmultisig); + + // Miniscript tests + + // Invalid checksum + CheckUnparsable("wsh(and_v(vc:andor(pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),pk_k(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn),and_v(v:older(1),pk_k(L4o2kDvXXDRH2VS9uBnouScLduWt4dZnM25se7kvEjJeQ285en2A))),after(10)))#abcdef12", "wsh(and_v(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)))#abcdef12", "Provided checksum 'abcdef12' does not match computed checksum 'tyzp6a7p'"); + // Only p2wsh context is valid + CheckUnparsable("sh(and_v(vc:andor(pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),pk_k(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn),and_v(v:older(1),pk_k(L4o2kDvXXDRH2VS9uBnouScLduWt4dZnM25se7kvEjJeQ285en2A))),after(10)))", "sh(and_v(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)))", "Miniscript expressions can only be used in wsh"); + CheckUnparsable("tr(and_v(vc:andor(pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),pk_k(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn),and_v(v:older(1),pk_k(L4o2kDvXXDRH2VS9uBnouScLduWt4dZnM25se7kvEjJeQ285en2A))),after(10)))", "tr(and_v(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)))", "tr(): key 'and_v(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10))' is not valid"); + CheckUnparsable("raw(and_v(vc:andor(pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),pk_k(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn),and_v(v:older(1),pk_k(L4o2kDvXXDRH2VS9uBnouScLduWt4dZnM25se7kvEjJeQ285en2A))),after(10)))", "sh(and_v(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)))", "Miniscript expressions can only be used in wsh"); + CheckUnparsable("", "tr(034D2224bbbbbbbbbbcbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb40,{{{{{{{{{{{{{{{{{{{{{{multi(1,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/967808'/9,xprvA1RpRA33e1JQ7ifknakTFNpgXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/968/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/3/4/5/58/55/2/5/58/58/2/5/5/5/8/5/2/8/5/85/2/8/2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/8/5/8/5/4/5/585/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/8/2/5/8/5/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/58/58/2/0/8/5/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/5/8/5/8/24/5/58/52/5/8/5/2/8/24/5/58/588/246/8/5/2/8/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/5/4/5/58/55/58/2/5/8/55/2/5/8/58/555/58/2/5/8/4//2/5/58/5w/2/5/8/5/2/4/5/58/5558'/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/8/2/5/8/5/5/8/58/2/5/58/58/2/5/8/9/588/2/58/2/5/8/5/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/82/5/8/5/5/58/52/6/8/5/2/8/{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{}{{{{{{{{{DDD2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8588/246/8/5/2DLDDDDDDDbbD3DDDD/8/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/3/4/5/58/55/2/5/58/58/2/5/5/5/8/5/2/8/5/85/2/8/2/5/8D)/5/2/5/58/58/2/5/58/58/58/588/2/58/2/5/8/5/25/58/58/2/5/58/58/2/5/8/9/588/2/58/2/6780,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFW/8/5/2/5/58678008')", "'multi(1,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/967808'/9,xprvA1RpRA33e1JQ7ifknakTFNpgXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/968/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/3/4/5/58/55/2/5/58/58/2/5/5/5/8/5/2/8/5/85/2/8/2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/8/5/8/5/4/5/585/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/8/2/5/8/5/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/58/58/2/0/8/5/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/5/8/5/8/24/5/58/52/5/8/5/2/8/24/5/58/588/246/8/5/2/8/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/5/4/5/58/55/58/2/5/8/55/2/5/8/58/555/58/2/5/8/4//2/5/58/5w/2/5/8/5/2/4/5/58/5558'/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/8/2/5/8/5/5/8/58/2/5/58/58/2/5/8/9/588/2/58/2/5/8/5/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/82/5/8/5/5/58/52/6/8/5/2/8/{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{}{{{{{{{{{DDD2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8588/246/8/5/2DLDDDDDDDbbD3DDDD/8/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/3/4/5/58/55/2/5/58/58/2/5/5/5/8/5/2/8/5/85/2/8/2/5/8D)/5/2/5/58/58/2/5/58/58/58/588/2/58/2/5/8/5/25/58/58/2/5/58/58/2/5/8/9/588/2/58/2/6780,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFW/8/5/2/5/58678008'' is not a valid descriptor function"); + // Insane at top level + CheckUnparsable("wsh(and_b(vc:andor(pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),pk_k(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn),and_v(v:older(1),pk_k(L4o2kDvXXDRH2VS9uBnouScLduWt4dZnM25se7kvEjJeQ285en2A))),after(10)))", "wsh(and_b(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)))", "and_b(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)) is invalid"); + // Invalid sub + CheckUnparsable("wsh(and_v(vc:andor(v:pk_k(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),pk_k(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn),and_v(v:older(1),pk_k(L4o2kDvXXDRH2VS9uBnouScLduWt4dZnM25se7kvEjJeQ285en2A))),after(10)))", "wsh(and_v(vc:andor(v:pk_k(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)))", "v:pk_k(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204) is invalid"); + // Insane subs + CheckUnparsable("wsh(or_i(older(1),pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd)))", "wsh(or_i(older(1),pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)))", "or_i(older(1),pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)) is not sane: witnesses without signature exist"); + CheckUnparsable("wsh(or_b(sha256(cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),s:pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)))", "wsh(or_b(sha256(cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),s:pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)))", "or_b(sha256(cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),s:pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)) is not sane: malleable witnesses exist"); + CheckUnparsable("wsh(and_b(and_b(older(1),a:older(100000000)),s:pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd)))", "wsh(and_b(and_b(older(1),a:older(100000000)),s:pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)))", "and_b(older(1),a:older(100000000)) is not sane: contains mixes of timelocks expressed in blocks and seconds"); + CheckUnparsable("wsh(and_b(or_b(pkh(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),s:pk(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn)),s:pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd)))", "wsh(and_b(or_b(pkh(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),s:pk(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0)),s:pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)))", "and_b(or_b(pkh(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),s:pk(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0)),s:pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)) is not sane: contains duplicate public keys"); + // Valid with extended keys. + Check("wsh(and_v(v:ripemd160(095ff41131e5946f3c85f79e44adbcf8e27e080e),multi(1,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0)))", "wsh(and_v(v:ripemd160(095ff41131e5946f3c85f79e44adbcf8e27e080e),multi(1,xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0)))", "wsh(and_v(v:ripemd160(095ff41131e5946f3c85f79e44adbcf8e27e080e),multi(1,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0)))", "wsh(and_v(v:ripemd160(095ff41131e5946f3c85f79e44adbcf8e27e080e),multi(1,xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0)))", UNSOLVABLE, {{"0020acf425291b98a1d7e0d4690139442abc289175be32ef1f75945e339924246d73"}}, OutputType::BECH32, {{},{0}}); + // Valid under sh(wsh()) and with a mix of xpubs and raw keys + Check("sh(wsh(thresh(1,pkh(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),a:and_n(multi(1,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0),n:older(2)))))", "sh(wsh(thresh(1,pkh(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),a:and_n(multi(1,xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0),n:older(2)))))", "sh(wsh(thresh(1,pkh(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),a:and_n(multi(1,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0),n:older(2)))))", "sh(wsh(thresh(1,pkh(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),a:and_n(multi(1,xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0),n:older(2)))))", UNSOLVABLE | MIXED_PUBKEYS, {{"a914767e9119ff3b3ac0cb6dcfe21de1842ccf85f1c487"}}, OutputType::P2SH_SEGWIT, {{},{0}}); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/fuzz/mempool_utils.h b/src/test/fuzz/mempool_utils.h new file mode 100644 index 0000000000..bfe12e30ba --- /dev/null +++ b/src/test/fuzz/mempool_utils.h @@ -0,0 +1,19 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_FUZZ_MEMPOOL_UTILS_H +#define BITCOIN_TEST_FUZZ_MEMPOOL_UTILS_H + +#include <validation.h> + +class DummyChainState final : public CChainState +{ +public: + void SetMempool(CTxMemPool* mempool) + { + m_mempool = mempool; + } +}; + +#endif // BITCOIN_TEST_FUZZ_MEMPOOL_UTILS_H diff --git a/src/test/fuzz/net.cpp b/src/test/fuzz/net.cpp index 4981287152..741810f6a2 100644 --- a/src/test/fuzz/net.cpp +++ b/src/test/fuzz/net.cpp @@ -68,7 +68,6 @@ FUZZ_TARGET_INIT(net, initialize_net) (void)node.GetAddrLocal(); (void)node.GetId(); (void)node.GetLocalNonce(); - (void)node.GetLocalServices(); const int ref_count = node.GetRefCount(); assert(ref_count >= 0); (void)node.GetCommonVersion(); diff --git a/src/test/fuzz/node_eviction.cpp b/src/test/fuzz/node_eviction.cpp index 6a363f00f7..e27b254580 100644 --- a/src/test/fuzz/node_eviction.cpp +++ b/src/test/fuzz/node_eviction.cpp @@ -32,6 +32,8 @@ FUZZ_TARGET(node_eviction) /*prefer_evict=*/fuzzed_data_provider.ConsumeBool(), /*m_is_local=*/fuzzed_data_provider.ConsumeBool(), /*m_network=*/fuzzed_data_provider.PickValueInArray(ALL_NETWORKS), + /*m_noban=*/fuzzed_data_provider.ConsumeBool(), + /*m_conn_type=*/fuzzed_data_provider.PickValueInArray(ALL_CONNECTION_TYPES), }); } // Make a copy since eviction_candidates may be in some valid but otherwise diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 1763cd8af3..272c9e6cdc 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -80,8 +80,7 @@ void fuzz_target(FuzzBufferType buffer, const std::string& LIMIT_TO_MESSAGE_TYPE CNode& p2p_node = *ConsumeNodeAsUniquePtr(fuzzed_data_provider).release(); connman.AddTestNode(p2p_node); - g_setup->m_node.peerman->InitializeNode(&p2p_node); - FillNode(fuzzed_data_provider, connman, *g_setup->m_node.peerman, p2p_node); + FillNode(fuzzed_data_provider, connman, p2p_node); const auto mock_time = ConsumeTime(fuzzed_data_provider); SetMockTime(mock_time); diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index e1c11e1afd..12e682416c 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -46,8 +46,7 @@ FUZZ_TARGET_INIT(process_messages, initialize_process_messages) peers.push_back(ConsumeNodeAsUniquePtr(fuzzed_data_provider, i).release()); CNode& p2p_node = *peers.back(); - g_setup->m_node.peerman->InitializeNode(&p2p_node); - FillNode(fuzzed_data_provider, connman, *g_setup->m_node.peerman, p2p_node); + FillNode(fuzzed_data_provider, connman, p2p_node); connman.AddTestNode(p2p_node); } diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index e4e83c3f32..26913a41d2 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -159,6 +159,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ "signrawtransactionwithkey", "submitblock", "submitheader", + "submitpackage", "syncwithvalidationinterfacequeue", "testmempoolaccept", "uptime", diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index 2d88ee295b..63fbf0516a 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -8,6 +8,7 @@ #include <node/miner.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/mempool_utils.h> #include <test/fuzz/util.h> #include <test/util/mining.h> #include <test/util/script.h> @@ -34,15 +35,6 @@ struct MockedTxPool : public CTxMemPool { } }; -class DummyChainState final : public CChainState -{ -public: - void SetMempool(CTxMemPool* mempool) - { - m_mempool = mempool; - } -}; - void initialize_tx_pool() { static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); diff --git a/src/test/fuzz/txorphan.cpp b/src/test/fuzz/txorphan.cpp new file mode 100644 index 0000000000..cc20f89bbf --- /dev/null +++ b/src/test/fuzz/txorphan.cpp @@ -0,0 +1,149 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <consensus/amount.h> +#include <consensus/validation.h> +#include <net_processing.h> +#include <node/eviction.h> +#include <policy/policy.h> +#include <primitives/transaction.h> +#include <script/script.h> +#include <sync.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/setup_common.h> +#include <txorphanage.h> +#include <uint256.h> +#include <util/check.h> +#include <util/time.h> + +#include <algorithm> +#include <cstdint> +#include <memory> +#include <set> +#include <utility> +#include <vector> + +void initialize_orphanage() +{ + static const auto testing_setup = MakeNoLogFileContext(); +} + +FUZZ_TARGET_INIT(txorphan, initialize_orphanage) +{ + FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + SetMockTime(ConsumeTime(fuzzed_data_provider)); + + TxOrphanage orphanage; + std::set<uint256> orphan_work_set; + std::vector<COutPoint> outpoints; + // initial outpoints used to construct transactions later + for (uint8_t i = 0; i < 4; i++) { + outpoints.emplace_back(uint256{i}, 0); + } + // if true, allow duplicate input when constructing tx + const bool duplicate_input = fuzzed_data_provider.ConsumeBool(); + + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10 * DEFAULT_MAX_ORPHAN_TRANSACTIONS) + { + // construct transaction + const CTransactionRef tx = [&] { + CMutableTransaction tx_mut; + const auto num_in = fuzzed_data_provider.ConsumeIntegralInRange<uint32_t>(1, outpoints.size()); + const auto num_out = fuzzed_data_provider.ConsumeIntegralInRange<uint32_t>(1, outpoints.size()); + // pick unique outpoints from outpoints as input + for (uint32_t i = 0; i < num_in; i++) { + auto& prevout = PickValue(fuzzed_data_provider, outpoints); + tx_mut.vin.emplace_back(prevout); + // pop the picked outpoint if duplicate input is not allowed + if (!duplicate_input) { + std::swap(prevout, outpoints.back()); + outpoints.pop_back(); + } + } + // output amount will not affect txorphanage + for (uint32_t i = 0; i < num_out; i++) { + tx_mut.vout.emplace_back(CAmount{0}, CScript{}); + } + // restore previously poped outpoints + for (auto& in : tx_mut.vin) { + outpoints.push_back(in.prevout); + } + const auto new_tx = MakeTransactionRef(tx_mut); + // add newly constructed transaction to outpoints + for (uint32_t i = 0; i < num_out; i++) { + outpoints.emplace_back(new_tx->GetHash(), i); + } + return new_tx; + }(); + + // trigger orphanage functions + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10 * DEFAULT_MAX_ORPHAN_TRANSACTIONS) + { + NodeId peer_id = fuzzed_data_provider.ConsumeIntegral<NodeId>(); + + CallOneOf( + fuzzed_data_provider, + [&] { + LOCK(g_cs_orphans); + orphanage.AddChildrenToWorkSet(*tx, orphan_work_set); + }, + [&] { + bool have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); + { + LOCK(g_cs_orphans); + bool get_tx = orphanage.GetTx(tx->GetHash()).first != nullptr; + Assert(have_tx == get_tx); + } + }, + [&] { + bool have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); + // AddTx should return false if tx is too big or already have it + // tx weight is unknown, we only check when tx is already in orphanage + { + LOCK(g_cs_orphans); + bool add_tx = orphanage.AddTx(tx, peer_id); + // have_tx == true -> add_tx == false + Assert(!have_tx || !add_tx); + } + have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); + { + LOCK(g_cs_orphans); + bool add_tx = orphanage.AddTx(tx, peer_id); + // if have_tx is still false, it must be too big + Assert(!have_tx == (GetTransactionWeight(*tx) > MAX_STANDARD_TX_WEIGHT)); + Assert(!have_tx || !add_tx); + } + }, + [&] { + bool have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); + // EraseTx should return 0 if m_orphans doesn't have the tx + { + LOCK(g_cs_orphans); + Assert(have_tx == orphanage.EraseTx(tx->GetHash())); + } + have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); + // have_tx should be false and EraseTx should fail + { + LOCK(g_cs_orphans); + Assert(!have_tx && !orphanage.EraseTx(tx->GetHash())); + } + }, + [&] { + LOCK(g_cs_orphans); + orphanage.EraseForPeer(peer_id); + }, + [&] { + // test mocktime and expiry + SetMockTime(ConsumeTime(fuzzed_data_provider)); + auto size_before = orphanage.Size(); + auto limit = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); + auto n_evicted = WITH_LOCK(g_cs_orphans, return orphanage.LimitOrphans(limit)); + Assert(size_before - n_evicted <= limit); + Assert(orphanage.Size() <= limit); + }); + } + } +} diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index 4b893c648e..fabcea22c3 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -289,57 +289,15 @@ bool FuzzedSock::IsConnected(std::string& errmsg) const return false; } -void FillNode(FuzzedDataProvider& fuzzed_data_provider, ConnmanTestMsg& connman, PeerManager& peerman, CNode& node) noexcept +void FillNode(FuzzedDataProvider& fuzzed_data_provider, ConnmanTestMsg& connman, CNode& node) noexcept { - const bool successfully_connected{fuzzed_data_provider.ConsumeBool()}; - const ServiceFlags remote_services = ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS); - const NetPermissionFlags permission_flags = ConsumeWeakEnum(fuzzed_data_provider, ALL_NET_PERMISSION_FLAGS); - const int32_t version = fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(MIN_PEER_PROTO_VERSION, std::numeric_limits<int32_t>::max()); - const bool relay_txs{fuzzed_data_provider.ConsumeBool()}; - - const CNetMsgMaker mm{0}; - - CSerializedNetMsg msg_version{ - mm.Make(NetMsgType::VERSION, - version, // - Using<CustomUintFormatter<8>>(remote_services), // - int64_t{}, // dummy time - int64_t{}, // ignored service bits - CService{}, // dummy - int64_t{}, // ignored service bits - CService{}, // ignored - uint64_t{1}, // dummy nonce - std::string{}, // dummy subver - int32_t{}, // dummy starting_height - relay_txs), - }; - - (void)connman.ReceiveMsgFrom(node, msg_version); - node.fPauseSend = false; - connman.ProcessMessagesOnce(node); - { - LOCK(node.cs_sendProcessing); - peerman.SendMessages(&node); - } - if (node.fDisconnect) return; - assert(node.nVersion == version); - assert(node.GetCommonVersion() == std::min(version, PROTOCOL_VERSION)); - assert(node.nServices == remote_services); - CNodeStateStats statestats; - assert(peerman.GetNodeStateStats(node.GetId(), statestats)); - assert(statestats.m_relay_txs == (relay_txs && !node.IsBlockOnlyConn())); - node.m_permissionFlags = permission_flags; - if (successfully_connected) { - CSerializedNetMsg msg_verack{mm.Make(NetMsgType::VERACK)}; - (void)connman.ReceiveMsgFrom(node, msg_verack); - node.fPauseSend = false; - connman.ProcessMessagesOnce(node); - { - LOCK(node.cs_sendProcessing); - peerman.SendMessages(&node); - } - assert(node.fSuccessfullyConnected == true); - } + connman.Handshake(node, + /*successfully_connected=*/fuzzed_data_provider.ConsumeBool(), + /*remote_services=*/ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS), + /*local_services=*/ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS), + /*permission_flags=*/ConsumeWeakEnum(fuzzed_data_provider, ALL_NET_PERMISSION_FLAGS), + /*version=*/fuzzed_data_provider.ConsumeIntegralInRange<int32_t>(MIN_PEER_PROTO_VERSION, std::numeric_limits<int32_t>::max()), + /*relay_txs=*/fuzzed_data_provider.ConsumeBool()); } CAmount ConsumeMoney(FuzzedDataProvider& fuzzed_data_provider, const std::optional<CAmount>& max) noexcept diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index f6a4e5570f..b0893e7926 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -296,7 +296,6 @@ template <bool ReturnUniquePtr = false> auto ConsumeNode(FuzzedDataProvider& fuzzed_data_provider, const std::optional<NodeId>& node_id_in = std::nullopt) noexcept { const NodeId node_id = node_id_in.value_or(fuzzed_data_provider.ConsumeIntegralInRange<NodeId>(0, std::numeric_limits<NodeId>::max())); - const ServiceFlags local_services = ConsumeWeakEnum(fuzzed_data_provider, ALL_SERVICE_FLAGS); const auto sock = std::make_shared<FuzzedSock>(fuzzed_data_provider); const CAddress address = ConsumeAddress(fuzzed_data_provider); const uint64_t keyed_net_group = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); @@ -307,7 +306,6 @@ auto ConsumeNode(FuzzedDataProvider& fuzzed_data_provider, const std::optional<N const bool inbound_onion{conn_type == ConnectionType::INBOUND ? fuzzed_data_provider.ConsumeBool() : false}; if constexpr (ReturnUniquePtr) { return std::make_unique<CNode>(node_id, - local_services, sock, address, keyed_net_group, @@ -318,7 +316,6 @@ auto ConsumeNode(FuzzedDataProvider& fuzzed_data_provider, const std::optional<N inbound_onion); } else { return CNode{node_id, - local_services, sock, address, keyed_net_group, @@ -331,7 +328,7 @@ auto ConsumeNode(FuzzedDataProvider& fuzzed_data_provider, const std::optional<N } inline std::unique_ptr<CNode> ConsumeNodeAsUniquePtr(FuzzedDataProvider& fdp, const std::optional<NodeId>& node_id_in = std::nullopt) { return ConsumeNode<true>(fdp, node_id_in); } -void FillNode(FuzzedDataProvider& fuzzed_data_provider, ConnmanTestMsg& connman, PeerManager& peerman, CNode& node) noexcept; +void FillNode(FuzzedDataProvider& fuzzed_data_provider, ConnmanTestMsg& connman, CNode& node) noexcept; class FuzzedFileProvider { diff --git a/src/test/fuzz/validation_load_mempool.cpp b/src/test/fuzz/validation_load_mempool.cpp index 9532610f8d..90c1a71d9f 100644 --- a/src/test/fuzz/validation_load_mempool.cpp +++ b/src/test/fuzz/validation_load_mempool.cpp @@ -2,10 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <kernel/mempool_persist.h> + #include <chainparamsbase.h> #include <mempool_args.h> +#include <node/mempool_persist_args.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/mempool_utils.h> #include <test/fuzz/util.h> #include <test/util/setup_common.h> #include <txmempool.h> @@ -15,6 +19,10 @@ #include <cstdint> #include <vector> +using kernel::DumpMempool; + +using node::MempoolPath; + namespace { const TestingSetup* g_setup; } // namespace @@ -33,9 +41,12 @@ FUZZ_TARGET_INIT(validation_load_mempool, initialize_validation_load_mempool) CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; + auto& chainstate{static_cast<DummyChainState&>(g_setup->m_node.chainman->ActiveChainstate())}; + chainstate.SetMempool(&pool); + auto fuzzed_fopen = [&](const fs::path&, const char*) { return fuzzed_file_provider.open(); }; - (void)LoadMempool(pool, g_setup->m_node.chainman->ActiveChainstate(), fuzzed_fopen); - (void)DumpMempool(pool, fuzzed_fopen, true); + (void)chainstate.LoadMempool(MempoolPath(g_setup->m_args), fuzzed_fopen); + (void)DumpMempool(pool, MempoolPath(g_setup->m_args), fuzzed_fopen, true); } diff --git a/src/test/miniscript_tests.cpp b/src/test/miniscript_tests.cpp index 3877fea907..95e8476b77 100644 --- a/src/test/miniscript_tests.cpp +++ b/src/test/miniscript_tests.cpp @@ -111,11 +111,17 @@ struct KeyConverter { assert(it != g_testdata->pkmap.end()); return it->second; } + + std::optional<std::string> ToString(const Key& key) const { + return HexStr(ToPKBytes(key)); + } }; //! Singleton instance of KeyConverter. const KeyConverter CONVERTER{}; +// https://github.com/llvm/llvm-project/issues/53444 +// NOLINTNEXTLINE(misc-unused-using-decls) using miniscript::operator"" _mst; enum TestMode : int { @@ -276,7 +282,7 @@ BOOST_AUTO_TEST_CASE(fixed_tests) // (for now) have 'd:' be 'u'. This tests we can't use a 'd:' wrapper for a thresh, which requires // its subs to all be 'u' (taken from https://github.com/rust-bitcoin/rust-miniscript/discussions/341). const auto ms_minimalif = miniscript::FromString("thresh(3,c:pk_k(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),sc:pk_k(03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556),sc:pk_k(0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798),sdv:older(32))", CONVERTER); - BOOST_CHECK(!ms_minimalif); + BOOST_CHECK(ms_minimalif && !ms_minimalif->IsValid()); // A Miniscript with duplicate keys is not sane const auto ms_dup1 = miniscript::FromString("and_v(v:pk(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),pk(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65))", CONVERTER); BOOST_CHECK(ms_dup1); @@ -290,6 +296,15 @@ BOOST_AUTO_TEST_CASE(fixed_tests) // Same when the duplicates are on different levels in the tree const auto ms_dup4 = miniscript::FromString("thresh(2,pkh(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),s:pk(03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556),a:and_b(dv:older(1),s:pk(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65)))", CONVERTER); BOOST_CHECK(ms_dup4 && !ms_dup4->IsSane() && !ms_dup4->CheckDuplicateKey()); + // Test we find the first insane sub closer to be a leaf node. This fragment is insane for two reasons: + // 1. It can be spent without a signature + // 2. It contains timelock mixes + // We'll report the timelock mix error, as it's "deeper" (closer to be a leaf node) than the "no 's' property" + // error is. + const auto ms_ins = miniscript::FromString("or_i(and_b(after(1),a:after(1000000000)),pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204))", CONVERTER); + BOOST_CHECK(ms_ins && ms_ins->IsValid() && !ms_ins->IsSane()); + const auto insane_sub = ms_ins->FindInsaneSub(); + BOOST_CHECK(insane_sub && *insane_sub->ToString(CONVERTER) == "and_b(after(1),a:after(1000000000))"); // Timelock tests Test("after(100)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // only heightlock diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 115c4b9b24..f2eaa0179f 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -58,7 +58,6 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) std::string pszDest; std::unique_ptr<CNode> pnode1 = std::make_unique<CNode>(id++, - NODE_NETWORK, /*sock=*/nullptr, addr, /*nKeyedNetGroupIn=*/0, @@ -77,7 +76,6 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) BOOST_CHECK_EQUAL(pnode1->ConnectedThroughNetwork(), Network::NET_IPV4); std::unique_ptr<CNode> pnode2 = std::make_unique<CNode>(id++, - NODE_NETWORK, /*sock=*/nullptr, addr, /*nKeyedNetGroupIn=*/1, @@ -96,7 +94,6 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) BOOST_CHECK_EQUAL(pnode2->ConnectedThroughNetwork(), Network::NET_IPV4); std::unique_ptr<CNode> pnode3 = std::make_unique<CNode>(id++, - NODE_NETWORK, /*sock=*/nullptr, addr, /*nKeyedNetGroupIn=*/0, @@ -115,7 +112,6 @@ BOOST_AUTO_TEST_CASE(cnode_simple_test) BOOST_CHECK_EQUAL(pnode3->ConnectedThroughNetwork(), Network::NET_IPV4); std::unique_ptr<CNode> pnode4 = std::make_unique<CNode>(id++, - NODE_NETWORK, /*sock=*/nullptr, addr, /*nKeyedNetGroupIn=*/1, @@ -629,7 +625,6 @@ BOOST_AUTO_TEST_CASE(ipv4_peer_with_ipv6_addrMe_test) ipv4AddrPeer.s_addr = 0xa0b0c001; CAddress addr = CAddress(CService(ipv4AddrPeer, 7777), NODE_NETWORK); std::unique_ptr<CNode> pnode = std::make_unique<CNode>(/*id=*/0, - NODE_NETWORK, /*sock=*/nullptr, addr, /*nKeyedNetGroupIn=*/0, @@ -648,7 +643,7 @@ BOOST_AUTO_TEST_CASE(ipv4_peer_with_ipv6_addrMe_test) pnode->SetAddrLocal(addrLocal); // before patch, this causes undefined behavior detectable with clang's -fsanitize=memory - GetLocalAddrForPeer(&*pnode); + GetLocalAddrForPeer(*pnode); // suppress no-checks-run warning; if this test fails, it's by triggering a sanitizer BOOST_CHECK(1); @@ -675,19 +670,15 @@ BOOST_AUTO_TEST_CASE(get_local_addr_for_peer_port) const uint16_t bind_port = 20001; m_node.args->ForceSetArg("-bind", strprintf("3.4.5.6:%u", bind_port)); - const uint32_t current_time = static_cast<uint32_t>(GetAdjustedTime()); - SetMockTime(current_time); - // Our address:port as seen from the peer, completely different from the above. in_addr peer_us_addr; peer_us_addr.s_addr = htonl(0x02030405); - const CAddress peer_us{CService{peer_us_addr, 20002}, NODE_NETWORK, current_time}; + const CService peer_us{peer_us_addr, 20002}; // Create a peer with a routable IPv4 address (outbound). in_addr peer_out_in_addr; peer_out_in_addr.s_addr = htonl(0x01020304); CNode peer_out{/*id=*/0, - /*nLocalServicesIn=*/NODE_NETWORK, /*sock=*/nullptr, /*addrIn=*/CAddress{CService{peer_out_in_addr, 8333}, NODE_NETWORK}, /*nKeyedNetGroupIn=*/0, @@ -700,16 +691,15 @@ BOOST_AUTO_TEST_CASE(get_local_addr_for_peer_port) peer_out.SetAddrLocal(peer_us); // Without the fix peer_us:8333 is chosen instead of the proper peer_us:bind_port. - auto chosen_local_addr = GetLocalAddrForPeer(&peer_out); + auto chosen_local_addr = GetLocalAddrForPeer(peer_out); BOOST_REQUIRE(chosen_local_addr); - const CAddress expected{CService{peer_us_addr, bind_port}, NODE_NETWORK, current_time}; + const CService expected{peer_us_addr, bind_port}; BOOST_CHECK(*chosen_local_addr == expected); // Create a peer with a routable IPv4 address (inbound). in_addr peer_in_in_addr; peer_in_in_addr.s_addr = htonl(0x05060708); CNode peer_in{/*id=*/0, - /*nLocalServicesIn=*/NODE_NETWORK, /*sock=*/nullptr, /*addrIn=*/CAddress{CService{peer_in_in_addr, 8333}, NODE_NETWORK}, /*nKeyedNetGroupIn=*/0, @@ -722,7 +712,7 @@ BOOST_AUTO_TEST_CASE(get_local_addr_for_peer_port) peer_in.SetAddrLocal(peer_us); // Without the fix peer_us:8333 is chosen instead of the proper peer_us:peer_us.GetPort(). - chosen_local_addr = GetLocalAddrForPeer(&peer_in); + chosen_local_addr = GetLocalAddrForPeer(peer_in); BOOST_REQUIRE(chosen_local_addr); BOOST_CHECK(*chosen_local_addr == peer_us); @@ -837,7 +827,6 @@ BOOST_AUTO_TEST_CASE(initial_advertise_from_version_message) in_addr peer_in_addr; peer_in_addr.s_addr = htonl(0x01020304); CNode peer{/*id=*/0, - /*nLocalServicesIn=*/NODE_NETWORK, /*sock=*/nullptr, /*addrIn=*/CAddress{CService{peer_in_addr, 8333}, NODE_NETWORK}, /*nKeyedNetGroupIn=*/0, @@ -857,7 +846,7 @@ BOOST_AUTO_TEST_CASE(initial_advertise_from_version_message) *static_cast<TestChainState*>(&m_node.chainman->ActiveChainstate()); chainstate.JumpOutOfIbd(); - m_node.peerman->InitializeNode(&peer); + m_node.peerman->InitializeNode(peer, NODE_NETWORK); std::atomic<bool> interrupt_dummy{false}; std::chrono::microseconds time_received_dummy{0}; diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index 15213f826b..62c7ddb673 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -4,6 +4,7 @@ #include <chainparams.h> #include <index/txindex.h> +#include <interfaces/chain.h> #include <script/standard.h> #include <test/util/setup_common.h> #include <util/time.h> @@ -15,7 +16,7 @@ BOOST_AUTO_TEST_SUITE(txindex_tests) BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) { - TxIndex txindex(1 << 20, true); + TxIndex txindex(interfaces::MakeChain(m_node), 1 << 20, true); CTransactionRef tx_disk; uint256 block_hash; @@ -28,7 +29,7 @@ BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) // BlockUntilSyncedToCurrentChain should return false before txindex is started. BOOST_CHECK(!txindex.BlockUntilSyncedToCurrentChain()); - BOOST_REQUIRE(txindex.Start(m_node.chainman->ActiveChainstate())); + BOOST_REQUIRE(txindex.Start()); // Allow tx index to catch up with the block index. constexpr int64_t timeout_ms = 10 * 1000; diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index 62b770753a..223db16c6c 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -5,11 +5,71 @@ #include <test/util/net.h> #include <chainparams.h> +#include <node/eviction.h> #include <net.h> +#include <net_processing.h> +#include <netmessagemaker.h> #include <span.h> #include <vector> +void ConnmanTestMsg::Handshake(CNode& node, + bool successfully_connected, + ServiceFlags remote_services, + ServiceFlags local_services, + NetPermissionFlags permission_flags, + int32_t version, + bool relay_txs) +{ + auto& peerman{static_cast<PeerManager&>(*m_msgproc)}; + auto& connman{*this}; + const CNetMsgMaker mm{0}; + + peerman.InitializeNode(node, local_services); + + CSerializedNetMsg msg_version{ + mm.Make(NetMsgType::VERSION, + version, // + Using<CustomUintFormatter<8>>(remote_services), // + int64_t{}, // dummy time + int64_t{}, // ignored service bits + CService{}, // dummy + int64_t{}, // ignored service bits + CService{}, // ignored + uint64_t{1}, // dummy nonce + std::string{}, // dummy subver + int32_t{}, // dummy starting_height + relay_txs), + }; + + (void)connman.ReceiveMsgFrom(node, msg_version); + node.fPauseSend = false; + connman.ProcessMessagesOnce(node); + { + LOCK(node.cs_sendProcessing); + peerman.SendMessages(&node); + } + if (node.fDisconnect) return; + assert(node.nVersion == version); + assert(node.GetCommonVersion() == std::min(version, PROTOCOL_VERSION)); + CNodeStateStats statestats; + assert(peerman.GetNodeStateStats(node.GetId(), statestats)); + assert(statestats.m_relay_txs == (relay_txs && !node.IsBlockOnlyConn())); + assert(statestats.their_services == remote_services); + node.m_permissionFlags = permission_flags; + if (successfully_connected) { + CSerializedNetMsg msg_verack{mm.Make(NetMsgType::VERACK)}; + (void)connman.ReceiveMsgFrom(node, msg_verack); + node.fPauseSend = false; + connman.ProcessMessagesOnce(node); + { + LOCK(node.cs_sendProcessing); + peerman.SendMessages(&node); + } + assert(node.fSuccessfullyConnected == true); + } +} + void ConnmanTestMsg::NodeReceiveMsgBytes(CNode& node, Span<const uint8_t> msg_bytes, bool& complete) const { assert(node.ReceiveMsgBytes(msg_bytes, complete)); @@ -58,6 +118,8 @@ std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(int n_candida /*prefer_evict=*/random_context.randbool(), /*m_is_local=*/random_context.randbool(), /*m_network=*/ALL_NETWORKS[random_context.randrange(ALL_NETWORKS.size())], + /*m_noban=*/false, + /*m_conn_type=*/ConnectionType::INBOUND, }); } return candidates; diff --git a/src/test/util/net.h b/src/test/util/net.h index c5dbaeca3e..7f61a03d27 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -6,6 +6,7 @@ #define BITCOIN_TEST_UTIL_NET_H #include <compat.h> +#include <node/eviction.h> #include <netaddress.h> #include <net.h> #include <util/sock.h> @@ -38,6 +39,14 @@ struct ConnmanTestMsg : public CConnman { m_nodes.clear(); } + void Handshake(CNode& node, + bool successfully_connected, + ServiceFlags remote_services, + ServiceFlags local_services, + NetPermissionFlags permission_flags, + int32_t version, + bool relay_txs); + void ProcessMessagesOnce(CNode& node) { m_msgproc->ProcessMessages(&node, flagInterruptMsgProc); } void NodeReceiveMsgBytes(CNode& node, Span<const uint8_t> msg_bytes, bool& complete) const; diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 0c9e880d67..67984721a3 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -36,7 +36,6 @@ #include <timedata.h> #include <txdb.h> #include <txmempool.h> -#include <util/designator.h> #include <util/strencodings.h> #include <util/string.h> #include <util/thread.h> @@ -55,8 +54,6 @@ using node::BlockAssembler; using node::CalculateCacheSizes; -using node::fPruneMode; -using node::fReindex; using node::LoadChainstate; using node::NodeContext; using node::RegenerateCommitments; @@ -158,10 +155,10 @@ BasicTestingSetup::~BasicTestingSetup() CTxMemPool::Options MemPoolOptionsForTest(const NodeContext& node) { CTxMemPool::Options mempool_opts{ - Desig(estimator) node.fee_estimator.get(), + .estimator = node.fee_estimator.get(), // Default to always checking mempool regardless of // chainparams.DefaultConsistencyChecks for tests - Desig(check_ratio) 1, + .check_ratio = 1, }; ApplyArgsManOptions(*node.args, mempool_opts); return mempool_opts; @@ -184,8 +181,8 @@ ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::ve m_cache_sizes = CalculateCacheSizes(m_args); const ChainstateManager::Options chainman_opts{ - chainparams, - GetAdjustedTime, + .chainparams = chainparams, + .adjusted_time_callback = GetAdjustedTime, }; m_node.chainman = std::make_unique<ChainstateManager>(chainman_opts); m_node.chainman->m_blockman.m_block_tree_db = std::make_unique<CBlockTreeDB>(m_cache_sizes.block_tree_db, true); @@ -219,25 +216,20 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const // instead of unit tests, but for now we need these here. RegisterAllCoreRPCCommands(tableRPC); - auto maybe_load_error = LoadChainstate(fReindex.load(), - *Assert(m_node.chainman.get()), - Assert(m_node.mempool.get()), - fPruneMode, - m_args.GetBoolArg("-reindex-chainstate", false), - m_cache_sizes.block_tree_db, - m_cache_sizes.coins_db, - m_cache_sizes.coins, - /*block_tree_db_in_memory=*/true, - /*coins_db_in_memory=*/true); - assert(!maybe_load_error.has_value()); - - auto maybe_verify_error = VerifyLoadedChainstate( - *Assert(m_node.chainman), - fReindex.load(), - m_args.GetBoolArg("-reindex-chainstate", false), - m_args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS), - m_args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL)); - assert(!maybe_verify_error.has_value()); + node::ChainstateLoadOptions options; + options.mempool = Assert(m_node.mempool.get()); + options.block_tree_db_in_memory = true; + options.coins_db_in_memory = true; + options.reindex = node::fReindex; + options.reindex_chainstate = m_args.GetBoolArg("-reindex-chainstate", false); + options.prune = node::fPruneMode; + options.check_blocks = m_args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); + options.check_level = m_args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL); + auto [status, error] = LoadChainstate(*Assert(m_node.chainman), m_cache_sizes, options); + assert(status == node::ChainstateLoadStatus::SUCCESS); + + std::tie(status, error) = VerifyLoadedChainstate(*Assert(m_node.chainman), options); + assert(status == node::ChainstateLoadStatus::SUCCESS); BlockValidationState state; if (!m_node.chainman->ActiveChainstate().ActivateBestChain(state)) { diff --git a/src/test/util/wallet.cpp b/src/test/util/wallet.cpp index 52aaeabccf..7a00ac9e1f 100644 --- a/src/test/util/wallet.cpp +++ b/src/test/util/wallet.cpp @@ -20,11 +20,10 @@ const std::string ADDRESS_BCRT1_UNSPENDABLE = "bcrt1qqqqqqqqqqqqqqqqqqqqqqqqqqqq std::string getnewaddress(CWallet& w) { constexpr auto output_type = OutputType::BECH32; - CTxDestination dest; - bilingual_str error; - if (!w.GetNewDestination(output_type, "", dest, error)) assert(false); + auto op_dest = w.GetNewDestination(output_type, ""); + assert(op_dest.HasRes()); - return EncodeDestination(dest); + return EncodeDestination(op_dest.GetObj()); } #endif // ENABLE_WALLET diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 102de74389..78adb521ac 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -24,8 +24,8 @@ BOOST_FIXTURE_TEST_SUITE(validation_chainstate_tests, TestingSetup) BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) { const ChainstateManager::Options chainman_opts{ - Params(), - GetAdjustedTime, + .chainparams = Params(), + .adjusted_time_callback = GetAdjustedTime, }; ChainstateManager manager{chainman_opts}; diff --git a/src/txdb.h b/src/txdb.h index faa543b412..a04596f7bb 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -8,6 +8,7 @@ #include <coins.h> #include <dbwrapper.h> +#include <sync.h> #include <memory> #include <optional> diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 69ae9fed99..7eff6bdbe3 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -458,6 +458,7 @@ CTxMemPool::CTxMemPool(const Options& opts) minerPolicyEstimator{opts.estimator}, m_max_size_bytes{opts.max_size_bytes}, m_expiry{opts.expiry}, + m_full_rbf{opts.full_rbf}, m_limits{opts.limits} { _clear(); //lock free clear @@ -1209,14 +1210,14 @@ void CTxMemPool::GetTransactionAncestry(const uint256& txid, size_t& ancestors, } } -bool CTxMemPool::IsLoaded() const +bool CTxMemPool::GetLoadTried() const { LOCK(cs); - return m_is_loaded; + return m_load_tried; } -void CTxMemPool::SetIsLoaded(bool loaded) +void CTxMemPool::SetLoadTried(bool load_tried) { LOCK(cs); - m_is_loaded = loaded; + m_load_tried = load_tried; } diff --git a/src/txmempool.h b/src/txmempool.h index f44e78fde5..d7d308038c 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -451,7 +451,7 @@ protected: void trackPackageRemoved(const CFeeRate& rate) EXCLUSIVE_LOCKS_REQUIRED(cs); - bool m_is_loaded GUARDED_BY(cs){false}; + bool m_load_tried GUARDED_BY(cs){false}; CFeeRate GetMinFee(size_t sizelimit) const; @@ -568,6 +568,7 @@ public: const int64_t m_max_size_bytes; const std::chrono::seconds m_expiry; + const bool m_full_rbf; using Limits = kernel::MemPoolLimits; @@ -727,11 +728,17 @@ public: */ void GetTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants, size_t* ancestorsize = nullptr, CAmount* ancestorfees = nullptr) const; - /** @returns true if the mempool is fully loaded */ - bool IsLoaded() const; + /** + * @returns true if we've made an attempt to load the mempool regardless of + * whether the attempt was successful or not + */ + bool GetLoadTried() const; - /** Sets the current loaded state */ - void SetIsLoaded(bool loaded); + /** + * Set whether or not we've made an attempt to load the mempool (regardless + * of whether the attempt was successful or not) + */ + void SetLoadTried(bool load_tried); unsigned long size() const { diff --git a/src/univalue/include/univalue.h b/src/univalue/include/univalue.h index 22be0311e8..dff544f96f 100644 --- a/src/univalue/include/univalue.h +++ b/src/univalue/include/univalue.h @@ -82,12 +82,14 @@ public: bool isArray() const { return (typ == VARR); } bool isObject() const { return (typ == VOBJ); } - bool push_back(const UniValue& val); - bool push_backV(const std::vector<UniValue>& vec); + void push_back(const UniValue& val); + void push_backV(const std::vector<UniValue>& vec); + template <class It> + void push_backV(It first, It last); void __pushKV(const std::string& key, const UniValue& val); - bool pushKV(const std::string& key, const UniValue& val); - bool pushKVs(const UniValue& obj); + void pushKV(const std::string& key, const UniValue& val); + void pushKVs(const UniValue& obj); std::string write(unsigned int prettyIndent = 0, unsigned int indentLevel = 0) const; @@ -104,6 +106,7 @@ private: std::vector<std::string> keys; std::vector<UniValue> values; + void checkType(const VType& expected) const; bool findKey(const std::string& key, size_t& retIdx) const; void writeArray(unsigned int prettyIndent, unsigned int indentLevel, std::string& s) const; void writeObject(unsigned int prettyIndent, unsigned int indentLevel, std::string& s) const; @@ -114,19 +117,7 @@ public: const std::vector<std::string>& getKeys() const; const std::vector<UniValue>& getValues() const; template <typename Int> - auto getInt() const - { - static_assert(std::is_integral<Int>::value); - if (typ != VNUM) { - throw std::runtime_error("JSON value is not an integer as expected"); - } - Int result; - const auto [first_nonmatching, error_condition] = std::from_chars(val.data(), val.data() + val.size(), result); - if (first_nonmatching != val.data() + val.size() || error_condition != std::errc{}) { - throw std::runtime_error("JSON integer out of range"); - } - return result; - } + Int getInt() const; bool get_bool() const; const std::string& get_str() const; double get_real() const; @@ -137,6 +128,26 @@ public: friend const UniValue& find_value( const UniValue& obj, const std::string& name); }; +template <class It> +void UniValue::push_backV(It first, It last) +{ + checkType(VARR); + values.insert(values.end(), first, last); +} + +template <typename Int> +Int UniValue::getInt() const +{ + static_assert(std::is_integral<Int>::value); + checkType(VNUM); + Int result; + const auto [first_nonmatching, error_condition] = std::from_chars(val.data(), val.data() + val.size(), result); + if (first_nonmatching != val.data() + val.size() || error_condition != std::errc{}) { + throw std::runtime_error("JSON integer out of range"); + } + return result; +} + enum jtokentype { JTOK_ERR = -1, JTOK_NONE = 0, // eof diff --git a/src/univalue/lib/univalue.cpp b/src/univalue/lib/univalue.cpp index 3553995c28..051bc8eba6 100644 --- a/src/univalue/lib/univalue.cpp +++ b/src/univalue/lib/univalue.cpp @@ -108,53 +108,46 @@ bool UniValue::setObject() return true; } -bool UniValue::push_back(const UniValue& val_) +void UniValue::push_back(const UniValue& val_) { - if (typ != VARR) - return false; + checkType(VARR); values.push_back(val_); - return true; } -bool UniValue::push_backV(const std::vector<UniValue>& vec) +void UniValue::push_backV(const std::vector<UniValue>& vec) { - if (typ != VARR) - return false; + checkType(VARR); values.insert(values.end(), vec.begin(), vec.end()); - - return true; } void UniValue::__pushKV(const std::string& key, const UniValue& val_) { + checkType(VOBJ); + keys.push_back(key); values.push_back(val_); } -bool UniValue::pushKV(const std::string& key, const UniValue& val_) +void UniValue::pushKV(const std::string& key, const UniValue& val_) { - if (typ != VOBJ) - return false; + checkType(VOBJ); size_t idx; if (findKey(key, idx)) values[idx] = val_; else __pushKV(key, val_); - return true; } -bool UniValue::pushKVs(const UniValue& obj) +void UniValue::pushKVs(const UniValue& obj) { - if (typ != VOBJ || obj.typ != VOBJ) - return false; + checkType(VOBJ); + obj.checkType(VOBJ); for (size_t i = 0; i < obj.keys.size(); i++) __pushKV(obj.keys[i], obj.values.at(i)); - - return true; } void UniValue::getObjMap(std::map<std::string,UniValue>& kv) const @@ -221,6 +214,14 @@ const UniValue& UniValue::operator[](size_t index) const return values.at(index); } +void UniValue::checkType(const VType& expected) const +{ + if (typ != expected) { + throw std::runtime_error{"JSON value of type " + std::string{uvTypeName(typ)} + " is not of expected type " + + std::string{uvTypeName(expected)}}; + } +} + const char *uvTypeName(UniValue::VType t) { switch (t) { diff --git a/src/univalue/lib/univalue_get.cpp b/src/univalue/lib/univalue_get.cpp index 9bbdb1fe69..5c58f388dd 100644 --- a/src/univalue/lib/univalue_get.cpp +++ b/src/univalue/lib/univalue_get.cpp @@ -46,8 +46,7 @@ bool ParseDouble(const std::string& str, double *out) const std::vector<std::string>& UniValue::getKeys() const { - if (typ != VOBJ) - throw std::runtime_error("JSON value is not an object as expected"); + checkType(VOBJ); return keys; } @@ -60,22 +59,19 @@ const std::vector<UniValue>& UniValue::getValues() const bool UniValue::get_bool() const { - if (typ != VBOOL) - throw std::runtime_error("JSON value is not a boolean as expected"); + checkType(VBOOL); return getBool(); } const std::string& UniValue::get_str() const { - if (typ != VSTR) - throw std::runtime_error("JSON value is not a string as expected"); + checkType(VSTR); return getValStr(); } double UniValue::get_real() const { - if (typ != VNUM) - throw std::runtime_error("JSON value is not a number as expected"); + checkType(VNUM); double retval; if (!ParseDouble(getValStr(), &retval)) throw std::runtime_error("JSON double out of range"); @@ -84,15 +80,12 @@ double UniValue::get_real() const const UniValue& UniValue::get_obj() const { - if (typ != VOBJ) - throw std::runtime_error("JSON value is not an object as expected"); + checkType(VOBJ); return *this; } const UniValue& UniValue::get_array() const { - if (typ != VARR) - throw std::runtime_error("JSON value is not an array as expected"); + checkType(VARR); return *this; } - diff --git a/src/univalue/sources.mk b/src/univalue/sources.mk index e156216378..5e4d9c3831 100644 --- a/src/univalue/sources.mk +++ b/src/univalue/sources.mk @@ -1,12 +1,8 @@ -# - All variables are namespaced with UNIVALUE_ to avoid colliding with -# downstream makefiles. # - All Variables ending in _HEADERS or _SOURCES confuse automake, so the # _INT postfix is applied. # - Convenience variables, for example a UNIVALUE_TEST_DIR should not be used # as they interfere with automatic dependency generation -# - The %reldir% is the relative path from the Makefile.am. This allows -# downstreams to use these variables without having to manually account for -# the path change. +# - The %reldir% is the relative path from the Makefile.am. UNIVALUE_INCLUDE_DIR_INT = %reldir%/include @@ -29,9 +25,6 @@ UNIVALUE_TEST_UNITESTER_INT += %reldir%/test/unitester.cpp UNIVALUE_TEST_JSON_INT = UNIVALUE_TEST_JSON_INT += %reldir%/test/test_json.cpp -UNIVALUE_TEST_NO_NUL_INT = -UNIVALUE_TEST_NO_NUL_INT += %reldir%/test/no_nul.cpp - UNIVALUE_TEST_OBJECT_INT = UNIVALUE_TEST_OBJECT_INT += %reldir%/test/object.cpp diff --git a/src/univalue/test/.gitignore b/src/univalue/test/.gitignore index 7b27cf0da2..5812c96b14 100644 --- a/src/univalue/test/.gitignore +++ b/src/univalue/test/.gitignore @@ -2,7 +2,6 @@ object unitester test_json -no_nul *.trs *.log diff --git a/src/univalue/test/no_nul.cpp b/src/univalue/test/no_nul.cpp deleted file mode 100644 index 3a7a727abb..0000000000 --- a/src/univalue/test/no_nul.cpp +++ /dev/null @@ -1,8 +0,0 @@ -#include <univalue.h> - -int main (int argc, char *argv[]) -{ - char buf[] = "___[1,2,3]___"; - UniValue val; - return val.read(buf + 3, 7) ? 0 : 1; -} diff --git a/src/univalue/test/object.cpp b/src/univalue/test/object.cpp index 8a35bf914d..cf8c29ec67 100644 --- a/src/univalue/test/object.cpp +++ b/src/univalue/test/object.cpp @@ -13,9 +13,6 @@ #include <string> #include <vector> -#define BOOST_FIXTURE_TEST_SUITE(a, b) -#define BOOST_AUTO_TEST_CASE(funcName) void funcName() -#define BOOST_AUTO_TEST_SUITE_END() #define BOOST_CHECK(expr) assert(expr) #define BOOST_CHECK_EQUAL(v1, v2) assert((v1) == (v2)) #define BOOST_CHECK_THROW(stmt, excMatch) { \ @@ -35,9 +32,7 @@ } \ } -BOOST_FIXTURE_TEST_SUITE(univalue_tests, BasicTestingSetup) - -BOOST_AUTO_TEST_CASE(univalue_constructor) +void univalue_constructor() { UniValue v1; BOOST_CHECK(v1.isNull()); @@ -85,7 +80,17 @@ BOOST_AUTO_TEST_CASE(univalue_constructor) BOOST_CHECK_EQUAL(v9.getValStr(), "zappa"); } -BOOST_AUTO_TEST_CASE(univalue_typecheck) +void univalue_push_throw() +{ + UniValue j; + BOOST_CHECK_THROW(j.push_back(1), std::runtime_error); + BOOST_CHECK_THROW(j.push_backV({1}), std::runtime_error); + BOOST_CHECK_THROW(j.__pushKV("k", 1), std::runtime_error); + BOOST_CHECK_THROW(j.pushKV("k", 1), std::runtime_error); + BOOST_CHECK_THROW(j.pushKVs({}), std::runtime_error); +} + +void univalue_typecheck() { UniValue v1; BOOST_CHECK(v1.setNumStr("1")); @@ -134,7 +139,7 @@ BOOST_AUTO_TEST_CASE(univalue_typecheck) BOOST_CHECK_THROW(vals[1].get_bool(), std::runtime_error); } -BOOST_AUTO_TEST_CASE(univalue_set) +void univalue_set() { UniValue v(UniValue::VSTR, "foo"); v.clear(); @@ -193,18 +198,18 @@ BOOST_AUTO_TEST_CASE(univalue_set) BOOST_CHECK(v.isNull()); } -BOOST_AUTO_TEST_CASE(univalue_array) +void univalue_array() { UniValue arr(UniValue::VARR); UniValue v((int64_t)1023LL); - BOOST_CHECK(arr.push_back(v)); + arr.push_back(v); std::string vStr("zippy"); - BOOST_CHECK(arr.push_back(vStr)); + arr.push_back(vStr); const char *s = "pippy"; - BOOST_CHECK(arr.push_back(s)); + arr.push_back(s); std::vector<UniValue> vec; v.setStr("boing"); @@ -213,13 +218,13 @@ BOOST_AUTO_TEST_CASE(univalue_array) v.setStr("going"); vec.push_back(v); - BOOST_CHECK(arr.push_backV(vec)); + arr.push_backV(vec); - BOOST_CHECK(arr.push_back((uint64_t) 400ULL)); - BOOST_CHECK(arr.push_back((int64_t) -400LL)); - BOOST_CHECK(arr.push_back((int) -401)); - BOOST_CHECK(arr.push_back(-40.1)); - BOOST_CHECK(arr.push_back(true)); + arr.push_back(uint64_t{400ULL}); + arr.push_back(int64_t{-400LL}); + arr.push_back(int{-401}); + arr.push_back(-40.1); + arr.push_back(true); BOOST_CHECK_EQUAL(arr.empty(), false); BOOST_CHECK_EQUAL(arr.size(), 10); @@ -252,7 +257,7 @@ BOOST_AUTO_TEST_CASE(univalue_array) BOOST_CHECK_EQUAL(arr.size(), 0); } -BOOST_AUTO_TEST_CASE(univalue_object) +void univalue_object() { UniValue obj(UniValue::VOBJ); std::string strKey, strVal; @@ -260,39 +265,39 @@ BOOST_AUTO_TEST_CASE(univalue_object) strKey = "age"; v.setInt(100); - BOOST_CHECK(obj.pushKV(strKey, v)); + obj.pushKV(strKey, v); strKey = "first"; strVal = "John"; - BOOST_CHECK(obj.pushKV(strKey, strVal)); + obj.pushKV(strKey, strVal); strKey = "last"; - const char *cVal = "Smith"; - BOOST_CHECK(obj.pushKV(strKey, cVal)); + const char* cVal = "Smith"; + obj.pushKV(strKey, cVal); strKey = "distance"; - BOOST_CHECK(obj.pushKV(strKey, (int64_t) 25)); + obj.pushKV(strKey, int64_t{25}); strKey = "time"; - BOOST_CHECK(obj.pushKV(strKey, (uint64_t) 3600)); + obj.pushKV(strKey, uint64_t{3600}); strKey = "calories"; - BOOST_CHECK(obj.pushKV(strKey, (int) 12)); + obj.pushKV(strKey, int{12}); strKey = "temperature"; - BOOST_CHECK(obj.pushKV(strKey, (double) 90.012)); + obj.pushKV(strKey, double{90.012}); strKey = "moon"; - BOOST_CHECK(obj.pushKV(strKey, true)); + obj.pushKV(strKey, true); strKey = "spoon"; - BOOST_CHECK(obj.pushKV(strKey, false)); + obj.pushKV(strKey, false); UniValue obj2(UniValue::VOBJ); - BOOST_CHECK(obj2.pushKV("cat1", 9000)); - BOOST_CHECK(obj2.pushKV("cat2", 12345)); + obj2.pushKV("cat1", 9000); + obj2.pushKV("cat2", 12345); - BOOST_CHECK(obj.pushKVs(obj2)); + obj.pushKVs(obj2); BOOST_CHECK_EQUAL(obj.empty(), false); BOOST_CHECK_EQUAL(obj.size(), 11); @@ -371,7 +376,7 @@ BOOST_AUTO_TEST_CASE(univalue_object) static const char *json1 = "[1.10000000,{\"key1\":\"str\\u0000\",\"key2\":800,\"key3\":{\"name\":\"martian http://test.com\"}}]"; -BOOST_AUTO_TEST_CASE(univalue_readwrite) +void univalue_readwrite() { UniValue v; BOOST_CHECK(v.read(json1)); @@ -414,11 +419,10 @@ BOOST_AUTO_TEST_CASE(univalue_readwrite) BOOST_CHECK(!v.read("{} 42")); } -BOOST_AUTO_TEST_SUITE_END() - int main (int argc, char *argv[]) { univalue_constructor(); + univalue_push_throw(); univalue_typecheck(); univalue_set(); univalue_array(); diff --git a/src/univalue/test/unitester.cpp b/src/univalue/test/unitester.cpp index 94c149b39f..6344a5a0ab 100644 --- a/src/univalue/test/unitester.cpp +++ b/src/univalue/test/unitester.cpp @@ -149,6 +149,13 @@ void unescape_unicode_test() assert(val[0].get_str() == "\xf0\x9d\x85\xa1"); } +void no_nul_test() +{ + char buf[] = "___[1,2,3]___"; + UniValue val; + assert(val.read(buf + 3, 7)); +} + int main (int argc, char *argv[]) { for (const auto& f: filenames) { @@ -156,6 +163,7 @@ int main (int argc, char *argv[]) } unescape_unicode_test(); + no_nul_test(); return 0; } diff --git a/src/util/asmap.cpp b/src/util/asmap.cpp index b1aa598453..33258d9962 100644 --- a/src/util/asmap.cpp +++ b/src/util/asmap.cpp @@ -8,10 +8,13 @@ #include <crypto/common.h> #include <fs.h> #include <logging.h> +#include <serialize.h> #include <streams.h> +#include <algorithm> #include <cassert> -#include <map> +#include <cstdio> +#include <utility> #include <vector> namespace { diff --git a/src/util/bip32.cpp b/src/util/bip32.cpp index 4c7e948368..39e43eeb31 100644 --- a/src/util/bip32.cpp +++ b/src/util/bip32.cpp @@ -2,12 +2,15 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <sstream> -#include <stdio.h> #include <tinyformat.h> #include <util/bip32.h> #include <util/strencodings.h> +#include <algorithm> +#include <cstdint> +#include <cstdio> +#include <sstream> + bool ParseHDKeypath(const std::string& keypath_str, std::vector<uint32_t>& keypath) { diff --git a/src/util/bip32.h b/src/util/bip32.h index aa4eac3791..0872bc88de 100644 --- a/src/util/bip32.h +++ b/src/util/bip32.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_UTIL_BIP32_H #define BITCOIN_UTIL_BIP32_H +#include <cstdint> #include <string> #include <vector> diff --git a/src/util/bytevectorhash.cpp b/src/util/bytevectorhash.cpp index 9054db4759..6d777613e6 100644 --- a/src/util/bytevectorhash.cpp +++ b/src/util/bytevectorhash.cpp @@ -6,6 +6,8 @@ #include <random.h> #include <util/bytevectorhash.h> +#include <vector> + ByteVectorHash::ByteVectorHash() : m_k0(GetRand<uint64_t>()), m_k1(GetRand<uint64_t>()) diff --git a/src/util/bytevectorhash.h b/src/util/bytevectorhash.h index b88c17460b..c2322b8daf 100644 --- a/src/util/bytevectorhash.h +++ b/src/util/bytevectorhash.h @@ -5,7 +5,8 @@ #ifndef BITCOIN_UTIL_BYTEVECTORHASH_H #define BITCOIN_UTIL_BYTEVECTORHASH_H -#include <stdint.h> +#include <cstdint> +#include <cstddef> #include <vector> /** diff --git a/src/util/designator.h b/src/util/designator.h deleted file mode 100644 index 3670b11e00..0000000000 --- a/src/util/designator.h +++ /dev/null @@ -1,21 +0,0 @@ -// Copyright (c) 2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_UTIL_DESIGNATOR_H -#define BITCOIN_UTIL_DESIGNATOR_H - -/** - * Designated initializers can be used to avoid ordering mishaps in aggregate - * initialization. However, they do not prevent uninitialized members. The - * checks can be disabled by defining DISABLE_DESIGNATED_INITIALIZER_ERRORS. - * This should only be needed on MSVC 2019. MSVC 2022 supports them with the - * option "/std:c++20" - */ -#ifndef DISABLE_DESIGNATED_INITIALIZER_ERRORS -#define Desig(field_name) .field_name = -#else -#define Desig(field_name) -#endif - -#endif // BITCOIN_UTIL_DESIGNATOR_H diff --git a/src/util/error.cpp b/src/util/error.cpp index af8cbd0353..33a35a6d59 100644 --- a/src/util/error.cpp +++ b/src/util/error.cpp @@ -5,9 +5,11 @@ #include <util/error.h> #include <tinyformat.h> -#include <util/system.h> #include <util/translation.h> +#include <cassert> +#include <string> + bilingual_str TransactionErrorString(const TransactionError err) { switch (err) { @@ -35,6 +37,8 @@ bilingual_str TransactionErrorString(const TransactionError err) return Untranslated("External signer not found"); case TransactionError::EXTERNAL_SIGNER_FAILED: return Untranslated("External signer failed to sign"); + case TransactionError::INVALID_PACKAGE: + return Untranslated("Transaction rejected due to invalid package"); // no default case, so the compiler can warn about missing cases } assert(false); diff --git a/src/util/error.h b/src/util/error.h index 4cc35eb1fd..0429de651a 100644 --- a/src/util/error.h +++ b/src/util/error.h @@ -32,6 +32,7 @@ enum class TransactionError { MAX_FEE_EXCEEDED, EXTERNAL_SIGNER_NOT_FOUND, EXTERNAL_SIGNER_FAILED, + INVALID_PACKAGE, }; bilingual_str TransactionErrorString(const TransactionError error); diff --git a/src/util/hasher.cpp b/src/util/hasher.cpp index c21941eb88..a80f20c894 100644 --- a/src/util/hasher.cpp +++ b/src/util/hasher.cpp @@ -2,11 +2,11 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <crypto/siphash.h> #include <random.h> +#include <span.h> #include <util/hasher.h> -#include <limits> - SaltedTxidHasher::SaltedTxidHasher() : k0(GetRand<uint64_t>()), k1(GetRand<uint64_t>()) {} SaltedOutpointHasher::SaltedOutpointHasher() : k0(GetRand<uint64_t>()), k1(GetRand<uint64_t>()) {} diff --git a/src/util/hasher.h b/src/util/hasher.h index 3d24a4d23c..426b8990e6 100644 --- a/src/util/hasher.h +++ b/src/util/hasher.h @@ -5,10 +5,16 @@ #ifndef BITCOIN_UTIL_HASHER_H #define BITCOIN_UTIL_HASHER_H +#include <crypto/common.h> #include <crypto/siphash.h> #include <primitives/transaction.h> #include <uint256.h> +#include <cstdint> +#include <cstring> + +template <typename C> class Span; + class SaltedTxidHasher { private: diff --git a/src/util/message.cpp b/src/util/message.cpp index f58876f915..d395c4b0bc 100644 --- a/src/util/message.cpp +++ b/src/util/message.cpp @@ -3,16 +3,20 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <hash.h> // For CHashWriter -#include <key.h> // For CKey -#include <key_io.h> // For DecodeDestination() -#include <pubkey.h> // For CPubKey -#include <script/standard.h> // For CTxDestination, IsValidDestination(), PKHash -#include <serialize.h> // For SER_GETHASH +#include <hash.h> +#include <key.h> +#include <key_io.h> +#include <pubkey.h> +#include <script/standard.h> +#include <serialize.h> +#include <uint256.h> #include <util/message.h> -#include <util/strencodings.h> // For DecodeBase64() +#include <util/strencodings.h> +#include <cassert> +#include <optional> #include <string> +#include <variant> #include <vector> /** diff --git a/src/util/message.h b/src/util/message.h index b31c5f5761..1b7febe60a 100644 --- a/src/util/message.h +++ b/src/util/message.h @@ -6,11 +6,12 @@ #ifndef BITCOIN_UTIL_MESSAGE_H #define BITCOIN_UTIL_MESSAGE_H -#include <key.h> // For CKey #include <uint256.h> #include <string> +class CKey; + extern const std::string MESSAGE_MAGIC; /** The result of a signed message verification. diff --git a/src/util/moneystr.cpp b/src/util/moneystr.cpp index 8c4bc6e6f4..d9e6cef600 100644 --- a/src/util/moneystr.cpp +++ b/src/util/moneystr.cpp @@ -10,6 +10,7 @@ #include <util/strencodings.h> #include <util/string.h> +#include <cstdint> #include <optional> std::string FormatMoney(const CAmount n) diff --git a/src/util/readwritefile.cpp b/src/util/readwritefile.cpp index 628e6a3980..3ec08119e7 100644 --- a/src/util/readwritefile.cpp +++ b/src/util/readwritefile.cpp @@ -5,8 +5,9 @@ #include <fs.h> +#include <algorithm> +#include <cstdio> #include <limits> -#include <stdio.h> #include <string> #include <utility> diff --git a/src/util/result.h b/src/util/result.h new file mode 100644 index 0000000000..2f586a4c9b --- /dev/null +++ b/src/util/result.h @@ -0,0 +1,49 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_UTIL_RESULT_H +#define BITCOIN_UTIL_RESULT_H + +#include <util/translation.h> + +#include <variant> + +/* + * 'BResult' is a generic class useful for wrapping a return object + * (in case of success) or propagating the error cause. +*/ +template<class T> +class BResult { +private: + std::variant<bilingual_str, T> m_variant; + +public: + BResult() : m_variant{Untranslated("")} {} + BResult(T obj) : m_variant{std::move(obj)} {} + BResult(bilingual_str error) : m_variant{std::move(error)} {} + + /* Whether the function succeeded or not */ + bool HasRes() const { return std::holds_alternative<T>(m_variant); } + + /* In case of success, the result object */ + const T& GetObj() const { + assert(HasRes()); + return std::get<T>(m_variant); + } + T ReleaseObj() + { + assert(HasRes()); + return std::move(std::get<T>(m_variant)); + } + + /* In case of failure, the error cause */ + const bilingual_str& GetError() const { + assert(!HasRes()); + return std::get<bilingual_str>(m_variant); + } + + explicit operator bool() const { return HasRes(); } +}; + +#endif // BITCOIN_UTIL_RESULT_H diff --git a/src/util/serfloat.h b/src/util/serfloat.h index 4d912b0176..343ccb9d3a 100644 --- a/src/util/serfloat.h +++ b/src/util/serfloat.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_UTIL_SERFLOAT_H #define BITCOIN_UTIL_SERFLOAT_H -#include <stdint.h> +#include <cstdint> /* Encode a double using the IEEE 754 binary64 format. All NaNs are encoded as x86/ARM's * positive quiet NaN with payload 0. */ diff --git a/src/util/spanparsing.cpp b/src/util/spanparsing.cpp index 8614bd1176..565c867e18 100644 --- a/src/util/spanparsing.cpp +++ b/src/util/spanparsing.cpp @@ -6,8 +6,9 @@ #include <span.h> +#include <algorithm> +#include <cstddef> #include <string> -#include <vector> namespace spanparsing { diff --git a/src/util/strencodings.cpp b/src/util/strencodings.cpp index 675fe7d2d7..303e19beec 100644 --- a/src/util/strencodings.cpp +++ b/src/util/strencodings.cpp @@ -3,17 +3,18 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <span.h> #include <util/strencodings.h> -#include <util/string.h> - -#include <tinyformat.h> #include <algorithm> #include <array> -#include <cstdlib> +#include <cassert> #include <cstring> #include <limits> #include <optional> +#include <ostream> +#include <string> +#include <vector> static const std::string CHARS_ALPHA_NUM = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789"; diff --git a/src/util/strencodings.h b/src/util/strencodings.h index 9a96bbe67b..14867b21b2 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -13,11 +13,14 @@ #include <util/string.h> #include <charconv> +#include <cstddef> #include <cstdint> -#include <iterator> #include <limits> #include <optional> #include <string> +#include <string_view> +#include <system_error> +#include <type_traits> #include <vector> /** Used by SanitizeString() */ diff --git a/src/util/string.cpp b/src/util/string.cpp index d05222e8b8..dff782c330 100644 --- a/src/util/string.cpp +++ b/src/util/string.cpp @@ -6,6 +6,8 @@ #include <boost/algorithm/string/replace.hpp> +#include <string> + void ReplaceAll(std::string& in_out, std::string_view search, std::string_view substitute) { boost::replace_all(in_out, search, substitute); diff --git a/src/util/syserror.cpp b/src/util/syserror.cpp index 391ddd3560..5270f55366 100644 --- a/src/util/syserror.cpp +++ b/src/util/syserror.cpp @@ -10,6 +10,7 @@ #include <util/syserror.h> #include <cstring> +#include <string> std::string SysErrorString(int err) { diff --git a/src/util/system.cpp b/src/util/system.cpp index 140d51a1cc..f6f2828fc8 100644 --- a/src/util/system.cpp +++ b/src/util/system.cpp @@ -55,13 +55,6 @@ #else -#ifdef _MSC_VER -#pragma warning(disable:4786) -#pragma warning(disable:4804) -#pragma warning(disable:4805) -#pragma warning(disable:4717) -#endif - #include <codecvt> #include <io.h> /* for _commit */ diff --git a/src/util/thread.cpp b/src/util/thread.cpp index 14be668685..f9f427ba20 100644 --- a/src/util/thread.cpp +++ b/src/util/thread.cpp @@ -9,6 +9,7 @@ #include <util/threadnames.h> #include <exception> +#include <functional> void util::TraceThread(const char* thread_name, std::function<void()> thread_func) { diff --git a/src/util/time.cpp b/src/util/time.cpp index 7d9d6bcff1..2cafc55c69 100644 --- a/src/util/time.cpp +++ b/src/util/time.cpp @@ -8,16 +8,19 @@ #endif #include <compat.h> +#include <tinyformat.h> #include <util/time.h> - #include <util/check.h> -#include <atomic> #include <boost/date_time/posix_time/posix_time.hpp> + +#include <atomic> +#include <chrono> #include <ctime> +#include <locale> #include <thread> - -#include <tinyformat.h> +#include <sstream> +#include <string> void UninterruptibleSleep(const std::chrono::microseconds& n) { std::this_thread::sleep_for(n); } diff --git a/src/util/time.h b/src/util/time.h index 0f87d66c2e..fc49f23ce3 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -9,7 +9,7 @@ #include <compat.h> #include <chrono> -#include <stdint.h> +#include <cstdint> #include <string> using namespace std::chrono_literals; @@ -24,6 +24,7 @@ struct NodeClock : public std::chrono::system_clock { }; using NodeSeconds = std::chrono::time_point<NodeClock, std::chrono::seconds>; +using SteadyClock = std::chrono::steady_clock; using SteadySeconds = std::chrono::time_point<std::chrono::steady_clock, std::chrono::seconds>; using SteadyMilliseconds = std::chrono::time_point<std::chrono::steady_clock, std::chrono::milliseconds>; using SteadyMicroseconds = std::chrono::time_point<std::chrono::steady_clock, std::chrono::microseconds>; diff --git a/src/util/translation.h b/src/util/translation.h index aee601d9c1..07b7f43c8a 100644 --- a/src/util/translation.h +++ b/src/util/translation.h @@ -6,7 +6,9 @@ #define BITCOIN_UTIL_TRANSLATION_H #include <tinyformat.h> + #include <functional> +#include <string> /** * Bilingual messages: diff --git a/src/util/url.cpp b/src/util/url.cpp index e42d93bce8..ea9323e666 100644 --- a/src/util/url.cpp +++ b/src/util/url.cpp @@ -5,7 +5,8 @@ #include <util/url.h> #include <event2/http.h> -#include <stdlib.h> + +#include <cstdlib> #include <string> std::string urlDecode(const std::string &urlEncoded) { diff --git a/src/util/vector.h b/src/util/vector.h index dab65ded2a..ed745affe5 100644 --- a/src/util/vector.h +++ b/src/util/vector.h @@ -7,6 +7,7 @@ #include <initializer_list> #include <type_traits> +#include <utility> #include <vector> /** Construct a vector with the specified elements. diff --git a/src/validation.cpp b/src/validation.cpp index 36a9083117..9b39350fed 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -5,6 +5,9 @@ #include <validation.h> +#include <kernel/coinstats.h> +#include <kernel/mempool_persist.h> + #include <arith_uint256.h> #include <chain.h> #include <chainparams.h> @@ -17,8 +20,8 @@ #include <consensus/validation.h> #include <cuckoocache.h> #include <flatfile.h> +#include <fs.h> #include <hash.h> -#include <kernel/coinstats.h> #include <logging.h> #include <logging/timer.h> #include <node/blockstorage.h> @@ -47,12 +50,14 @@ #include <util/rbf.h> #include <util/strencodings.h> #include <util/system.h> +#include <util/time.h> #include <util/trace.h> #include <util/translation.h> #include <validationinterface.h> #include <warnings.h> #include <algorithm> +#include <chrono> #include <deque> #include <numeric> #include <optional> @@ -61,8 +66,9 @@ using kernel::CCoinsStats; using kernel::CoinStatsHashType; using kernel::ComputeUTXOStats; +using kernel::LoadMempool; -using node::BLOCKFILE_CHUNK_SIZE; +using fsbridge::FopenFn; using node::BlockManager; using node::BlockMap; using node::CBlockIndexHeightOnlyComparator; @@ -70,11 +76,8 @@ using node::CBlockIndexWorkComparator; using node::fImporting; using node::fPruneMode; using node::fReindex; -using node::nPruneTarget; -using node::OpenBlockFile; using node::ReadBlockFromDisk; using node::SnapshotMetadata; -using node::UNDOFILE_CHUNK_SIZE; using node::UndoReadFromDisk; using node::UnlinkPrunedFiles; @@ -740,7 +743,10 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // Applications relying on first-seen mempool behavior should // check all unconfirmed ancestors; otherwise an opt-in ancestor // might be replaced, causing removal of this descendant. - if (!SignalsOptInRBF(*ptxConflicting)) { + // + // If replaceability signaling is ignored due to node setting, + // replacement is always allowed. + if (!m_pool.m_full_rbf && !SignalsOptInRBF(*ptxConflicting)) { return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "txn-mempool-conflict"); } @@ -3858,13 +3864,11 @@ void PruneBlockFilesManual(CChainState& active_chainstate, int nManualPruneHeigh } } -void CChainState::LoadMempool(const ArgsManager& args) +void CChainState::LoadMempool(const fs::path& load_path, FopenFn mockable_fopen_function) { if (!m_mempool) return; - if (args.GetBoolArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { - ::LoadMempool(*m_mempool, *this); - } - m_mempool->SetIsLoaded(!ShutdownRequested()); + ::LoadMempool(*m_mempool, load_path, *this, mockable_fopen_function); + m_mempool->SetLoadTried(!ShutdownRequested()); } bool CChainState::LoadChainTip() @@ -4635,153 +4639,6 @@ bool CChainState::ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) return ret; } -static const uint64_t MEMPOOL_DUMP_VERSION = 1; - -bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mockable_fopen_function) -{ - int64_t nExpiryTimeout = std::chrono::seconds{pool.m_expiry}.count(); - FILE* filestr{mockable_fopen_function(gArgs.GetDataDirNet() / "mempool.dat", "rb")}; - CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); - if (file.IsNull()) { - LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); - return false; - } - - int64_t count = 0; - int64_t expired = 0; - int64_t failed = 0; - int64_t already_there = 0; - int64_t unbroadcast = 0; - int64_t nNow = GetTime(); - - try { - uint64_t version; - file >> version; - if (version != MEMPOOL_DUMP_VERSION) { - return false; - } - uint64_t num; - file >> num; - while (num) { - --num; - CTransactionRef tx; - int64_t nTime; - int64_t nFeeDelta; - file >> tx; - file >> nTime; - file >> nFeeDelta; - - CAmount amountdelta = nFeeDelta; - if (amountdelta) { - pool.PrioritiseTransaction(tx->GetHash(), amountdelta); - } - if (nTime > nNow - nExpiryTimeout) { - LOCK(cs_main); - const auto& accepted = AcceptToMemoryPool(active_chainstate, tx, nTime, /*bypass_limits=*/false, /*test_accept=*/false); - if (accepted.m_result_type == MempoolAcceptResult::ResultType::VALID) { - ++count; - } else { - // mempool may contain the transaction already, e.g. from - // wallet(s) having loaded it while we were processing - // mempool transactions; consider these as valid, instead of - // failed, but mark them as 'already there' - if (pool.exists(GenTxid::Txid(tx->GetHash()))) { - ++already_there; - } else { - ++failed; - } - } - } else { - ++expired; - } - if (ShutdownRequested()) - return false; - } - std::map<uint256, CAmount> mapDeltas; - file >> mapDeltas; - - for (const auto& i : mapDeltas) { - pool.PrioritiseTransaction(i.first, i.second); - } - - std::set<uint256> unbroadcast_txids; - file >> unbroadcast_txids; - unbroadcast = unbroadcast_txids.size(); - for (const auto& txid : unbroadcast_txids) { - // Ensure transactions were accepted to mempool then add to - // unbroadcast set. - if (pool.get(txid) != nullptr) pool.AddUnbroadcastTx(txid); - } - } catch (const std::exception& e) { - LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); - return false; - } - - LogPrintf("Imported mempool transactions from disk: %i succeeded, %i failed, %i expired, %i already there, %i waiting for initial broadcast\n", count, failed, expired, already_there, unbroadcast); - return true; -} - -bool DumpMempool(const CTxMemPool& pool, FopenFn mockable_fopen_function, bool skip_file_commit) -{ - int64_t start = GetTimeMicros(); - - std::map<uint256, CAmount> mapDeltas; - std::vector<TxMempoolInfo> vinfo; - std::set<uint256> unbroadcast_txids; - - static Mutex dump_mutex; - LOCK(dump_mutex); - - { - LOCK(pool.cs); - for (const auto &i : pool.mapDeltas) { - mapDeltas[i.first] = i.second; - } - vinfo = pool.infoAll(); - unbroadcast_txids = pool.GetUnbroadcastTxs(); - } - - int64_t mid = GetTimeMicros(); - - try { - FILE* filestr{mockable_fopen_function(gArgs.GetDataDirNet() / "mempool.dat.new", "wb")}; - if (!filestr) { - return false; - } - - CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); - - uint64_t version = MEMPOOL_DUMP_VERSION; - file << version; - - file << (uint64_t)vinfo.size(); - for (const auto& i : vinfo) { - file << *(i.tx); - file << int64_t{count_seconds(i.m_time)}; - file << int64_t{i.nFeeDelta}; - mapDeltas.erase(i.tx->GetHash()); - } - - file << mapDeltas; - - LogPrintf("Writing %d unbroadcast transactions to disk.\n", unbroadcast_txids.size()); - file << unbroadcast_txids; - - if (!skip_file_commit && !FileCommit(file.Get())) - throw std::runtime_error("FileCommit failed"); - file.fclose(); - if (!RenameOver(gArgs.GetDataDirNet() / "mempool.dat.new", gArgs.GetDataDirNet() / "mempool.dat")) { - throw std::runtime_error("Rename failed"); - } - int64_t last = GetTimeMicros(); - LogPrintf("Dumped mempool: %gs to copy, %gs to dump\n", (mid-start)*MICRO, (last-mid)*MICRO); - } catch (const std::exception& e) { - LogPrintf("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); - return false; - } - return true; -} - //! Guess how far we are in the verification process at the given block index //! require cs_main if pindex has not been validated yet (because nChainTx might be unset) double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex *pindex) { diff --git a/src/validation.h b/src/validation.h index cce268ee8e..a44dbd9c7a 100644 --- a/src/validation.h +++ b/src/validation.h @@ -68,8 +68,6 @@ static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static constexpr bool DEFAULT_COINSTATSINDEX{false}; static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; -/** Default for -persistmempool */ -static const bool DEFAULT_PERSIST_MEMPOOL = true; /** Default for -stopatheight */ static const int DEFAULT_STOPATHEIGHT = 0; /** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ActiveChain().Tip() will not be pruned. */ @@ -679,7 +677,7 @@ public: void CheckBlockIndex(); /** Load the persisted mempool from disk */ - void LoadMempool(const ArgsManager& args); + void LoadMempool(const fs::path& load_path, fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen); /** Update the chain tip based on database information, i.e. CoinsTip()'s best block. */ bool LoadChainTip() EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -834,7 +832,7 @@ private: friend CChainState; public: - using Options = ChainstateManagerOpts; + using Options = kernel::ChainstateManagerOpts; explicit ChainstateManager(const Options& opts) : m_chainparams{opts.chainparams}, @@ -1014,14 +1012,6 @@ bool DeploymentEnabled(const ChainstateManager& chainman, DEP dep) return DeploymentEnabled(chainman.GetConsensus(), dep); } -using FopenFn = std::function<FILE*(const fs::path&, const char*)>; - -/** Dump the mempool to disk. */ -bool DumpMempool(const CTxMemPool& pool, FopenFn mockable_fopen_function = fsbridge::fopen, bool skip_file_commit = false); - -/** Load the mempool from disk. */ -bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mockable_fopen_function = fsbridge::fopen); - /** * Return the expected assumeutxo value for a given height, if one exists. * diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index f8230f7a1d..dbd768a758 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -315,12 +315,6 @@ BerkeleyBatch::BerkeleyBatch(BerkeleyDatabase& database, const bool read_only, b env = database.env.get(); pdb = database.m_db.get(); strFile = fs::PathToString(database.m_filename); - if (!Exists(std::string("version"))) { - bool fTmp = fReadOnly; - fReadOnly = false; - Write(std::string("version"), CLIENT_VERSION); - fReadOnly = fTmp; - } } void BerkeleyDatabase::Open() diff --git a/src/wallet/coinselection.cpp b/src/wallet/coinselection.cpp index 07df8d9fc8..49e6bac462 100644 --- a/src/wallet/coinselection.cpp +++ b/src/wallet/coinselection.cpp @@ -104,9 +104,6 @@ std::optional<SelectionResult> SelectCoinsBnB(std::vector<OutputGroup>& utxo_poo if (curr_waste <= best_waste) { best_selection = curr_selection; best_waste = curr_waste; - if (best_waste == 0) { - break; - } } curr_waste -= (curr_value - selection_target); // Remove the excess value as we will be selecting different coins now backtrack = true; diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index 5e70ed4a30..43fdcee48d 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -219,19 +219,18 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo new_coin_control.m_min_depth = 1; constexpr int RANDOM_CHANGE_POSITION = -1; - bilingual_str fail_reason; - FeeCalculation fee_calc_out; - std::optional<CreatedTransactionResult> txr = CreateTransaction(wallet, recipients, RANDOM_CHANGE_POSITION, fail_reason, new_coin_control, fee_calc_out, false); - if (!txr) { - errors.push_back(Untranslated("Unable to create transaction.") + Untranslated(" ") + fail_reason); + auto res = CreateTransaction(wallet, recipients, RANDOM_CHANGE_POSITION, new_coin_control, false); + if (!res) { + errors.push_back(Untranslated("Unable to create transaction.") + Untranslated(" ") + res.GetError()); return Result::WALLET_ERROR; } + const auto& txr = res.GetObj(); // Write back new fee if successful - new_fee = txr->fee; + new_fee = txr.fee; // Write back transaction - mtx = CMutableTransaction(*txr->tx); + mtx = CMutableTransaction(*txr.tx); return Result::OK; } diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index f54b2c83d2..23f91d9b3a 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -146,11 +146,10 @@ public: void abortRescan() override { m_wallet->AbortRescan(); } bool backupWallet(const std::string& filename) override { return m_wallet->BackupWallet(filename); } std::string getWalletName() override { return m_wallet->GetName(); } - bool getNewDestination(const OutputType type, const std::string label, CTxDestination& dest) override + BResult<CTxDestination> getNewDestination(const OutputType type, const std::string label) override { LOCK(m_wallet->cs_wallet); - bilingual_str error; - return m_wallet->GetNewDestination(type, label, dest, error); + return m_wallet->GetNewDestination(type, label); } bool getPubKey(const CScript& script, const CKeyID& address, CPubKey& pub_key) override { @@ -191,29 +190,27 @@ public: std::string* purpose) override { LOCK(m_wallet->cs_wallet); - auto it = m_wallet->m_address_book.find(dest); - if (it == m_wallet->m_address_book.end() || it->second.IsChange()) { - return false; - } + const auto& entry = m_wallet->FindAddressBookEntry(dest, /*allow_change=*/false); + if (!entry) return false; // addr not found if (name) { - *name = it->second.GetLabel(); + *name = entry->GetLabel(); } if (is_mine) { *is_mine = m_wallet->IsMine(dest); } if (purpose) { - *purpose = it->second.purpose; + *purpose = entry->purpose; } return true; } - std::vector<WalletAddress> getAddresses() override + std::vector<WalletAddress> getAddresses() const override { LOCK(m_wallet->cs_wallet); std::vector<WalletAddress> result; - for (const auto& item : m_wallet->m_address_book) { - if (item.second.IsChange()) continue; - result.emplace_back(item.first, m_wallet->IsMine(item.first), item.second.GetLabel(), item.second.purpose); - } + m_wallet->ForEachAddrBookEntry([&](const CTxDestination& dest, const std::string& label, const std::string& purpose, bool is_change) EXCLUSIVE_LOCKS_REQUIRED(m_wallet->cs_wallet) { + if (is_change) return; + result.emplace_back(dest, m_wallet->IsMine(dest), label, purpose); + }); return result; } std::vector<std::string> getAddressReceiveRequests() override { @@ -252,22 +249,21 @@ public: LOCK(m_wallet->cs_wallet); return m_wallet->ListLockedCoins(outputs); } - CTransactionRef createTransaction(const std::vector<CRecipient>& recipients, + BResult<CTransactionRef> createTransaction(const std::vector<CRecipient>& recipients, const CCoinControl& coin_control, bool sign, int& change_pos, - CAmount& fee, - bilingual_str& fail_reason) override + CAmount& fee) override { LOCK(m_wallet->cs_wallet); - FeeCalculation fee_calc_out; - std::optional<CreatedTransactionResult> txr = CreateTransaction(*m_wallet, recipients, change_pos, - fail_reason, coin_control, fee_calc_out, sign); - if (!txr) return {}; - fee = txr->fee; - change_pos = txr->change_pos; + const auto& res = CreateTransaction(*m_wallet, recipients, change_pos, + coin_control, sign); + if (!res) return res.GetError(); + const auto& txr = res.GetObj(); + fee = txr.fee; + change_pos = txr.change_pos; - return txr->tx; + return txr.tx; } void commitTransaction(CTransactionRef tx, WalletValueMap value_map, @@ -573,11 +569,13 @@ public: options.require_existing = true; return MakeWallet(m_context, LoadWallet(m_context, name, true /* load_on_start */, options, status, error, warnings)); } - std::unique_ptr<Wallet> restoreWallet(const fs::path& backup_file, const std::string& wallet_name, bilingual_str& error, std::vector<bilingual_str>& warnings) override + BResult<std::unique_ptr<Wallet>> restoreWallet(const fs::path& backup_file, const std::string& wallet_name, std::vector<bilingual_str>& warnings) override { DatabaseStatus status; - - return MakeWallet(m_context, RestoreWallet(m_context, backup_file, wallet_name, /*load_on_start=*/true, status, error, warnings)); + bilingual_str error; + BResult<std::unique_ptr<Wallet>> wallet{MakeWallet(m_context, RestoreWallet(m_context, backup_file, wallet_name, /*load_on_start=*/true, status, error, warnings))}; + if (!wallet) return error; + return wallet; } std::string getWalletDir() override { diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index 8de4017371..d3303c0b1f 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -130,12 +130,10 @@ CAmount CachedTxGetCredit(const CWallet& wallet, const CWalletTx& wtx, const ism return 0; CAmount credit = 0; - if (filter & ISMINE_SPENDABLE) { + const isminefilter get_amount_filter{filter & ISMINE_ALL}; + if (get_amount_filter) { // GetBalance can assume transactions in mapWallet won't change - credit += GetCachableAmount(wallet, wtx, CWalletTx::CREDIT, ISMINE_SPENDABLE); - } - if (filter & ISMINE_WATCH_ONLY) { - credit += GetCachableAmount(wallet, wtx, CWalletTx::CREDIT, ISMINE_WATCH_ONLY); + credit += GetCachableAmount(wallet, wtx, CWalletTx::CREDIT, get_amount_filter); } return credit; } @@ -146,11 +144,9 @@ CAmount CachedTxGetDebit(const CWallet& wallet, const CWalletTx& wtx, const ismi return 0; CAmount debit = 0; - if (filter & ISMINE_SPENDABLE) { - debit += GetCachableAmount(wallet, wtx, CWalletTx::DEBIT, ISMINE_SPENDABLE); - } - if (filter & ISMINE_WATCH_ONLY) { - debit += GetCachableAmount(wallet, wtx, CWalletTx::DEBIT, ISMINE_WATCH_ONLY); + const isminefilter get_amount_filter{filter & ISMINE_ALL}; + if (get_amount_filter) { + debit += GetCachableAmount(wallet, wtx, CWalletTx::DEBIT, get_amount_filter); } return debit; } diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index f25ad59528..9428d049de 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -58,13 +58,12 @@ RPCHelpMan getnewaddress() output_type = parsed.value(); } - CTxDestination dest; - bilingual_str error; - if (!pwallet->GetNewDestination(output_type, label, dest, error)) { - throw JSONRPCError(RPC_WALLET_KEYPOOL_RAN_OUT, error.original); + auto op_dest = pwallet->GetNewDestination(output_type, label); + if (!op_dest) { + throw JSONRPCError(RPC_WALLET_KEYPOOL_RAN_OUT, op_dest.GetError().original); } - return EncodeDestination(dest); + return EncodeDestination(op_dest.GetObj()); }, }; } @@ -106,12 +105,11 @@ RPCHelpMan getrawchangeaddress() output_type = parsed.value(); } - CTxDestination dest; - bilingual_str error; - if (!pwallet->GetNewChangeDestination(output_type, dest, error)) { - throw JSONRPCError(RPC_WALLET_KEYPOOL_RAN_OUT, error.original); + auto op_dest = pwallet->GetNewChangeDestination(output_type); + if (!op_dest) { + throw JSONRPCError(RPC_WALLET_KEYPOOL_RAN_OUT, op_dest.GetError().original); } - return EncodeDestination(dest); + return EncodeDestination(op_dest.GetObj()); }, }; } @@ -637,17 +635,6 @@ RPCHelpMan getaddressinfo() }; } -/** Convert CAddressBookData to JSON record. */ -static UniValue AddressBookDataToJSON(const CAddressBookData& data, const bool verbose) -{ - UniValue ret(UniValue::VOBJ); - if (verbose) { - ret.pushKV("name", data.GetLabel()); - } - ret.pushKV("purpose", data.purpose); - return ret; -} - RPCHelpMan getaddressesbylabel() { return RPCHelpMan{"getaddressesbylabel", @@ -680,10 +667,10 @@ RPCHelpMan getaddressesbylabel() // Find all addresses that have the given label UniValue ret(UniValue::VOBJ); std::set<std::string> addresses; - for (const std::pair<const CTxDestination, CAddressBookData>& item : pwallet->m_address_book) { - if (item.second.IsChange()) continue; - if (item.second.GetLabel() == label) { - std::string address = EncodeDestination(item.first); + pwallet->ForEachAddrBookEntry([&](const CTxDestination& _dest, const std::string& _label, const std::string& _purpose, bool _is_change) { + if (_is_change) return; + if (_label == label) { + std::string address = EncodeDestination(_dest); // CWallet::m_address_book is not expected to contain duplicate // address strings, but build a separate set as a precaution just in // case it does. @@ -693,9 +680,11 @@ RPCHelpMan getaddressesbylabel() // and since duplicate addresses are unexpected (checked with // std::set in O(log(N))), UniValue::__pushKV is used instead, // which currently is O(1). - ret.__pushKV(address, AddressBookDataToJSON(item.second, false)); + UniValue value(UniValue::VOBJ); + value.pushKV("purpose", _purpose); + ret.__pushKV(address, value); } - } + }); if (ret.empty()) { throw JSONRPCError(RPC_WALLET_INVALID_LABEL_NAME, std::string("No addresses with label " + label)); @@ -742,13 +731,7 @@ RPCHelpMan listlabels() } // Add to a set to sort by label name, then insert into Univalue array - std::set<std::string> label_set; - for (const std::pair<const CTxDestination, CAddressBookData>& entry : pwallet->m_address_book) { - if (entry.second.IsChange()) continue; - if (purpose.empty() || entry.second.purpose == purpose) { - label_set.insert(entry.second.GetLabel()); - } - } + std::set<std::string> label_set = pwallet->ListAddrBookLabels(purpose); UniValue ret(UniValue::VARR); for (const std::string& name : label_set) { diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 5f1673fb12..62aba4f3f2 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -100,11 +100,13 @@ RPCHelpMan importprivkey() "Hint: use importmulti to import more than one private key.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported key exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" + "The rescan parameter can be set to false if the key was never used to create transactions. If it is set to false,\n" + "but the key was used to create transactions, rescanwallet needs to be called with the appropriate block range.\n" "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"privkey", RPCArg::Type::STR, RPCArg::Optional::NO, "The private key (see dumpprivkey)"}, {"label", RPCArg::Type::STR, RPCArg::DefaultHint{"current label if address exists, otherwise \"\""}, "An optional label"}, - {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Rescan the wallet for transactions"}, + {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions."}, }, RPCResult{RPCResult::Type::NONE, "", ""}, RPCExamples{ @@ -201,6 +203,8 @@ RPCHelpMan importaddress() "\nAdds an address or script (in hex) that can be watched as if it were in your wallet but cannot be used to spend. Requires a new wallet backup.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported address exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" + "The rescan parameter can be set to false if the key was never used to create transactions. If it is set to false,\n" + "but the key was used to create transactions, rescanwallet needs to be called with the appropriate block range.\n" "If you have the full public key, you should call importpubkey instead of this.\n" "Hint: use importmulti to import more than one address.\n" "\nNote: If you import a non-standard raw script in hex form, outputs sending to it will be treated\n" @@ -210,7 +214,7 @@ RPCHelpMan importaddress() { {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The Bitcoin address (or hex-encoded script)"}, {"label", RPCArg::Type::STR, RPCArg::Default{""}, "An optional label"}, - {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Rescan the wallet for transactions"}, + {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions."}, {"p2sh", RPCArg::Type::BOOL, RPCArg::Default{false}, "Add the P2SH version of the script as well"}, }, RPCResult{RPCResult::Type::NONE, "", ""}, @@ -401,11 +405,13 @@ RPCHelpMan importpubkey() "Hint: use importmulti to import more than one public key.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported pubkey exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" + "The rescan parameter can be set to false if the key was never used to create transactions. If it is set to false,\n" + "but the key was used to create transactions, rescanwallet needs to be called with the appropriate block range.\n" "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"pubkey", RPCArg::Type::STR, RPCArg::Optional::NO, "The hex-encoded public key"}, {"label", RPCArg::Type::STR, RPCArg::Default{""}, "An optional label"}, - {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Rescan the wallet for transactions"}, + {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions."}, }, RPCResult{RPCResult::Type::NONE, "", ""}, RPCExamples{ @@ -484,7 +490,7 @@ RPCHelpMan importwallet() { return RPCHelpMan{"importwallet", "\nImports keys from a wallet dump file (see dumpwallet). Requires a new wallet backup to include imported keys.\n" - "Note: Use \"getwalletinfo\" to query the scanning progress.\n", + "Note: Blockchain and Mempool will be rescanned after a successful import. Use \"getwalletinfo\" to query the scanning progress.\n", { {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The wallet file"}, }, @@ -1250,6 +1256,8 @@ RPCHelpMan importmulti() "Conversely, if all the private keys are provided and the address/script is spendable, the watchonly option must be set to false, or a warning will be returned.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported keys, addresses or scripts exist but related transactions are still missing.\n" + "The rescan parameter can be set to false if the key was never used to create transactions. If it is set to false,\n" + "but the key was used to create transactions, rescanwallet needs to be called with the appropriate block range.\n" "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"requests", RPCArg::Type::ARR, RPCArg::Optional::NO, "Data to be imported", @@ -1291,7 +1299,7 @@ RPCHelpMan importmulti() "\"requests\""}, {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", { - {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Stating if should rescan the blockchain after all imports"}, + {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions after all imports."}, }, "\"options\""}, }, @@ -1593,7 +1601,7 @@ RPCHelpMan importdescriptors() " Use the string \"now\" to substitute the current synced blockchain time.\n" " \"now\" can be specified to bypass scanning, for outputs which are known to never have been used, and\n" " 0 can be specified to scan the entire blockchain. Blocks up to 2 hours before the earliest timestamp\n" - " of all descriptors being imported will be scanned.", + "of all descriptors being imported will be scanned as well as the mempool.", /*oneline_description=*/"", {"timestamp | \"now\"", "integer / string"} }, {"internal", RPCArg::Type::BOOL, RPCArg::Default{false}, "Whether matching outputs should be treated as not incoming payments (e.g. change)"}, diff --git a/src/wallet/rpc/coins.cpp b/src/wallet/rpc/coins.cpp index ad59cc94ff..a9fff95882 100644 --- a/src/wallet/rpc/coins.cpp +++ b/src/wallet/rpc/coins.cpp @@ -18,10 +18,10 @@ namespace wallet { static CAmount GetReceived(const CWallet& wallet, const UniValue& params, bool by_label) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { - std::set<CTxDestination> addresses; + std::vector<CTxDestination> addresses; if (by_label) { // Get the set of addresses assigned to label - addresses = wallet.GetLabelAddresses(LabelFromValue(params[0])); + addresses = wallet.ListAddrBookAddresses(CWallet::AddrBookFilter{LabelFromValue(params[0])}); if (addresses.empty()) throw JSONRPCError(RPC_WALLET_ERROR, "Label not found in wallet"); } else { // Get the address @@ -29,7 +29,7 @@ static CAmount GetReceived(const CWallet& wallet, const UniValue& params, bool b if (!IsValidDestination(dest)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Invalid Bitcoin address"); } - addresses.insert(dest); + addresses.emplace_back(dest); } // Filter by own scripts only diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index 0e8cee5db8..83e23497cb 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -156,18 +156,16 @@ UniValue SendMoney(CWallet& wallet, const CCoinControl &coin_control, std::vecto // Send constexpr int RANDOM_CHANGE_POSITION = -1; - bilingual_str error; - FeeCalculation fee_calc_out; - std::optional<CreatedTransactionResult> txr = CreateTransaction(wallet, recipients, RANDOM_CHANGE_POSITION, error, coin_control, fee_calc_out, true); - if (!txr) { - throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, error.original); + auto res = CreateTransaction(wallet, recipients, RANDOM_CHANGE_POSITION, coin_control, true); + if (!res) { + throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, res.GetError().original); } - CTransactionRef tx = txr->tx; + const CTransactionRef& tx = res.GetObj().tx; wallet.CommitTransaction(tx, std::move(map_value), {} /* orderForm */); if (verbose) { UniValue entry(UniValue::VOBJ); entry.pushKV("txid", tx->GetHash().GetHex()); - entry.pushKV("fee_reason", StringForFeeReason(fee_calc_out.reason)); + entry.pushKV("fee_reason", StringForFeeReason(res.GetObj().fee_calc.reason)); return entry; } return tx->GetHash().GetHex(); @@ -1635,7 +1633,7 @@ RPCHelpMan walletcreatefundedpsbt() }, true ); - UniValue options = request.params[3]; + UniValue options{request.params[3].isNull() ? UniValue::VOBJ : request.params[3]}; CAmount fee; int change_position; diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index fae9bf3ea5..0b52dcc001 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -85,14 +85,12 @@ static UniValue ListReceived(const CWallet& wallet, const UniValue& params, cons filter |= ISMINE_WATCH_ONLY; } - bool has_filtered_address = false; - CTxDestination filtered_address = CNoDestination(); + std::optional<CTxDestination> filtered_address{std::nullopt}; if (!by_label && !params[3].isNull() && !params[3].get_str().empty()) { if (!IsValidDestinationString(params[3].get_str())) { throw JSONRPCError(RPC_WALLET_ERROR, "address_filter parameter was invalid"); } filtered_address = DecodeDestination(params[3].get_str()); - has_filtered_address = true; } // Tally @@ -106,23 +104,21 @@ static UniValue ListReceived(const CWallet& wallet, const UniValue& params, cons // Coinbase with less than 1 confirmation is no longer in the main chain if ((wtx.IsCoinBase() && (nDepth < 1)) - || (wallet.IsTxImmatureCoinBase(wtx) && !include_immature_coinbase)) - { + || (wallet.IsTxImmatureCoinBase(wtx) && !include_immature_coinbase)) { continue; } - for (const CTxOut& txout : wtx.tx->vout) - { + for (const CTxOut& txout : wtx.tx->vout) { CTxDestination address; if (!ExtractDestination(txout.scriptPubKey, address)) continue; - if (has_filtered_address && !(filtered_address == address)) { + if (filtered_address && !(filtered_address == address)) { continue; } isminefilter mine = wallet.IsMine(address); - if(!(mine & filter)) + if (!(mine & filter)) continue; tallyitem& item = mapTally[address]; @@ -138,70 +134,55 @@ static UniValue ListReceived(const CWallet& wallet, const UniValue& params, cons UniValue ret(UniValue::VARR); std::map<std::string, tallyitem> label_tally; - // Create m_address_book iterator - // If we aren't filtering, go from begin() to end() - auto start = wallet.m_address_book.begin(); - auto end = wallet.m_address_book.end(); - // If we are filtering, find() the applicable entry - if (has_filtered_address) { - start = wallet.m_address_book.find(filtered_address); - if (start != end) { - end = std::next(start); - } - } + const auto& func = [&](const CTxDestination& address, const std::string& label, const std::string& purpose, bool is_change) { + if (is_change) return; // no change addresses - for (auto item_it = start; item_it != end; ++item_it) - { - if (item_it->second.IsChange()) continue; - const CTxDestination& address = item_it->first; - const std::string& label = item_it->second.GetLabel(); auto it = mapTally.find(address); if (it == mapTally.end() && !fIncludeEmpty) - continue; + return; CAmount nAmount = 0; int nConf = std::numeric_limits<int>::max(); bool fIsWatchonly = false; - if (it != mapTally.end()) - { + if (it != mapTally.end()) { nAmount = (*it).second.nAmount; nConf = (*it).second.nConf; fIsWatchonly = (*it).second.fIsWatchonly; } - if (by_label) - { + if (by_label) { tallyitem& _item = label_tally[label]; _item.nAmount += nAmount; _item.nConf = std::min(_item.nConf, nConf); _item.fIsWatchonly = fIsWatchonly; - } - else - { + } else { UniValue obj(UniValue::VOBJ); - if(fIsWatchonly) - obj.pushKV("involvesWatchonly", true); + if (fIsWatchonly) obj.pushKV("involvesWatchonly", true); obj.pushKV("address", EncodeDestination(address)); obj.pushKV("amount", ValueFromAmount(nAmount)); obj.pushKV("confirmations", (nConf == std::numeric_limits<int>::max() ? 0 : nConf)); obj.pushKV("label", label); UniValue transactions(UniValue::VARR); - if (it != mapTally.end()) - { - for (const uint256& _item : (*it).second.txids) - { + if (it != mapTally.end()) { + for (const uint256& _item : (*it).second.txids) { transactions.push_back(_item.GetHex()); } } obj.pushKV("txids", transactions); ret.push_back(obj); } + }; + + if (filtered_address) { + const auto& entry = wallet.FindAddressBookEntry(*filtered_address, /*allow_change=*/false); + if (entry) func(*filtered_address, entry->GetLabel(), entry->purpose, /*is_change=*/false); + } else { + // No filtered addr, walk-through the addressbook entry + wallet.ForEachAddrBookEntry(func); } - if (by_label) - { - for (const auto& entry : label_tally) - { + if (by_label) { + for (const auto& entry : label_tally) { CAmount nAmount = entry.second.nAmount; int nConf = entry.second.nConf; UniValue obj(UniValue::VOBJ); @@ -329,11 +310,12 @@ static void MaybePushAddress(UniValue & entry, const CTxDestination &dest) * @param wtx The wallet transaction. * @param nMinDepth The minimum confirmation depth. * @param fLong Whether to include the JSON version of the transaction. - * @param ret The UniValue into which the result is stored. + * @param ret The vector into which the result is stored. * @param filter_ismine The "is mine" filter flags. * @param filter_label Optional label string to filter incoming transactions. */ -static void ListTransactions(const CWallet& wallet, const CWalletTx& wtx, int nMinDepth, bool fLong, UniValue& ret, const isminefilter& filter_ismine, const std::string* filter_label) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) +template <class Vec> +static void ListTransactions(const CWallet& wallet, const CWalletTx& wtx, int nMinDepth, bool fLong, Vec& ret, const isminefilter& filter_ismine, const std::string* filter_label) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { CAmount nFee; std::list<COutputEntry> listReceived; @@ -519,8 +501,7 @@ RPCHelpMan listtransactions() if (nFrom < 0) throw JSONRPCError(RPC_INVALID_PARAMETER, "Negative from"); - UniValue ret(UniValue::VARR); - + std::vector<UniValue> ret; { LOCK(pwallet->cs_wallet); @@ -542,9 +523,9 @@ RPCHelpMan listtransactions() if ((nFrom + nCount) > (int)ret.size()) nCount = ret.size() - nFrom; - const std::vector<UniValue>& txs = ret.getValues(); + auto txs_rev_it{std::make_move_iterator(ret.rend())}; UniValue result{UniValue::VARR}; - result.push_backV({ txs.rend() - nFrom - nCount, txs.rend() - nFrom }); // Return oldest to newest + result.push_backV(txs_rev_it - nFrom - nCount, txs_rev_it - nFrom); // Return oldest to newest return result; }, }; @@ -908,7 +889,7 @@ RPCHelpMan rescanblockchain() } CWallet::ScanResult result = - pwallet->ScanForWalletTransactions(start_block, start_height, stop_height, reserver, true /* fUpdate */); + pwallet->ScanForWalletTransactions(start_block, start_height, stop_height, reserver, /*fUpdate=*/true, /*save_progress=*/false); switch (result.status) { case CWallet::ScanResult::SUCCESS: break; diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 1fec82a485..bba31dfe0b 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -21,26 +21,22 @@ namespace wallet { //! Value for the first BIP 32 hardened derivation. Can be used as a bit mask and as a value. See BIP 32 for more details. const uint32_t BIP32_HARDENED_KEY_LIMIT = 0x80000000; -bool LegacyScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestination& dest, bilingual_str& error) +BResult<CTxDestination> LegacyScriptPubKeyMan::GetNewDestination(const OutputType type) { if (LEGACY_OUTPUT_TYPES.count(type) == 0) { - error = _("Error: Legacy wallets only support the \"legacy\", \"p2sh-segwit\", and \"bech32\" address types"); - return false; + return _("Error: Legacy wallets only support the \"legacy\", \"p2sh-segwit\", and \"bech32\" address types");; } assert(type != OutputType::BECH32M); LOCK(cs_KeyStore); - error.clear(); // Generate a new key that is added to wallet CPubKey new_key; if (!GetKeyFromPool(new_key, type)) { - error = _("Error: Keypool ran out, please call keypoolrefill first"); - return false; + return _("Error: Keypool ran out, please call keypoolrefill first"); } LearnRelatedScripts(new_key, type); - dest = GetDestinationForKey(new_key, type); - return true; + return GetDestinationForKey(new_key, type); } typedef std::vector<unsigned char> valtype; @@ -1658,12 +1654,11 @@ std::set<CKeyID> LegacyScriptPubKeyMan::GetKeys() const return set_address; } -bool DescriptorScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDestination& dest, bilingual_str& error) +BResult<CTxDestination> DescriptorScriptPubKeyMan::GetNewDestination(const OutputType type) { // Returns true if this descriptor supports getting new addresses. Conditions where we may be unable to fetch them (e.g. locked) are caught later if (!CanGetAddresses()) { - error = _("No addresses available"); - return false; + return _("No addresses available"); } { LOCK(cs_desc_man); @@ -1681,15 +1676,14 @@ bool DescriptorScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDest std::vector<CScript> scripts_temp; if (m_wallet_descriptor.range_end <= m_max_cached_index && !TopUp(1)) { // We can't generate anymore keys - error = _("Error: Keypool ran out, please call keypoolrefill first"); - return false; + return _("Error: Keypool ran out, please call keypoolrefill first"); } if (!m_wallet_descriptor.descriptor->ExpandFromCache(m_wallet_descriptor.next_index, m_wallet_descriptor.cache, scripts_temp, out_keys)) { // We can't generate anymore keys - error = _("Error: Keypool ran out, please call keypoolrefill first"); - return false; + return _("Error: Keypool ran out, please call keypoolrefill first"); } + CTxDestination dest; std::optional<OutputType> out_script_type = m_wallet_descriptor.descriptor->GetOutputType(); if (out_script_type && out_script_type == type) { ExtractDestination(scripts_temp[0], dest); @@ -1698,7 +1692,7 @@ bool DescriptorScriptPubKeyMan::GetNewDestination(const OutputType type, CTxDest } m_wallet_descriptor.next_index++; WalletBatch(m_storage.GetDatabase()).WriteDescriptor(GetID(), m_wallet_descriptor); - return true; + return dest; } } @@ -1769,9 +1763,14 @@ bool DescriptorScriptPubKeyMan::Encrypt(const CKeyingMaterial& master_key, Walle bool DescriptorScriptPubKeyMan::GetReservedDestination(const OutputType type, bool internal, CTxDestination& address, int64_t& index, CKeyPool& keypool, bilingual_str& error) { LOCK(cs_desc_man); - bool result = GetNewDestination(type, address, error); + auto op_dest = GetNewDestination(type); index = m_wallet_descriptor.next_index - 1; - return result; + if (op_dest) { + address = op_dest.GetObj(); + } else { + error = op_dest.GetError(); + } + return op_dest.HasRes(); } void DescriptorScriptPubKeyMan::ReturnDestination(int64_t index, bool internal, const CTxDestination& addr) diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index ad924791af..eebc05330f 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -11,6 +11,7 @@ #include <script/standard.h> #include <util/error.h> #include <util/message.h> +#include <util/result.h> #include <util/time.h> #include <wallet/crypter.h> #include <wallet/ismine.h> @@ -171,7 +172,7 @@ protected: public: explicit ScriptPubKeyMan(WalletStorage& storage) : m_storage(storage) {} virtual ~ScriptPubKeyMan() {}; - virtual bool GetNewDestination(const OutputType type, CTxDestination& dest, bilingual_str& error) { return false; } + virtual BResult<CTxDestination> GetNewDestination(const OutputType type) { return Untranslated("Not supported"); } virtual isminetype IsMine(const CScript& script) const { return ISMINE_NO; } //! Check that the given decryption key is valid for this ScriptPubKeyMan, i.e. it decrypts all of the keys handled by it. @@ -359,7 +360,7 @@ private: public: using ScriptPubKeyMan::ScriptPubKeyMan; - bool GetNewDestination(const OutputType type, CTxDestination& dest, bilingual_str& error) override; + BResult<CTxDestination> GetNewDestination(const OutputType type) override; isminetype IsMine(const CScript& script) const override; bool CheckDecryptionKey(const CKeyingMaterial& master_key, bool accept_no_keys = false) override; @@ -567,7 +568,7 @@ public: mutable RecursiveMutex cs_desc_man; - bool GetNewDestination(const OutputType type, CTxDestination& dest, bilingual_str& error) override; + BResult<CTxDestination> GetNewDestination(const OutputType type) override; isminetype IsMine(const CScript& script) const override; bool CheckDecryptionKey(const CKeyingMaterial& master_key, bool accept_no_keys = false) override; diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 5799a9ff2a..9be29c4709 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -27,25 +27,20 @@ using interfaces::FoundBlock; namespace wallet { static constexpr size_t OUTPUT_GROUP_MAX_ENTRIES{100}; -int GetTxSpendSize(const CWallet& wallet, const CWalletTx& wtx, unsigned int out, bool use_max_sig) -{ - return CalculateMaximumSignedInputSize(wtx.tx->vout[out], &wallet, use_max_sig); -} - -int CalculateMaximumSignedInputSize(const CTxOut& txout, const SigningProvider* provider, bool use_max_sig) +int CalculateMaximumSignedInputSize(const CTxOut& txout, const COutPoint outpoint, const SigningProvider* provider, const CCoinControl* coin_control) { CMutableTransaction txn; - txn.vin.push_back(CTxIn(COutPoint())); - if (!provider || !DummySignInput(*provider, txn.vin[0], txout, use_max_sig)) { + txn.vin.push_back(CTxIn(outpoint)); + if (!provider || !DummySignInput(*provider, txn.vin[0], txout, coin_control)) { return -1; } return GetVirtualTransactionInputSize(txn.vin[0]); } -int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* wallet, bool use_max_sig) +int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* wallet, const CCoinControl* coin_control) { const std::unique_ptr<SigningProvider> provider = wallet->GetSolvingProvider(txout.scriptPubKey); - return CalculateMaximumSignedInputSize(txout, provider.get(), use_max_sig); + return CalculateMaximumSignedInputSize(txout, COutPoint(), provider.get(), coin_control); } // txouts needs to be in the order of tx.vin @@ -198,7 +193,7 @@ CoinsResult AvailableCoins(const CWallet& wallet, // Filter by spendable outputs only if (!spendable && only_spendable) continue; - int input_bytes = GetTxSpendSize(wallet, wtx, i, (coinControl && coinControl->fAllowWatchOnly)); + int input_bytes = CalculateMaximumSignedInputSize(output, COutPoint(), provider.get(), coinControl); result.coins.emplace_back(outpoint, output, nDepth, input_bytes, spendable, solvable, safeTx, wtx.GetTxTime(), tx_from_me, feerate); result.total_amount += output.nValue; @@ -289,8 +284,9 @@ std::map<CTxDestination, std::vector<COutput>> ListCoins(const CWallet& wallet) ) { CTxDestination address; if (ExtractDestination(FindNonChangeParentOutput(wallet, *wtx.tx, output.n).scriptPubKey, address)) { + const auto out = wtx.tx->vout.at(output.n); result[address].emplace_back( - COutPoint(wtx.GetHash(), output.n), wtx.tx->vout.at(output.n), depth, GetTxSpendSize(wallet, wtx, output.n), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ false, wtx.GetTxTime(), CachedTxIsFromMe(wallet, wtx, ISMINE_ALL)); + COutPoint(wtx.GetHash(), output.n), out, depth, CalculateMaximumSignedInputSize(out, &wallet, /*coin_control=*/nullptr), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ false, wtx.GetTxTime(), CachedTxIsFromMe(wallet, wtx, ISMINE_ALL)); } } } @@ -397,10 +393,13 @@ std::optional<SelectionResult> AttemptSelection(const CWallet& wallet, const CAm // The knapsack solver has some legacy behavior where it will spend dust outputs. We retain this behavior, so don't filter for positive only here. std::vector<OutputGroup> all_groups = GroupOutputs(wallet, coins, coin_selection_params, eligibility_filter, false /* positive_only */); + CAmount target_with_change = nTargetValue; // While nTargetValue includes the transaction fees for non-input things, it does not include the fee for creating a change output. - // So we need to include that for KnapsackSolver as well, as we are expecting to create a change output. - if (auto knapsack_result{KnapsackSolver(all_groups, nTargetValue + coin_selection_params.m_change_fee, - coin_selection_params.m_min_change_target, coin_selection_params.rng_fast)}) { + // So we need to include that for KnapsackSolver and SRD as well, as we are expecting to create a change output. + if (!coin_selection_params.m_subtract_fee_outputs) { + target_with_change += coin_selection_params.m_change_fee; + } + if (auto knapsack_result{KnapsackSolver(all_groups, target_with_change, coin_selection_params.m_min_change_target, coin_selection_params.rng_fast)}) { knapsack_result->ComputeAndSetWaste(coin_selection_params.m_cost_of_change); results.push_back(*knapsack_result); } @@ -409,7 +408,7 @@ std::optional<SelectionResult> AttemptSelection(const CWallet& wallet, const CAm // barely meets the target. Just use the lower bound change target instead of the randomly // generated one, since SRD will result in a random change amount anyway; avoid making the // target needlessly large. - const CAmount srd_target = nTargetValue + coin_selection_params.m_change_fee + CHANGE_LOWER; + const CAmount srd_target = target_with_change + CHANGE_LOWER; if (auto srd_result{SelectCoinsSRD(positive_groups, srd_target, coin_selection_params.rng_fast)}) { srd_result->ComputeAndSetWaste(coin_selection_params.m_cost_of_change); results.push_back(*srd_result); @@ -447,14 +446,14 @@ std::optional<SelectionResult> SelectCoins(const CWallet& wallet, const std::vec if (ptr_wtx->tx->vout.size() <= outpoint.n) { return std::nullopt; } - input_bytes = GetTxSpendSize(wallet, *ptr_wtx, outpoint.n, false); txout = ptr_wtx->tx->vout.at(outpoint.n); + input_bytes = CalculateMaximumSignedInputSize(txout, &wallet, &coin_control); } else { // The input is external. We did not find the tx in mapWallet. if (!coin_control.GetExternalOutput(outpoint, txout)) { return std::nullopt; } - input_bytes = CalculateMaximumSignedInputSize(txout, &coin_control.m_external_provider, /*use_max_sig=*/true); + input_bytes = CalculateMaximumSignedInputSize(txout, outpoint, &coin_control.m_external_provider, &coin_control); } // If available, override calculated size with coin control specified size if (coin_control.HasInputWeight(outpoint)) { @@ -653,19 +652,16 @@ static void DiscourageFeeSniping(CMutableTransaction& tx, FastRandomContext& rng } } -static std::optional<CreatedTransactionResult> CreateTransactionInternal( +static BResult<CreatedTransactionResult> CreateTransactionInternal( CWallet& wallet, const std::vector<CRecipient>& vecSend, int change_pos, - bilingual_str& error, const CCoinControl& coin_control, - FeeCalculation& fee_calc_out, bool sign) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { AssertLockHeld(wallet.cs_wallet); // out variables, to be packed into returned result structure - CTransactionRef tx; CAmount nFeeRet; int nChangePosInOut = change_pos; @@ -694,6 +690,7 @@ static std::optional<CreatedTransactionResult> CreateTransactionInternal( // Create change script that will be used if we need change CScript scriptChange; + bilingual_str error; // possible error str // coin control: send change to custom address if (!std::get_if<CNoDestination>(&coin_control.destChange)) { @@ -741,13 +738,11 @@ static std::optional<CreatedTransactionResult> CreateTransactionInternal( // Do not, ever, assume that it's fine to change the fee rate if the user has explicitly // provided one if (coin_control.m_feerate && coin_selection_params.m_effective_feerate > *coin_control.m_feerate) { - error = strprintf(_("Fee rate (%s) is lower than the minimum fee rate setting (%s)"), coin_control.m_feerate->ToString(FeeEstimateMode::SAT_VB), coin_selection_params.m_effective_feerate.ToString(FeeEstimateMode::SAT_VB)); - return std::nullopt; + return strprintf(_("Fee rate (%s) is lower than the minimum fee rate setting (%s)"), coin_control.m_feerate->ToString(FeeEstimateMode::SAT_VB), coin_selection_params.m_effective_feerate.ToString(FeeEstimateMode::SAT_VB)); } if (feeCalc.reason == FeeReason::FALLBACK && !wallet.m_allow_fallback_fee) { // eventually allow a fallback fee - error = _("Fee estimation failed. Fallbackfee is disabled. Wait a few blocks or enable -fallbackfee."); - return std::nullopt; + return _("Fee estimation failed. Fallbackfee is disabled. Wait a few blocks or enable -fallbackfee."); } // Calculate the cost of change @@ -760,7 +755,8 @@ static std::optional<CreatedTransactionResult> CreateTransactionInternal( // vouts to the payees if (!coin_selection_params.m_subtract_fee_outputs) { - coin_selection_params.tx_noinputs_size = 11; // Static vsize overhead + outputs vsize. 4 nVersion, 4 nLocktime, 1 input count, 1 output count, 1 witness overhead (dummy, flag, stack size) + coin_selection_params.tx_noinputs_size = 10; // Static vsize overhead + outputs vsize. 4 nVersion, 4 nLocktime, 1 input count, 1 witness overhead (dummy, flag, stack size) + coin_selection_params.tx_noinputs_size += GetSizeOfCompactSize(vecSend.size()); // bytes for output count } for (const auto& recipient : vecSend) { @@ -771,10 +767,8 @@ static std::optional<CreatedTransactionResult> CreateTransactionInternal( coin_selection_params.tx_noinputs_size += ::GetSerializeSize(txout, PROTOCOL_VERSION); } - if (IsDust(txout, wallet.chain().relayDustFee())) - { - error = _("Transaction amount too small"); - return std::nullopt; + if (IsDust(txout, wallet.chain().relayDustFee())) { + return _("Transaction amount too small"); } txNew.vout.push_back(txout); } @@ -795,8 +789,7 @@ static std::optional<CreatedTransactionResult> CreateTransactionInternal( // Choose coins to use std::optional<SelectionResult> result = SelectCoins(wallet, res_available_coins.coins, /*nTargetValue=*/selection_target, coin_control, coin_selection_params); if (!result) { - error = _("Insufficient funds"); - return std::nullopt; + return _("Insufficient funds"); } TRACE5(coin_selection, selected_coins, wallet.GetName().c_str(), GetAlgorithmName(result->m_algo).c_str(), result->m_target, result->GetWaste(), result->GetSelectedValue()); @@ -810,10 +803,8 @@ static std::optional<CreatedTransactionResult> CreateTransactionInternal( // Insert change txn at random position: nChangePosInOut = rng_fast.randrange(txNew.vout.size() + 1); } - else if ((unsigned int)nChangePosInOut > txNew.vout.size()) - { - error = _("Transaction change output index out of range"); - return std::nullopt; + else if ((unsigned int)nChangePosInOut > txNew.vout.size()) { + return _("Transaction change output index out of range"); } assert(nChangePosInOut != -1); @@ -840,8 +831,7 @@ static std::optional<CreatedTransactionResult> CreateTransactionInternal( TxSize tx_sizes = CalculateMaximumSignedTxSize(CTransaction(txNew), &wallet, &coin_control); int nBytes = tx_sizes.vsize; if (nBytes == -1) { - error = _("Missing solving data for estimating transaction size"); - return std::nullopt; + return _("Missing solving data for estimating transaction size"); } nFeeRet = coin_selection_params.m_effective_feerate.GetFee(nBytes); @@ -901,11 +891,10 @@ static std::optional<CreatedTransactionResult> CreateTransactionInternal( // Error if this output is reduced to be below dust if (IsDust(txout, wallet.chain().relayDustFee())) { if (txout.nValue < 0) { - error = _("The transaction amount is too small to pay the fee"); + return _("The transaction amount is too small to pay the fee"); } else { - error = _("The transaction amount is too small to send after the fee has been deducted"); + return _("The transaction amount is too small to send after the fee has been deducted"); } - return std::nullopt; } } ++i; @@ -915,42 +904,37 @@ static std::optional<CreatedTransactionResult> CreateTransactionInternal( // Give up if change keypool ran out and change is required if (scriptChange.empty() && nChangePosInOut != -1) { - return std::nullopt; + return error; } if (sign && !wallet.SignTransaction(txNew)) { - error = _("Signing transaction failed"); - return std::nullopt; + return _("Signing transaction failed"); } // Return the constructed transaction data. - tx = MakeTransactionRef(std::move(txNew)); + CTransactionRef tx = MakeTransactionRef(std::move(txNew)); // Limit size if ((sign && GetTransactionWeight(*tx) > MAX_STANDARD_TX_WEIGHT) || (!sign && tx_sizes.weight > MAX_STANDARD_TX_WEIGHT)) { - error = _("Transaction too large"); - return std::nullopt; + return _("Transaction too large"); } if (nFeeRet > wallet.m_default_max_tx_fee) { - error = TransactionErrorString(TransactionError::MAX_FEE_EXCEEDED); - return std::nullopt; + return TransactionErrorString(TransactionError::MAX_FEE_EXCEEDED); } if (gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS)) { // Lastly, ensure this tx will pass the mempool's chain limits if (!wallet.chain().checkChainLimits(tx)) { - error = _("Transaction has too long of a mempool chain"); - return std::nullopt; + return _("Transaction has too long of a mempool chain"); } } // Before we return success, we assume any change key will be used to prevent // accidental re-use. reservedest.KeepDestination(); - fee_calc_out = feeCalc; wallet.WalletLogPrintf("Fee Calculation: Fee:%d Bytes:%u Tgt:%d (requested %d) Reason:\"%s\" Decay %.5f: Estimation: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out) Fail: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out)\n", nFeeRet, nBytes, feeCalc.returnedTarget, feeCalc.desiredTarget, StringForFeeReason(feeCalc.reason), feeCalc.est.decay, @@ -960,52 +944,50 @@ static std::optional<CreatedTransactionResult> CreateTransactionInternal( feeCalc.est.fail.start, feeCalc.est.fail.end, (feeCalc.est.fail.totalConfirmed + feeCalc.est.fail.inMempool + feeCalc.est.fail.leftMempool) > 0.0 ? 100 * feeCalc.est.fail.withinTarget / (feeCalc.est.fail.totalConfirmed + feeCalc.est.fail.inMempool + feeCalc.est.fail.leftMempool) : 0.0, feeCalc.est.fail.withinTarget, feeCalc.est.fail.totalConfirmed, feeCalc.est.fail.inMempool, feeCalc.est.fail.leftMempool); - return CreatedTransactionResult(tx, nFeeRet, nChangePosInOut); + return CreatedTransactionResult(tx, nFeeRet, nChangePosInOut, feeCalc); } -std::optional<CreatedTransactionResult> CreateTransaction( +BResult<CreatedTransactionResult> CreateTransaction( CWallet& wallet, const std::vector<CRecipient>& vecSend, int change_pos, - bilingual_str& error, const CCoinControl& coin_control, - FeeCalculation& fee_calc_out, bool sign) { if (vecSend.empty()) { - error = _("Transaction must have at least one recipient"); - return std::nullopt; + return _("Transaction must have at least one recipient"); } if (std::any_of(vecSend.cbegin(), vecSend.cend(), [](const auto& recipient){ return recipient.nAmount < 0; })) { - error = _("Transaction amounts must not be negative"); - return std::nullopt; + return _("Transaction amounts must not be negative"); } LOCK(wallet.cs_wallet); - std::optional<CreatedTransactionResult> txr_ungrouped = CreateTransactionInternal(wallet, vecSend, change_pos, error, coin_control, fee_calc_out, sign); - TRACE4(coin_selection, normal_create_tx_internal, wallet.GetName().c_str(), txr_ungrouped.has_value(), - txr_ungrouped.has_value() ? txr_ungrouped->fee : 0, txr_ungrouped.has_value() ? txr_ungrouped->change_pos : 0); - if (!txr_ungrouped) return std::nullopt; + auto res = CreateTransactionInternal(wallet, vecSend, change_pos, coin_control, sign); + TRACE4(coin_selection, normal_create_tx_internal, wallet.GetName().c_str(), res.HasRes(), + res ? res.GetObj().fee : 0, res ? res.GetObj().change_pos : 0); + if (!res) return res; + const auto& txr_ungrouped = res.GetObj(); // try with avoidpartialspends unless it's enabled already - if (txr_ungrouped->fee > 0 /* 0 means non-functional fee rate estimation */ && wallet.m_max_aps_fee > -1 && !coin_control.m_avoid_partial_spends) { + if (txr_ungrouped.fee > 0 /* 0 means non-functional fee rate estimation */ && wallet.m_max_aps_fee > -1 && !coin_control.m_avoid_partial_spends) { TRACE1(coin_selection, attempting_aps_create_tx, wallet.GetName().c_str()); CCoinControl tmp_cc = coin_control; tmp_cc.m_avoid_partial_spends = true; - bilingual_str error2; // fired and forgotten; if an error occurs, we discard the results - std::optional<CreatedTransactionResult> txr_grouped = CreateTransactionInternal(wallet, vecSend, change_pos, error2, tmp_cc, fee_calc_out, sign); + auto res_tx_grouped = CreateTransactionInternal(wallet, vecSend, change_pos, tmp_cc, sign); + // Helper optional class for now + std::optional<CreatedTransactionResult> txr_grouped{res_tx_grouped.HasRes() ? std::make_optional(res_tx_grouped.GetObj()) : std::nullopt}; // if fee of this alternative one is within the range of the max fee, we use this one - const bool use_aps{txr_grouped.has_value() ? (txr_grouped->fee <= txr_ungrouped->fee + wallet.m_max_aps_fee) : false}; + const bool use_aps{txr_grouped.has_value() ? (txr_grouped->fee <= txr_ungrouped.fee + wallet.m_max_aps_fee) : false}; TRACE5(coin_selection, aps_create_tx_internal, wallet.GetName().c_str(), use_aps, txr_grouped.has_value(), txr_grouped.has_value() ? txr_grouped->fee : 0, txr_grouped.has_value() ? txr_grouped->change_pos : 0); if (txr_grouped) { wallet.WalletLogPrintf("Fee non-grouped = %lld, grouped = %lld, using %s\n", - txr_ungrouped->fee, txr_grouped->fee, use_aps ? "grouped" : "non-grouped"); - if (use_aps) return txr_grouped; + txr_ungrouped.fee, txr_grouped->fee, use_aps ? "grouped" : "non-grouped"); + if (use_aps) return res_tx_grouped; } } - return txr_ungrouped; + return res; } bool FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, CCoinControl coinControl) @@ -1041,12 +1023,15 @@ bool FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& nFeeRet, } } - FeeCalculation fee_calc_out; - std::optional<CreatedTransactionResult> txr = CreateTransaction(wallet, vecSend, nChangePosInOut, error, coinControl, fee_calc_out, false); - if (!txr) return false; - CTransactionRef tx_new = txr->tx; - nFeeRet = txr->fee; - nChangePosInOut = txr->change_pos; + auto res = CreateTransaction(wallet, vecSend, nChangePosInOut, coinControl, false); + if (!res) { + error = res.GetError(); + return false; + } + const auto& txr = res.GetObj(); + CTransactionRef tx_new = txr.tx; + nFeeRet = txr.fee; + nChangePosInOut = txr.change_pos; if (nChangePosInOut != -1) { tx.vout.insert(tx.vout.begin() + nChangePosInOut, tx_new->vout[nChangePosInOut]); diff --git a/src/wallet/spend.h b/src/wallet/spend.h index cba42d6fae..ab0ff1ee58 100644 --- a/src/wallet/spend.h +++ b/src/wallet/spend.h @@ -6,6 +6,8 @@ #define BITCOIN_WALLET_SPEND_H #include <consensus/amount.h> +#include <policy/fees.h> // for FeeCalculation +#include <util/result.h> #include <wallet/coinselection.h> #include <wallet/transaction.h> #include <wallet/wallet.h> @@ -14,23 +16,16 @@ namespace wallet { /** Get the marginal bytes if spending the specified output from this transaction. - * use_max_sig indicates whether to use the maximum sized, 72 byte signature when calculating the - * size of the input spend. This should only be set when watch-only outputs are allowed */ -int GetTxSpendSize(const CWallet& wallet, const CWalletTx& wtx, unsigned int out, bool use_max_sig = false); - -//Get the marginal bytes of spending the specified output -int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* pwallet, bool use_max_sig = false); -int CalculateMaximumSignedInputSize(const CTxOut& txout, const SigningProvider* pwallet, bool use_max_sig = false); - + * Use CoinControl to determine whether to expect signature grinding when calculating the size of the input spend. */ +int CalculateMaximumSignedInputSize(const CTxOut& txout, const CWallet* pwallet, const CCoinControl* coin_control = nullptr); +int CalculateMaximumSignedInputSize(const CTxOut& txout, const COutPoint outpoint, const SigningProvider* pwallet, const CCoinControl* coin_control = nullptr); struct TxSize { int64_t vsize{-1}; int64_t weight{-1}; }; -/** Calculate the size of the transaction assuming all signatures are max size -* Use DummySignatureCreator, which inserts 71 byte signatures everywhere. -* NOTE: this requires that all inputs must be in mapWallet (eg the tx should -* be AllInputsMine). */ +/** Calculate the size of the transaction using CoinControl to determine + * whether to expect signature grinding when calculating the size of the input spend. */ TxSize CalculateMaximumSignedTxSize(const CTransaction& tx, const CWallet* wallet, const std::vector<CTxOut>& txouts, const CCoinControl* coin_control = nullptr); TxSize CalculateMaximumSignedTxSize(const CTransaction& tx, const CWallet* wallet, const CCoinControl* coin_control = nullptr) EXCLUSIVE_LOCKS_REQUIRED(wallet->cs_wallet); @@ -107,10 +102,11 @@ struct CreatedTransactionResult { CTransactionRef tx; CAmount fee; + FeeCalculation fee_calc; int change_pos; - CreatedTransactionResult(CTransactionRef tx, CAmount fee, int change_pos) - : tx(tx), fee(fee), change_pos(change_pos) {} + CreatedTransactionResult(CTransactionRef _tx, CAmount _fee, int _change_pos, const FeeCalculation& _fee_calc) + : tx(_tx), fee(_fee), fee_calc(_fee_calc), change_pos(_change_pos) {} }; /** @@ -118,7 +114,7 @@ struct CreatedTransactionResult * selected by SelectCoins(); Also create the change output, when needed * @note passing change_pos as -1 will result in setting a random position */ -std::optional<CreatedTransactionResult> CreateTransaction(CWallet& wallet, const std::vector<CRecipient>& vecSend, int change_pos, bilingual_str& error, const CCoinControl& coin_control, FeeCalculation& fee_calc_out, bool sign = true); +BResult<CreatedTransactionResult> CreateTransaction(CWallet& wallet, const std::vector<CRecipient>& vecSend, int change_pos, const CCoinControl& coin_control, bool sign = true); /** * Insert additional inputs into the transaction by diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index d6f47e9954..a418105ee1 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -74,11 +74,9 @@ static void add_coin(std::vector<COutput>& coins, CWallet& wallet, const CAmount tx.vout.resize(nInput + 1); tx.vout[nInput].nValue = nValue; if (spendable) { - CTxDestination dest; - bilingual_str error; - const bool destination_ok = wallet.GetNewDestination(OutputType::BECH32, "", dest, error); - assert(destination_ok); - tx.vout[nInput].scriptPubKey = GetScriptForDestination(dest); + auto op_dest = wallet.GetNewDestination(OutputType::BECH32, ""); + assert(op_dest.HasRes()); + tx.vout[nInput].scriptPubKey = GetScriptForDestination(op_dest.GetObj()); } uint256 txid = tx.GetHash(); @@ -86,7 +84,8 @@ static void add_coin(std::vector<COutput>& coins, CWallet& wallet, const CAmount auto ret = wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(txid), std::forward_as_tuple(MakeTransactionRef(std::move(tx)), TxStateInactive{})); assert(ret.second); CWalletTx& wtx = (*ret.first).second; - coins.emplace_back(COutPoint(wtx.GetHash(), nInput), wtx.tx->vout.at(nInput), nAge, GetTxSpendSize(wallet, wtx, nInput), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, wtx.GetTxTime(), fIsFromMe, feerate); + const auto& txout = wtx.tx->vout.at(nInput); + coins.emplace_back(COutPoint(wtx.GetHash(), nInput), txout, nAge, CalculateMaximumSignedInputSize(txout, &wallet, /*coin_control=*/nullptr), /*spendable=*/ true, /*solvable=*/ true, /*safe=*/ true, wtx.GetTxTime(), fIsFromMe, feerate); } /** Check if SelectionResult a is equivalent to SelectionResult b. @@ -198,8 +197,8 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) expected_result.Clear(); // Select 5 Cent - add_coin(4 * CENT, 4, expected_result); - add_coin(1 * CENT, 1, expected_result); + add_coin(3 * CENT, 3, expected_result); + add_coin(2 * CENT, 2, expected_result); const auto result3 = SelectCoinsBnB(GroupCoins(utxo_pool), 5 * CENT, 0.5 * CENT); BOOST_CHECK(result3); BOOST_CHECK(EquivalentResult(expected_result, *result3)); @@ -224,8 +223,9 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) // Select 10 Cent add_coin(5 * CENT, 5, utxo_pool); - add_coin(5 * CENT, 5, expected_result); add_coin(4 * CENT, 4, expected_result); + add_coin(3 * CENT, 3, expected_result); + add_coin(2 * CENT, 2, expected_result); add_coin(1 * CENT, 1, expected_result); const auto result5 = SelectCoinsBnB(GroupCoins(utxo_pool), 10 * CENT, 0.5 * CENT); BOOST_CHECK(result5); @@ -865,7 +865,23 @@ BOOST_AUTO_TEST_CASE(waste_test) const CAmount new_target{in_amt - fee * 2 - fee_diff * 2}; add_coin(1 * COIN, 1, selection, fee, fee + fee_diff); add_coin(2 * COIN, 2, selection, fee, fee + fee_diff); - BOOST_CHECK_EQUAL(0, GetSelectionWaste(selection, /* change cost */ 0, new_target)); + BOOST_CHECK_EQUAL(0, GetSelectionWaste(selection, /*change_cost=*/ 0, new_target)); + selection.clear(); + + // Negative waste when the long term fee is greater than the current fee and the selected value == target + const CAmount exact_target1{3 * COIN - 2 * fee}; + const CAmount target_waste1{-2 * fee_diff}; // = (2 * fee) - (2 * (fee + fee_diff)) + add_coin(1 * COIN, 1, selection, fee, fee + fee_diff); + add_coin(2 * COIN, 2, selection, fee, fee + fee_diff); + BOOST_CHECK_EQUAL(target_waste1, GetSelectionWaste(selection, /*change_cost=*/ 0, exact_target1)); + selection.clear(); + + // Negative waste when the long term fee is greater than the current fee and change_cost < - (inputs * (fee - long_term_fee)) + const CAmount large_fee_diff{90}; + const CAmount target_waste2{-2 * large_fee_diff + change_cost}; // = (2 * fee) - (2 * (fee + large_fee_diff)) + change_cost + add_coin(1 * COIN, 1, selection, fee, fee + large_fee_diff); + add_coin(2 * COIN, 2, selection, fee, fee + large_fee_diff); + BOOST_CHECK_EQUAL(target_waste2, GetSelectionWaste(selection, change_cost, target)); } BOOST_AUTO_TEST_CASE(effective_value_test) diff --git a/src/wallet/test/fuzz/notifications.cpp b/src/wallet/test/fuzz/notifications.cpp index 9089c8ff46..5c173773e4 100644 --- a/src/wallet/test/fuzz/notifications.cpp +++ b/src/wallet/test/fuzz/notifications.cpp @@ -69,15 +69,14 @@ struct FuzzedWallet { CScript GetScriptPubKey(FuzzedDataProvider& fuzzed_data_provider) { auto type{fuzzed_data_provider.PickValueInArray(OUTPUT_TYPES)}; - CTxDestination dest; - bilingual_str error; + BResult<CTxDestination> op_dest; if (fuzzed_data_provider.ConsumeBool()) { - assert(wallet->GetNewDestination(type, "", dest, error)); + op_dest = wallet->GetNewDestination(type, ""); } else { - assert(wallet->GetNewChangeDestination(type, dest, error)); + op_dest = wallet->GetNewChangeDestination(type); } - assert(error.empty()); - return GetScriptForDestination(dest); + assert(op_dest.HasRes()); + return GetScriptForDestination(op_dest.GetObj()); } }; @@ -138,8 +137,13 @@ FUZZ_TARGET_INIT(wallet_notifications, initialize_setup) block.vtx.emplace_back(MakeTransactionRef(tx)); } // Mine block - a.wallet->blockConnected(block, chain.size()); - b.wallet->blockConnected(block, chain.size()); + const uint256& hash = block.GetHash(); + interfaces::BlockInfo info{hash}; + info.prev_hash = &block.hashPrevBlock; + info.height = chain.size(); + info.data = █ + a.wallet->blockConnected(info); + b.wallet->blockConnected(info); // Store the coins for the next block Coins coins_new; for (const auto& tx : block.vtx) { @@ -155,8 +159,13 @@ FUZZ_TARGET_INIT(wallet_notifications, initialize_setup) auto& [coins, block]{chain.back()}; if (block.vtx.empty()) return; // Can only disconnect if the block was submitted first // Disconnect block - a.wallet->blockDisconnected(block, chain.size() - 1); - b.wallet->blockDisconnected(block, chain.size() - 1); + const uint256& hash = block.GetHash(); + interfaces::BlockInfo info{hash}; + info.prev_hash = &block.hashPrevBlock; + info.height = chain.size() - 1; + info.data = █ + a.wallet->blockDisconnected(info); + b.wallet->blockDisconnected(info); chain.pop_back(); }); auto& [coins, first_block]{chain.front()}; diff --git a/src/wallet/test/spend_tests.cpp b/src/wallet/test/spend_tests.cpp index bdc148afb4..53c3b5d2ae 100644 --- a/src/wallet/test/spend_tests.cpp +++ b/src/wallet/test/spend_tests.cpp @@ -28,19 +28,18 @@ BOOST_FIXTURE_TEST_CASE(SubtractFee, TestChain100Setup) auto check_tx = [&wallet](CAmount leftover_input_amount) { CRecipient recipient{GetScriptForRawPubKey({}), 50 * COIN - leftover_input_amount, true /* subtract fee */}; constexpr int RANDOM_CHANGE_POSITION = -1; - bilingual_str error; CCoinControl coin_control; coin_control.m_feerate.emplace(10000); coin_control.fOverrideFeeRate = true; // We need to use a change type with high cost of change so that the leftover amount will be dropped to fee instead of added as a change output coin_control.m_change_type = OutputType::LEGACY; - FeeCalculation fee_calc; - std::optional<CreatedTransactionResult> txr = CreateTransaction(*wallet, {recipient}, RANDOM_CHANGE_POSITION, error, coin_control, fee_calc); - BOOST_CHECK(txr.has_value()); - BOOST_CHECK_EQUAL(txr->tx->vout.size(), 1); - BOOST_CHECK_EQUAL(txr->tx->vout[0].nValue, recipient.nAmount + leftover_input_amount - txr->fee); - BOOST_CHECK_GT(txr->fee, 0); - return txr->fee; + auto res = CreateTransaction(*wallet, {recipient}, RANDOM_CHANGE_POSITION, coin_control); + BOOST_CHECK(res); + const auto& txr = res.GetObj(); + BOOST_CHECK_EQUAL(txr.tx->vout.size(), 1); + BOOST_CHECK_EQUAL(txr.tx->vout[0].nValue, recipient.nAmount + leftover_input_amount - txr.fee); + BOOST_CHECK_GT(txr.fee, 0); + return txr.fee; }; // Send full input amount to recipient, check that only nonzero fee is diff --git a/src/wallet/test/util.cpp b/src/wallet/test/util.cpp index aa3121511d..ab72721f9d 100644 --- a/src/wallet/test/util.cpp +++ b/src/wallet/test/util.cpp @@ -38,7 +38,7 @@ std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cc } WalletRescanReserver reserver(*wallet); reserver.reserve(); - CWallet::ScanResult result = wallet->ScanForWalletTransactions(cchain.Genesis()->GetBlockHash(), 0 /* start_height */, {} /* max_height */, reserver, false /* update */); + CWallet::ScanResult result = wallet->ScanForWalletTransactions(cchain.Genesis()->GetBlockHash(), /*start_height=*/0, /*max_height=*/{}, reserver, /*fUpdate=*/false, /*save_progress=*/false); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); BOOST_CHECK_EQUAL(result.last_scanned_block, cchain.Tip()->GetBlockHash()); BOOST_CHECK_EQUAL(*result.last_scanned_height, cchain.Height()); diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 0c03b2f4a2..7b0906c0a8 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -4,7 +4,6 @@ #include <wallet/wallet.h> -#include <any> #include <future> #include <memory> #include <stdint.h> @@ -13,7 +12,6 @@ #include <interfaces/chain.h> #include <key_io.h> #include <node/blockstorage.h> -#include <node/context.h> #include <policy/policy.h> #include <rpc/server.h> #include <test/util/logging.h> @@ -55,9 +53,6 @@ static const std::shared_ptr<CWallet> TestLoadWallet(WalletContext& context) auto database = MakeWalletDatabase("", options, status, error); auto wallet = CWallet::Create(context, "", std::move(database), options.create_flags, error, warnings); NotifyWalletLoaded(context, wallet); - if (context.chain) { - wallet->postInitProcess(); - } return wallet; } @@ -112,7 +107,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions({} /* start_block */, 0 /* start_height */, {} /* max_height */, reserver, false /* update */); + CWallet::ScanResult result = wallet.ScanForWalletTransactions(/*start_block=*/{}, /*start_height=*/0, /*max_height=*/{}, reserver, /*fUpdate=*/false, /*save_progress=*/false); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); BOOST_CHECK(result.last_failed_block.IsNull()); BOOST_CHECK(result.last_scanned_block.IsNull()); @@ -123,7 +118,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) // Verify ScanForWalletTransactions picks up transactions in both the old // and new block files. { - CWallet wallet(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); + CWallet wallet(m_node.chain.get(), "", m_args, CreateMockWalletDatabase()); { LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); @@ -131,13 +126,28 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) } AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(wallet); + std::chrono::steady_clock::time_point fake_time; + reserver.setNow([&] { fake_time += 60s; return fake_time; }); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); + + { + CBlockLocator locator; + BOOST_CHECK(!WalletBatch{wallet.GetDatabase()}.ReadBestBlock(locator)); + BOOST_CHECK(locator.IsNull()); + } + + CWallet::ScanResult result = wallet.ScanForWalletTransactions(/*start_block=*/oldTip->GetBlockHash(), /*start_height=*/oldTip->nHeight, /*max_height=*/{}, reserver, /*fUpdate=*/false, /*save_progress=*/true); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::SUCCESS); BOOST_CHECK(result.last_failed_block.IsNull()); BOOST_CHECK_EQUAL(result.last_scanned_block, newTip->GetBlockHash()); BOOST_CHECK_EQUAL(*result.last_scanned_height, newTip->nHeight); BOOST_CHECK_EQUAL(GetBalance(wallet).m_mine_immature, 100 * COIN); + + { + CBlockLocator locator; + BOOST_CHECK(WalletBatch{wallet.GetDatabase()}.ReadBestBlock(locator)); + BOOST_CHECK(!locator.IsNull()); + } } // Prune the older block file. @@ -161,7 +171,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); + CWallet::ScanResult result = wallet.ScanForWalletTransactions(/*start_block=*/oldTip->GetBlockHash(), /*start_height=*/oldTip->nHeight, /*max_height=*/{}, reserver, /*fUpdate=*/false, /*save_progress=*/false); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); BOOST_CHECK_EQUAL(result.last_failed_block, oldTip->GetBlockHash()); BOOST_CHECK_EQUAL(result.last_scanned_block, newTip->GetBlockHash()); @@ -188,7 +198,7 @@ BOOST_FIXTURE_TEST_CASE(scan_for_wallet_transactions, TestChain100Setup) AddKey(wallet, coinbaseKey); WalletRescanReserver reserver(wallet); reserver.reserve(); - CWallet::ScanResult result = wallet.ScanForWalletTransactions(oldTip->GetBlockHash(), oldTip->nHeight, {} /* max_height */, reserver, false /* update */); + CWallet::ScanResult result = wallet.ScanForWalletTransactions(/*start_block=*/oldTip->GetBlockHash(), /*start_height=*/oldTip->nHeight, /*max_height=*/{}, reserver, /*fUpdate=*/false, /*save_progress=*/false); BOOST_CHECK_EQUAL(result.status, CWallet::ScanResult::FAILURE); BOOST_CHECK_EQUAL(result.last_failed_block, newTip->GetBlockHash()); BOOST_CHECK(result.last_scanned_block.IsNull()); @@ -521,14 +531,12 @@ public: CWalletTx& AddTx(CRecipient recipient) { CTransactionRef tx; - bilingual_str error; CCoinControl dummy; - FeeCalculation fee_calc_out; { constexpr int RANDOM_CHANGE_POSITION = -1; - std::optional<CreatedTransactionResult> txr = CreateTransaction(*wallet, {recipient}, RANDOM_CHANGE_POSITION, error, dummy, fee_calc_out); - BOOST_CHECK(txr.has_value()); - tx = txr->tx; + auto res = CreateTransaction(*wallet, {recipient}, RANDOM_CHANGE_POSITION, dummy); + BOOST_CHECK(res); + tx = res.GetObj().tx; } wallet->CommitTransaction(tx, {}, {}); CMutableTransaction blocktx; @@ -614,9 +622,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) wallet->SetMinVersion(FEATURE_LATEST); wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); BOOST_CHECK(!wallet->TopUpKeyPool(1000)); - CTxDestination dest; - bilingual_str error; - BOOST_CHECK(!wallet->GetNewDestination(OutputType::BECH32, "", dest, error)); + BOOST_CHECK(!wallet->GetNewDestination(OutputType::BECH32, "")); } { const std::shared_ptr<CWallet> wallet = std::make_shared<CWallet>(m_node.chain.get(), "", m_args, CreateDummyWalletDatabase()); @@ -624,9 +630,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet->SetMinVersion(FEATURE_LATEST); wallet->SetWalletFlag(WALLET_FLAG_DISABLE_PRIVATE_KEYS); - CTxDestination dest; - bilingual_str error; - BOOST_CHECK(!wallet->GetNewDestination(OutputType::BECH32, "", dest, error)); + BOOST_CHECK(!wallet->GetNewDestination(OutputType::BECH32, "")); } } @@ -761,6 +765,7 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) // being blocked wallet = TestLoadWallet(context); BOOST_CHECK(rescan_completed); + // AddToWallet events for block_tx and mempool_tx BOOST_CHECK_EQUAL(addtx_count, 2); { LOCK(wallet->cs_wallet); @@ -773,6 +778,8 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) // transactionAddedToMempool events are processed promise.set_value(); SyncWithValidationInterfaceQueue(); + // AddToWallet events for block_tx and mempool_tx events are counted a + // second time as the notificaiton queue is processed BOOST_CHECK_EQUAL(addtx_count, 4); @@ -796,7 +803,7 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) SyncWithValidationInterfaceQueue(); }); wallet = TestLoadWallet(context); - BOOST_CHECK_EQUAL(addtx_count, 4); + BOOST_CHECK_EQUAL(addtx_count, 2); { LOCK(wallet->cs_wallet); BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_tx.GetHash()), 1U); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 041481559b..54a3221e2d 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1314,30 +1314,31 @@ void CWallet::transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRe } } -void CWallet::blockConnected(const CBlock& block, int height) +void CWallet::blockConnected(const interfaces::BlockInfo& block) { - const uint256& block_hash = block.GetHash(); + assert(block.data); LOCK(cs_wallet); - m_last_block_processed_height = height; - m_last_block_processed = block_hash; - for (size_t index = 0; index < block.vtx.size(); index++) { - SyncTransaction(block.vtx[index], TxStateConfirmed{block_hash, height, static_cast<int>(index)}); - transactionRemovedFromMempool(block.vtx[index], MemPoolRemovalReason::BLOCK, 0 /* mempool_sequence */); + m_last_block_processed_height = block.height; + m_last_block_processed = block.hash; + for (size_t index = 0; index < block.data->vtx.size(); index++) { + SyncTransaction(block.data->vtx[index], TxStateConfirmed{block.hash, block.height, static_cast<int>(index)}); + transactionRemovedFromMempool(block.data->vtx[index], MemPoolRemovalReason::BLOCK, 0 /* mempool_sequence */); } } -void CWallet::blockDisconnected(const CBlock& block, int height) +void CWallet::blockDisconnected(const interfaces::BlockInfo& block) { + assert(block.data); LOCK(cs_wallet); // At block disconnection, this will change an abandoned transaction to // be unconfirmed, whether or not the transaction is added back to the mempool. // User may have to call abandontransaction again. It may be addressed in the // future with a stickier abandoned state or even removing abandontransaction call. - m_last_block_processed_height = height - 1; - m_last_block_processed = block.hashPrevBlock; - for (const CTransactionRef& ptx : block.vtx) { + m_last_block_processed_height = block.height - 1; + m_last_block_processed = *Assert(block.prev_hash); + for (const CTransactionRef& ptx : Assert(block.data)->vtx) { SyncTransaction(ptx, TxStateInactive{}); } } @@ -1504,13 +1505,16 @@ bool CWallet::AddWalletFlags(uint64_t flags) } // Helper for producing a max-sized low-S low-R signature (eg 71 bytes) -// or a max-sized low-S signature (e.g. 72 bytes) if use_max_sig is true -bool DummySignInput(const SigningProvider& provider, CTxIn &tx_in, const CTxOut &txout, bool use_max_sig) +// or a max-sized low-S signature (e.g. 72 bytes) depending on coin_control +bool DummySignInput(const SigningProvider& provider, CTxIn &tx_in, const CTxOut &txout, const CCoinControl* coin_control) { // Fill in dummy signatures for fee calculation. const CScript& scriptPubKey = txout.scriptPubKey; SignatureData sigdata; + // Use max sig if watch only inputs were used or if this particular input is an external input + // to ensure a sufficient fee is attained for the requested feerate. + const bool use_max_sig = coin_control && (coin_control->fAllowWatchOnly || coin_control->IsExternalSelected(tx_in.prevout)); if (!ProduceSignature(provider, use_max_sig ? DUMMY_MAXIMUM_SIGNATURE_CREATOR : DUMMY_SIGNATURE_CREATOR, scriptPubKey, sigdata)) { return false; } @@ -1577,12 +1581,9 @@ bool CWallet::DummySignTx(CMutableTransaction &txNew, const std::vector<CTxOut> nIn++; continue; } - // Use max sig if watch only inputs were used or if this particular input is an external input - // to ensure a sufficient fee is attained for the requested feerate. - const bool use_max_sig = coin_control && (coin_control->fAllowWatchOnly || coin_control->IsExternalSelected(txin.prevout)); const std::unique_ptr<SigningProvider> provider = GetSolvingProvider(txout.scriptPubKey); - if (!provider || !DummySignInput(*provider, txin, txout, use_max_sig)) { - if (!coin_control || !DummySignInput(coin_control->m_external_provider, txin, txout, use_max_sig)) { + if (!provider || !DummySignInput(*provider, txin, txout, coin_control)) { + if (!coin_control || !DummySignInput(coin_control->m_external_provider, txin, txout, coin_control)) { return false; } } @@ -1665,7 +1666,7 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r if (start) { // TODO: this should take into account failure by ScanResult::USER_ABORT - ScanResult result = ScanForWalletTransactions(start_block, start_height, {} /* max_height */, reserver, update); + ScanResult result = ScanForWalletTransactions(start_block, start_height, /*max_height=*/{}, reserver, /*fUpdate=*/update, /*save_progress=*/false); if (result.status == ScanResult::FAILURE) { int64_t time_max; CHECK_NONFATAL(chain().findBlock(result.last_failed_block, FoundBlock().maxTime(time_max))); @@ -1678,7 +1679,8 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r /** * Scan the block chain (starting in start_block) for transactions * from or to us. If fUpdate is true, found transactions that already - * exist in the wallet will be updated. + * exist in the wallet will be updated. If max_height is not set, the + * mempool will be scanned as well. * * @param[in] start_block Scan starting block. If block is not on the active * chain, the scan will return SUCCESS immediately. @@ -1696,12 +1698,11 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r * the main chain after to the addition of any new keys you want to detect * transactions for. */ -CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_block, int start_height, std::optional<int> max_height, const WalletRescanReserver& reserver, bool fUpdate) +CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_block, int start_height, std::optional<int> max_height, const WalletRescanReserver& reserver, bool fUpdate, const bool save_progress) { - using Clock = std::chrono::steady_clock; - constexpr auto LOG_INTERVAL{60s}; - auto current_time{Clock::now()}; - auto start_time{Clock::now()}; + constexpr auto INTERVAL_TIME{60s}; + auto current_time{reserver.now()}; + auto start_time{reserver.now()}; assert(reserver.isReserved()); @@ -1728,8 +1729,10 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc if (block_height % 100 == 0 && progress_end - progress_begin > 0.0) { ShowProgress(strprintf("%s " + _("Rescanning…").translated, GetDisplayName()), std::max(1, std::min(99, (int)(m_scanning_progress * 100)))); } - if (Clock::now() >= current_time + LOG_INTERVAL) { - current_time = Clock::now(); + + bool next_interval = reserver.now() >= current_time + INTERVAL_TIME; + if (next_interval) { + current_time = reserver.now(); WalletLogPrintf("Still rescanning. At block %d. Progress=%f\n", block_height, progress_current); } @@ -1759,6 +1762,16 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc // scan succeeded, record block as most recent successfully scanned result.last_scanned_block = block_hash; result.last_scanned_height = block_height; + + if (save_progress && next_interval) { + CBlockLocator loc = m_chain->getActiveChainLocator(block_hash); + + if (!loc.IsNull()) { + WalletLogPrintf("Saving scan progress %d.\n", block_height); + WalletBatch batch(GetDatabase()); + batch.WriteBestBlock(loc); + } + } } else { // could not scan block, keep scanning but record this block as the most recent failure result.last_failed_block = block_hash; @@ -1788,6 +1801,10 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc } } } + if (!max_height) { + WalletLogPrintf("Scanning current mempool transactions.\n"); + WITH_LOCK(cs_wallet, chain().requestMempoolTransactions(*this)); + } ShowProgress(strprintf("%s " + _("Rescanning…").translated, GetDisplayName()), 100); // hide progress dialog in GUI if (block_height && fAbortRescan) { WalletLogPrintf("Rescan aborted at block %d. Progress=%f\n", block_height, progress_current); @@ -1796,8 +1813,7 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc WalletLogPrintf("Rescan interrupted by shutdown request at block %d. Progress=%f\n", block_height, progress_current); result.status = ScanResult::USER_ABORT; } else { - auto duration_milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(Clock::now() - start_time); - WalletLogPrintf("Rescan completed in %15dms\n", duration_milliseconds.count()); + WalletLogPrintf("Rescan completed in %15dms\n", Ticks<std::chrono::milliseconds>(reserver.now() - start_time)); } return result; } @@ -1969,7 +1985,6 @@ TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& comp if (n_signed) { *n_signed = 0; } - const PrecomputedTransactionData txdata = PrecomputePSBTData(psbtx); LOCK(cs_wallet); // Get all of the previous transactions for (unsigned int i = 0; i < psbtx.tx->vin.size(); ++i) { @@ -1993,6 +2008,8 @@ TransactionError CWallet::FillPSBT(PartiallySignedTransaction& psbtx, bool& comp } } + const PrecomputedTransactionData txdata = PrecomputePSBTData(psbtx); + // Fill in information from ScriptPubKeyMans for (ScriptPubKeyMan* spk_man : GetAllScriptPubKeyMans()) { int n_signed_this_spkm = 0; @@ -2307,37 +2324,36 @@ bool CWallet::TopUpKeyPool(unsigned int kpSize) return res; } -bool CWallet::GetNewDestination(const OutputType type, const std::string label, CTxDestination& dest, bilingual_str& error) +BResult<CTxDestination> CWallet::GetNewDestination(const OutputType type, const std::string label) { LOCK(cs_wallet); - error.clear(); - bool result = false; auto spk_man = GetScriptPubKeyMan(type, false /* internal */); - if (spk_man) { - spk_man->TopUp(); - result = spk_man->GetNewDestination(type, dest, error); - } else { - error = strprintf(_("Error: No %s addresses available."), FormatOutputType(type)); + if (!spk_man) { + return strprintf(_("Error: No %s addresses available."), FormatOutputType(type)); } - if (result) { - SetAddressBook(dest, label, "receive"); + + spk_man->TopUp(); + auto op_dest = spk_man->GetNewDestination(type); + if (op_dest) { + SetAddressBook(op_dest.GetObj(), label, "receive"); } - return result; + return op_dest; } -bool CWallet::GetNewChangeDestination(const OutputType type, CTxDestination& dest, bilingual_str& error) +BResult<CTxDestination> CWallet::GetNewChangeDestination(const OutputType type) { LOCK(cs_wallet); - error.clear(); + CTxDestination dest; + bilingual_str error; ReserveDestination reservedest(this, type); if (!reservedest.GetReservedDestination(dest, true, error)) { - return false; + return error; } reservedest.KeepDestination(); - return true; + return dest; } std::optional<int64_t> CWallet::GetOldestKeyPoolTime() const @@ -2368,21 +2384,45 @@ void CWallet::MarkDestinationsDirty(const std::set<CTxDestination>& destinations } } -std::set<CTxDestination> CWallet::GetLabelAddresses(const std::string& label) const +void CWallet::ForEachAddrBookEntry(const ListAddrBookFunc& func) const { AssertLockHeld(cs_wallet); - std::set<CTxDestination> result; - for (const std::pair<const CTxDestination, CAddressBookData>& item : m_address_book) - { - if (item.second.IsChange()) continue; - const CTxDestination& address = item.first; - const std::string& strName = item.second.GetLabel(); - if (strName == label) - result.insert(address); + for (const std::pair<const CTxDestination, CAddressBookData>& item : m_address_book) { + const auto& entry = item.second; + func(item.first, entry.GetLabel(), entry.purpose, entry.IsChange()); } +} + +std::vector<CTxDestination> CWallet::ListAddrBookAddresses(const std::optional<AddrBookFilter>& _filter) const +{ + AssertLockHeld(cs_wallet); + std::vector<CTxDestination> result; + AddrBookFilter filter = _filter ? *_filter : AddrBookFilter(); + ForEachAddrBookEntry([&result, &filter](const CTxDestination& dest, const std::string& label, const std::string& purpose, bool is_change) { + // Filter by change + if (filter.ignore_change && is_change) return; + // Filter by label + if (filter.m_op_label && *filter.m_op_label != label) return; + // All good + result.emplace_back(dest); + }); return result; } +std::set<std::string> CWallet::ListAddrBookLabels(const std::string& purpose) const +{ + AssertLockHeld(cs_wallet); + std::set<std::string> label_set; + ForEachAddrBookEntry([&](const CTxDestination& _dest, const std::string& _label, + const std::string& _purpose, bool _is_change) { + if (_is_change) return; + if (purpose.empty() || _purpose == purpose) { + label_set.insert(_label); + } + }); + return label_set; +} + bool ReserveDestination::GetReservedDestination(CTxDestination& dest, bool internal, bilingual_str& error) { m_spk_man = pwallet->GetScriptPubKeyMan(type, internal); @@ -3011,20 +3051,31 @@ bool CWallet::AttachChain(const std::shared_ptr<CWallet>& walletInstance, interf if (tip_height && *tip_height != rescan_height) { - if (chain.havePruned()) { + // Technically we could execute the code below in any case, but performing the + // `while` loop below can make startup very slow, so only check blocks on disk + // if necessary. + if (chain.havePruned() || chain.hasAssumedValidChain()) { int block_height = *tip_height; while (block_height > 0 && chain.haveBlockOnDisk(block_height - 1) && rescan_height != block_height) { --block_height; } if (rescan_height != block_height) { - // We can't rescan beyond non-pruned blocks, stop and throw an error. + // We can't rescan beyond blocks we don't have data for, stop and throw an error. // This might happen if a user uses an old wallet within a pruned node // or if they ran -disablewallet for a longer time, then decided to re-enable // Exit early and print an error. + // It also may happen if an assumed-valid chain is in use and therefore not + // all block data is available. // If a block is pruned after this check, we will load the wallet, // but fail the rescan with a generic error. - error = _("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)"); + + error = chain.hasAssumedValidChain() ? + _( + "Assumed-valid: last wallet synchronisation goes beyond " + "available block data. You need to wait for the background " + "validation chain to download more blocks.") : + _("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)"); return false; } } @@ -3045,7 +3096,7 @@ bool CWallet::AttachChain(const std::shared_ptr<CWallet>& walletInstance, interf { WalletRescanReserver reserver(*walletInstance); - if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(chain.getBlockHash(rescan_height), rescan_height, {} /* max height */, reserver, true /* update */).status)) { + if (!reserver.reserve() || (ScanResult::SUCCESS != walletInstance->ScanForWalletTransactions(chain.getBlockHash(rescan_height), rescan_height, /*max_height=*/{}, reserver, /*fUpdate=*/true, /*save_progress=*/true).status)) { error = _("Failed to rescan the wallet during initialization"); return false; } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index bf42b3da9e..15f5917040 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -15,6 +15,7 @@ #include <psbt.h> #include <tinyformat.h> #include <util/message.h> +#include <util/result.h> #include <util/strencodings.h> #include <util/string.h> #include <util/system.h> @@ -45,7 +46,6 @@ using LoadWalletFn = std::function<void(std::unique_ptr<interfaces::Wallet> wall class CScript; enum class FeeEstimateMode; -struct FeeCalculation; struct bilingual_str; namespace wallet { @@ -508,8 +508,8 @@ public: CWalletTx* AddToWallet(CTransactionRef tx, const TxState& state, const UpdateWalletTxFn& update_wtx=nullptr, bool fFlushOnClose=true, bool rescanning_old_block = false); bool LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void transactionAddedToMempool(const CTransactionRef& tx, uint64_t mempool_sequence) override; - void blockConnected(const CBlock& block, int height) override; - void blockDisconnected(const CBlock& block, int height) override; + void blockConnected(const interfaces::BlockInfo& block) override; + void blockDisconnected(const interfaces::BlockInfo& block) override; void updatedBlockTip() override; int64_t RescanFromTime(int64_t startTime, const WalletRescanReserver& reserver, bool update); @@ -528,7 +528,7 @@ public: //! USER_ABORT. uint256 last_failed_block; }; - ScanResult ScanForWalletTransactions(const uint256& start_block, int start_height, std::optional<int> max_height, const WalletRescanReserver& reserver, bool fUpdate); + ScanResult ScanForWalletTransactions(const uint256& start_block, int start_height, std::optional<int> max_height, const WalletRescanReserver& reserver, bool fUpdate, const bool save_progress); void transactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason, uint64_t mempool_sequence) override; void ReacceptWalletTransactions() EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); void ResendWalletTransactions(); @@ -635,7 +635,30 @@ public: std::optional<int64_t> GetOldestKeyPoolTime() const; - std::set<CTxDestination> GetLabelAddresses(const std::string& label) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + // Filter struct for 'ListAddrBookAddresses' + struct AddrBookFilter { + // Fetch addresses with the provided label + std::optional<std::string> m_op_label{std::nullopt}; + // Don't include change addresses by default + bool ignore_change{true}; + }; + + /** + * Filter and retrieve destinations stored in the addressbook + */ + std::vector<CTxDestination> ListAddrBookAddresses(const std::optional<AddrBookFilter>& filter) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + /** + * Retrieve all the known labels in the address book + */ + std::set<std::string> ListAddrBookLabels(const std::string& purpose) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); + + /** + * Walk-through the address book entries. + * Stops when the provided 'ListAddrBookFunc' returns false. + */ + using ListAddrBookFunc = std::function<void(const CTxDestination& dest, const std::string& label, const std::string& purpose, bool is_change)>; + void ForEachAddrBookEntry(const ListAddrBookFunc& func) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); /** * Marks all outputs in each one of the destinations dirty, so their cache is @@ -643,8 +666,8 @@ public: */ void MarkDestinationsDirty(const std::set<CTxDestination>& destinations) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - bool GetNewDestination(const OutputType type, const std::string label, CTxDestination& dest, bilingual_str& error); - bool GetNewChangeDestination(const OutputType type, CTxDestination& dest, bilingual_str& error); + BResult<CTxDestination> GetNewDestination(const OutputType type, const std::string label); + BResult<CTxDestination> GetNewChangeDestination(const OutputType type); isminetype IsMine(const CTxDestination& dest) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); isminetype IsMine(const CScript& script) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); @@ -896,8 +919,11 @@ void MaybeResendWalletTxs(WalletContext& context); class WalletRescanReserver { private: + using Clock = std::chrono::steady_clock; + using NowFn = std::function<Clock::time_point()>; CWallet& m_wallet; bool m_could_reserve; + NowFn m_now; public: explicit WalletRescanReserver(CWallet& w) : m_wallet(w), m_could_reserve(false) {} @@ -918,6 +944,10 @@ public: return (m_could_reserve && m_wallet.fScanningWallet); } + Clock::time_point now() const { return m_now ? m_now() : Clock::now(); }; + + void setNow(NowFn now) { m_now = std::move(now); } + ~WalletRescanReserver() { if (m_could_reserve) { @@ -932,7 +962,7 @@ bool AddWalletSetting(interfaces::Chain& chain, const std::string& wallet_name); //! Remove wallet name from persistent configuration so it will not be loaded on startup. bool RemoveWalletSetting(interfaces::Chain& chain, const std::string& wallet_name); -bool DummySignInput(const SigningProvider& provider, CTxIn &tx_in, const CTxOut &txout, bool use_max_sig); +bool DummySignInput(const SigningProvider& provider, CTxIn &tx_in, const CTxOut &txout, const CCoinControl* coin_control = nullptr); bool FillInputToWeight(CTxIn& txin, int64_t target_weight); } // namespace wallet diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index a04446c840..8afd3f416d 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -883,12 +883,10 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) if (result != DBErrors::LOAD_OK) return result; - // Last client version to open this wallet, was previously the file version number + // Last client version to open this wallet int last_client = CLIENT_VERSION; - m_batch->Read(DBKeys::VERSION, last_client); - - int wallet_version = pwallet->GetVersion(); - pwallet->WalletLogPrintf("Wallet File Version = %d\n", wallet_version > 0 ? wallet_version : last_client); + bool has_last_client = m_batch->Read(DBKeys::VERSION, last_client); + pwallet->WalletLogPrintf("Wallet file version = %d, last client version = %d\n", pwallet->GetVersion(), last_client); pwallet->WalletLogPrintf("Keys: %u plaintext, %u encrypted, %u w/ metadata, %u total. Unknown wallet records: %u\n", wss.nKeys, wss.nCKeys, wss.nKeyMeta, wss.nKeys + wss.nCKeys, wss.m_unknown_records); @@ -909,7 +907,7 @@ DBErrors WalletBatch::LoadWallet(CWallet* pwallet) if (wss.fIsEncrypted && (last_client == 40000 || last_client == 50000)) return DBErrors::NEED_REWRITE; - if (last_client < CLIENT_VERSION) // Update + if (!has_last_client || last_client != CLIENT_VERSION) // Update m_batch->Write(DBKeys::VERSION, CLIENT_VERSION); if (wss.fAnyUnordered) |