diff options
Diffstat (limited to 'src')
51 files changed, 843 insertions, 369 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index d293775fc3..02ec5c098e 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -177,6 +177,7 @@ BITCOIN_CORE_H = \ kernel/context.h \ kernel/mempool_limits.h \ kernel/mempool_options.h \ + kernel/mempool_persist.h \ key.h \ key_io.h \ logging.h \ @@ -198,6 +199,7 @@ BITCOIN_CORE_H = \ node/chainstate.h \ node/coin.h \ node/context.h \ + node/mempool_persist_args.h \ node/miner.h \ node/minisketchwrapper.h \ node/psbt.h \ @@ -366,6 +368,7 @@ libbitcoin_node_a_SOURCES = \ kernel/checks.cpp \ kernel/coinstats.cpp \ kernel/context.cpp \ + kernel/mempool_persist.cpp \ mapport.cpp \ mempool_args.cpp \ net.cpp \ @@ -379,6 +382,7 @@ libbitcoin_node_a_SOURCES = \ node/context.cpp \ node/eviction.cpp \ node/interfaces.cpp \ + node/mempool_persist_args.cpp \ node/miner.cpp \ node/minisketchwrapper.cpp \ node/psbt.cpp \ @@ -881,6 +885,7 @@ libbitcoinkernel_la_SOURCES = \ kernel/checks.cpp \ kernel/coinstats.cpp \ kernel/context.cpp \ + kernel/mempool_persist.cpp \ key.cpp \ logging.cpp \ node/blockstorage.cpp \ diff --git a/src/Makefile.test.include b/src/Makefile.test.include index b806b62d5b..d9195ad32e 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -373,7 +373,7 @@ endif $(AM_V_at)$(MAKE) $(AM_MAKEFLAGS) -C secp256k1 check if ENABLE_TESTS -UNIVALUE_TESTS = univalue/test/object univalue/test/unitester univalue/test/no_nul +UNIVALUE_TESTS = univalue/test/object univalue/test/unitester noinst_PROGRAMS += $(UNIVALUE_TESTS) TESTS += $(UNIVALUE_TESTS) @@ -382,11 +382,6 @@ univalue_test_unitester_LDADD = $(LIBUNIVALUE) univalue_test_unitester_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) -DJSON_TEST_SRC=\"$(srcdir)/$(UNIVALUE_TEST_DATA_DIR_INT)\" univalue_test_unitester_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) -univalue_test_no_nul_SOURCES = $(UNIVALUE_TEST_NO_NUL_INT) -univalue_test_no_nul_LDADD = $(LIBUNIVALUE) -univalue_test_no_nul_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) -univalue_test_no_nul_LDFLAGS = -static $(LIBTOOL_APP_LDFLAGS) - univalue_test_object_SOURCES = $(UNIVALUE_TEST_OBJECT_INT) univalue_test_object_LDADD = $(LIBUNIVALUE) univalue_test_object_CPPFLAGS = -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) diff --git a/src/Makefile.test_fuzz.include b/src/Makefile.test_fuzz.include index 8922dda3ad..b43816636f 100644 --- a/src/Makefile.test_fuzz.include +++ b/src/Makefile.test_fuzz.include @@ -10,6 +10,7 @@ EXTRA_LIBRARIES += \ TEST_FUZZ_H = \ test/fuzz/fuzz.h \ test/fuzz/FuzzedDataProvider.h \ + test/fuzz/mempool_utils.h \ test/fuzz/util.h libtest_fuzz_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(MINIUPNPC_CPPFLAGS) $(NATPMP_CPPFLAGS) $(EVENT_CFLAGS) $(EVENT_PTHREADS_CFLAGS) diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index 1817aa1a53..d53e917aba 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -78,8 +78,8 @@ int main(int argc, char* argv[]) // SETUP: Chainstate const ChainstateManager::Options chainman_opts{ - chainparams, - static_cast<int64_t(*)()>(GetTime), + .chainparams = chainparams, + .adjusted_time_callback = static_cast<int64_t (*)()>(GetTime), }; ChainstateManager chainman{chainman_opts}; @@ -9,6 +9,7 @@ #include <cstdio> #include <filesystem> +#include <functional> #include <iomanip> #include <ios> #include <ostream> @@ -199,6 +200,7 @@ bool create_directories(const std::filesystem::path& p, std::error_code& ec) = d /** Bridge operations to C stdio */ namespace fsbridge { + using FopenFn = std::function<FILE*(const fs::path&, const char*)>; FILE *fopen(const fs::path& p, const char *mode); /** diff --git a/src/init.cpp b/src/init.cpp index 816404a84f..97c823fe0c 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -10,6 +10,7 @@ #include <init.h> #include <kernel/checks.h> +#include <kernel/mempool_persist.h> #include <addrman.h> #include <banman.h> @@ -41,6 +42,7 @@ #include <node/chainstate.h> #include <node/context.h> #include <node/interface_ui.h> +#include <node/mempool_persist_args.h> #include <node/miner.h> #include <policy/feerate.h> #include <policy/fees.h> @@ -102,14 +104,19 @@ #include <zmq/zmqrpc.h> #endif +using kernel::DumpMempool; + using node::CacheSizes; using node::CalculateCacheSizes; using node::ChainstateLoadVerifyError; using node::ChainstateLoadingError; using node::CleanupBlockRevFiles; +using node::DEFAULT_PERSIST_MEMPOOL; using node::DEFAULT_PRINTPRIORITY; using node::DEFAULT_STOPAFTERBLOCKIMPORT; using node::LoadChainstate; +using node::MempoolPath; +using node::ShouldPersistMempool; using node::NodeContext; using node::ThreadImport; using node::VerifyLoadedChainstate; @@ -245,8 +252,8 @@ void Shutdown(NodeContext& node) node.addrman.reset(); node.netgroupman.reset(); - if (node.mempool && node.mempool->IsLoaded() && node.args->GetBoolArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { - DumpMempool(*node.mempool); + if (node.mempool && node.mempool->GetLoadTried() && ShouldPersistMempool(*node.args)) { + DumpMempool(*node.mempool, MempoolPath(*node.args)); } // Drop transactions we were still watching, and record fee estimations. @@ -1440,8 +1447,8 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) node.mempool = std::make_unique<CTxMemPool>(mempool_opts); const ChainstateManager::Options chainman_opts{ - chainparams, - GetAdjustedTime, + .chainparams = chainparams, + .adjusted_time_callback = GetAdjustedTime, }; node.chainman = std::make_unique<ChainstateManager>(chainman_opts); ChainstateManager& chainman = *node.chainman; @@ -1669,7 +1676,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } chainman.m_load_block = std::thread(&util::TraceThread, "loadblk", [=, &chainman, &args] { - ThreadImport(chainman, vImportFiles, args); + ThreadImport(chainman, vImportFiles, args, ShouldPersistMempool(args) ? MempoolPath(args) : fs::path{}); }); // Wait for genesis block to be processed diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index df9e55874f..ff994b8edc 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -287,6 +287,9 @@ public: //! to be prepared to handle this by ignoring notifications about unknown //! removed transactions and already added new transactions. virtual void requestMempoolTransactions(Notifications& notifications) = 0; + + //! Return true if an assumed-valid chain is in use. + virtual bool hasAssumedValidChain() = 0; }; //! Interface to let node manage chain clients (wallets, or maybe tools for diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index fe198c999b..e29fefae56 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -329,7 +329,7 @@ public: virtual std::string getWalletDir() = 0; //! Restore backup wallet - virtual std::unique_ptr<Wallet> restoreWallet(const fs::path& backup_file, const std::string& wallet_name, bilingual_str& error, std::vector<bilingual_str>& warnings) = 0; + virtual BResult<std::unique_ptr<Wallet>> restoreWallet(const fs::path& backup_file, const std::string& wallet_name, std::vector<bilingual_str>& warnings) = 0; //! Return available wallets in wallet directory. virtual std::vector<std::string> listWalletDir() = 0; diff --git a/src/kernel/chainstatemanager_opts.h b/src/kernel/chainstatemanager_opts.h index 575d94e2e9..510a1f9edc 100644 --- a/src/kernel/chainstatemanager_opts.h +++ b/src/kernel/chainstatemanager_opts.h @@ -10,6 +10,8 @@ class CChainParams; +namespace kernel { + /** * An options struct for `ChainstateManager`, more ergonomically referred to as * `ChainstateManager::Options` due to the using-declaration in @@ -20,4 +22,6 @@ struct ChainstateManagerOpts { const std::function<int64_t()> adjusted_time_callback{nullptr}; }; +} // namespace kernel + #endif // BITCOIN_KERNEL_CHAINSTATEMANAGER_OPTS_H diff --git a/src/kernel/mempool_persist.cpp b/src/kernel/mempool_persist.cpp new file mode 100644 index 0000000000..1a1cf2bbdc --- /dev/null +++ b/src/kernel/mempool_persist.cpp @@ -0,0 +1,189 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <kernel/mempool_persist.h> + +#include <clientversion.h> +#include <consensus/amount.h> +#include <fs.h> +#include <logging.h> +#include <primitives/transaction.h> +#include <serialize.h> +#include <shutdown.h> +#include <streams.h> +#include <sync.h> +#include <txmempool.h> +#include <uint256.h> +#include <util/system.h> +#include <util/time.h> +#include <validation.h> + +#include <chrono> +#include <cstdint> +#include <cstdio> +#include <exception> +#include <functional> +#include <map> +#include <memory> +#include <set> +#include <stdexcept> +#include <utility> +#include <vector> + +using fsbridge::FopenFn; + +namespace kernel { + +static const uint64_t MEMPOOL_DUMP_VERSION = 1; + +bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, CChainState& active_chainstate, FopenFn mockable_fopen_function) +{ + if (load_path.empty()) return false; + + FILE* filestr{mockable_fopen_function(load_path, "rb")}; + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + if (file.IsNull()) { + LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); + return false; + } + + int64_t count = 0; + int64_t expired = 0; + int64_t failed = 0; + int64_t already_there = 0; + int64_t unbroadcast = 0; + auto now = NodeClock::now(); + + try { + uint64_t version; + file >> version; + if (version != MEMPOOL_DUMP_VERSION) { + return false; + } + uint64_t num; + file >> num; + while (num) { + --num; + CTransactionRef tx; + int64_t nTime; + int64_t nFeeDelta; + file >> tx; + file >> nTime; + file >> nFeeDelta; + + CAmount amountdelta = nFeeDelta; + if (amountdelta) { + pool.PrioritiseTransaction(tx->GetHash(), amountdelta); + } + if (nTime > TicksSinceEpoch<std::chrono::seconds>(now - pool.m_expiry)) { + LOCK(cs_main); + const auto& accepted = AcceptToMemoryPool(active_chainstate, tx, nTime, /*bypass_limits=*/false, /*test_accept=*/false); + if (accepted.m_result_type == MempoolAcceptResult::ResultType::VALID) { + ++count; + } else { + // mempool may contain the transaction already, e.g. from + // wallet(s) having loaded it while we were processing + // mempool transactions; consider these as valid, instead of + // failed, but mark them as 'already there' + if (pool.exists(GenTxid::Txid(tx->GetHash()))) { + ++already_there; + } else { + ++failed; + } + } + } else { + ++expired; + } + if (ShutdownRequested()) + return false; + } + std::map<uint256, CAmount> mapDeltas; + file >> mapDeltas; + + for (const auto& i : mapDeltas) { + pool.PrioritiseTransaction(i.first, i.second); + } + + std::set<uint256> unbroadcast_txids; + file >> unbroadcast_txids; + unbroadcast = unbroadcast_txids.size(); + for (const auto& txid : unbroadcast_txids) { + // Ensure transactions were accepted to mempool then add to + // unbroadcast set. + if (pool.get(txid) != nullptr) pool.AddUnbroadcastTx(txid); + } + } catch (const std::exception& e) { + LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); + return false; + } + + LogPrintf("Imported mempool transactions from disk: %i succeeded, %i failed, %i expired, %i already there, %i waiting for initial broadcast\n", count, failed, expired, already_there, unbroadcast); + return true; +} + +bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mockable_fopen_function, bool skip_file_commit) +{ + auto start = SteadyClock::now(); + + std::map<uint256, CAmount> mapDeltas; + std::vector<TxMempoolInfo> vinfo; + std::set<uint256> unbroadcast_txids; + + static Mutex dump_mutex; + LOCK(dump_mutex); + + { + LOCK(pool.cs); + for (const auto &i : pool.mapDeltas) { + mapDeltas[i.first] = i.second; + } + vinfo = pool.infoAll(); + unbroadcast_txids = pool.GetUnbroadcastTxs(); + } + + auto mid = SteadyClock::now(); + + try { + FILE* filestr{mockable_fopen_function(dump_path + ".new", "wb")}; + if (!filestr) { + return false; + } + + CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); + + uint64_t version = MEMPOOL_DUMP_VERSION; + file << version; + + file << (uint64_t)vinfo.size(); + for (const auto& i : vinfo) { + file << *(i.tx); + file << int64_t{count_seconds(i.m_time)}; + file << int64_t{i.nFeeDelta}; + mapDeltas.erase(i.tx->GetHash()); + } + + file << mapDeltas; + + LogPrintf("Writing %d unbroadcast transactions to disk.\n", unbroadcast_txids.size()); + file << unbroadcast_txids; + + if (!skip_file_commit && !FileCommit(file.Get())) + throw std::runtime_error("FileCommit failed"); + file.fclose(); + if (!RenameOver(dump_path + ".new", dump_path)) { + throw std::runtime_error("Rename failed"); + } + auto last = SteadyClock::now(); + + LogPrintf("Dumped mempool: %gs to copy, %gs to dump\n", + Ticks<SecondsDouble>(mid - start), + Ticks<SecondsDouble>(last - mid)); + } catch (const std::exception& e) { + LogPrintf("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); + return false; + } + return true; +} + +} // namespace kernel diff --git a/src/kernel/mempool_persist.h b/src/kernel/mempool_persist.h new file mode 100644 index 0000000000..9a15ec6dca --- /dev/null +++ b/src/kernel/mempool_persist.h @@ -0,0 +1,28 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_KERNEL_MEMPOOL_PERSIST_H +#define BITCOIN_KERNEL_MEMPOOL_PERSIST_H + +#include <fs.h> + +class CChainState; +class CTxMemPool; + +namespace kernel { + +/** Dump the mempool to disk. */ +bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, + fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen, + bool skip_file_commit = false); + +/** Load the mempool from disk. */ +bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, + CChainState& active_chainstate, + fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen); + +} // namespace kernel + + +#endif // BITCOIN_KERNEL_MEMPOOL_PERSIST_H diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 1ea082eb26..74d1bf44d2 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -375,7 +375,7 @@ struct Peer { std::deque<CInv> m_getdata_requests GUARDED_BY(m_getdata_requests_mutex); /** Time of the last getheaders message to this peer */ - std::atomic<NodeClock::time_point> m_last_getheaders_timestamp{NodeSeconds{}}; + NodeClock::time_point m_last_getheaders_timestamp{}; explicit Peer(NodeId id, ServiceFlags our_services) : m_id{id} @@ -2317,7 +2317,7 @@ bool PeerManagerImpl::MaybeSendGetHeaders(CNode& pfrom, const CBlockLocator& loc // Only allow a new getheaders message to go out if we don't have a recent // one already in-flight - if (current_time - peer.m_last_getheaders_timestamp.load() > HEADERS_RESPONSE_TIME) { + if (current_time - peer.m_last_getheaders_timestamp > HEADERS_RESPONSE_TIME) { m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, locator, uint256())); peer.m_last_getheaders_timestamp = current_time; return true; @@ -4004,7 +4004,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Assume that this is in response to any outstanding getheaders // request we may have sent, and clear out the time of our last request - peer->m_last_getheaders_timestamp.store(NodeSeconds{}); + peer->m_last_getheaders_timestamp = {}; std::vector<CBlockHeader> headers; diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index cadafcaa8d..103f4f0d7f 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -823,7 +823,7 @@ struct CImportingNow { } }; -void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args) +void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args, const fs::path& mempool_path) { SetSyscallSandboxPolicy(SyscallSandboxPolicy::INITIALIZATION_LOAD_BLOCKS); ScheduleBatchPriority(); @@ -893,6 +893,6 @@ void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFile return; } } // End scope of CImportingNow - chainman.ActiveChainstate().LoadMempool(args); + chainman.ActiveChainstate().LoadMempool(mempool_path); } } // namespace node diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index e017f3f427..9b76371aae 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -211,7 +211,7 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, c bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); -void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args); +void ThreadImport(ChainstateManager& chainman, std::vector<fs::path> vImportFiles, const ArgsManager& args, const fs::path& mempool_path); } // namespace node #endif // BITCOIN_NODE_BLOCKSTORAGE_H diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 93aa9e526b..46e0efb9b6 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -775,6 +775,11 @@ public: notifications.transactionAddedToMempool(entry.GetSharedTx(), 0 /* mempool_sequence */); } } + bool hasAssumedValidChain() override + { + return Assert(m_node.chainman)->IsSnapshotActive(); + } + NodeContext& m_node; }; } // namespace diff --git a/src/node/mempool_persist_args.cpp b/src/node/mempool_persist_args.cpp new file mode 100644 index 0000000000..4e775869c6 --- /dev/null +++ b/src/node/mempool_persist_args.cpp @@ -0,0 +1,23 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <node/mempool_persist_args.h> + +#include <fs.h> +#include <util/system.h> +#include <validation.h> + +namespace node { + +bool ShouldPersistMempool(const ArgsManager& argsman) +{ + return argsman.GetBoolArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL); +} + +fs::path MempoolPath(const ArgsManager& argsman) +{ + return argsman.GetDataDirNet() / "mempool.dat"; +} + +} // namespace node diff --git a/src/node/mempool_persist_args.h b/src/node/mempool_persist_args.h new file mode 100644 index 0000000000..f719ec62ab --- /dev/null +++ b/src/node/mempool_persist_args.h @@ -0,0 +1,25 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_MEMPOOL_PERSIST_ARGS_H +#define BITCOIN_NODE_MEMPOOL_PERSIST_ARGS_H + +#include <fs.h> + +class ArgsManager; + +namespace node { + +/** + * Default for -persistmempool, indicating whether the node should attempt to + * automatically load the mempool on start and save to disk on shutdown + */ +static constexpr bool DEFAULT_PERSIST_MEMPOOL{true}; + +bool ShouldPersistMempool(const ArgsManager& argsman); +fs::path MempoolPath(const ArgsManager& argsman); + +} // namespace node + +#endif // BITCOIN_NODE_MEMPOOL_PERSIST_ARGS_H diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index 11140c5da9..01d84624e8 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -391,9 +391,10 @@ void RestoreWalletActivity::restore(const fs::path& backup_file, const std::stri tr("Restoring Wallet <b>%1</b>…").arg(name.toHtmlEscaped())); QTimer::singleShot(0, worker(), [this, backup_file, wallet_name] { - std::unique_ptr<interfaces::Wallet> wallet = node().walletLoader().restoreWallet(backup_file, wallet_name, m_error_message, m_warning_message); + auto wallet{node().walletLoader().restoreWallet(backup_file, wallet_name, m_warning_message)}; - if (wallet) m_wallet_model = m_wallet_controller->getOrCreateWallet(std::move(wallet)); + m_error_message = wallet ? bilingual_str{} : wallet.GetError(); + if (wallet) m_wallet_model = m_wallet_controller->getOrCreateWallet(wallet.ReleaseObj()); QTimer::singleShot(0, this, &RestoreWalletActivity::finish); }); diff --git a/src/qt/walletframe.cpp b/src/qt/walletframe.cpp index 6b38e207d3..8dc97e66a2 100644 --- a/src/qt/walletframe.cpp +++ b/src/qt/walletframe.cpp @@ -213,6 +213,14 @@ void WalletFrame::gotoLoadPSBT(bool from_clipboard) } std::ifstream in{filename.toLocal8Bit().data(), std::ios::binary}; data.assign(std::istream_iterator<unsigned char>{in}, {}); + + // Some psbt files may be base64 strings in the file rather than binary data + std::string b64_str{data.begin(), data.end()}; + b64_str.erase(b64_str.find_last_not_of(" \t\n\r\f\v") + 1); // Trim trailing whitespace + auto b64_dec = DecodeBase64(b64_str); + if (b64_dec.has_value()) { + data = b64_dec.value(); + } } std::string error; diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index b80cdce3f0..52d5eaaa50 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -617,7 +617,7 @@ const RPCResult getblock_vin{ {RPCResult::Type::OBJ, "scriptPubKey", "", { {RPCResult::Type::STR, "asm", "The asm"}, - {RPCResult::Type::STR, "hex", "The hex"}, + {RPCResult::Type::STR_HEX, "hex", "The hex"}, {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, {RPCResult::Type::STR, "type", "The type (one of: " + GetAllOutputTypes() + ")"}, }}, @@ -1014,9 +1014,9 @@ static RPCHelpMan gettxout() {RPCResult::Type::NUM, "confirmations", "The number of confirmations"}, {RPCResult::Type::STR_AMOUNT, "value", "The transaction value in " + CURRENCY_UNIT}, {RPCResult::Type::OBJ, "scriptPubKey", "", { - {RPCResult::Type::STR, "asm", ""}, + {RPCResult::Type::STR, "asm", "The asm"}, {RPCResult::Type::STR, "desc", "Inferred descriptor for the output"}, - {RPCResult::Type::STR_HEX, "hex", ""}, + {RPCResult::Type::STR_HEX, "hex", "The hex"}, {RPCResult::Type::STR, "type", "The type, eg pubkeyhash"}, {RPCResult::Type::STR, "address", /*optional=*/true, "The Bitcoin address (only if a well-defined address exists)"}, }}, diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 84d43e7818..3b53ec82e4 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -5,9 +5,12 @@ #include <rpc/blockchain.h> +#include <kernel/mempool_persist.h> + #include <chainparams.h> #include <core_io.h> #include <fs.h> +#include <node/mempool_persist_args.h> #include <policy/rbf.h> #include <policy/settings.h> #include <primitives/transaction.h> @@ -18,7 +21,11 @@ #include <univalue.h> #include <util/moneystr.h> +using kernel::DumpMempool; + using node::DEFAULT_MAX_RAW_TX_FEE_RATE; +using node::MempoolPath; +using node::ShouldPersistMempool; using node::NodeContext; static RPCHelpMan sendrawtransaction() @@ -653,7 +660,7 @@ UniValue MempoolInfoToJSON(const CTxMemPool& pool) // Make sure this call is atomic in the pool. LOCK(pool.cs); UniValue ret(UniValue::VOBJ); - ret.pushKV("loaded", pool.IsLoaded()); + ret.pushKV("loaded", pool.GetLoadTried()); ret.pushKV("size", (int64_t)pool.size()); ret.pushKV("bytes", (int64_t)pool.GetTotalTxSize()); ret.pushKV("usage", (int64_t)pool.DynamicMemoryUsage()); @@ -717,16 +724,18 @@ static RPCHelpMan savemempool() const ArgsManager& args{EnsureAnyArgsman(request.context)}; const CTxMemPool& mempool = EnsureAnyMemPool(request.context); - if (!mempool.IsLoaded()) { + if (!mempool.GetLoadTried()) { throw JSONRPCError(RPC_MISC_ERROR, "The mempool was not loaded yet"); } - if (!DumpMempool(mempool)) { + const fs::path& dump_path = MempoolPath(args); + + if (!DumpMempool(mempool, dump_path)) { throw JSONRPCError(RPC_MISC_ERROR, "Unable to dump mempool to disk"); } UniValue ret(UniValue::VOBJ); - ret.pushKV("filename", fs::path((args.GetDataDirNet() / "mempool.dat")).u8string()); + ret.pushKV("filename", dump_path.u8string()); return ret; }, diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 5ca921ae2f..43e0e6bc2d 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -730,7 +730,7 @@ const RPCResult decodepsbt_inputs{ {RPCResult::Type::OBJ, "final_scriptSig", /*optional=*/true, "", { {RPCResult::Type::STR, "asm", "The asm"}, - {RPCResult::Type::STR, "hex", "The hex"}, + {RPCResult::Type::STR_HEX, "hex", "The hex"}, }}, {RPCResult::Type::ARR, "final_scriptwitness", /*optional=*/true, "", { diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index ca0170c84b..34a4da74f8 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -6,6 +6,7 @@ #include <key_io.h> #include <pubkey.h> +#include <script/miniscript.h> #include <script/script.h> #include <script/standard.h> @@ -161,6 +162,20 @@ public: virtual ~PubkeyProvider() = default; + /** Compare two public keys represented by this provider. + * Used by the Miniscript descriptors to check for duplicate keys in the script. + */ + bool operator<(PubkeyProvider& other) const { + CPubKey a, b; + SigningProvider dummy; + KeyOriginInfo dummy_info; + + GetPubKey(0, dummy, a, dummy_info); + other.GetPubKey(0, dummy, b, dummy_info); + + return a < b; + } + /** Derive a public key. * read_cache is the cache to read keys from (if not nullptr) * write_cache is the cache to write keys to (if not nullptr) @@ -493,12 +508,12 @@ public: /** Base class for all Descriptor implementations. */ class DescriptorImpl : public Descriptor { - //! Public key arguments for this descriptor (size 1 for PK, PKH, WPKH; any size for Multisig). +protected: + //! Public key arguments for this descriptor (size 1 for PK, PKH, WPKH; any size for WSH and Multisig). const std::vector<std::unique_ptr<PubkeyProvider>> m_pubkey_args; //! The string name of the descriptor function. const std::string m_name; -protected: //! The sub-descriptor arguments (empty for everything but SH and WSH). //! In doc/descriptors.m this is referred to as SCRIPT expressions sh(SCRIPT) //! and wsh(SCRIPT), and distinct from KEY expressions and ADDR expressions. @@ -563,7 +578,7 @@ public: return true; } - bool ToStringHelper(const SigningProvider* arg, std::string& out, const StringType type, const DescriptorCache* cache = nullptr) const + virtual bool ToStringHelper(const SigningProvider* arg, std::string& out, const StringType type, const DescriptorCache* cache = nullptr) const { std::string extra = ToStringExtra(); size_t pos = extra.size() > 0 ? 1 : 0; @@ -917,6 +932,89 @@ public: bool IsSingleType() const final { return true; } }; +/* We instantiate Miniscript here with a simple integer as key type. + * The value of these key integers are an index in the + * DescriptorImpl::m_pubkey_args vector. + */ + +/** + * The context for converting a Miniscript descriptor into a Script. + */ +class ScriptMaker { + //! Keys contained in the Miniscript (the evaluation of DescriptorImpl::m_pubkey_args). + const std::vector<CPubKey>& m_keys; + +public: + ScriptMaker(const std::vector<CPubKey>& keys LIFETIMEBOUND) : m_keys(keys) {} + + std::vector<unsigned char> ToPKBytes(uint32_t key) const { + return {m_keys[key].begin(), m_keys[key].end()}; + } + + std::vector<unsigned char> ToPKHBytes(uint32_t key) const { + auto id = m_keys[key].GetID(); + return {id.begin(), id.end()}; + } +}; + +/** + * The context for converting a Miniscript descriptor to its textual form. + */ +class StringMaker { + //! To convert private keys for private descriptors. + const SigningProvider* m_arg; + //! Keys contained in the Miniscript (a reference to DescriptorImpl::m_pubkey_args). + const std::vector<std::unique_ptr<PubkeyProvider>>& m_pubkeys; + //! Whether to serialize keys as private or public. + bool m_private; + +public: + StringMaker(const SigningProvider* arg LIFETIMEBOUND, const std::vector<std::unique_ptr<PubkeyProvider>>& pubkeys LIFETIMEBOUND, bool priv) + : m_arg(arg), m_pubkeys(pubkeys), m_private(priv) {} + + std::optional<std::string> ToString(uint32_t key) const + { + std::string ret; + if (m_private) { + if (!m_pubkeys[key]->ToPrivateString(*m_arg, ret)) return {}; + } else { + ret = m_pubkeys[key]->ToString(); + } + return ret; + } +}; + +class MiniscriptDescriptor final : public DescriptorImpl +{ +private: + miniscript::NodeRef<uint32_t> m_node; + +protected: + std::vector<CScript> MakeScripts(const std::vector<CPubKey>& keys, Span<const CScript> scripts, + FlatSigningProvider& provider) const override + { + for (const auto& key : keys) provider.pubkeys.emplace(key.GetID(), key); + return Vector(m_node->ToScript(ScriptMaker(keys))); + } + +public: + MiniscriptDescriptor(std::vector<std::unique_ptr<PubkeyProvider>> providers, miniscript::NodeRef<uint32_t> node) + : DescriptorImpl(std::move(providers), "?"), m_node(std::move(node)) {} + + bool ToStringHelper(const SigningProvider* arg, std::string& out, const StringType type, + const DescriptorCache* cache = nullptr) const override + { + if (const auto res = m_node->ToString(StringMaker(arg, m_pubkey_args, type == StringType::PRIVATE))) { + out = *res; + return true; + } + return false; + } + + bool IsSolvable() const override { return false; } // For now, mark these descriptors as non-solvable (as we don't have signing logic for them). + bool IsSingleType() const final { return true; } +}; + //////////////////////////////////////////////////////////////////////////// // Parser // //////////////////////////////////////////////////////////////////////////// @@ -1058,6 +1156,94 @@ std::unique_ptr<PubkeyProvider> ParsePubkey(uint32_t key_exp_index, const Span<c return std::make_unique<OriginPubkeyProvider>(key_exp_index, std::move(info), std::move(provider)); } +std::unique_ptr<PubkeyProvider> InferPubkey(const CPubKey& pubkey, ParseScriptContext, const SigningProvider& provider) +{ + std::unique_ptr<PubkeyProvider> key_provider = std::make_unique<ConstPubkeyProvider>(0, pubkey, false); + KeyOriginInfo info; + if (provider.GetKeyOrigin(pubkey.GetID(), info)) { + return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider)); + } + return key_provider; +} + +std::unique_ptr<PubkeyProvider> InferXOnlyPubkey(const XOnlyPubKey& xkey, ParseScriptContext ctx, const SigningProvider& provider) +{ + unsigned char full_key[CPubKey::COMPRESSED_SIZE] = {0x02}; + std::copy(xkey.begin(), xkey.end(), full_key + 1); + CPubKey pubkey(full_key); + std::unique_ptr<PubkeyProvider> key_provider = std::make_unique<ConstPubkeyProvider>(0, pubkey, true); + KeyOriginInfo info; + if (provider.GetKeyOriginByXOnly(xkey, info)) { + return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider)); + } + return key_provider; +} + +/** + * The context for parsing a Miniscript descriptor (either from Script or from its textual representation). + */ +struct KeyParser { + //! The Key type is an index in DescriptorImpl::m_pubkey_args + using Key = uint32_t; + //! Must not be nullptr if parsing from string. + FlatSigningProvider* m_out; + //! Must not be nullptr if parsing from Script. + const SigningProvider* m_in; + //! List of keys contained in the Miniscript. + mutable std::vector<std::unique_ptr<PubkeyProvider>> m_keys; + //! Used to detect key parsing errors within a Miniscript. + mutable std::string m_key_parsing_error; + + KeyParser(FlatSigningProvider* out LIFETIMEBOUND, const SigningProvider* in LIFETIMEBOUND) : m_out(out), m_in(in) {} + + bool KeyCompare(const Key& a, const Key& b) const { + return *m_keys.at(a) < *m_keys.at(b); + } + + template<typename I> std::optional<Key> FromString(I begin, I end) const + { + assert(m_out); + Key key = m_keys.size(); + auto pk = ParsePubkey(key, {&*begin, &*end}, ParseScriptContext::P2WSH, *m_out, m_key_parsing_error); + if (!pk) return {}; + m_keys.push_back(std::move(pk)); + return key; + } + + std::optional<std::string> ToString(const Key& key) const + { + return m_keys.at(key)->ToString(); + } + + template<typename I> std::optional<Key> FromPKBytes(I begin, I end) const + { + assert(m_in); + CPubKey pubkey(begin, end); + if (pubkey.IsValid()) { + Key key = m_keys.size(); + m_keys.push_back(InferPubkey(pubkey, ParseScriptContext::P2WSH, *m_in)); + return key; + } + return {}; + } + + template<typename I> std::optional<Key> FromPKHBytes(I begin, I end) const + { + assert(end - begin == 20); + assert(m_in); + uint160 hash; + std::copy(begin, end, hash.begin()); + CKeyID keyid(hash); + CPubKey pubkey; + if (m_in->GetPubKey(keyid, pubkey)) { + Key key = m_keys.size(); + m_keys.push_back(InferPubkey(pubkey, ParseScriptContext::P2WSH, *m_in)); + return key; + } + return {}; + } +}; + /** Parse a script in a particular context. */ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const char>& sp, ParseScriptContext ctx, FlatSigningProvider& out, std::string& error) { @@ -1279,6 +1465,45 @@ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const error = "Can only have raw() at top level"; return nullptr; } + // Process miniscript expressions. + { + KeyParser parser(&out, nullptr); + auto node = miniscript::FromString(std::string(expr.begin(), expr.end()), parser); + if (node) { + if (ctx != ParseScriptContext::P2WSH) { + error = "Miniscript expressions can only be used in wsh"; + return nullptr; + } + if (parser.m_key_parsing_error != "") { + error = std::move(parser.m_key_parsing_error); + return nullptr; + } + if (!node->IsSane()) { + // Try to find the first insane sub for better error reporting. + auto insane_node = node.get(); + if (const auto sub = node->FindInsaneSub()) insane_node = sub; + if (const auto str = insane_node->ToString(parser)) error = *str; + if (!insane_node->IsValid()) { + error += " is invalid"; + } else { + error += " is not sane"; + if (!insane_node->IsNonMalleable()) { + error += ": malleable witnesses exist"; + } else if (insane_node == node.get() && !insane_node->NeedsSignature()) { + error += ": witnesses without signature exist"; + } else if (!insane_node->CheckTimeLocksMix()) { + error += ": contains mixes of timelocks expressed in blocks and seconds"; + } else if (!insane_node->CheckDuplicateKey()) { + error += ": contains duplicate public keys"; + } else if (!insane_node->ValidSatisfactions()) { + error += ": needs witnesses that may exceed resource limits"; + } + } + return nullptr; + } + return std::make_unique<MiniscriptDescriptor>(std::move(parser.m_keys), std::move(node)); + } + } if (ctx == ParseScriptContext::P2SH) { error = "A function is needed within P2SH"; return nullptr; @@ -1290,29 +1515,6 @@ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const return nullptr; } -std::unique_ptr<PubkeyProvider> InferPubkey(const CPubKey& pubkey, ParseScriptContext, const SigningProvider& provider) -{ - std::unique_ptr<PubkeyProvider> key_provider = std::make_unique<ConstPubkeyProvider>(0, pubkey, false); - KeyOriginInfo info; - if (provider.GetKeyOrigin(pubkey.GetID(), info)) { - return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider)); - } - return key_provider; -} - -std::unique_ptr<PubkeyProvider> InferXOnlyPubkey(const XOnlyPubKey& xkey, ParseScriptContext ctx, const SigningProvider& provider) -{ - unsigned char full_key[CPubKey::COMPRESSED_SIZE] = {0x02}; - std::copy(xkey.begin(), xkey.end(), full_key + 1); - CPubKey pubkey(full_key); - std::unique_ptr<PubkeyProvider> key_provider = std::make_unique<ConstPubkeyProvider>(0, pubkey, true); - KeyOriginInfo info; - if (provider.GetKeyOriginByXOnly(xkey, info)) { - return std::make_unique<OriginPubkeyProvider>(0, std::move(info), std::move(key_provider)); - } - return key_provider; -} - std::unique_ptr<DescriptorImpl> InferMultiA(const CScript& script, ParseScriptContext ctx, const SigningProvider& provider) { auto match = MatchMultiA(script); @@ -1426,6 +1628,14 @@ std::unique_ptr<DescriptorImpl> InferScript(const CScript& script, ParseScriptCo } } + if (ctx == ParseScriptContext::P2WSH) { + KeyParser parser(nullptr, &provider); + auto node = miniscript::FromScript(script, parser); + if (node && node->IsSane()) { + return std::make_unique<MiniscriptDescriptor>(std::move(parser.m_keys), std::move(node)); + } + } + CTxDestination dest; if (ExtractDestination(script, dest)) { if (GetScriptForDestination(dest) == script) { diff --git a/src/script/miniscript.h b/src/script/miniscript.h index 2c239c2678..f783d1dafc 100644 --- a/src/script/miniscript.h +++ b/src/script/miniscript.h @@ -429,6 +429,21 @@ private: )); } + /** Like TreeEval, but without downfn or State type. + * upfn takes (const Node&, Span<Result>) and returns Result. */ + template<typename Result, typename UpFn> + Result TreeEval(UpFn upfn) const + { + struct DummyState {}; + return std::move(*TreeEvalMaybe<Result>(DummyState{}, + [](DummyState, const Node&, size_t) { return DummyState{}; }, + [&upfn](DummyState, const Node& node, Span<Result> subs) { + Result res{upfn(node, subs)}; + return std::optional<Result>(std::move(res)); + } + )); + } + /** Compare two miniscript subtrees, using a non-recursive algorithm. */ friend int Compare(const Node<Key>& node1, const Node<Key>& node2) { @@ -818,6 +833,15 @@ public: //! Return the expression type. Type GetType() const { return typ; } + //! Find an insane subnode which has no insane children. Nullptr if there is none. + const Node* FindInsaneSub() const { + return TreeEval<const Node*>([](const Node& node, Span<const Node*> subs) -> const Node* { + for (auto& sub: subs) if (sub) return sub; + if (!node.IsSaneSubexpression()) return &node; + return nullptr; + }); + } + //! Check whether this node is valid at all. bool IsValid() const { return !(GetType() == ""_mst) && ScriptSize() <= MAX_STANDARD_P2WSH_SCRIPT_SIZE; } @@ -953,7 +977,11 @@ void BuildBack(const Ctx& ctx, Fragment nt, std::vector<NodeRef<Key>>& construct } } -//! Parse a miniscript from its textual descriptor form. +/** + * Parse a miniscript from its textual descriptor form. + * This does not check whether the script is valid, let alone sane. The caller is expected to use + * the `IsValidTopLevel()` and `IsSaneTopLevel()` to check for these properties on the node. + */ template<typename Key, typename Ctx> inline NodeRef<Key> Parse(Span<const char> in, const Ctx& ctx) { @@ -1255,9 +1283,7 @@ inline NodeRef<Key> Parse(Span<const char> in, const Ctx& ctx) // Sanity checks on the produced miniscript assert(constructed.size() == 1); if (in.size() > 0) return {}; - const NodeRef<Key> tl_node = std::move(constructed.front()); - if (!tl_node->IsValidTopLevel()) return {}; - return tl_node; + return std::move(constructed.front()); } /** Decode a script into opcode/push pairs. diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 63c86a896d..a8c666079d 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -141,14 +141,13 @@ void DoCheck(const std::string& prv, const std::string& pub, const std::string& } else { parse_priv = Parse(prv, keys_priv, error); } + BOOST_CHECK_MESSAGE(parse_priv, error); if (replace_apostrophe_with_h_in_pub) { parse_pub = Parse(UseHInsteadOfApostrophe(pub), keys_pub, error); } else { parse_pub = Parse(pub, keys_pub, error); } - - BOOST_CHECK(parse_priv); - BOOST_CHECK(parse_pub); + BOOST_CHECK_MESSAGE(parse_pub, error); // Check that the correct OutputType is inferred BOOST_CHECK(parse_priv->GetOutputType() == type); @@ -161,8 +160,8 @@ void DoCheck(const std::string& prv, const std::string& pub, const std::string& // Check that both versions serialize back to the public version. std::string pub1 = parse_priv->ToString(); std::string pub2 = parse_pub->ToString(); - BOOST_CHECK(EqualDescriptor(pub, pub1)); - BOOST_CHECK(EqualDescriptor(pub, pub2)); + BOOST_CHECK_MESSAGE(EqualDescriptor(pub, pub1), "Private ser: " + pub1 + " Public desc: " + pub); + BOOST_CHECK_MESSAGE(EqualDescriptor(pub, pub2), "Public ser: " + pub2 + " Public desc: " + pub); // Check that both can be serialized with private key back to the private version, but not without private key. if (!(flags & MISSING_PRIVKEYS)) { @@ -486,6 +485,29 @@ Check("sh(wsh(multi(20,KzoAz5CanayRKex3fSLQ2BwJpN7U52gZvxMyk78nDMHuqrUxuSJy,KwGN } nonminimalmultisig << std::vector<unsigned char>{4} << OP_CHECKMULTISIG; CheckInferRaw(nonminimalmultisig); + + // Miniscript tests + + // Invalid checksum + CheckUnparsable("wsh(and_v(vc:andor(pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),pk_k(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn),and_v(v:older(1),pk_k(L4o2kDvXXDRH2VS9uBnouScLduWt4dZnM25se7kvEjJeQ285en2A))),after(10)))#abcdef12", "wsh(and_v(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)))#abcdef12", "Provided checksum 'abcdef12' does not match computed checksum 'tyzp6a7p'"); + // Only p2wsh context is valid + CheckUnparsable("sh(and_v(vc:andor(pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),pk_k(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn),and_v(v:older(1),pk_k(L4o2kDvXXDRH2VS9uBnouScLduWt4dZnM25se7kvEjJeQ285en2A))),after(10)))", "sh(and_v(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)))", "Miniscript expressions can only be used in wsh"); + CheckUnparsable("tr(and_v(vc:andor(pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),pk_k(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn),and_v(v:older(1),pk_k(L4o2kDvXXDRH2VS9uBnouScLduWt4dZnM25se7kvEjJeQ285en2A))),after(10)))", "tr(and_v(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)))", "tr(): key 'and_v(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10))' is not valid"); + CheckUnparsable("raw(and_v(vc:andor(pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),pk_k(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn),and_v(v:older(1),pk_k(L4o2kDvXXDRH2VS9uBnouScLduWt4dZnM25se7kvEjJeQ285en2A))),after(10)))", "sh(and_v(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)))", "Miniscript expressions can only be used in wsh"); + CheckUnparsable("", "tr(034D2224bbbbbbbbbbcbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb40,{{{{{{{{{{{{{{{{{{{{{{multi(1,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/967808'/9,xprvA1RpRA33e1JQ7ifknakTFNpgXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/968/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/3/4/5/58/55/2/5/58/58/2/5/5/5/8/5/2/8/5/85/2/8/2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/8/5/8/5/4/5/585/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/8/2/5/8/5/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/58/58/2/0/8/5/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/5/8/5/8/24/5/58/52/5/8/5/2/8/24/5/58/588/246/8/5/2/8/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/5/4/5/58/55/58/2/5/8/55/2/5/8/58/555/58/2/5/8/4//2/5/58/5w/2/5/8/5/2/4/5/58/5558'/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/8/2/5/8/5/5/8/58/2/5/58/58/2/5/8/9/588/2/58/2/5/8/5/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/82/5/8/5/5/58/52/6/8/5/2/8/{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{}{{{{{{{{{DDD2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8588/246/8/5/2DLDDDDDDDbbD3DDDD/8/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/3/4/5/58/55/2/5/58/58/2/5/5/5/8/5/2/8/5/85/2/8/2/5/8D)/5/2/5/58/58/2/5/58/58/58/588/2/58/2/5/8/5/25/58/58/2/5/58/58/2/5/8/9/588/2/58/2/6780,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFW/8/5/2/5/58678008')", "'multi(1,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/967808'/9,xprvA1RpRA33e1JQ7ifknakTFNpgXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/968/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/3/4/5/58/55/2/5/58/58/2/5/5/5/8/5/2/8/5/85/2/8/2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/8/5/8/5/4/5/585/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/8/2/5/8/5/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/58/58/2/0/8/5/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/5/8/5/8/24/5/58/52/5/8/5/2/8/24/5/58/588/246/8/5/2/8/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/5/4/5/58/55/58/2/5/8/55/2/5/8/58/555/58/2/5/8/4//2/5/58/5w/2/5/8/5/2/4/5/58/5558'/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/8/2/5/8/5/5/8/58/2/5/58/58/2/5/8/9/588/2/58/2/5/8/5/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/82/5/8/5/5/58/52/6/8/5/2/8/{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{}{{{{{{{{{DDD2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8588/246/8/5/2DLDDDDDDDbbD3DDDD/8/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/3/4/5/58/55/2/5/58/58/2/5/5/5/8/5/2/8/5/85/2/8/2/5/8D)/5/2/5/58/58/2/5/58/58/58/588/2/58/2/5/8/5/25/58/58/2/5/58/58/2/5/8/9/588/2/58/2/6780,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFW/8/5/2/5/58678008'' is not a valid descriptor function"); + // Insane at top level + CheckUnparsable("wsh(and_b(vc:andor(pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),pk_k(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn),and_v(v:older(1),pk_k(L4o2kDvXXDRH2VS9uBnouScLduWt4dZnM25se7kvEjJeQ285en2A))),after(10)))", "wsh(and_b(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)))", "and_b(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)) is invalid"); + // Invalid sub + CheckUnparsable("wsh(and_v(vc:andor(v:pk_k(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),pk_k(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn),and_v(v:older(1),pk_k(L4o2kDvXXDRH2VS9uBnouScLduWt4dZnM25se7kvEjJeQ285en2A))),after(10)))", "wsh(and_v(vc:andor(v:pk_k(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)))", "v:pk_k(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204) is invalid"); + // Insane subs + CheckUnparsable("wsh(or_i(older(1),pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd)))", "wsh(or_i(older(1),pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)))", "or_i(older(1),pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)) is not sane: witnesses without signature exist"); + CheckUnparsable("wsh(or_b(sha256(cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),s:pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)))", "wsh(or_b(sha256(cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),s:pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)))", "or_b(sha256(cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),s:pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)) is not sane: malleable witnesses exist"); + CheckUnparsable("wsh(and_b(and_b(older(1),a:older(100000000)),s:pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd)))", "wsh(and_b(and_b(older(1),a:older(100000000)),s:pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)))", "and_b(older(1),a:older(100000000)) is not sane: contains mixes of timelocks expressed in blocks and seconds"); + CheckUnparsable("wsh(and_b(or_b(pkh(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),s:pk(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn)),s:pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd)))", "wsh(and_b(or_b(pkh(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),s:pk(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0)),s:pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)))", "and_b(or_b(pkh(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),s:pk(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0)),s:pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204)) is not sane: contains duplicate public keys"); + // Valid with extended keys. + Check("wsh(and_v(v:ripemd160(095ff41131e5946f3c85f79e44adbcf8e27e080e),multi(1,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0)))", "wsh(and_v(v:ripemd160(095ff41131e5946f3c85f79e44adbcf8e27e080e),multi(1,xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0)))", "wsh(and_v(v:ripemd160(095ff41131e5946f3c85f79e44adbcf8e27e080e),multi(1,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0)))", "wsh(and_v(v:ripemd160(095ff41131e5946f3c85f79e44adbcf8e27e080e),multi(1,xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0)))", UNSOLVABLE, {{"0020acf425291b98a1d7e0d4690139442abc289175be32ef1f75945e339924246d73"}}, OutputType::BECH32, {{},{0}}); + // Valid under sh(wsh()) and with a mix of xpubs and raw keys + Check("sh(wsh(thresh(1,pkh(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),a:and_n(multi(1,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0),n:older(2)))))", "sh(wsh(thresh(1,pkh(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),a:and_n(multi(1,xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0),n:older(2)))))", "sh(wsh(thresh(1,pkh(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),a:and_n(multi(1,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc,xprv9uPDJpEQgRQfDcW7BkF7eTya6RPxXeJCqCJGHuCJ4GiRVLzkTXBAJMu2qaMWPrS7AANYqdq6vcBcBUdJCVVFceUvJFjaPdGZ2y9WACViL4L/0),n:older(2)))))", "sh(wsh(thresh(1,pkh(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),a:and_n(multi(1,xpub6ERApfZwUNrhLCkDtcHTcxd75RbzS1ed54G1LkBUHQVHQKqhMkhgbmJbZRkrgZw4koxb5JaHWkY4ALHY2grBGRjaDMzQLcgJvLJuZZvRcEL,xpub68NZiKmJWnxxS6aaHmn81bvJeTESw724CRDs6HbuccFQN9Ku14VQrADWgqbhhTHBaohPX4CjNLf9fq9MYo6oDaPPLPxSb7gwQN3ih19Zm4Y/0),n:older(2)))))", UNSOLVABLE | MIXED_PUBKEYS, {{"a914767e9119ff3b3ac0cb6dcfe21de1842ccf85f1c487"}}, OutputType::P2SH_SEGWIT, {{},{0}}); } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/fuzz/mempool_utils.h b/src/test/fuzz/mempool_utils.h new file mode 100644 index 0000000000..bfe12e30ba --- /dev/null +++ b/src/test/fuzz/mempool_utils.h @@ -0,0 +1,19 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_FUZZ_MEMPOOL_UTILS_H +#define BITCOIN_TEST_FUZZ_MEMPOOL_UTILS_H + +#include <validation.h> + +class DummyChainState final : public CChainState +{ +public: + void SetMempool(CTxMemPool* mempool) + { + m_mempool = mempool; + } +}; + +#endif // BITCOIN_TEST_FUZZ_MEMPOOL_UTILS_H diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index 2d88ee295b..63fbf0516a 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -8,6 +8,7 @@ #include <node/miner.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/mempool_utils.h> #include <test/fuzz/util.h> #include <test/util/mining.h> #include <test/util/script.h> @@ -34,15 +35,6 @@ struct MockedTxPool : public CTxMemPool { } }; -class DummyChainState final : public CChainState -{ -public: - void SetMempool(CTxMemPool* mempool) - { - m_mempool = mempool; - } -}; - void initialize_tx_pool() { static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); diff --git a/src/test/fuzz/txorphan.cpp b/src/test/fuzz/txorphan.cpp index d318baa6a2..b18d783259 100644 --- a/src/test/fuzz/txorphan.cpp +++ b/src/test/fuzz/txorphan.cpp @@ -3,8 +3,10 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/amount.h> -#include <net.h> +#include <consensus/validation.h> #include <net_processing.h> +#include <node/eviction.h> +#include <policy/policy.h> #include <primitives/transaction.h> #include <script/script.h> #include <sync.h> @@ -99,16 +101,20 @@ FUZZ_TARGET_INIT(txorphan, initialize_orphanage) [&] { bool have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); // AddTx should return false if tx is too big or already have it + // tx weight is unknown, we only check when tx is already in orphanage { LOCK(g_cs_orphans); - Assert(have_tx != orphanage.AddTx(tx, peer_id)); + bool add_tx = orphanage.AddTx(tx, peer_id); + // have_tx == true -> add_tx == false + Assert(!have_tx || !add_tx); } have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); - // tx should already be added since it will not be too big in the test - // have_tx should be true and AddTx should fail { LOCK(g_cs_orphans); - Assert(have_tx && !orphanage.AddTx(tx, peer_id)); + bool add_tx = orphanage.AddTx(tx, peer_id); + // if have_tx is still false, it must be too big + Assert(!have_tx == GetTransactionWeight(*tx) > MAX_STANDARD_TX_WEIGHT); + Assert(!have_tx || !add_tx); } }, [&] { diff --git a/src/test/fuzz/validation_load_mempool.cpp b/src/test/fuzz/validation_load_mempool.cpp index 9532610f8d..90c1a71d9f 100644 --- a/src/test/fuzz/validation_load_mempool.cpp +++ b/src/test/fuzz/validation_load_mempool.cpp @@ -2,10 +2,14 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <kernel/mempool_persist.h> + #include <chainparamsbase.h> #include <mempool_args.h> +#include <node/mempool_persist_args.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/mempool_utils.h> #include <test/fuzz/util.h> #include <test/util/setup_common.h> #include <txmempool.h> @@ -15,6 +19,10 @@ #include <cstdint> #include <vector> +using kernel::DumpMempool; + +using node::MempoolPath; + namespace { const TestingSetup* g_setup; } // namespace @@ -33,9 +41,12 @@ FUZZ_TARGET_INIT(validation_load_mempool, initialize_validation_load_mempool) CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; + auto& chainstate{static_cast<DummyChainState&>(g_setup->m_node.chainman->ActiveChainstate())}; + chainstate.SetMempool(&pool); + auto fuzzed_fopen = [&](const fs::path&, const char*) { return fuzzed_file_provider.open(); }; - (void)LoadMempool(pool, g_setup->m_node.chainman->ActiveChainstate(), fuzzed_fopen); - (void)DumpMempool(pool, fuzzed_fopen, true); + (void)chainstate.LoadMempool(MempoolPath(g_setup->m_args), fuzzed_fopen); + (void)DumpMempool(pool, MempoolPath(g_setup->m_args), fuzzed_fopen, true); } diff --git a/src/test/miniscript_tests.cpp b/src/test/miniscript_tests.cpp index 3877fea907..bc5c49ef63 100644 --- a/src/test/miniscript_tests.cpp +++ b/src/test/miniscript_tests.cpp @@ -111,6 +111,10 @@ struct KeyConverter { assert(it != g_testdata->pkmap.end()); return it->second; } + + std::optional<std::string> ToString(const Key& key) const { + return HexStr(ToPKBytes(key)); + } }; //! Singleton instance of KeyConverter. @@ -276,7 +280,7 @@ BOOST_AUTO_TEST_CASE(fixed_tests) // (for now) have 'd:' be 'u'. This tests we can't use a 'd:' wrapper for a thresh, which requires // its subs to all be 'u' (taken from https://github.com/rust-bitcoin/rust-miniscript/discussions/341). const auto ms_minimalif = miniscript::FromString("thresh(3,c:pk_k(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),sc:pk_k(03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556),sc:pk_k(0279be667ef9dcbbac55a06295ce870b07029bfcdb2dce28d959f2815b16f81798),sdv:older(32))", CONVERTER); - BOOST_CHECK(!ms_minimalif); + BOOST_CHECK(ms_minimalif && !ms_minimalif->IsValid()); // A Miniscript with duplicate keys is not sane const auto ms_dup1 = miniscript::FromString("and_v(v:pk(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),pk(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65))", CONVERTER); BOOST_CHECK(ms_dup1); @@ -290,6 +294,15 @@ BOOST_AUTO_TEST_CASE(fixed_tests) // Same when the duplicates are on different levels in the tree const auto ms_dup4 = miniscript::FromString("thresh(2,pkh(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65),s:pk(03fff97bd5755eeea420453a14355235d382f6472f8568a18b2f057a1460297556),a:and_b(dv:older(1),s:pk(03d30199d74fb5a22d47b6e054e2f378cedacffcb89904a61d75d0dbd407143e65)))", CONVERTER); BOOST_CHECK(ms_dup4 && !ms_dup4->IsSane() && !ms_dup4->CheckDuplicateKey()); + // Test we find the first insane sub closer to be a leaf node. This fragment is insane for two reasons: + // 1. It can be spent without a signature + // 2. It contains timelock mixes + // We'll report the timelock mix error, as it's "deeper" (closer to be a leaf node) than the "no 's' property" + // error is. + const auto ms_ins = miniscript::FromString("or_i(and_b(after(1),a:after(1000000000)),pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204))", CONVERTER); + BOOST_CHECK(ms_ins && ms_ins->IsValid() && !ms_ins->IsSane()); + const auto insane_sub = ms_ins->FindInsaneSub(); + BOOST_CHECK(insane_sub && *insane_sub->ToString(CONVERTER) == "and_b(after(1),a:after(1000000000))"); // Timelock tests Test("after(100)", "?", TESTMODE_VALID | TESTMODE_NONMAL); // only heightlock diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index de53499088..0fba9258f1 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -183,8 +183,8 @@ ChainTestingSetup::ChainTestingSetup(const std::string& chainName, const std::ve m_cache_sizes = CalculateCacheSizes(m_args); const ChainstateManager::Options chainman_opts{ - chainparams, - GetAdjustedTime, + .chainparams = chainparams, + .adjusted_time_callback = GetAdjustedTime, }; m_node.chainman = std::make_unique<ChainstateManager>(chainman_opts); m_node.chainman->m_blockman.m_block_tree_db = std::make_unique<CBlockTreeDB>(m_cache_sizes.block_tree_db, true); diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 102de74389..78adb521ac 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -24,8 +24,8 @@ BOOST_FIXTURE_TEST_SUITE(validation_chainstate_tests, TestingSetup) BOOST_AUTO_TEST_CASE(validation_chainstate_resize_caches) { const ChainstateManager::Options chainman_opts{ - Params(), - GetAdjustedTime, + .chainparams = Params(), + .adjusted_time_callback = GetAdjustedTime, }; ChainstateManager manager{chainman_opts}; diff --git a/src/txmempool.cpp b/src/txmempool.cpp index aeaa10034e..7eff6bdbe3 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -1210,14 +1210,14 @@ void CTxMemPool::GetTransactionAncestry(const uint256& txid, size_t& ancestors, } } -bool CTxMemPool::IsLoaded() const +bool CTxMemPool::GetLoadTried() const { LOCK(cs); - return m_is_loaded; + return m_load_tried; } -void CTxMemPool::SetIsLoaded(bool loaded) +void CTxMemPool::SetLoadTried(bool load_tried) { LOCK(cs); - m_is_loaded = loaded; + m_load_tried = load_tried; } diff --git a/src/txmempool.h b/src/txmempool.h index 6e37f59f2e..d7d308038c 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -451,7 +451,7 @@ protected: void trackPackageRemoved(const CFeeRate& rate) EXCLUSIVE_LOCKS_REQUIRED(cs); - bool m_is_loaded GUARDED_BY(cs){false}; + bool m_load_tried GUARDED_BY(cs){false}; CFeeRate GetMinFee(size_t sizelimit) const; @@ -728,11 +728,17 @@ public: */ void GetTransactionAncestry(const uint256& txid, size_t& ancestors, size_t& descendants, size_t* ancestorsize = nullptr, CAmount* ancestorfees = nullptr) const; - /** @returns true if the mempool is fully loaded */ - bool IsLoaded() const; + /** + * @returns true if we've made an attempt to load the mempool regardless of + * whether the attempt was successful or not + */ + bool GetLoadTried() const; - /** Sets the current loaded state */ - void SetIsLoaded(bool loaded); + /** + * Set whether or not we've made an attempt to load the mempool (regardless + * of whether the attempt was successful or not) + */ + void SetLoadTried(bool load_tried); unsigned long size() const { diff --git a/src/univalue/include/univalue.h b/src/univalue/include/univalue.h index 7f9a6aaffd..5cb8268472 100644 --- a/src/univalue/include/univalue.h +++ b/src/univalue/include/univalue.h @@ -82,14 +82,14 @@ public: bool isArray() const { return (typ == VARR); } bool isObject() const { return (typ == VOBJ); } - bool push_back(const UniValue& val); - bool push_backV(const std::vector<UniValue>& vec); + void push_back(const UniValue& val); + void push_backV(const std::vector<UniValue>& vec); template <class It> - bool push_backV(It first, It last); + void push_backV(It first, It last); void __pushKV(const std::string& key, const UniValue& val); - bool pushKV(const std::string& key, const UniValue& val); - bool pushKVs(const UniValue& obj); + void pushKV(const std::string& key, const UniValue& val); + void pushKVs(const UniValue& obj); std::string write(unsigned int prettyIndent = 0, unsigned int indentLevel = 0) const; @@ -140,11 +140,10 @@ public: }; template <class It> -bool UniValue::push_backV(It first, It last) +void UniValue::push_backV(It first, It last) { - if (typ != VARR) return false; + if (typ != VARR) throw std::runtime_error{"JSON value is not an array as expected"}; values.insert(values.end(), first, last); - return true; } enum jtokentype { diff --git a/src/univalue/lib/univalue.cpp b/src/univalue/lib/univalue.cpp index 3553995c28..ac6f31a5a9 100644 --- a/src/univalue/lib/univalue.cpp +++ b/src/univalue/lib/univalue.cpp @@ -108,53 +108,45 @@ bool UniValue::setObject() return true; } -bool UniValue::push_back(const UniValue& val_) +void UniValue::push_back(const UniValue& val_) { - if (typ != VARR) - return false; + if (typ != VARR) throw std::runtime_error{"JSON value is not an array as expected"}; values.push_back(val_); - return true; } -bool UniValue::push_backV(const std::vector<UniValue>& vec) +void UniValue::push_backV(const std::vector<UniValue>& vec) { - if (typ != VARR) - return false; + if (typ != VARR) throw std::runtime_error{"JSON value is not an array as expected"}; values.insert(values.end(), vec.begin(), vec.end()); - - return true; } void UniValue::__pushKV(const std::string& key, const UniValue& val_) { + if (typ != VOBJ) throw std::runtime_error{"JSON value is not an object as expected"}; + keys.push_back(key); values.push_back(val_); } -bool UniValue::pushKV(const std::string& key, const UniValue& val_) +void UniValue::pushKV(const std::string& key, const UniValue& val_) { - if (typ != VOBJ) - return false; + if (typ != VOBJ) throw std::runtime_error{"JSON value is not an object as expected"}; size_t idx; if (findKey(key, idx)) values[idx] = val_; else __pushKV(key, val_); - return true; } -bool UniValue::pushKVs(const UniValue& obj) +void UniValue::pushKVs(const UniValue& obj) { - if (typ != VOBJ || obj.typ != VOBJ) - return false; + if (typ != VOBJ || obj.typ != VOBJ) throw std::runtime_error{"JSON value is not an object as expected"}; for (size_t i = 0; i < obj.keys.size(); i++) __pushKV(obj.keys[i], obj.values.at(i)); - - return true; } void UniValue::getObjMap(std::map<std::string,UniValue>& kv) const diff --git a/src/univalue/sources.mk b/src/univalue/sources.mk index e156216378..5e4d9c3831 100644 --- a/src/univalue/sources.mk +++ b/src/univalue/sources.mk @@ -1,12 +1,8 @@ -# - All variables are namespaced with UNIVALUE_ to avoid colliding with -# downstream makefiles. # - All Variables ending in _HEADERS or _SOURCES confuse automake, so the # _INT postfix is applied. # - Convenience variables, for example a UNIVALUE_TEST_DIR should not be used # as they interfere with automatic dependency generation -# - The %reldir% is the relative path from the Makefile.am. This allows -# downstreams to use these variables without having to manually account for -# the path change. +# - The %reldir% is the relative path from the Makefile.am. UNIVALUE_INCLUDE_DIR_INT = %reldir%/include @@ -29,9 +25,6 @@ UNIVALUE_TEST_UNITESTER_INT += %reldir%/test/unitester.cpp UNIVALUE_TEST_JSON_INT = UNIVALUE_TEST_JSON_INT += %reldir%/test/test_json.cpp -UNIVALUE_TEST_NO_NUL_INT = -UNIVALUE_TEST_NO_NUL_INT += %reldir%/test/no_nul.cpp - UNIVALUE_TEST_OBJECT_INT = UNIVALUE_TEST_OBJECT_INT += %reldir%/test/object.cpp diff --git a/src/univalue/test/.gitignore b/src/univalue/test/.gitignore index 7b27cf0da2..5812c96b14 100644 --- a/src/univalue/test/.gitignore +++ b/src/univalue/test/.gitignore @@ -2,7 +2,6 @@ object unitester test_json -no_nul *.trs *.log diff --git a/src/univalue/test/no_nul.cpp b/src/univalue/test/no_nul.cpp deleted file mode 100644 index 3a7a727abb..0000000000 --- a/src/univalue/test/no_nul.cpp +++ /dev/null @@ -1,8 +0,0 @@ -#include <univalue.h> - -int main (int argc, char *argv[]) -{ - char buf[] = "___[1,2,3]___"; - UniValue val; - return val.read(buf + 3, 7) ? 0 : 1; -} diff --git a/src/univalue/test/object.cpp b/src/univalue/test/object.cpp index 8a35bf914d..cf8c29ec67 100644 --- a/src/univalue/test/object.cpp +++ b/src/univalue/test/object.cpp @@ -13,9 +13,6 @@ #include <string> #include <vector> -#define BOOST_FIXTURE_TEST_SUITE(a, b) -#define BOOST_AUTO_TEST_CASE(funcName) void funcName() -#define BOOST_AUTO_TEST_SUITE_END() #define BOOST_CHECK(expr) assert(expr) #define BOOST_CHECK_EQUAL(v1, v2) assert((v1) == (v2)) #define BOOST_CHECK_THROW(stmt, excMatch) { \ @@ -35,9 +32,7 @@ } \ } -BOOST_FIXTURE_TEST_SUITE(univalue_tests, BasicTestingSetup) - -BOOST_AUTO_TEST_CASE(univalue_constructor) +void univalue_constructor() { UniValue v1; BOOST_CHECK(v1.isNull()); @@ -85,7 +80,17 @@ BOOST_AUTO_TEST_CASE(univalue_constructor) BOOST_CHECK_EQUAL(v9.getValStr(), "zappa"); } -BOOST_AUTO_TEST_CASE(univalue_typecheck) +void univalue_push_throw() +{ + UniValue j; + BOOST_CHECK_THROW(j.push_back(1), std::runtime_error); + BOOST_CHECK_THROW(j.push_backV({1}), std::runtime_error); + BOOST_CHECK_THROW(j.__pushKV("k", 1), std::runtime_error); + BOOST_CHECK_THROW(j.pushKV("k", 1), std::runtime_error); + BOOST_CHECK_THROW(j.pushKVs({}), std::runtime_error); +} + +void univalue_typecheck() { UniValue v1; BOOST_CHECK(v1.setNumStr("1")); @@ -134,7 +139,7 @@ BOOST_AUTO_TEST_CASE(univalue_typecheck) BOOST_CHECK_THROW(vals[1].get_bool(), std::runtime_error); } -BOOST_AUTO_TEST_CASE(univalue_set) +void univalue_set() { UniValue v(UniValue::VSTR, "foo"); v.clear(); @@ -193,18 +198,18 @@ BOOST_AUTO_TEST_CASE(univalue_set) BOOST_CHECK(v.isNull()); } -BOOST_AUTO_TEST_CASE(univalue_array) +void univalue_array() { UniValue arr(UniValue::VARR); UniValue v((int64_t)1023LL); - BOOST_CHECK(arr.push_back(v)); + arr.push_back(v); std::string vStr("zippy"); - BOOST_CHECK(arr.push_back(vStr)); + arr.push_back(vStr); const char *s = "pippy"; - BOOST_CHECK(arr.push_back(s)); + arr.push_back(s); std::vector<UniValue> vec; v.setStr("boing"); @@ -213,13 +218,13 @@ BOOST_AUTO_TEST_CASE(univalue_array) v.setStr("going"); vec.push_back(v); - BOOST_CHECK(arr.push_backV(vec)); + arr.push_backV(vec); - BOOST_CHECK(arr.push_back((uint64_t) 400ULL)); - BOOST_CHECK(arr.push_back((int64_t) -400LL)); - BOOST_CHECK(arr.push_back((int) -401)); - BOOST_CHECK(arr.push_back(-40.1)); - BOOST_CHECK(arr.push_back(true)); + arr.push_back(uint64_t{400ULL}); + arr.push_back(int64_t{-400LL}); + arr.push_back(int{-401}); + arr.push_back(-40.1); + arr.push_back(true); BOOST_CHECK_EQUAL(arr.empty(), false); BOOST_CHECK_EQUAL(arr.size(), 10); @@ -252,7 +257,7 @@ BOOST_AUTO_TEST_CASE(univalue_array) BOOST_CHECK_EQUAL(arr.size(), 0); } -BOOST_AUTO_TEST_CASE(univalue_object) +void univalue_object() { UniValue obj(UniValue::VOBJ); std::string strKey, strVal; @@ -260,39 +265,39 @@ BOOST_AUTO_TEST_CASE(univalue_object) strKey = "age"; v.setInt(100); - BOOST_CHECK(obj.pushKV(strKey, v)); + obj.pushKV(strKey, v); strKey = "first"; strVal = "John"; - BOOST_CHECK(obj.pushKV(strKey, strVal)); + obj.pushKV(strKey, strVal); strKey = "last"; - const char *cVal = "Smith"; - BOOST_CHECK(obj.pushKV(strKey, cVal)); + const char* cVal = "Smith"; + obj.pushKV(strKey, cVal); strKey = "distance"; - BOOST_CHECK(obj.pushKV(strKey, (int64_t) 25)); + obj.pushKV(strKey, int64_t{25}); strKey = "time"; - BOOST_CHECK(obj.pushKV(strKey, (uint64_t) 3600)); + obj.pushKV(strKey, uint64_t{3600}); strKey = "calories"; - BOOST_CHECK(obj.pushKV(strKey, (int) 12)); + obj.pushKV(strKey, int{12}); strKey = "temperature"; - BOOST_CHECK(obj.pushKV(strKey, (double) 90.012)); + obj.pushKV(strKey, double{90.012}); strKey = "moon"; - BOOST_CHECK(obj.pushKV(strKey, true)); + obj.pushKV(strKey, true); strKey = "spoon"; - BOOST_CHECK(obj.pushKV(strKey, false)); + obj.pushKV(strKey, false); UniValue obj2(UniValue::VOBJ); - BOOST_CHECK(obj2.pushKV("cat1", 9000)); - BOOST_CHECK(obj2.pushKV("cat2", 12345)); + obj2.pushKV("cat1", 9000); + obj2.pushKV("cat2", 12345); - BOOST_CHECK(obj.pushKVs(obj2)); + obj.pushKVs(obj2); BOOST_CHECK_EQUAL(obj.empty(), false); BOOST_CHECK_EQUAL(obj.size(), 11); @@ -371,7 +376,7 @@ BOOST_AUTO_TEST_CASE(univalue_object) static const char *json1 = "[1.10000000,{\"key1\":\"str\\u0000\",\"key2\":800,\"key3\":{\"name\":\"martian http://test.com\"}}]"; -BOOST_AUTO_TEST_CASE(univalue_readwrite) +void univalue_readwrite() { UniValue v; BOOST_CHECK(v.read(json1)); @@ -414,11 +419,10 @@ BOOST_AUTO_TEST_CASE(univalue_readwrite) BOOST_CHECK(!v.read("{} 42")); } -BOOST_AUTO_TEST_SUITE_END() - int main (int argc, char *argv[]) { univalue_constructor(); + univalue_push_throw(); univalue_typecheck(); univalue_set(); univalue_array(); diff --git a/src/univalue/test/unitester.cpp b/src/univalue/test/unitester.cpp index 94c149b39f..6344a5a0ab 100644 --- a/src/univalue/test/unitester.cpp +++ b/src/univalue/test/unitester.cpp @@ -149,6 +149,13 @@ void unescape_unicode_test() assert(val[0].get_str() == "\xf0\x9d\x85\xa1"); } +void no_nul_test() +{ + char buf[] = "___[1,2,3]___"; + UniValue val; + assert(val.read(buf + 3, 7)); +} + int main (int argc, char *argv[]) { for (const auto& f: filenames) { @@ -156,6 +163,7 @@ int main (int argc, char *argv[]) } unescape_unicode_test(); + no_nul_test(); return 0; } diff --git a/src/util/result.h b/src/util/result.h index dcf5edaa5b..2f586a4c9b 100644 --- a/src/util/result.h +++ b/src/util/result.h @@ -6,6 +6,7 @@ #define BITCOIN_UTIL_RESULT_H #include <util/translation.h> + #include <variant> /* @@ -18,9 +19,9 @@ private: std::variant<bilingual_str, T> m_variant; public: - BResult() : m_variant(Untranslated("")) {} - BResult(const T& _obj) : m_variant(_obj) {} - BResult(const bilingual_str& error) : m_variant(error) {} + BResult() : m_variant{Untranslated("")} {} + BResult(T obj) : m_variant{std::move(obj)} {} + BResult(bilingual_str error) : m_variant{std::move(error)} {} /* Whether the function succeeded or not */ bool HasRes() const { return std::holds_alternative<T>(m_variant); } @@ -30,6 +31,11 @@ public: assert(HasRes()); return std::get<T>(m_variant); } + T ReleaseObj() + { + assert(HasRes()); + return std::move(std::get<T>(m_variant)); + } /* In case of failure, the error cause */ const bilingual_str& GetError() const { diff --git a/src/util/time.h b/src/util/time.h index 9df69a953c..fc49f23ce3 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -24,6 +24,7 @@ struct NodeClock : public std::chrono::system_clock { }; using NodeSeconds = std::chrono::time_point<NodeClock, std::chrono::seconds>; +using SteadyClock = std::chrono::steady_clock; using SteadySeconds = std::chrono::time_point<std::chrono::steady_clock, std::chrono::seconds>; using SteadyMilliseconds = std::chrono::time_point<std::chrono::steady_clock, std::chrono::milliseconds>; using SteadyMicroseconds = std::chrono::time_point<std::chrono::steady_clock, std::chrono::microseconds>; diff --git a/src/validation.cpp b/src/validation.cpp index 4c694a2c21..6840753cd4 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -5,6 +5,9 @@ #include <validation.h> +#include <kernel/coinstats.h> +#include <kernel/mempool_persist.h> + #include <arith_uint256.h> #include <chain.h> #include <chainparams.h> @@ -17,8 +20,8 @@ #include <consensus/validation.h> #include <cuckoocache.h> #include <flatfile.h> +#include <fs.h> #include <hash.h> -#include <kernel/coinstats.h> #include <logging.h> #include <logging/timer.h> #include <node/blockstorage.h> @@ -47,12 +50,14 @@ #include <util/rbf.h> #include <util/strencodings.h> #include <util/system.h> +#include <util/time.h> #include <util/trace.h> #include <util/translation.h> #include <validationinterface.h> #include <warnings.h> #include <algorithm> +#include <chrono> #include <deque> #include <numeric> #include <optional> @@ -61,7 +66,9 @@ using kernel::CCoinsStats; using kernel::CoinStatsHashType; using kernel::ComputeUTXOStats; +using kernel::LoadMempool; +using fsbridge::FopenFn; using node::BLOCKFILE_CHUNK_SIZE; using node::BlockManager; using node::BlockMap; @@ -3861,13 +3868,11 @@ void PruneBlockFilesManual(CChainState& active_chainstate, int nManualPruneHeigh } } -void CChainState::LoadMempool(const ArgsManager& args) +void CChainState::LoadMempool(const fs::path& load_path, FopenFn mockable_fopen_function) { if (!m_mempool) return; - if (args.GetBoolArg("-persistmempool", DEFAULT_PERSIST_MEMPOOL)) { - ::LoadMempool(*m_mempool, *this); - } - m_mempool->SetIsLoaded(!ShutdownRequested()); + ::LoadMempool(*m_mempool, load_path, *this, mockable_fopen_function); + m_mempool->SetLoadTried(!ShutdownRequested()); } bool CChainState::LoadChainTip() @@ -4638,153 +4643,6 @@ bool CChainState::ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) return ret; } -static const uint64_t MEMPOOL_DUMP_VERSION = 1; - -bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mockable_fopen_function) -{ - int64_t nExpiryTimeout = std::chrono::seconds{pool.m_expiry}.count(); - FILE* filestr{mockable_fopen_function(gArgs.GetDataDirNet() / "mempool.dat", "rb")}; - CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); - if (file.IsNull()) { - LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); - return false; - } - - int64_t count = 0; - int64_t expired = 0; - int64_t failed = 0; - int64_t already_there = 0; - int64_t unbroadcast = 0; - int64_t nNow = GetTime(); - - try { - uint64_t version; - file >> version; - if (version != MEMPOOL_DUMP_VERSION) { - return false; - } - uint64_t num; - file >> num; - while (num) { - --num; - CTransactionRef tx; - int64_t nTime; - int64_t nFeeDelta; - file >> tx; - file >> nTime; - file >> nFeeDelta; - - CAmount amountdelta = nFeeDelta; - if (amountdelta) { - pool.PrioritiseTransaction(tx->GetHash(), amountdelta); - } - if (nTime > nNow - nExpiryTimeout) { - LOCK(cs_main); - const auto& accepted = AcceptToMemoryPool(active_chainstate, tx, nTime, /*bypass_limits=*/false, /*test_accept=*/false); - if (accepted.m_result_type == MempoolAcceptResult::ResultType::VALID) { - ++count; - } else { - // mempool may contain the transaction already, e.g. from - // wallet(s) having loaded it while we were processing - // mempool transactions; consider these as valid, instead of - // failed, but mark them as 'already there' - if (pool.exists(GenTxid::Txid(tx->GetHash()))) { - ++already_there; - } else { - ++failed; - } - } - } else { - ++expired; - } - if (ShutdownRequested()) - return false; - } - std::map<uint256, CAmount> mapDeltas; - file >> mapDeltas; - - for (const auto& i : mapDeltas) { - pool.PrioritiseTransaction(i.first, i.second); - } - - std::set<uint256> unbroadcast_txids; - file >> unbroadcast_txids; - unbroadcast = unbroadcast_txids.size(); - for (const auto& txid : unbroadcast_txids) { - // Ensure transactions were accepted to mempool then add to - // unbroadcast set. - if (pool.get(txid) != nullptr) pool.AddUnbroadcastTx(txid); - } - } catch (const std::exception& e) { - LogPrintf("Failed to deserialize mempool data on disk: %s. Continuing anyway.\n", e.what()); - return false; - } - - LogPrintf("Imported mempool transactions from disk: %i succeeded, %i failed, %i expired, %i already there, %i waiting for initial broadcast\n", count, failed, expired, already_there, unbroadcast); - return true; -} - -bool DumpMempool(const CTxMemPool& pool, FopenFn mockable_fopen_function, bool skip_file_commit) -{ - int64_t start = GetTimeMicros(); - - std::map<uint256, CAmount> mapDeltas; - std::vector<TxMempoolInfo> vinfo; - std::set<uint256> unbroadcast_txids; - - static Mutex dump_mutex; - LOCK(dump_mutex); - - { - LOCK(pool.cs); - for (const auto &i : pool.mapDeltas) { - mapDeltas[i.first] = i.second; - } - vinfo = pool.infoAll(); - unbroadcast_txids = pool.GetUnbroadcastTxs(); - } - - int64_t mid = GetTimeMicros(); - - try { - FILE* filestr{mockable_fopen_function(gArgs.GetDataDirNet() / "mempool.dat.new", "wb")}; - if (!filestr) { - return false; - } - - CAutoFile file(filestr, SER_DISK, CLIENT_VERSION); - - uint64_t version = MEMPOOL_DUMP_VERSION; - file << version; - - file << (uint64_t)vinfo.size(); - for (const auto& i : vinfo) { - file << *(i.tx); - file << int64_t{count_seconds(i.m_time)}; - file << int64_t{i.nFeeDelta}; - mapDeltas.erase(i.tx->GetHash()); - } - - file << mapDeltas; - - LogPrintf("Writing %d unbroadcast transactions to disk.\n", unbroadcast_txids.size()); - file << unbroadcast_txids; - - if (!skip_file_commit && !FileCommit(file.Get())) - throw std::runtime_error("FileCommit failed"); - file.fclose(); - if (!RenameOver(gArgs.GetDataDirNet() / "mempool.dat.new", gArgs.GetDataDirNet() / "mempool.dat")) { - throw std::runtime_error("Rename failed"); - } - int64_t last = GetTimeMicros(); - LogPrintf("Dumped mempool: %gs to copy, %gs to dump\n", (mid-start)*MICRO, (last-mid)*MICRO); - } catch (const std::exception& e) { - LogPrintf("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); - return false; - } - return true; -} - //! Guess how far we are in the verification process at the given block index //! require cs_main if pindex has not been validated yet (because nChainTx might be unset) double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex *pindex) { diff --git a/src/validation.h b/src/validation.h index 0e27e117fa..a21d9e3a28 100644 --- a/src/validation.h +++ b/src/validation.h @@ -68,8 +68,6 @@ static const bool DEFAULT_CHECKPOINTS_ENABLED = true; static const bool DEFAULT_TXINDEX = false; static constexpr bool DEFAULT_COINSTATSINDEX{false}; static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; -/** Default for -persistmempool */ -static const bool DEFAULT_PERSIST_MEMPOOL = true; /** Default for -stopatheight */ static const int DEFAULT_STOPATHEIGHT = 0; /** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ActiveChain().Tip() will not be pruned. */ @@ -679,7 +677,7 @@ public: void CheckBlockIndex(); /** Load the persisted mempool from disk */ - void LoadMempool(const ArgsManager& args); + void LoadMempool(const fs::path& load_path, fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen); /** Update the chain tip based on database information, i.e. CoinsTip()'s best block. */ bool LoadChainTip() EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -834,7 +832,7 @@ private: friend CChainState; public: - using Options = ChainstateManagerOpts; + using Options = kernel::ChainstateManagerOpts; explicit ChainstateManager(const Options& opts) : m_chainparams{opts.chainparams}, @@ -1014,14 +1012,6 @@ bool DeploymentEnabled(const ChainstateManager& chainman, DEP dep) return DeploymentEnabled(chainman.GetConsensus(), dep); } -using FopenFn = std::function<FILE*(const fs::path&, const char*)>; - -/** Dump the mempool to disk. */ -bool DumpMempool(const CTxMemPool& pool, FopenFn mockable_fopen_function = fsbridge::fopen, bool skip_file_commit = false); - -/** Load the mempool from disk. */ -bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mockable_fopen_function = fsbridge::fopen); - /** * Return the expected assumeutxo value for a given height, if one exists. * diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index 547d972c8d..23f91d9b3a 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -569,11 +569,13 @@ public: options.require_existing = true; return MakeWallet(m_context, LoadWallet(m_context, name, true /* load_on_start */, options, status, error, warnings)); } - std::unique_ptr<Wallet> restoreWallet(const fs::path& backup_file, const std::string& wallet_name, bilingual_str& error, std::vector<bilingual_str>& warnings) override + BResult<std::unique_ptr<Wallet>> restoreWallet(const fs::path& backup_file, const std::string& wallet_name, std::vector<bilingual_str>& warnings) override { DatabaseStatus status; - - return MakeWallet(m_context, RestoreWallet(m_context, backup_file, wallet_name, /*load_on_start=*/true, status, error, warnings)); + bilingual_str error; + BResult<std::unique_ptr<Wallet>> wallet{MakeWallet(m_context, RestoreWallet(m_context, backup_file, wallet_name, /*load_on_start=*/true, status, error, warnings))}; + if (!wallet) return error; + return wallet; } std::string getWalletDir() override { diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index 8de4017371..d3303c0b1f 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -130,12 +130,10 @@ CAmount CachedTxGetCredit(const CWallet& wallet, const CWalletTx& wtx, const ism return 0; CAmount credit = 0; - if (filter & ISMINE_SPENDABLE) { + const isminefilter get_amount_filter{filter & ISMINE_ALL}; + if (get_amount_filter) { // GetBalance can assume transactions in mapWallet won't change - credit += GetCachableAmount(wallet, wtx, CWalletTx::CREDIT, ISMINE_SPENDABLE); - } - if (filter & ISMINE_WATCH_ONLY) { - credit += GetCachableAmount(wallet, wtx, CWalletTx::CREDIT, ISMINE_WATCH_ONLY); + credit += GetCachableAmount(wallet, wtx, CWalletTx::CREDIT, get_amount_filter); } return credit; } @@ -146,11 +144,9 @@ CAmount CachedTxGetDebit(const CWallet& wallet, const CWalletTx& wtx, const ismi return 0; CAmount debit = 0; - if (filter & ISMINE_SPENDABLE) { - debit += GetCachableAmount(wallet, wtx, CWalletTx::DEBIT, ISMINE_SPENDABLE); - } - if (filter & ISMINE_WATCH_ONLY) { - debit += GetCachableAmount(wallet, wtx, CWalletTx::DEBIT, ISMINE_WATCH_ONLY); + const isminefilter get_amount_filter{filter & ISMINE_ALL}; + if (get_amount_filter) { + debit += GetCachableAmount(wallet, wtx, CWalletTx::DEBIT, get_amount_filter); } return debit; } diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 5f1673fb12..62aba4f3f2 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -100,11 +100,13 @@ RPCHelpMan importprivkey() "Hint: use importmulti to import more than one private key.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported key exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" + "The rescan parameter can be set to false if the key was never used to create transactions. If it is set to false,\n" + "but the key was used to create transactions, rescanwallet needs to be called with the appropriate block range.\n" "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"privkey", RPCArg::Type::STR, RPCArg::Optional::NO, "The private key (see dumpprivkey)"}, {"label", RPCArg::Type::STR, RPCArg::DefaultHint{"current label if address exists, otherwise \"\""}, "An optional label"}, - {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Rescan the wallet for transactions"}, + {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions."}, }, RPCResult{RPCResult::Type::NONE, "", ""}, RPCExamples{ @@ -201,6 +203,8 @@ RPCHelpMan importaddress() "\nAdds an address or script (in hex) that can be watched as if it were in your wallet but cannot be used to spend. Requires a new wallet backup.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported address exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" + "The rescan parameter can be set to false if the key was never used to create transactions. If it is set to false,\n" + "but the key was used to create transactions, rescanwallet needs to be called with the appropriate block range.\n" "If you have the full public key, you should call importpubkey instead of this.\n" "Hint: use importmulti to import more than one address.\n" "\nNote: If you import a non-standard raw script in hex form, outputs sending to it will be treated\n" @@ -210,7 +214,7 @@ RPCHelpMan importaddress() { {"address", RPCArg::Type::STR, RPCArg::Optional::NO, "The Bitcoin address (or hex-encoded script)"}, {"label", RPCArg::Type::STR, RPCArg::Default{""}, "An optional label"}, - {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Rescan the wallet for transactions"}, + {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions."}, {"p2sh", RPCArg::Type::BOOL, RPCArg::Default{false}, "Add the P2SH version of the script as well"}, }, RPCResult{RPCResult::Type::NONE, "", ""}, @@ -401,11 +405,13 @@ RPCHelpMan importpubkey() "Hint: use importmulti to import more than one public key.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported pubkey exists but related transactions are still missing, leading to temporarily incorrect/bogus balances and unspent outputs until rescan completes.\n" + "The rescan parameter can be set to false if the key was never used to create transactions. If it is set to false,\n" + "but the key was used to create transactions, rescanwallet needs to be called with the appropriate block range.\n" "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"pubkey", RPCArg::Type::STR, RPCArg::Optional::NO, "The hex-encoded public key"}, {"label", RPCArg::Type::STR, RPCArg::Default{""}, "An optional label"}, - {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Rescan the wallet for transactions"}, + {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions."}, }, RPCResult{RPCResult::Type::NONE, "", ""}, RPCExamples{ @@ -484,7 +490,7 @@ RPCHelpMan importwallet() { return RPCHelpMan{"importwallet", "\nImports keys from a wallet dump file (see dumpwallet). Requires a new wallet backup to include imported keys.\n" - "Note: Use \"getwalletinfo\" to query the scanning progress.\n", + "Note: Blockchain and Mempool will be rescanned after a successful import. Use \"getwalletinfo\" to query the scanning progress.\n", { {"filename", RPCArg::Type::STR, RPCArg::Optional::NO, "The wallet file"}, }, @@ -1250,6 +1256,8 @@ RPCHelpMan importmulti() "Conversely, if all the private keys are provided and the address/script is spendable, the watchonly option must be set to false, or a warning will be returned.\n" "\nNote: This call can take over an hour to complete if rescan is true, during that time, other rpc calls\n" "may report that the imported keys, addresses or scripts exist but related transactions are still missing.\n" + "The rescan parameter can be set to false if the key was never used to create transactions. If it is set to false,\n" + "but the key was used to create transactions, rescanwallet needs to be called with the appropriate block range.\n" "Note: Use \"getwalletinfo\" to query the scanning progress.\n", { {"requests", RPCArg::Type::ARR, RPCArg::Optional::NO, "Data to be imported", @@ -1291,7 +1299,7 @@ RPCHelpMan importmulti() "\"requests\""}, {"options", RPCArg::Type::OBJ, RPCArg::Optional::OMITTED_NAMED_ARG, "", { - {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Stating if should rescan the blockchain after all imports"}, + {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions after all imports."}, }, "\"options\""}, }, @@ -1593,7 +1601,7 @@ RPCHelpMan importdescriptors() " Use the string \"now\" to substitute the current synced blockchain time.\n" " \"now\" can be specified to bypass scanning, for outputs which are known to never have been used, and\n" " 0 can be specified to scan the entire blockchain. Blocks up to 2 hours before the earliest timestamp\n" - " of all descriptors being imported will be scanned.", + "of all descriptors being imported will be scanned as well as the mempool.", /*oneline_description=*/"", {"timestamp | \"now\"", "integer / string"} }, {"internal", RPCArg::Type::BOOL, RPCArg::Default{false}, "Whether matching outputs should be treated as not incoming payments (e.g. change)"}, diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index 22c1f2bdc2..83e23497cb 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -1633,7 +1633,7 @@ RPCHelpMan walletcreatefundedpsbt() }, true ); - UniValue options = request.params[3]; + UniValue options{request.params[3].isNull() ? UniValue::VOBJ : request.params[3]}; CAmount fee; int change_position; diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 81c01e5023..7b0906c0a8 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -53,9 +53,6 @@ static const std::shared_ptr<CWallet> TestLoadWallet(WalletContext& context) auto database = MakeWalletDatabase("", options, status, error); auto wallet = CWallet::Create(context, "", std::move(database), options.create_flags, error, warnings); NotifyWalletLoaded(context, wallet); - if (context.chain) { - wallet->postInitProcess(); - } return wallet; } @@ -768,6 +765,7 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) // being blocked wallet = TestLoadWallet(context); BOOST_CHECK(rescan_completed); + // AddToWallet events for block_tx and mempool_tx BOOST_CHECK_EQUAL(addtx_count, 2); { LOCK(wallet->cs_wallet); @@ -780,6 +778,8 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) // transactionAddedToMempool events are processed promise.set_value(); SyncWithValidationInterfaceQueue(); + // AddToWallet events for block_tx and mempool_tx events are counted a + // second time as the notificaiton queue is processed BOOST_CHECK_EQUAL(addtx_count, 4); @@ -803,7 +803,7 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) SyncWithValidationInterfaceQueue(); }); wallet = TestLoadWallet(context); - BOOST_CHECK_EQUAL(addtx_count, 4); + BOOST_CHECK_EQUAL(addtx_count, 2); { LOCK(wallet->cs_wallet); BOOST_CHECK_EQUAL(wallet->mapWallet.count(block_tx.GetHash()), 1U); diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 475627c76c..890e171913 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1678,7 +1678,8 @@ int64_t CWallet::RescanFromTime(int64_t startTime, const WalletRescanReserver& r /** * Scan the block chain (starting in start_block) for transactions * from or to us. If fUpdate is true, found transactions that already - * exist in the wallet will be updated. + * exist in the wallet will be updated. If max_height is not set, the + * mempool will be scanned as well. * * @param[in] start_block Scan starting block. If block is not on the active * chain, the scan will return SUCCESS immediately. @@ -1799,6 +1800,10 @@ CWallet::ScanResult CWallet::ScanForWalletTransactions(const uint256& start_bloc } } } + if (!max_height) { + WalletLogPrintf("Scanning current mempool transactions.\n"); + WITH_LOCK(cs_wallet, chain().requestMempoolTransactions(*this)); + } ShowProgress(strprintf("%s " + _("Rescanning…").translated, GetDisplayName()), 100); // hide progress dialog in GUI if (block_height && fAbortRescan) { WalletLogPrintf("Rescan aborted at block %d. Progress=%f\n", block_height, progress_current); @@ -3044,20 +3049,31 @@ bool CWallet::AttachChain(const std::shared_ptr<CWallet>& walletInstance, interf if (tip_height && *tip_height != rescan_height) { - if (chain.havePruned()) { + // Technically we could execute the code below in any case, but performing the + // `while` loop below can make startup very slow, so only check blocks on disk + // if necessary. + if (chain.havePruned() || chain.hasAssumedValidChain()) { int block_height = *tip_height; while (block_height > 0 && chain.haveBlockOnDisk(block_height - 1) && rescan_height != block_height) { --block_height; } if (rescan_height != block_height) { - // We can't rescan beyond non-pruned blocks, stop and throw an error. + // We can't rescan beyond blocks we don't have data for, stop and throw an error. // This might happen if a user uses an old wallet within a pruned node // or if they ran -disablewallet for a longer time, then decided to re-enable // Exit early and print an error. + // It also may happen if an assumed-valid chain is in use and therefore not + // all block data is available. // If a block is pruned after this check, we will load the wallet, // but fail the rescan with a generic error. - error = _("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)"); + + error = chain.hasAssumedValidChain() ? + _( + "Assumed-valid: last wallet synchronisation goes beyond " + "available block data. You need to wait for the background " + "validation chain to download more blocks.") : + _("Prune: last wallet synchronisation goes beyond pruned data. You need to -reindex (download the whole blockchain again in case of pruned node)"); return false; } } |