From 6d242ff1e9ca02fd8f5cb3ffe82dfb48a52366cc Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Wed, 26 Jun 2024 18:57:14 +0000 Subject: kernel: remove mempool_persist.cpp DumpMempool/LoadMempool are not necessary for the kernel --- src/Makefile.am | 1 - 1 file changed, 1 deletion(-) diff --git a/src/Makefile.am b/src/Makefile.am index 0c47a737d0..64ed261fcc 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -950,7 +950,6 @@ libbitcoinkernel_la_SOURCES = \ kernel/context.cpp \ kernel/cs_main.cpp \ kernel/disconnected_transactions.cpp \ - kernel/mempool_persist.cpp \ kernel/mempool_removal_reason.cpp \ logging.cpp \ node/blockstorage.cpp \ -- cgit v1.2.3 From f1478c05458562a9bef5c2ba43959d758e7b4745 Mon Sep 17 00:00:00 2001 From: Cory Fields Date: Wed, 26 Jun 2024 18:58:46 +0000 Subject: mempool: move LoadMempool/DumpMempool to node --- src/Makefile.am | 4 +- src/init.cpp | 6 +- src/kernel/mempool_persist.cpp | 221 ------------------------------ src/kernel/mempool_persist.h | 34 ----- src/node/mempool_persist.cpp | 221 ++++++++++++++++++++++++++++++ src/node/mempool_persist.h | 34 +++++ src/rpc/mempool.cpp | 8 +- src/test/fuzz/validation_load_mempool.cpp | 6 +- 8 files changed, 267 insertions(+), 267 deletions(-) delete mode 100644 src/kernel/mempool_persist.cpp delete mode 100644 src/kernel/mempool_persist.h create mode 100644 src/node/mempool_persist.cpp create mode 100644 src/node/mempool_persist.h diff --git a/src/Makefile.am b/src/Makefile.am index 64ed261fcc..52d4aae893 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -192,7 +192,6 @@ BITCOIN_CORE_H = \ kernel/mempool_entry.h \ kernel/mempool_limits.h \ kernel/mempool_options.h \ - kernel/mempool_persist.h \ kernel/mempool_removal_reason.h \ kernel/messagestartchars.h \ kernel/notifications_interface.h \ @@ -228,6 +227,7 @@ BITCOIN_CORE_H = \ node/interface_ui.h \ node/kernel_notifications.h \ node/mempool_args.h \ + node/mempool_persist.h \ node/mempool_persist_args.h \ node/miner.h \ node/mini_miner.h \ @@ -413,7 +413,6 @@ libbitcoin_node_a_SOURCES = \ kernel/context.cpp \ kernel/cs_main.cpp \ kernel/disconnected_transactions.cpp \ - kernel/mempool_persist.cpp \ kernel/mempool_removal_reason.cpp \ mapport.cpp \ net.cpp \ @@ -435,6 +434,7 @@ libbitcoin_node_a_SOURCES = \ node/interfaces.cpp \ node/kernel_notifications.cpp \ node/mempool_args.cpp \ + node/mempool_persist.cpp \ node/mempool_persist_args.cpp \ node/miner.cpp \ node/mini_miner.cpp \ diff --git a/src/init.cpp b/src/init.cpp index c6ef62372e..18f1ff6da8 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -8,7 +8,6 @@ #include #include -#include #include #include @@ -51,6 +50,7 @@ #include #include #include +#include #include #include #include @@ -119,8 +119,6 @@ using common::AmountErrMsg; using common::InvalidPortErrMsg; using common::ResolveErrMsg; -using kernel::DumpMempool; -using kernel::LoadMempool; using kernel::ValidationCacheSizes; using node::ApplyArgsManOptions; @@ -130,6 +128,8 @@ using node::CalculateCacheSizes; using node::DEFAULT_PERSIST_MEMPOOL; using node::DEFAULT_PRINTPRIORITY; using node::DEFAULT_STOPATHEIGHT; +using node::DumpMempool; +using node::LoadMempool; using node::KernelNotifications; using node::LoadChainstate; using node::MempoolPath; diff --git a/src/kernel/mempool_persist.cpp b/src/kernel/mempool_persist.cpp deleted file mode 100644 index 53028a45ae..0000000000 --- a/src/kernel/mempool_persist.cpp +++ /dev/null @@ -1,221 +0,0 @@ -// Copyright (c) 2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -using fsbridge::FopenFn; - -namespace kernel { - -static const uint64_t MEMPOOL_DUMP_VERSION_NO_XOR_KEY{1}; -static const uint64_t MEMPOOL_DUMP_VERSION{2}; - -bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active_chainstate, ImportMempoolOptions&& opts) -{ - if (load_path.empty()) return false; - - AutoFile file{opts.mockable_fopen_function(load_path, "rb")}; - if (file.IsNull()) { - LogInfo("Failed to open mempool file. Continuing anyway.\n"); - return false; - } - - int64_t count = 0; - int64_t expired = 0; - int64_t failed = 0; - int64_t already_there = 0; - int64_t unbroadcast = 0; - const auto now{NodeClock::now()}; - - try { - uint64_t version; - file >> version; - std::vector xor_key; - if (version == MEMPOOL_DUMP_VERSION_NO_XOR_KEY) { - // Leave XOR-key empty - } else if (version == MEMPOOL_DUMP_VERSION) { - file >> xor_key; - } else { - return false; - } - file.SetXor(xor_key); - uint64_t total_txns_to_load; - file >> total_txns_to_load; - uint64_t txns_tried = 0; - LogInfo("Loading %u mempool transactions from file...\n", total_txns_to_load); - int next_tenth_to_report = 0; - while (txns_tried < total_txns_to_load) { - const int percentage_done(100.0 * txns_tried / total_txns_to_load); - if (next_tenth_to_report < percentage_done / 10) { - LogInfo("Progress loading mempool transactions from file: %d%% (tried %u, %u remaining)\n", - percentage_done, txns_tried, total_txns_to_load - txns_tried); - next_tenth_to_report = percentage_done / 10; - } - ++txns_tried; - - CTransactionRef tx; - int64_t nTime; - int64_t nFeeDelta; - file >> TX_WITH_WITNESS(tx); - file >> nTime; - file >> nFeeDelta; - - if (opts.use_current_time) { - nTime = TicksSinceEpoch(now); - } - - CAmount amountdelta = nFeeDelta; - if (amountdelta && opts.apply_fee_delta_priority) { - pool.PrioritiseTransaction(tx->GetHash(), amountdelta); - } - if (nTime > TicksSinceEpoch(now - pool.m_opts.expiry)) { - LOCK(cs_main); - const auto& accepted = AcceptToMemoryPool(active_chainstate, tx, nTime, /*bypass_limits=*/false, /*test_accept=*/false); - if (accepted.m_result_type == MempoolAcceptResult::ResultType::VALID) { - ++count; - } else { - // mempool may contain the transaction already, e.g. from - // wallet(s) having loaded it while we were processing - // mempool transactions; consider these as valid, instead of - // failed, but mark them as 'already there' - if (pool.exists(GenTxid::Txid(tx->GetHash()))) { - ++already_there; - } else { - ++failed; - } - } - } else { - ++expired; - } - if (active_chainstate.m_chainman.m_interrupt) - return false; - } - std::map mapDeltas; - file >> mapDeltas; - - if (opts.apply_fee_delta_priority) { - for (const auto& i : mapDeltas) { - pool.PrioritiseTransaction(i.first, i.second); - } - } - - std::set unbroadcast_txids; - file >> unbroadcast_txids; - if (opts.apply_unbroadcast_set) { - unbroadcast = unbroadcast_txids.size(); - for (const auto& txid : unbroadcast_txids) { - // Ensure transactions were accepted to mempool then add to - // unbroadcast set. - if (pool.get(txid) != nullptr) pool.AddUnbroadcastTx(txid); - } - } - } catch (const std::exception& e) { - LogInfo("Failed to deserialize mempool data on file: %s. Continuing anyway.\n", e.what()); - return false; - } - - LogInfo("Imported mempool transactions from file: %i succeeded, %i failed, %i expired, %i already there, %i waiting for initial broadcast\n", count, failed, expired, already_there, unbroadcast); - return true; -} - -bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mockable_fopen_function, bool skip_file_commit) -{ - auto start = SteadyClock::now(); - - std::map mapDeltas; - std::vector vinfo; - std::set unbroadcast_txids; - - static Mutex dump_mutex; - LOCK(dump_mutex); - - { - LOCK(pool.cs); - for (const auto &i : pool.mapDeltas) { - mapDeltas[i.first] = i.second; - } - vinfo = pool.infoAll(); - unbroadcast_txids = pool.GetUnbroadcastTxs(); - } - - auto mid = SteadyClock::now(); - - AutoFile file{mockable_fopen_function(dump_path + ".new", "wb")}; - if (file.IsNull()) { - return false; - } - - try { - const uint64_t version{pool.m_opts.persist_v1_dat ? MEMPOOL_DUMP_VERSION_NO_XOR_KEY : MEMPOOL_DUMP_VERSION}; - file << version; - - std::vector xor_key(8); - if (!pool.m_opts.persist_v1_dat) { - FastRandomContext{}.fillrand(xor_key); - file << xor_key; - } - file.SetXor(xor_key); - - uint64_t mempool_transactions_to_write(vinfo.size()); - file << mempool_transactions_to_write; - LogInfo("Writing %u mempool transactions to file...\n", mempool_transactions_to_write); - for (const auto& i : vinfo) { - file << TX_WITH_WITNESS(*(i.tx)); - file << int64_t{count_seconds(i.m_time)}; - file << int64_t{i.nFeeDelta}; - mapDeltas.erase(i.tx->GetHash()); - } - - file << mapDeltas; - - LogInfo("Writing %d unbroadcast transactions to file.\n", unbroadcast_txids.size()); - file << unbroadcast_txids; - - if (!skip_file_commit && !FileCommit(file.Get())) - throw std::runtime_error("FileCommit failed"); - file.fclose(); - if (!RenameOver(dump_path + ".new", dump_path)) { - throw std::runtime_error("Rename failed"); - } - auto last = SteadyClock::now(); - - LogInfo("Dumped mempool: %.3fs to copy, %.3fs to dump, %d bytes dumped to file\n", - Ticks(mid - start), - Ticks(last - mid), - fs::file_size(dump_path)); - } catch (const std::exception& e) { - LogInfo("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); - return false; - } - return true; -} - -} // namespace kernel diff --git a/src/kernel/mempool_persist.h b/src/kernel/mempool_persist.h deleted file mode 100644 index e124a8eadf..0000000000 --- a/src/kernel/mempool_persist.h +++ /dev/null @@ -1,34 +0,0 @@ -// Copyright (c) 2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_KERNEL_MEMPOOL_PERSIST_H -#define BITCOIN_KERNEL_MEMPOOL_PERSIST_H - -#include - -class Chainstate; -class CTxMemPool; - -namespace kernel { - -/** Dump the mempool to a file. */ -bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, - fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen, - bool skip_file_commit = false); - -struct ImportMempoolOptions { - fsbridge::FopenFn mockable_fopen_function{fsbridge::fopen}; - bool use_current_time{false}; - bool apply_fee_delta_priority{true}; - bool apply_unbroadcast_set{true}; -}; -/** Import the file and attempt to add its contents to the mempool. */ -bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, - Chainstate& active_chainstate, - ImportMempoolOptions&& opts); - -} // namespace kernel - - -#endif // BITCOIN_KERNEL_MEMPOOL_PERSIST_H diff --git a/src/node/mempool_persist.cpp b/src/node/mempool_persist.cpp new file mode 100644 index 0000000000..a265c2e12d --- /dev/null +++ b/src/node/mempool_persist.cpp @@ -0,0 +1,221 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using fsbridge::FopenFn; + +namespace node { + +static const uint64_t MEMPOOL_DUMP_VERSION_NO_XOR_KEY{1}; +static const uint64_t MEMPOOL_DUMP_VERSION{2}; + +bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active_chainstate, ImportMempoolOptions&& opts) +{ + if (load_path.empty()) return false; + + AutoFile file{opts.mockable_fopen_function(load_path, "rb")}; + if (file.IsNull()) { + LogInfo("Failed to open mempool file. Continuing anyway.\n"); + return false; + } + + int64_t count = 0; + int64_t expired = 0; + int64_t failed = 0; + int64_t already_there = 0; + int64_t unbroadcast = 0; + const auto now{NodeClock::now()}; + + try { + uint64_t version; + file >> version; + std::vector xor_key; + if (version == MEMPOOL_DUMP_VERSION_NO_XOR_KEY) { + // Leave XOR-key empty + } else if (version == MEMPOOL_DUMP_VERSION) { + file >> xor_key; + } else { + return false; + } + file.SetXor(xor_key); + uint64_t total_txns_to_load; + file >> total_txns_to_load; + uint64_t txns_tried = 0; + LogInfo("Loading %u mempool transactions from file...\n", total_txns_to_load); + int next_tenth_to_report = 0; + while (txns_tried < total_txns_to_load) { + const int percentage_done(100.0 * txns_tried / total_txns_to_load); + if (next_tenth_to_report < percentage_done / 10) { + LogInfo("Progress loading mempool transactions from file: %d%% (tried %u, %u remaining)\n", + percentage_done, txns_tried, total_txns_to_load - txns_tried); + next_tenth_to_report = percentage_done / 10; + } + ++txns_tried; + + CTransactionRef tx; + int64_t nTime; + int64_t nFeeDelta; + file >> TX_WITH_WITNESS(tx); + file >> nTime; + file >> nFeeDelta; + + if (opts.use_current_time) { + nTime = TicksSinceEpoch(now); + } + + CAmount amountdelta = nFeeDelta; + if (amountdelta && opts.apply_fee_delta_priority) { + pool.PrioritiseTransaction(tx->GetHash(), amountdelta); + } + if (nTime > TicksSinceEpoch(now - pool.m_opts.expiry)) { + LOCK(cs_main); + const auto& accepted = AcceptToMemoryPool(active_chainstate, tx, nTime, /*bypass_limits=*/false, /*test_accept=*/false); + if (accepted.m_result_type == MempoolAcceptResult::ResultType::VALID) { + ++count; + } else { + // mempool may contain the transaction already, e.g. from + // wallet(s) having loaded it while we were processing + // mempool transactions; consider these as valid, instead of + // failed, but mark them as 'already there' + if (pool.exists(GenTxid::Txid(tx->GetHash()))) { + ++already_there; + } else { + ++failed; + } + } + } else { + ++expired; + } + if (active_chainstate.m_chainman.m_interrupt) + return false; + } + std::map mapDeltas; + file >> mapDeltas; + + if (opts.apply_fee_delta_priority) { + for (const auto& i : mapDeltas) { + pool.PrioritiseTransaction(i.first, i.second); + } + } + + std::set unbroadcast_txids; + file >> unbroadcast_txids; + if (opts.apply_unbroadcast_set) { + unbroadcast = unbroadcast_txids.size(); + for (const auto& txid : unbroadcast_txids) { + // Ensure transactions were accepted to mempool then add to + // unbroadcast set. + if (pool.get(txid) != nullptr) pool.AddUnbroadcastTx(txid); + } + } + } catch (const std::exception& e) { + LogInfo("Failed to deserialize mempool data on file: %s. Continuing anyway.\n", e.what()); + return false; + } + + LogInfo("Imported mempool transactions from file: %i succeeded, %i failed, %i expired, %i already there, %i waiting for initial broadcast\n", count, failed, expired, already_there, unbroadcast); + return true; +} + +bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mockable_fopen_function, bool skip_file_commit) +{ + auto start = SteadyClock::now(); + + std::map mapDeltas; + std::vector vinfo; + std::set unbroadcast_txids; + + static Mutex dump_mutex; + LOCK(dump_mutex); + + { + LOCK(pool.cs); + for (const auto &i : pool.mapDeltas) { + mapDeltas[i.first] = i.second; + } + vinfo = pool.infoAll(); + unbroadcast_txids = pool.GetUnbroadcastTxs(); + } + + auto mid = SteadyClock::now(); + + AutoFile file{mockable_fopen_function(dump_path + ".new", "wb")}; + if (file.IsNull()) { + return false; + } + + try { + const uint64_t version{pool.m_opts.persist_v1_dat ? MEMPOOL_DUMP_VERSION_NO_XOR_KEY : MEMPOOL_DUMP_VERSION}; + file << version; + + std::vector xor_key(8); + if (!pool.m_opts.persist_v1_dat) { + FastRandomContext{}.fillrand(xor_key); + file << xor_key; + } + file.SetXor(xor_key); + + uint64_t mempool_transactions_to_write(vinfo.size()); + file << mempool_transactions_to_write; + LogInfo("Writing %u mempool transactions to file...\n", mempool_transactions_to_write); + for (const auto& i : vinfo) { + file << TX_WITH_WITNESS(*(i.tx)); + file << int64_t{count_seconds(i.m_time)}; + file << int64_t{i.nFeeDelta}; + mapDeltas.erase(i.tx->GetHash()); + } + + file << mapDeltas; + + LogInfo("Writing %d unbroadcast transactions to file.\n", unbroadcast_txids.size()); + file << unbroadcast_txids; + + if (!skip_file_commit && !FileCommit(file.Get())) + throw std::runtime_error("FileCommit failed"); + file.fclose(); + if (!RenameOver(dump_path + ".new", dump_path)) { + throw std::runtime_error("Rename failed"); + } + auto last = SteadyClock::now(); + + LogInfo("Dumped mempool: %.3fs to copy, %.3fs to dump, %d bytes dumped to file\n", + Ticks(mid - start), + Ticks(last - mid), + fs::file_size(dump_path)); + } catch (const std::exception& e) { + LogInfo("Failed to dump mempool: %s. Continuing anyway.\n", e.what()); + return false; + } + return true; +} + +} // namespace node diff --git a/src/node/mempool_persist.h b/src/node/mempool_persist.h new file mode 100644 index 0000000000..7c5754a90c --- /dev/null +++ b/src/node/mempool_persist.h @@ -0,0 +1,34 @@ +// Copyright (c) 2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_MEMPOOL_PERSIST_H +#define BITCOIN_NODE_MEMPOOL_PERSIST_H + +#include + +class Chainstate; +class CTxMemPool; + +namespace node { + +/** Dump the mempool to a file. */ +bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, + fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen, + bool skip_file_commit = false); + +struct ImportMempoolOptions { + fsbridge::FopenFn mockable_fopen_function{fsbridge::fopen}; + bool use_current_time{false}; + bool apply_fee_delta_priority{true}; + bool apply_unbroadcast_set{true}; +}; +/** Import the file and attempt to add its contents to the mempool. */ +bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, + Chainstate& active_chainstate, + ImportMempoolOptions&& opts); + +} // namespace node + + +#endif // BITCOIN_NODE_MEMPOOL_PERSIST_H diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index fd11f6cfeb..b67d272b65 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -5,7 +5,7 @@ #include -#include +#include #include #include @@ -27,7 +27,7 @@ #include -using kernel::DumpMempool; +using node::DumpMempool; using node::DEFAULT_MAX_BURN_AMOUNT; using node::DEFAULT_MAX_RAW_TX_FEE_RATE; @@ -759,13 +759,13 @@ static RPCHelpMan importmempool() const UniValue& use_current_time{request.params[1]["use_current_time"]}; const UniValue& apply_fee_delta{request.params[1]["apply_fee_delta_priority"]}; const UniValue& apply_unbroadcast{request.params[1]["apply_unbroadcast_set"]}; - kernel::ImportMempoolOptions opts{ + node::ImportMempoolOptions opts{ .use_current_time = use_current_time.isNull() ? true : use_current_time.get_bool(), .apply_fee_delta_priority = apply_fee_delta.isNull() ? false : apply_fee_delta.get_bool(), .apply_unbroadcast_set = apply_unbroadcast.isNull() ? false : apply_unbroadcast.get_bool(), }; - if (!kernel::LoadMempool(mempool, load_path, chainstate, std::move(opts))) { + if (!node::LoadMempool(mempool, load_path, chainstate, std::move(opts))) { throw JSONRPCError(RPC_MISC_ERROR, "Unable to import mempool file, see debug.log for details."); } diff --git a/src/test/fuzz/validation_load_mempool.cpp b/src/test/fuzz/validation_load_mempool.cpp index 51140ae039..c70d9ddf1e 100644 --- a/src/test/fuzz/validation_load_mempool.cpp +++ b/src/test/fuzz/validation_load_mempool.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include +#include #include #include @@ -21,8 +21,8 @@ #include #include -using kernel::DumpMempool; -using kernel::LoadMempool; +using node::DumpMempool; +using node::LoadMempool; using node::MempoolPath; -- cgit v1.2.3