Merge bitcoin/bitcoin#28530: tests, bug fix: DisconnectedBlockTransactions rewrite followups

9b3da70bd06b45482e7211aa95637a72bd115553 [test] DisconnectedBlockTransactions::DynamicMemoryUsage (glozow)
b2d04479647af64ad7cf5ebfb6175251b2f6b72e bugfix: correct DisconnectedBlockTransactions memory usage (stickies-v)
f4254e209801d6a790b5f0c251c0b32154a4e3cc assume duplicate transactions are not added to `iters_by_txid` (ismaelsadeeq)
29eb219c1247993378fce06c8f71aab20736c237 move only: move implementation code to disconnected_transactions.cpp (ismaelsadeeq)
81dfeddea70ae5feeaf79062585c2ff9f33c0ca3 refactor: update `MAX_DISCONNECTED_TX_POOL` from kb to bytes (ismaelsadeeq)

Pull request description:

  This PR is a follow-up to fix review comments and a bugfix from #28385

  The PR

  - Updated `DisconnectedBlockTransactions`'s `MAX_DISCONNECTED_TX_POOL` from kb to bytes.
  - Moved `DisconnectedBlockTransactions` implementation code to `kernel/disconnected_transactions.cpp`.
  - `AddTransactionsFromBlock` now assume duplicate transactions are not passed by asserting after inserting each transaction to `iters_by_txid`.
  - Included a Bug fix: In the current master we are underestimating the memory usage of `DisconnectedBlockTransactions`.

      * When adding and subtracting `cachedInnerUsage` we call `RecursiveDynamicUsage` with `CTransaction` which invokes this [`RecursiveDynamicUsage(const CTransaction& tx)`](https://github.com/bitcoin/bitcoin/blob/6e721c923c87abdb8d99674093d08d8c17bc52c2/src/core_memusage.h#L32) version of `RecursiveDynamicUsage`, the output of that call only account for the memory usage of the inputs and outputs of the `CTransaction`, this omits the memory usage of the `CTransaction` object and the control block.
      * This PR fixes this bug by calling `RecursiveDynamicUsage` with `CTransactionRef` when adding and subtracting `cachedInnerUsage` which invokes [`RecursiveDynamicUsage(const std::shared_ptr<X>& p)`](https://github.com/bitcoin/bitcoin/blob/6e721c923c87abdb8d99674093d08d8c17bc52c2/src/core_memusage.h#L67) version of `RecursiveDynamicUsage` the output of the calculation accounts for the` CTransaction` object, the control blocks, inputs and outputs memory usage.
      * see  [comment ](https://github.com/bitcoin/bitcoin/pull/28385#discussion_r1322948452)
  - Added test for DisconnectedBlockTransactions memory limit.

ACKs for top commit:
  stickies-v:
    ACK 9b3da70bd06b45482e7211aa95637a72bd115553 - nice work!
  BrandonOdiwuor:
    re ACK 9b3da70bd06b45482e7211aa95637a72bd115553
  glozow:
    ACK 9b3da70bd06b45482e7211aa95637a72bd115553

Tree-SHA512: 69b9595d09f4d0209038f97081d790cea92ccf63efb94e9e372749979fcbe527f7f17a8e454720cedd12021be0c8e11cf99874625d3dafd9ec602b12dbeb4098

2 files changed, 102 insertions, 73 deletions

diff --git a/src/kernel/disconnected_transactions.cpp b/src/kernel/disconnected_transactions.cpp
new file mode 100644
index 0000000000..f865fed688
--- /dev/null
+++ b/src/kernel/disconnected_transactions.cpp
@@ -0,0 +1,90 @@
+// Copyright (c) 2023 The Bitcoin Core developers
+// Distributed under the MIT software license, see the accompanying
+// file COPYING or http://www.opensource.org/licenses/mit-license.php.
+
+#include <kernel/disconnected_transactions.h>
+
+#include <assert.h>
+#include <core_memusage.h>
+#include <memusage.h>
+#include <primitives/transaction.h>
+#include <util/hasher.h>
+
+#include <memory>
+#include <utility>
+
+// It's almost certainly a logic bug if we don't clear out queuedTx before
+// destruction, as we add to it while disconnecting blocks, and then we
+// need to re-process remaining transactions to ensure mempool consistency.
+// For now, assert() that we've emptied out this object on destruction.
+// This assert() can always be removed if the reorg-processing code were
+// to be refactored such that this assumption is no longer true (for
+// instance if there was some other way we cleaned up the mempool after a
+// reorg, besides draining this object).
+DisconnectedBlockTransactions::~DisconnectedBlockTransactions()
+{
+    assert(queuedTx.empty());
+    assert(iters_by_txid.empty());
+    assert(cachedInnerUsage == 0);
+}
+
+std::vector<CTransactionRef> DisconnectedBlockTransactions::LimitMemoryUsage()
+{
+    std::vector<CTransactionRef> evicted;
+
+    while (!queuedTx.empty() && DynamicMemoryUsage() > m_max_mem_usage) {
+        evicted.emplace_back(queuedTx.front());
+        cachedInnerUsage -= RecursiveDynamicUsage(queuedTx.front());
+        iters_by_txid.erase(queuedTx.front()->GetHash());
+        queuedTx.pop_front();
+    }
+    return evicted;
+}
+
+size_t DisconnectedBlockTransactions::DynamicMemoryUsage() const
+{
+    return cachedInnerUsage + memusage::DynamicUsage(iters_by_txid) + memusage::DynamicUsage(queuedTx);
+}
+
+[[nodiscard]] std::vector<CTransactionRef> DisconnectedBlockTransactions::AddTransactionsFromBlock(const std::vector<CTransactionRef>& vtx)
+{
+    iters_by_txid.reserve(iters_by_txid.size() + vtx.size());
+    for (auto block_it = vtx.rbegin(); block_it != vtx.rend(); ++block_it) {
+        auto it = queuedTx.insert(queuedTx.end(), *block_it);
+        auto [_, inserted] = iters_by_txid.emplace((*block_it)->GetHash(), it);
+        assert(inserted); // callers may never pass multiple transactions with the same txid
+        cachedInnerUsage += RecursiveDynamicUsage(*block_it);
+    }
+    return LimitMemoryUsage();
+}
+
+void DisconnectedBlockTransactions::removeForBlock(const std::vector<CTransactionRef>& vtx)
+{
+    // Short-circuit in the common case of a block being added to the tip
+    if (queuedTx.empty()) {
+        return;
+    }
+    for (const auto& tx : vtx) {
+        auto iter = iters_by_txid.find(tx->GetHash());
+        if (iter != iters_by_txid.end()) {
+            auto list_iter = iter->second;
+            iters_by_txid.erase(iter);
+            cachedInnerUsage -= RecursiveDynamicUsage(*list_iter);
+            queuedTx.erase(list_iter);
+        }
+    }
+}
+
+void DisconnectedBlockTransactions::clear()
+{
+    cachedInnerUsage = 0;
+    iters_by_txid.clear();
+    queuedTx.clear();
+}
+
+std::list<CTransactionRef> DisconnectedBlockTransactions::take()
+{
+    std::list<CTransactionRef> ret = std::move(queuedTx);
+    clear();
+    return ret;
+}
diff --git a/src/kernel/disconnected_transactions.h b/src/kernel/disconnected_transactions.h
index 7db39ba5ca..401ec435e6 100644
--- a/src/kernel/disconnected_transactions.h
+++ b/src/kernel/disconnected_transactions.h
@@ -5,8 +5,6 @@
 #ifndef BITCOIN_KERNEL_DISCONNECTED_TRANSACTIONS_H
 #define BITCOIN_KERNEL_DISCONNECTED_TRANSACTIONS_H
 
-#include <core_memusage.h>
-#include <memusage.h>
 #include <primitives/transaction.h>
 #include <util/hasher.h>
 
@@ -14,8 +12,8 @@
 #include <unordered_map>
 #include <vector>
 
-/** Maximum kilobytes for transactions to store for processing during reorg */
-static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20'000;
+/** Maximum bytes for transactions to store for processing during reorg */
+static const unsigned int MAX_DISCONNECTED_TX_POOL_BYTES{20'000'000};
 /**
  * DisconnectedBlockTransactions
 
@@ -38,8 +36,7 @@ static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20'000;
  */
 class DisconnectedBlockTransactions {
 private:
-    /** Cached dynamic memory usage for the CTransactions (memory for the shared pointers is
-     * included in the container calculations). */
+    /** Cached dynamic memory usage for the `CTransactionRef`s */
     uint64_t cachedInnerUsage = 0;
     const size_t m_max_mem_usage;
     std::list<CTransactionRef> queuedTx;
@@ -47,39 +44,15 @@ private:
     std::unordered_map<uint256, TxList::iterator, SaltedTxidHasher> iters_by_txid;
 
     /** Trim the earliest-added entries until we are within memory bounds. */
-    std::vector<CTransactionRef> LimitMemoryUsage()
-    {
-        std::vector<CTransactionRef> evicted;
-
-        while (!queuedTx.empty() && DynamicMemoryUsage() > m_max_mem_usage) {
-            evicted.emplace_back(queuedTx.front());
-            cachedInnerUsage -= RecursiveDynamicUsage(*queuedTx.front());
-            iters_by_txid.erase(queuedTx.front()->GetHash());
-            queuedTx.pop_front();
-        }
-        return evicted;
-    }
+    std::vector<CTransactionRef> LimitMemoryUsage();
 
 public:
-    DisconnectedBlockTransactions(size_t max_mem_usage) : m_max_mem_usage{max_mem_usage} {}
+    DisconnectedBlockTransactions(size_t max_mem_usage)
+        : m_max_mem_usage{max_mem_usage} {}
 
-    // It's almost certainly a logic bug if we don't clear out queuedTx before
-    // destruction, as we add to it while disconnecting blocks, and then we
-    // need to re-process remaining transactions to ensure mempool consistency.
-    // For now, assert() that we've emptied out this object on destruction.
-    // This assert() can always be removed if the reorg-processing code were
-    // to be refactored such that this assumption is no longer true (for
-    // instance if there was some other way we cleaned up the mempool after a
-    // reorg, besides draining this object).
-    ~DisconnectedBlockTransactions() {
-        assert(queuedTx.empty());
-        assert(iters_by_txid.empty());
-        assert(cachedInnerUsage == 0);
-    }
+    ~DisconnectedBlockTransactions();
 
-    size_t DynamicMemoryUsage() const {
-        return cachedInnerUsage + memusage::DynamicUsage(iters_by_txid) + memusage::DynamicUsage(queuedTx);
-    }
+    size_t DynamicMemoryUsage() const;
 
     /** Add transactions from the block, iterating through vtx in reverse order. Callers should call
      * this function for blocks in descending order by block height.
@@ -88,50 +61,16 @@ public:
      * corresponding entry in iters_by_txid.
      * @returns vector of transactions that were evicted for size-limiting.
      */
-    [[nodiscard]] std::vector<CTransactionRef> AddTransactionsFromBlock(const std::vector<CTransactionRef>& vtx)
-    {
-        iters_by_txid.reserve(iters_by_txid.size() + vtx.size());
-        for (auto block_it = vtx.rbegin(); block_it != vtx.rend(); ++block_it) {
-            auto it = queuedTx.insert(queuedTx.end(), *block_it);
-            iters_by_txid.emplace((*block_it)->GetHash(), it);
-            cachedInnerUsage += RecursiveDynamicUsage(**block_it);
-        }
-        return LimitMemoryUsage();
-    }
+    [[nodiscard]] std::vector<CTransactionRef> AddTransactionsFromBlock(const std::vector<CTransactionRef>& vtx);
 
     /** Remove any entries that are in this block. */
-    void removeForBlock(const std::vector<CTransactionRef>& vtx)
-    {
-        // Short-circuit in the common case of a block being added to the tip
-        if (queuedTx.empty()) {
-            return;
-        }
-        for (const auto& tx : vtx) {
-            auto iter = iters_by_txid.find(tx->GetHash());
-            if (iter != iters_by_txid.end()) {
-                auto list_iter = iter->second;
-                iters_by_txid.erase(iter);
-                cachedInnerUsage -= RecursiveDynamicUsage(**list_iter);
-                queuedTx.erase(list_iter);
-            }
-        }
-    }
+    void removeForBlock(const std::vector<CTransactionRef>& vtx);
 
     size_t size() const { return queuedTx.size(); }
 
-    void clear()
-    {
-        cachedInnerUsage = 0;
-        iters_by_txid.clear();
-        queuedTx.clear();
-    }
+    void clear();
 
     /** Clear all data structures and return the list of transactions. */
-    std::list<CTransactionRef> take()
-    {
-        std::list<CTransactionRef> ret = std::move(queuedTx);
-        clear();
-        return ret;
-    }
+    std::list<CTransactionRef> take();
 };
 #endif // BITCOIN_KERNEL_DISCONNECTED_TRANSACTIONS_H