aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorJames O'Beirne <james.obeirne@gmail.com>2019-09-16 15:01:12 -0400
committerJames O'Beirne <james.obeirne@pm.me>2020-07-01 14:44:28 -0400
commitf36aaa6392fdbdac6891d92202d3efeff98754f4 (patch)
tree4cebab3fa566f6f9dc53228b92264f1cda6095b1
parentb223111da2e0e9ceccef75df8a20252b0094b7bc (diff)
Add CChainState::ResizeCoinsCaches
Also adds CCoinsViewCache::ReallocateCache() to attempt to free memory that the cacheCoins's allocator may be hanging onto when downsizing the cache. Adds `CChainState::m_coins{tip,db}_cache_size_bytes` data members so that we can reference cache size on a per-chainstate basis for flushing.
Diffstat
-rw-r--r--src/coins.cpp8
-rw-r--r--src/coins.h7
-rw-r--r--src/init.cpp4
-rw-r--r--src/test/util/setup_common.cpp2
-rw-r--r--src/test/validation_chainstatemanager_tests.cpp11
-rw-r--r--src/test/validation_flush_tests.cpp2
-rw-r--r--src/validation.cpp41
-rw-r--r--src/validation.h14
8 files changed, 73 insertions, 16 deletions
diff --git a/src/coins.cpp b/src/coins.cpp
index 7b76c13f98..5de2ed7810 100644
--- a/src/coins.cpp
+++ b/src/coins.cpp
@@ -245,6 +245,14 @@ bool CCoinsViewCache::HaveInputs(const CTransaction& tx) const
return true;
}
+void CCoinsViewCache::ReallocateCache()
+{
+ // Cache should be empty when we're calling this.
+ assert(cacheCoins.size() == 0);
+ cacheCoins.~CCoinsMap();
+ ::new (&cacheCoins) CCoinsMap();
+}
+
static const size_t MIN_TRANSACTION_OUTPUT_WEIGHT = WITNESS_SCALE_FACTOR * ::GetSerializeSize(CTxOut(), PROTOCOL_VERSION);
static const size_t MAX_OUTPUTS_PER_BLOCK = MAX_BLOCK_WEIGHT / MIN_TRANSACTION_OUTPUT_WEIGHT;
diff --git a/src/coins.h b/src/coins.h
index a3f34bb0ee..a3e241ac90 100644
--- a/src/coins.h
+++ b/src/coins.h
@@ -318,6 +318,13 @@ public:
//! Check whether all prevouts of the transaction are present in the UTXO set represented by this view
bool HaveInputs(const CTransaction& tx) const;
+ //! Force a reallocation of the cache map. This is required when downsizing
+ //! the cache because the map's allocator may be hanging onto a lot of
+ //! memory despite having called .clear().
+ //!
+ //! See: https://stackoverflow.com/questions/42114044/how-to-release-unordered-map-memory
+ void ReallocateCache();
+
private:
/**
* @note this is marked const, but may actually append to `cacheCoins`, increasing
diff --git a/src/init.cpp b/src/init.cpp
index 37e6251295..4561a10870 100644
--- a/src/init.cpp
+++ b/src/init.cpp
@@ -1533,7 +1533,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node)
int64_t nCoinDBCache = std::min(nTotalCache / 2, (nTotalCache / 4) + (1 << 23)); // use 25%-50% of the remainder for disk cache
nCoinDBCache = std::min(nCoinDBCache, nMaxCoinsDBCache << 20); // cap total coins db cache
nTotalCache -= nCoinDBCache;
- nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache
+ int64_t nCoinCacheUsage = nTotalCache; // the rest goes to in-memory cache
int64_t nMempoolSizeMax = gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000;
LogPrintf("Cache configuration:\n");
LogPrintf("* Using %.1f MiB for block index database\n", nBlockTreeDBCache * (1.0 / 1024 / 1024));
@@ -1645,7 +1645,7 @@ bool AppInitMain(const util::Ref& context, NodeContext& node)
}
// The on-disk coinsdb is now in a good state, create the cache
- chainstate->InitCoinsCache();
+ chainstate->InitCoinsCache(nCoinCacheUsage);
assert(chainstate->CanFlushToDisk());
if (!is_coinsview_empty(chainstate)) {
diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp
index 3b7a7c8d12..83916fd080 100644
--- a/src/test/util/setup_common.cpp
+++ b/src/test/util/setup_common.cpp
@@ -139,7 +139,7 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const
::ChainstateActive().InitCoinsDB(
/* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false);
assert(!::ChainstateActive().CanFlushToDisk());
- ::ChainstateActive().InitCoinsCache();
+ ::ChainstateActive().InitCoinsCache(1 << 23);
assert(::ChainstateActive().CanFlushToDisk());
if (!LoadGenesisBlock(chainparams)) {
throw std::runtime_error("LoadGenesisBlock failed.");
diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp
index 0d149285ad..d16633c149 100644
--- a/src/test/validation_chainstatemanager_tests.cpp
+++ b/src/test/validation_chainstatemanager_tests.cpp
@@ -28,13 +28,11 @@ BOOST_AUTO_TEST_CASE(chainstatemanager)
// Create a legacy (IBD) chainstate.
//
- ENTER_CRITICAL_SECTION(cs_main);
- CChainState& c1 = manager.InitializeChainstate();
- LEAVE_CRITICAL_SECTION(cs_main);
+ CChainState& c1 = *WITH_LOCK(::cs_main, return &manager.InitializeChainstate());
chainstates.push_back(&c1);
c1.InitCoinsDB(
/* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false);
- WITH_LOCK(::cs_main, c1.InitCoinsCache());
+ WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23));
BOOST_CHECK(!manager.IsSnapshotActive());
BOOST_CHECK(!manager.IsSnapshotValidated());
@@ -57,12 +55,13 @@ BOOST_AUTO_TEST_CASE(chainstatemanager)
// Create a snapshot-based chainstate.
//
ENTER_CRITICAL_SECTION(cs_main);
- CChainState& c2 = manager.InitializeChainstate(GetRandHash());
+ CChainState& c2 = *WITH_LOCK(::cs_main,
+ return &manager.InitializeChainstate(GetRandHash()));
LEAVE_CRITICAL_SECTION(cs_main);
chainstates.push_back(&c2);
c2.InitCoinsDB(
/* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false);
- WITH_LOCK(::cs_main, c2.InitCoinsCache());
+ WITH_LOCK(::cs_main, c2.InitCoinsCache(1 << 23));
// Unlike c1, which doesn't have any blocks. Gets us different tip, height.
c2.LoadGenesisBlock(chainparams);
BlockValidationState _;
diff --git a/src/test/validation_flush_tests.cpp b/src/test/validation_flush_tests.cpp
index a863e3a4d5..7111fc3380 100644
--- a/src/test/validation_flush_tests.cpp
+++ b/src/test/validation_flush_tests.cpp
@@ -21,7 +21,7 @@ BOOST_AUTO_TEST_CASE(getcoinscachesizestate)
BlockManager blockman{};
CChainState chainstate{blockman};
chainstate.InitCoinsDB(/*cache_size_bytes*/ 1 << 10, /*in_memory*/ true, /*should_wipe*/ false);
- WITH_LOCK(::cs_main, chainstate.InitCoinsCache());
+ WITH_LOCK(::cs_main, chainstate.InitCoinsCache(1 << 10));
CTxMemPool tx_pool{};
constexpr bool is_64_bit = sizeof(void*) == 8;
diff --git a/src/validation.cpp b/src/validation.cpp
index dbdf5028fd..1d7d85b751 100644
--- a/src/validation.cpp
+++ b/src/validation.cpp
@@ -135,7 +135,6 @@ bool fPruneMode = false;
bool fRequireStandard = true;
bool fCheckBlockIndex = false;
bool fCheckpointsEnabled = DEFAULT_CHECKPOINTS_ENABLED;
-size_t nCoinCacheUsage = 5000 * 300;
uint64_t nPruneTarget = 0;
int64_t nMaxTipAge = DEFAULT_MAX_TIP_AGE;
@@ -1279,9 +1278,10 @@ void CChainState::InitCoinsDB(
leveldb_name, cache_size_bytes, in_memory, should_wipe);
}
-void CChainState::InitCoinsCache()
+void CChainState::InitCoinsCache(size_t cache_size_bytes)
{
assert(m_coins_views != nullptr);
+ m_coinstip_cache_size_bytes = cache_size_bytes;
m_coins_views->InitCache();
}
@@ -2228,7 +2228,7 @@ CoinsCacheSizeState CChainState::GetCoinsCacheSizeState(const CTxMemPool& tx_poo
{
return this->GetCoinsCacheSizeState(
tx_pool,
- nCoinCacheUsage,
+ m_coinstip_cache_size_bytes,
gArgs.GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000);
}
@@ -4300,7 +4300,7 @@ bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview,
}
}
// check level 3: check for inconsistencies during memory-only disconnect of tip blocks
- if (nCheckLevel >= 3 && (coins.DynamicMemoryUsage() + ::ChainstateActive().CoinsTip().DynamicMemoryUsage()) <= nCoinCacheUsage) {
+ if (nCheckLevel >= 3 && (coins.DynamicMemoryUsage() + ::ChainstateActive().CoinsTip().DynamicMemoryUsage()) <= ::ChainstateActive().m_coinstip_cache_size_bytes) {
assert(coins.GetBestBlock() == pindex->GetBlockHash());
DisconnectResult res = ::ChainstateActive().DisconnectBlock(block, pindex, coins);
if (res == DISCONNECT_FAILED) {
@@ -4965,6 +4965,39 @@ std::string CChainState::ToString()
tip ? tip->nHeight : -1, tip ? tip->GetBlockHash().ToString() : "null");
}
+bool CChainState::ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size)
+{
+ if (coinstip_size == m_coinstip_cache_size_bytes &&
+ coinsdb_size == m_coinsdb_cache_size_bytes) {
+ // Cache sizes are unchanged, no need to continue.
+ return true;
+ }
+ size_t old_coinstip_size = m_coinstip_cache_size_bytes;
+ m_coinstip_cache_size_bytes = coinstip_size;
+ m_coinsdb_cache_size_bytes = coinsdb_size;
+ CoinsDB().ResizeCache(coinsdb_size);
+
+ LogPrintf("[%s] resized coinsdb cache to %.1f MiB\n",
+ this->ToString(), coinsdb_size * (1.0 / 1024 / 1024));
+ LogPrintf("[%s] resized coinstip cache to %.1f MiB\n",
+ this->ToString(), coinstip_size * (1.0 / 1024 / 1024));
+
+ BlockValidationState state;
+ const CChainParams& chainparams = Params();
+
+ bool ret;
+
+ if (coinstip_size > old_coinstip_size) {
+ // Likely no need to flush if cache sizes have grown.
+ ret = FlushStateToDisk(chainparams, state, FlushStateMode::IF_NEEDED);
+ } else {
+ // Otherwise, flush state to disk and deallocate the in-memory coins map.
+ ret = FlushStateToDisk(chainparams, state, FlushStateMode::ALWAYS);
+ CoinsTip().ReallocateCache();
+ }
+ return ret;
+}
+
std::string CBlockFileInfo::ToString() const
{
return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast));
diff --git a/src/validation.h b/src/validation.h
index 8112e38704..b201403585 100644
--- a/src/validation.h
+++ b/src/validation.h
@@ -127,7 +127,6 @@ extern bool g_parallel_script_checks;
extern bool fRequireStandard;
extern bool fCheckBlockIndex;
extern bool fCheckpointsEnabled;
-extern size_t nCoinCacheUsage;
/** A fee rate smaller than this is considered zero fee (for relaying, mining and transaction creation) */
extern CFeeRate minRelayTxFee;
/** If the tip is older than this (in seconds), the node is considered to be in initial block download. */
@@ -519,7 +518,7 @@ public:
//! Initialize the in-memory coins cache (to be done after the health of the on-disk database
//! is verified).
- void InitCoinsCache() EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
+ void InitCoinsCache(size_t cache_size_bytes) EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
//! @returns whether or not the CoinsViews object has been fully initialized and we can
//! safely flush this object to disk.
@@ -568,6 +567,17 @@ public:
//! Destructs all objects related to accessing the UTXO set.
void ResetCoinsViews() { m_coins_views.reset(); }
+ //! The cache size of the on-disk coins view.
+ size_t m_coinsdb_cache_size_bytes{0};
+
+ //! The cache size of the in-memory coins view.
+ size_t m_coinstip_cache_size_bytes{0};
+
+ //! Resize the CoinsViews caches dynamically and flush state to disk.
+ //! @returns true unless an error occurred during the flush.
+ bool ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size)
+ EXCLUSIVE_LOCKS_REQUIRED(::cs_main);
+
/**
* Update the on-disk chain state.
* The caches and indexes are flushed depending on the mode we're called with
generated by cgit v1.2.3 (git 2.26.2) at 2025-03-21 07:52:30 +0000