aboutsummaryrefslogtreecommitdiff
path: root/src
diff options
context:
space:
mode:
Diffstat (limited to 'src')
-rw-r--r--src/chainparams.cpp3
-rw-r--r--src/chainparams.h2
-rw-r--r--src/init.cpp90
-rw-r--r--src/main.cpp278
-rw-r--r--src/main.h45
5 files changed, 383 insertions, 35 deletions
diff --git a/src/chainparams.cpp b/src/chainparams.cpp
index 2d4fde2476..589c7b5472 100644
--- a/src/chainparams.cpp
+++ b/src/chainparams.cpp
@@ -121,6 +121,7 @@ public:
vAlertPubKey = ParseHex("04fc9702847840aaf195de8442ebecedf5b095cdbb9bc716bda9110971b28a49e0ead8564ff0db22209e0374782c093bb899692d524e9d6a6956e7c5ecbcd68284");
nDefaultPort = 8333;
nMinerThreads = 0;
+ nPruneAfterHeight = 100000;
/**
* Build the genesis block. Note that the output of the genesis coinbase cannot
@@ -198,6 +199,7 @@ public:
vAlertPubKey = ParseHex("04302390343f91cc401d56d68b123028bf52e5fca1939df127f63c6467cdf9c8e2c14b61104cf817d0b780da337893ecc4aaff1309e536162dabbdb45200ca2b0a");
nDefaultPort = 18333;
nMinerThreads = 0;
+ nPruneAfterHeight = 1000;
//! Modify the testnet genesis block so the timestamp is valid for a later start.
genesis.nTime = 1296688602;
@@ -257,6 +259,7 @@ public:
consensus.hashGenesisBlock = genesis.GetHash();
nDefaultPort = 18444;
assert(consensus.hashGenesisBlock == uint256S("0x0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206"));
+ nPruneAfterHeight = 1000;
vFixedSeeds.clear(); //! Regtest mode doesn't have any fixed seeds.
vSeeds.clear(); //! Regtest mode doesn't have any DNS seeds.
diff --git a/src/chainparams.h b/src/chainparams.h
index 0692094478..5e974123dc 100644
--- a/src/chainparams.h
+++ b/src/chainparams.h
@@ -58,6 +58,7 @@ public:
bool DefaultConsistencyChecks() const { return fDefaultConsistencyChecks; }
/** Make standard checks */
bool RequireStandard() const { return fRequireStandard; }
+ int64_t PruneAfterHeight() const { return nPruneAfterHeight; }
/** Make miner stop after a block is found. In RPC, don't return until nGenProcLimit blocks are generated */
bool MineBlocksOnDemand() const { return fMineBlocksOnDemand; }
/** In the future use NetworkIDString() for RPC fields */
@@ -77,6 +78,7 @@ protected:
std::vector<unsigned char> vAlertPubKey;
int nDefaultPort;
int nMinerThreads;
+ uint64_t nPruneAfterHeight;
std::vector<CDNSSeedData> vSeeds;
std::vector<unsigned char> base58Prefixes[MAX_BASE58_TYPES];
std::string strNetworkID;
diff --git a/src/init.cpp b/src/init.cpp
index 665aebe1c5..b648a237bf 100644
--- a/src/init.cpp
+++ b/src/init.cpp
@@ -275,7 +275,12 @@ std::string HelpMessage(HelpMessageMode mode)
#ifndef WIN32
strUsage += HelpMessageOpt("-pid=<file>", strprintf(_("Specify pid file (default: %s)"), "bitcoind.pid"));
#endif
- strUsage += HelpMessageOpt("-reindex", _("Rebuild block chain index from current blk000??.dat files") + " " + _("on startup"));
+ strUsage += HelpMessageOpt("-prune=<n>", _("Reduce storage requirements by pruning (deleting) old blocks. This mode disables wallet support and is incompatible with -txindex.") + " " +
+ _("Warning: Reverting this setting requires re-downloading the entire blockchain.") + " " +
+ _("(default: 0 = disable pruning blocks,") + " " +
+ strprintf(_(">%u = target size in MiB to use for block files)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
+strUsage += HelpMessageOpt("-reindex", _("Rebuild block chain index from current blk000??.dat files") + " " + _("on startup"));
+
#if !defined(WIN32)
strUsage += HelpMessageOpt("-sysperms", _("Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)"));
#endif
@@ -352,7 +357,7 @@ std::string HelpMessage(HelpMessageMode mode)
strUsage += HelpMessageOpt("-flushwallet", strprintf(_("Run a thread to flush wallet periodically (default: %u)"), 1));
strUsage += HelpMessageOpt("-stopafterblockimport", strprintf(_("Stop running after importing blocks from disk (default: %u)"), 0));
}
- string debugCategories = "addrman, alert, bench, coindb, db, lock, rand, rpc, selectcoins, mempool, net, proxy"; // Don't translate these and qt below
+ string debugCategories = "addrman, alert, bench, coindb, db, lock, rand, rpc, selectcoins, mempool, net, proxy, prune"; // Don't translate these and qt below
if (mode == HMM_BITCOIN_QT)
debugCategories += ", qt";
strUsage += HelpMessageOpt("-debug=<category>", strprintf(_("Output debugging information (default: %u, supplying <category> is optional)"), 0) + ". " +
@@ -458,10 +463,33 @@ struct CImportingNow
}
};
+
+// If we're using -prune with -reindex, then delete block files that will be ignored by the
+// reindex. Since reindexing works by starting at block file 0 and looping until a blockfile
+// is missing, and since pruning works by deleting the oldest block file first, just check
+// for block file 0, and if it doesn't exist, delete all the block files in the
+// directory (since they won't be read by the reindex but will take up disk space).
+void DeleteAllBlockFiles()
+{
+ if (boost::filesystem::exists(GetBlockPosFilename(CDiskBlockPos(0, 0), "blk")))
+ return;
+
+ LogPrintf("Removing all blk?????.dat and rev?????.dat files for -reindex with -prune\n");
+ boost::filesystem::path blocksdir = GetDataDir() / "blocks";
+ for (boost::filesystem::directory_iterator it(blocksdir); it != boost::filesystem::directory_iterator(); it++) {
+ if (is_regular_file(*it)) {
+ if ((it->path().filename().string().length() == 12) &&
+ (it->path().filename().string().substr(8,4) == ".dat") &&
+ ((it->path().filename().string().substr(0,3) == "blk") ||
+ (it->path().filename().string().substr(0,3) == "rev")))
+ boost::filesystem::remove(it->path());
+ }
+ }
+}
+
void ThreadImport(std::vector<boost::filesystem::path> vImportFiles)
{
RenameThread("bitcoin-loadblk");
-
// -reindex
if (fReindex) {
CImportingNow imp;
@@ -674,6 +702,21 @@ bool AppInit2(boost::thread_group& threadGroup)
if (nFD - MIN_CORE_FILEDESCRIPTORS < nMaxConnections)
nMaxConnections = nFD - MIN_CORE_FILEDESCRIPTORS;
+ // if using block pruning, then disable txindex
+ // also disable the wallet (for now, until SPV support is implemented in wallet)
+ if (GetArg("-prune", 0)) {
+ if (GetBoolArg("-txindex", false))
+ return InitError(_("Prune mode is incompatible with -txindex."));
+#ifdef ENABLE_WALLET
+ if (!GetBoolArg("-disablewallet", false)) {
+ if (SoftSetBoolArg("-disablewallet", true))
+ LogPrintf("%s : parameter interaction: -prune -> setting -disablewallet=1\n", __func__);
+ else
+ return InitError(_("Can't run with a wallet in prune mode."));
+ }
+#endif
+ }
+
// ********************************************************* Step 3: parameter-to-internal-flags
fDebug = !mapMultiArgs["-debug"].empty();
@@ -710,6 +753,21 @@ bool AppInit2(boost::thread_group& threadGroup)
nScriptCheckThreads = MAX_SCRIPTCHECK_THREADS;
fServer = GetBoolArg("-server", false);
+
+ // block pruning; get the amount of disk space (in MB) to allot for block & undo files
+ int64_t nSignedPruneTarget = GetArg("-prune", 0) * 1024 * 1024;
+ if (nSignedPruneTarget < 0) {
+ return InitError(_("Prune cannot be configured with a negative value."));
+ }
+ nPruneTarget = (uint64_t) nSignedPruneTarget;
+ if (nPruneTarget) {
+ if (nPruneTarget < MIN_DISK_SPACE_FOR_BLOCK_FILES) {
+ return InitError(strprintf(_("Prune configured below the minimum of %d MB. Please use a higher number."), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024));
+ }
+ LogPrintf("Prune configured to target %uMiB on disk for block and undo files.\n", nPruneTarget / 1024 / 1024);
+ fPruneMode = true;
+ }
+
#ifdef ENABLE_WALLET
bool fDisableWallet = GetBoolArg("-disablewallet", false);
#endif
@@ -1030,8 +1088,12 @@ bool AppInit2(boost::thread_group& threadGroup)
pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview);
pcoinsTip = new CCoinsViewCache(pcoinscatcher);
- if (fReindex)
+ if (fReindex) {
pblocktree->WriteReindexing(true);
+ //If we're reindexing in prune mode, wipe away all our block and undo data files
+ if (fPruneMode)
+ DeleteAllBlockFiles();
+ }
if (!LoadBlockIndex()) {
strLoadError = _("Error loading block database");
@@ -1055,7 +1117,18 @@ bool AppInit2(boost::thread_group& threadGroup)
break;
}
+ // Check for changed -prune state. What we are concerned about is a user who has pruned blocks
+ // in the past, but is now trying to run unpruned.
+ if (fHavePruned && !fPruneMode) {
+ strLoadError = _("You need to rebuild the database using -reindex to go back to unpruned mode. This will redownload the entire blockchain");
+ break;
+ }
+
uiInterface.InitMessage(_("Verifying blocks..."));
+ if (fHavePruned && GetArg("-checkblocks", 288) > MIN_BLOCKS_TO_KEEP) {
+ LogPrintf("Prune: pruned datadir may not have more than %d blocks; -checkblocks=%d may fail\n",
+ MIN_BLOCKS_TO_KEEP, GetArg("-checkblocks", 288));
+ }
if (!CVerifyDB().VerifyDB(pcoinsdbview, GetArg("-checklevel", 3),
GetArg("-checkblocks", 288))) {
strLoadError = _("Corrupted block database detected");
@@ -1106,6 +1179,15 @@ bool AppInit2(boost::thread_group& threadGroup)
mempool.ReadFeeEstimates(est_filein);
fFeeEstimatesInitialized = true;
+ // if prune mode, unset NODE_NETWORK and prune block files
+ if (fPruneMode) {
+ LogPrintf("Unsetting NODE_NETWORK on prune mode\n");
+ nLocalServices &= ~NODE_NETWORK;
+ if (!fReindex) {
+ PruneAndFlush();
+ }
+ }
+
// ********************************************************* Step 8: load wallet
#ifdef ENABLE_WALLET
if (fDisableWallet) {
diff --git a/src/main.cpp b/src/main.cpp
index 4352c719a1..bf32ac91e7 100644
--- a/src/main.cpp
+++ b/src/main.cpp
@@ -52,9 +52,12 @@ int nScriptCheckThreads = 0;
bool fImporting = false;
bool fReindex = false;
bool fTxIndex = false;
+bool fHavePruned = false;
+bool fPruneMode = false;
bool fIsBareMultisigStd = true;
bool fCheckBlockIndex = false;
unsigned int nCoinCacheSize = 5000;
+uint64_t nPruneTarget = 0;
/** Fees smaller than this (in satoshi) are considered zero fee (for relaying and mining) */
CFeeRate minRelayTxFee = CFeeRate(1000);
@@ -110,17 +113,25 @@ namespace {
/**
* The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself and all ancestors) and
- * as good as our current tip or better. Entries may be failed, though.
+ * as good as our current tip or better. Entries may be failed, though, and pruning nodes may be
+ * missing the data for the block.
*/
set<CBlockIndex*, CBlockIndexWorkComparator> setBlockIndexCandidates;
/** Number of nodes with fSyncStarted. */
int nSyncStarted = 0;
- /** All pairs A->B, where A (or one if its ancestors) misses transactions, but B has transactions. */
+ /** All pairs A->B, where A (or one if its ancestors) misses transactions, but B has transactions.
+ * Pruned nodes may have entries where B is missing data.
+ */
multimap<CBlockIndex*, CBlockIndex*> mapBlocksUnlinked;
CCriticalSection cs_LastBlockFile;
std::vector<CBlockFileInfo> vinfoBlockFile;
int nLastBlockFile = 0;
+ /** Global flag to indicate we should check to see if there are
+ * block/undo files that should be deleted. Set on startup
+ * or if we allocate more file space when we're in prune mode
+ */
+ bool fCheckForPruning = false;
/**
* Every received block is assigned a unique and increasing identifier, so we
@@ -1849,6 +1860,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin
}
enum FlushStateMode {
+ FLUSH_STATE_NONE,
FLUSH_STATE_IF_NEEDED,
FLUSH_STATE_PERIODIC,
FLUSH_STATE_ALWAYS
@@ -1856,16 +1868,30 @@ enum FlushStateMode {
/**
* Update the on-disk chain state.
- * The caches and indexes are flushed if either they're too large, forceWrite is set, or
- * fast is not set and it's been a while since the last write.
+ * The caches and indexes are flushed depending on the mode we're called with
+ * if they're too large, if it's been a while since the last write,
+ * or always and in all cases if we're in prune mode and are deleting files.
*/
bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) {
- LOCK(cs_main);
+ LOCK2(cs_main, cs_LastBlockFile);
static int64_t nLastWrite = 0;
+ std::set<int> setFilesToPrune;
+ bool fFlushForPrune = false;
try {
+ if (fPruneMode && fCheckForPruning) {
+ FindFilesToPrune(setFilesToPrune);
+ if (!setFilesToPrune.empty()) {
+ fFlushForPrune = true;
+ if (!fHavePruned) {
+ pblocktree->WriteFlag("prunedblockfiles", true);
+ fHavePruned = true;
+ }
+ }
+ }
if ((mode == FLUSH_STATE_ALWAYS) ||
((mode == FLUSH_STATE_PERIODIC || mode == FLUSH_STATE_IF_NEEDED) && pcoinsTip->GetCacheSize() > nCoinCacheSize) ||
- (mode == FLUSH_STATE_PERIODIC && GetTimeMicros() > nLastWrite + DATABASE_WRITE_INTERVAL * 1000000)) {
+ (mode == FLUSH_STATE_PERIODIC && GetTimeMicros() > nLastWrite + DATABASE_WRITE_INTERVAL * 1000000) ||
+ fFlushForPrune) {
// Typical CCoins structures on disk are around 100 bytes in size.
// Pushing a new one to the database can cause it to be written
// twice (once in the log, and once in the tables). This is already
@@ -1893,9 +1919,16 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) {
return state.Abort("Files to write to block index database");
}
}
- // Finally flush the chainstate (which may refer to block index entries).
+ // Flush the chainstate (which may refer to block index entries).
if (!pcoinsTip->Flush())
return state.Abort("Failed to write to coin database");
+
+ // Finally remove any pruned files
+ if (fFlushForPrune) {
+ UnlinkPrunedFiles(setFilesToPrune);
+ fCheckForPruning = false;
+ }
+
// Update best block in wallet (so we can detect restored wallets).
if (mode != FLUSH_STATE_IF_NEEDED) {
GetMainSignals().SetBestChain(chainActive.GetLocator());
@@ -1913,6 +1946,12 @@ void FlushStateToDisk() {
FlushStateToDisk(state, FLUSH_STATE_ALWAYS);
}
+void PruneAndFlush() {
+ CValidationState state;
+ fCheckForPruning = true;
+ FlushStateToDisk(state, FLUSH_STATE_NONE);
+}
+
/** Update chainActive and related internal data structures. */
void static UpdateTip(CBlockIndex *pindexNew) {
chainActive.SetTip(pindexNew);
@@ -2083,15 +2122,29 @@ static CBlockIndex* FindMostWorkChain() {
CBlockIndex *pindexTest = pindexNew;
bool fInvalidAncestor = false;
while (pindexTest && !chainActive.Contains(pindexTest)) {
- assert(pindexTest->nStatus & BLOCK_HAVE_DATA);
assert(pindexTest->nChainTx || pindexTest->nHeight == 0);
- if (pindexTest->nStatus & BLOCK_FAILED_MASK) {
- // Candidate has an invalid ancestor, remove entire chain from the set.
- if (pindexBestInvalid == NULL || pindexNew->nChainWork > pindexBestInvalid->nChainWork)
+
+ // Pruned nodes may have entries in setBlockIndexCandidates for
+ // which block files have been deleted. Remove those as candidates
+ // for the most work chain if we come across them; we can't switch
+ // to a chain unless we have all the non-active-chain parent blocks.
+ bool fFailedChain = pindexTest->nStatus & BLOCK_FAILED_MASK;
+ bool fMissingData = !(pindexTest->nStatus & BLOCK_HAVE_DATA);
+ if (fFailedChain || fMissingData) {
+ // Candidate chain is not usable (either invalid or missing data)
+ if (fFailedChain && (pindexBestInvalid == NULL || pindexNew->nChainWork > pindexBestInvalid->nChainWork))
pindexBestInvalid = pindexNew;
CBlockIndex *pindexFailed = pindexNew;
+ // Remove the entire chain from the set.
while (pindexTest != pindexFailed) {
- pindexFailed->nStatus |= BLOCK_FAILED_CHILD;
+ if (fFailedChain) {
+ pindexFailed->nStatus |= BLOCK_FAILED_CHILD;
+ } else if (fMissingData) {
+ // If we're missing data, then add back to mapBlocksUnlinked,
+ // so that if the block arrives in the future we can try adding
+ // to setBlockIndexCandidates again.
+ mapBlocksUnlinked.insert(std::make_pair(pindexFailed->pprev, pindexFailed));
+ }
setBlockIndexCandidates.erase(pindexFailed);
pindexFailed = pindexFailed->pprev;
}
@@ -2219,7 +2272,9 @@ bool ActivateBestChain(CValidationState &state, CBlock *pblock) {
uint256 hashNewTip = pindexNewTip->GetBlockHash();
// Relay inventory, but don't relay old inventory during initial block download.
int nBlockEstimate = Checkpoints::GetTotalBlocksEstimate();
- {
+ // Don't relay blocks if pruning -- could cause a peer to try to download, resulting
+ // in a stalled download if the block file is pruned before the request.
+ if (nLocalServices & NODE_NETWORK) {
LOCK(cs_vNodes);
BOOST_FOREACH(CNode* pnode, vNodes)
if (chainActive.Height() > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : nBlockEstimate))
@@ -2419,6 +2474,8 @@ bool FindBlockPos(CValidationState &state, CDiskBlockPos &pos, unsigned int nAdd
unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE;
unsigned int nNewChunks = (vinfoBlockFile[nFile].nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE;
if (nNewChunks > nOldChunks) {
+ if (fPruneMode)
+ fCheckForPruning = true;
if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos)) {
FILE *file = OpenBlockFile(pos);
if (file) {
@@ -2450,6 +2507,8 @@ bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigne
unsigned int nOldChunks = (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE;
unsigned int nNewChunks = (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE;
if (nNewChunks > nOldChunks) {
+ if (fPruneMode)
+ fCheckForPruning = true;
if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos)) {
FILE *file = OpenUndoFile(pos);
if (file) {
@@ -2665,7 +2724,10 @@ bool AcceptBlock(CBlock& block, CValidationState& state, CBlockIndex** ppindex,
if (!AcceptBlockHeader(block, state, &pindex))
return false;
- if (pindex->nStatus & BLOCK_HAVE_DATA) {
+ // If we're pruning, ensure that we don't allow a peer to dump a copy
+ // of old blocks. But we might need blocks that are not on the main chain
+ // to handle a reorg, even if we've processed once.
+ if (pindex->nStatus & BLOCK_HAVE_DATA || chainActive.Contains(pindex)) {
// TODO: deal better with duplicate blocks.
// return state.DoS(20, error("AcceptBlock(): already have block %d %s", pindex->nHeight, pindex->GetBlockHash().ToString()), REJECT_DUPLICATE, "duplicate");
return true;
@@ -2698,6 +2760,9 @@ bool AcceptBlock(CBlock& block, CValidationState& state, CBlockIndex** ppindex,
return state.Abort(std::string("System error: ") + e.what());
}
+ if (fCheckForPruning)
+ FlushStateToDisk(state, FLUSH_STATE_NONE); // we just allocated more disk space for block files
+
return true;
}
@@ -2785,6 +2850,112 @@ bool AbortNode(const std::string &strMessage, const std::string &userMessage) {
return false;
}
+
+/**
+ * BLOCK PRUNING CODE
+ */
+
+/* Calculate the amount of disk space the block & undo files currently use */
+uint64_t CalculateCurrentUsage()
+{
+ uint64_t retval = 0;
+ BOOST_FOREACH(const CBlockFileInfo &file, vinfoBlockFile) {
+ retval += file.nSize + file.nUndoSize;
+ }
+ return retval;
+}
+
+/* Prune a block file (modify associated database entries)*/
+void PruneOneBlockFile(const int fileNumber)
+{
+ for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); ++it) {
+ CBlockIndex* pindex = it->second;
+ if (pindex->nFile == fileNumber) {
+ pindex->nStatus &= ~BLOCK_HAVE_DATA;
+ pindex->nStatus &= ~BLOCK_HAVE_UNDO;
+ pindex->nFile = 0;
+ pindex->nDataPos = 0;
+ pindex->nUndoPos = 0;
+ setDirtyBlockIndex.insert(pindex);
+
+ // Prune from mapBlocksUnlinked -- any block we prune would have
+ // to be downloaded again in order to consider its chain, at which
+ // point it would be considered as a candidate for
+ // mapBlocksUnlinked or setBlockIndexCandidates.
+ std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> range = mapBlocksUnlinked.equal_range(pindex->pprev);
+ while (range.first != range.second) {
+ std::multimap<CBlockIndex *, CBlockIndex *>::iterator it = range.first;
+ range.first++;
+ if (it->second == pindex) {
+ mapBlocksUnlinked.erase(it);
+ }
+ }
+ }
+ }
+
+ vinfoBlockFile[fileNumber].SetNull();
+ setDirtyFileInfo.insert(fileNumber);
+}
+
+
+void UnlinkPrunedFiles(std::set<int>& setFilesToPrune)
+{
+ for (set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) {
+ CDiskBlockPos pos(*it, 0);
+ boost::filesystem::remove(GetBlockPosFilename(pos, "blk"));
+ boost::filesystem::remove(GetBlockPosFilename(pos, "rev"));
+ LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it);
+ }
+}
+
+/* Calculate the block/rev files that should be deleted to remain under target*/
+void FindFilesToPrune(std::set<int>& setFilesToPrune)
+{
+ LOCK2(cs_main, cs_LastBlockFile);
+ if (chainActive.Tip() == NULL || nPruneTarget == 0) {
+ return;
+ }
+ if (chainActive.Tip()->nHeight <= Params().PruneAfterHeight()) {
+ return;
+ }
+
+ unsigned int nLastBlockWeMustKeep = chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP;
+ uint64_t nCurrentUsage = CalculateCurrentUsage();
+ // We don't check to prune until after we've allocated new space for files
+ // So we should leave a buffer under our target to account for another allocation
+ // before the next pruning.
+ uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE;
+ uint64_t nBytesToPrune;
+ int count=0;
+
+ if (nCurrentUsage + nBuffer >= nPruneTarget) {
+ for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) {
+ nBytesToPrune = vinfoBlockFile[fileNumber].nSize + vinfoBlockFile[fileNumber].nUndoSize;
+
+ if (vinfoBlockFile[fileNumber].nSize == 0)
+ continue;
+
+ if (nCurrentUsage + nBuffer < nPruneTarget) // are we below our target?
+ break;
+
+ // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip
+ if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeMustKeep)
+ break;
+
+ PruneOneBlockFile(fileNumber);
+ // Queue up the files for removal
+ setFilesToPrune.insert(fileNumber);
+ nCurrentUsage -= nBytesToPrune;
+ count++;
+ }
+ }
+
+ LogPrint("prune", "Prune: target=%dMiB actual=%dMiB diff=%dMiB min_must_keep=%d removed %d blk/rev pairs\n",
+ nPruneTarget/1024/1024, nCurrentUsage/1024/1024,
+ ((int64_t)nPruneTarget - (int64_t)nCurrentUsage)/1024/1024,
+ nLastBlockWeMustKeep, count);
+}
+
bool CheckDiskSpace(uint64_t nAdditionalBytes)
{
uint64_t nFreeBytesAvailable = boost::filesystem::space(GetDataDir()).available;
@@ -2872,7 +3043,9 @@ bool static LoadBlockIndexDB()
{
CBlockIndex* pindex = item.second;
pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex);
- if (pindex->nStatus & BLOCK_HAVE_DATA) {
+ // We can link the chain of blocks for which we've received transactions at some point.
+ // Pruned nodes may have deleted the block.
+ if (pindex->nTx > 0) {
if (pindex->pprev) {
if (pindex->pprev->nChainTx) {
pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx;
@@ -2929,6 +3102,11 @@ bool static LoadBlockIndexDB()
}
}
+ // Check whether we have ever pruned block & undo files
+ pblocktree->ReadFlag("prunedblockfiles", fHavePruned);
+ if (fHavePruned)
+ LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n");
+
// Check whether we need to continue reindexing
bool fReindexing = false;
pblocktree->ReadReindexing(fReindexing);
@@ -3069,6 +3247,7 @@ void UnloadBlockIndex()
delete entry.second;
}
mapBlockIndex.clear();
+ fHavePruned = false;
}
bool LoadBlockIndex()
@@ -3260,6 +3439,7 @@ void static CheckBlockIndex()
int nHeight = 0;
CBlockIndex* pindexFirstInvalid = NULL; // Oldest ancestor of pindex which is invalid.
CBlockIndex* pindexFirstMissing = NULL; // Oldest ancestor of pindex which does not have BLOCK_HAVE_DATA.
+ CBlockIndex* pindexFirstNeverProcessed = NULL; // Oldest ancestor of pindex for which nTx == 0.
CBlockIndex* pindexFirstNotTreeValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_TREE (regardless of being valid or not).
CBlockIndex* pindexFirstNotTransactionsValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_TRANSACTIONS (regardless of being valid or not).
CBlockIndex* pindexFirstNotChainValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_CHAIN (regardless of being valid or not).
@@ -3268,6 +3448,7 @@ void static CheckBlockIndex()
nNodes++;
if (pindexFirstInvalid == NULL && pindex->nStatus & BLOCK_FAILED_VALID) pindexFirstInvalid = pindex;
if (pindexFirstMissing == NULL && !(pindex->nStatus & BLOCK_HAVE_DATA)) pindexFirstMissing = pindex;
+ if (pindexFirstNeverProcessed == NULL && pindex->nTx == 0) pindexFirstNeverProcessed = pindex;
if (pindex->pprev != NULL && pindexFirstNotTreeValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TREE) pindexFirstNotTreeValid = pindex;
if (pindex->pprev != NULL && pindexFirstNotTransactionsValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) pindexFirstNotTransactionsValid = pindex;
if (pindex->pprev != NULL && pindexFirstNotChainValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_CHAIN) pindexFirstNotChainValid = pindex;
@@ -3279,12 +3460,21 @@ void static CheckBlockIndex()
assert(pindex->GetBlockHash() == consensusParams.hashGenesisBlock); // Genesis block's hash must match.
assert(pindex == chainActive.Genesis()); // The current active chain's genesis block must be this block.
}
- // HAVE_DATA is equivalent to VALID_TRANSACTIONS and equivalent to nTx > 0 (we stored the number of transactions in the block)
- assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0));
- assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0));
if (pindex->nChainTx == 0) assert(pindex->nSequenceId == 0); // nSequenceId can't be set for blocks that aren't linked
- // All parents having data is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to nChainTx being set.
- assert((pindexFirstMissing != NULL) == (pindex->nChainTx == 0)); // nChainTx == 0 is used to signal that all parent block's transaction data is available.
+ // VALID_TRANSACTIONS is equivalent to nTx > 0 for all nodes (whether or not pruning has occurred).
+ // HAVE_DATA is only equivalent to nTx > 0 (or VALID_TRANSACTIONS) if no pruning has occurred.
+ if (!fHavePruned) {
+ // If we've never pruned, then HAVE_DATA should be equivalent to nTx > 0
+ assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0));
+ assert(pindexFirstMissing == pindexFirstNeverProcessed);
+ } else {
+ // If we have pruned, then we can only say that HAVE_DATA implies nTx > 0
+ if (pindex->nStatus & BLOCK_HAVE_DATA) assert(pindex->nTx > 0);
+ }
+ if (pindex->nStatus & BLOCK_HAVE_UNDO) assert(pindex->nStatus & BLOCK_HAVE_DATA);
+ assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // This is pruning-independent.
+ // All parents having had data (at some point) is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to nChainTx being set.
+ assert((pindexFirstNeverProcessed != NULL) == (pindex->nChainTx == 0)); // nChainTx != 0 is used to signal that all parent blocks have been processed (but may have been pruned).
assert((pindexFirstNotTransactionsValid != NULL) == (pindex->nChainTx == 0));
assert(pindex->nHeight == nHeight); // nHeight must be consistent.
assert(pindex->pprev == NULL || pindex->nChainWork >= pindex->pprev->nChainWork); // For every block except the genesis block, the chainwork must be larger than the parent's.
@@ -3297,11 +3487,20 @@ void static CheckBlockIndex()
// Checks for not-invalid blocks.
assert((pindex->nStatus & BLOCK_FAILED_MASK) == 0); // The failed mask cannot be set for blocks without invalid parents.
}
- if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && pindexFirstMissing == NULL) {
- if (pindexFirstInvalid == NULL) { // If this block sorts at least as good as the current tip and is valid, it must be in setBlockIndexCandidates.
- assert(setBlockIndexCandidates.count(pindex));
+ if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && pindexFirstNeverProcessed == NULL) {
+ if (pindexFirstInvalid == NULL) {
+ // If this block sorts at least as good as the current tip and
+ // is valid and we have all data for its parents, it must be in
+ // setBlockIndexCandidates. chainActive.Tip() must also be there
+ // even if some data has been pruned.
+ if (pindexFirstMissing == NULL || pindex == chainActive.Tip()) {
+ assert(setBlockIndexCandidates.count(pindex));
+ }
+ // If some parent is missing, then it could be that this block was in
+ // setBlockIndexCandidates but had to be removed because of the missing data.
+ // In this case it must be in mapBlocksUnlinked -- see test below.
}
- } else { // If this block sorts worse than the current tip, it cannot be in setBlockIndexCandidates.
+ } else { // If this block sorts worse than the current tip or some ancestor's block has never been seen, it cannot be in setBlockIndexCandidates.
assert(setBlockIndexCandidates.count(pindex) == 0);
}
// Check whether this block is in mapBlocksUnlinked.
@@ -3315,12 +3514,28 @@ void static CheckBlockIndex()
}
rangeUnlinked.first++;
}
- if (pindex->pprev && pindex->nStatus & BLOCK_HAVE_DATA && pindexFirstMissing != NULL) {
- if (pindexFirstInvalid == NULL) { // If this block has block data available, some parent doesn't, and has no invalid parents, it must be in mapBlocksUnlinked.
- assert(foundInUnlinked);
+ if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed != NULL && pindexFirstInvalid == NULL) {
+ // If this block has block data available, some parent was never received, and has no invalid parents, it must be in mapBlocksUnlinked.
+ assert(foundInUnlinked);
+ }
+ if (!(pindex->nStatus & BLOCK_HAVE_DATA)) assert(!foundInUnlinked); // Can't be in mapBlocksUnlinked if we don't HAVE_DATA
+ if (pindexFirstMissing == NULL) assert(!foundInUnlinked); // We aren't missing data for any parent -- cannot be in mapBlocksUnlinked.
+ if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed == NULL && pindexFirstMissing != NULL) {
+ // We HAVE_DATA for this block, have received data for all parents at some point, but we're currently missing data for some parent.
+ assert(fHavePruned); // We must have pruned.
+ // This block may have entered mapBlocksUnlinked if:
+ // - it has a descendant that at some point had more work than the
+ // tip, and
+ // - we tried switching to that descendant but were missing
+ // data for some intermediate block between chainActive and the
+ // tip.
+ // So if this block is itself better than chainActive.Tip() and it wasn't in
+ // setBlockIndexCandidates, then it must be in mapBlocksUnlinked.
+ if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && setBlockIndexCandidates.count(pindex) == 0) {
+ if (pindexFirstInvalid == NULL) {
+ assert(foundInUnlinked);
+ }
}
- } else { // If this block does not have block data available, or all parents do, it cannot be in mapBlocksUnlinked.
- assert(!foundInUnlinked);
}
// assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow
// End: actual consistency checks.
@@ -3340,6 +3555,7 @@ void static CheckBlockIndex()
// If pindex was the first with a certain property, unset the corresponding variable.
if (pindex == pindexFirstInvalid) pindexFirstInvalid = NULL;
if (pindex == pindexFirstMissing) pindexFirstMissing = NULL;
+ if (pindex == pindexFirstNeverProcessed) pindexFirstNeverProcessed = NULL;
if (pindex == pindexFirstNotTreeValid) pindexFirstNotTreeValid = NULL;
if (pindex == pindexFirstNotTransactionsValid) pindexFirstNotTransactionsValid = NULL;
if (pindex == pindexFirstNotChainValid) pindexFirstNotChainValid = NULL;
@@ -3497,7 +3713,9 @@ void static ProcessGetData(CNode* pfrom)
}
}
}
- if (send)
+ // Pruned nodes may have deleted the block, so check whether
+ // it's available before trying to send.
+ if (send && (mi->second->nStatus & BLOCK_HAVE_DATA))
{
// Send block from disk
CBlock block;
diff --git a/src/main.h b/src/main.h
index bb6fd6f204..d34f6c30d1 100644
--- a/src/main.h
+++ b/src/main.h
@@ -132,6 +132,26 @@ extern CBlockIndex *pindexBestHeader;
/** Minimum disk space required - used in CheckDiskSpace() */
static const uint64_t nMinDiskSpace = 52428800;
+/** Pruning-related variables and constants */
+/** True if any block files have ever been pruned. */
+extern bool fHavePruned;
+/** True if we're running in -prune mode. */
+extern bool fPruneMode;
+/** Number of MiB of block files that we're trying to stay below. */
+extern uint64_t nPruneTarget;
+/** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of chainActive.Tip() will not be pruned. */
+static const signed int MIN_BLOCKS_TO_KEEP = 288;
+
+// Require that user allocate at least 550MB for block & undo files (blk???.dat and rev???.dat)
+// At 1MB per block, 288 blocks = 288MB.
+// Add 15% for Undo data = 331MB
+// Add 20% for Orphan block rate = 397MB
+// We want the low water mark after pruning to be at least 397 MB and since we prune in
+// full block file chunks, we need the high water mark which triggers the prune to be
+// one 128MB block file + added 15% undo data = 147MB greater for a total of 545MB
+// Setting the target to > than 550MB will make it likely we can respect the target.
+static const signed int MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024;
+
/** Register with a network node to receive its signals */
void RegisterNodeSignals(CNodeSignals& nodeSignals);
/** Unregister a network node */
@@ -186,6 +206,28 @@ bool GetTransaction(const uint256 &hash, CTransaction &tx, uint256 &hashBlock, b
bool ActivateBestChain(CValidationState &state, CBlock *pblock = NULL);
CAmount GetBlockValue(int nHeight, const CAmount& nFees);
+/**
+ * Prune block and undo files (blk???.dat and undo???.dat) so that the disk space used is less than a user-defined target.
+ * The user sets the target (in MB) on the command line or in config file. This will be run on startup and whenever new
+ * space is allocated in a block or undo file, staying below the target. Changing back to unpruned requires a reindex
+ * (which in this case means the blockchain must be re-downloaded.)
+ *
+ * Pruning functions are called from FlushStateToDisk when the global fCheckForPruning flag has been set.
+ * Block and undo files are deleted in lock-step (when blk00003.dat is deleted, so is rev00003.dat.)
+ * Pruning cannot take place until the longest chain is at least a certain length (100000 on mainnet, 1000 on testnet, 10 on regtest).
+ * Pruning will never delete a block within a defined distance (currently 288) from the active chain's tip.
+ * The block index is updated by unsetting HAVE_DATA and HAVE_UNDO for any blocks that were stored in the deleted files.
+ * A db flag records the fact that at least some block files have been pruned.
+ *
+ * @param[out] setFilesToPrune The set of file indices that can be unlinked will be returned
+ */
+void FindFilesToPrune(std::set<int>& setFilesToPrune);
+
+/**
+ * Actually unlink the specified files
+ */
+void UnlinkPrunedFiles(std::set<int>& setFilesToPrune);
+
/** Create a new block index entry for a given block hash */
CBlockIndex * InsertBlockIndex(uint256 hash);
/** Abort with a message */
@@ -196,7 +238,8 @@ bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats);
void Misbehaving(NodeId nodeid, int howmuch);
/** Flush all state, indexes and buffers to disk. */
void FlushStateToDisk();
-
+/** Prune block files and flush state to disk. */
+void PruneAndFlush();
/** (try to) add transaction to memory pool **/
bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree,