diff options
Diffstat (limited to 'src')
| -rw-r--r-- | src/chainparams.cpp | 3 | ||||
| -rw-r--r-- | src/chainparams.h | 2 | ||||
| -rw-r--r-- | src/init.cpp | 90 | ||||
| -rw-r--r-- | src/main.cpp | 278 | ||||
| -rw-r--r-- | src/main.h | 45 |
5 files changed, 383 insertions, 35 deletions
diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 2d4fde2476..589c7b5472 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -121,6 +121,7 @@ public: vAlertPubKey = ParseHex("04fc9702847840aaf195de8442ebecedf5b095cdbb9bc716bda9110971b28a49e0ead8564ff0db22209e0374782c093bb899692d524e9d6a6956e7c5ecbcd68284"); nDefaultPort = 8333; nMinerThreads = 0; + nPruneAfterHeight = 100000; /** * Build the genesis block. Note that the output of the genesis coinbase cannot @@ -198,6 +199,7 @@ public: vAlertPubKey = ParseHex("04302390343f91cc401d56d68b123028bf52e5fca1939df127f63c6467cdf9c8e2c14b61104cf817d0b780da337893ecc4aaff1309e536162dabbdb45200ca2b0a"); nDefaultPort = 18333; nMinerThreads = 0; + nPruneAfterHeight = 1000; //! Modify the testnet genesis block so the timestamp is valid for a later start. genesis.nTime = 1296688602; @@ -257,6 +259,7 @@ public: consensus.hashGenesisBlock = genesis.GetHash(); nDefaultPort = 18444; assert(consensus.hashGenesisBlock == uint256S("0x0f9188f13cb7b2c71f2a335e3a4fc328bf5beb436012afca590b1a11466e2206")); + nPruneAfterHeight = 1000; vFixedSeeds.clear(); //! Regtest mode doesn't have any fixed seeds. vSeeds.clear(); //! Regtest mode doesn't have any DNS seeds. diff --git a/src/chainparams.h b/src/chainparams.h index 0692094478..5e974123dc 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -58,6 +58,7 @@ public: bool DefaultConsistencyChecks() const { return fDefaultConsistencyChecks; } /** Make standard checks */ bool RequireStandard() const { return fRequireStandard; } + int64_t PruneAfterHeight() const { return nPruneAfterHeight; } /** Make miner stop after a block is found. In RPC, don't return until nGenProcLimit blocks are generated */ bool MineBlocksOnDemand() const { return fMineBlocksOnDemand; } /** In the future use NetworkIDString() for RPC fields */ @@ -77,6 +78,7 @@ protected: std::vector<unsigned char> vAlertPubKey; int nDefaultPort; int nMinerThreads; + uint64_t nPruneAfterHeight; std::vector<CDNSSeedData> vSeeds; std::vector<unsigned char> base58Prefixes[MAX_BASE58_TYPES]; std::string strNetworkID; diff --git a/src/init.cpp b/src/init.cpp index 665aebe1c5..b648a237bf 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -275,7 +275,12 @@ std::string HelpMessage(HelpMessageMode mode) #ifndef WIN32 strUsage += HelpMessageOpt("-pid=<file>", strprintf(_("Specify pid file (default: %s)"), "bitcoind.pid")); #endif - strUsage += HelpMessageOpt("-reindex", _("Rebuild block chain index from current blk000??.dat files") + " " + _("on startup")); + strUsage += HelpMessageOpt("-prune=<n>", _("Reduce storage requirements by pruning (deleting) old blocks. This mode disables wallet support and is incompatible with -txindex.") + " " + + _("Warning: Reverting this setting requires re-downloading the entire blockchain.") + " " + + _("(default: 0 = disable pruning blocks,") + " " + + strprintf(_(">%u = target size in MiB to use for block files)"), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024)); +strUsage += HelpMessageOpt("-reindex", _("Rebuild block chain index from current blk000??.dat files") + " " + _("on startup")); + #if !defined(WIN32) strUsage += HelpMessageOpt("-sysperms", _("Create new files with system default permissions, instead of umask 077 (only effective with disabled wallet functionality)")); #endif @@ -352,7 +357,7 @@ std::string HelpMessage(HelpMessageMode mode) strUsage += HelpMessageOpt("-flushwallet", strprintf(_("Run a thread to flush wallet periodically (default: %u)"), 1)); strUsage += HelpMessageOpt("-stopafterblockimport", strprintf(_("Stop running after importing blocks from disk (default: %u)"), 0)); } - string debugCategories = "addrman, alert, bench, coindb, db, lock, rand, rpc, selectcoins, mempool, net, proxy"; // Don't translate these and qt below + string debugCategories = "addrman, alert, bench, coindb, db, lock, rand, rpc, selectcoins, mempool, net, proxy, prune"; // Don't translate these and qt below if (mode == HMM_BITCOIN_QT) debugCategories += ", qt"; strUsage += HelpMessageOpt("-debug=<category>", strprintf(_("Output debugging information (default: %u, supplying <category> is optional)"), 0) + ". " + @@ -458,10 +463,33 @@ struct CImportingNow } }; + +// If we're using -prune with -reindex, then delete block files that will be ignored by the +// reindex. Since reindexing works by starting at block file 0 and looping until a blockfile +// is missing, and since pruning works by deleting the oldest block file first, just check +// for block file 0, and if it doesn't exist, delete all the block files in the +// directory (since they won't be read by the reindex but will take up disk space). +void DeleteAllBlockFiles() +{ + if (boost::filesystem::exists(GetBlockPosFilename(CDiskBlockPos(0, 0), "blk"))) + return; + + LogPrintf("Removing all blk?????.dat and rev?????.dat files for -reindex with -prune\n"); + boost::filesystem::path blocksdir = GetDataDir() / "blocks"; + for (boost::filesystem::directory_iterator it(blocksdir); it != boost::filesystem::directory_iterator(); it++) { + if (is_regular_file(*it)) { + if ((it->path().filename().string().length() == 12) && + (it->path().filename().string().substr(8,4) == ".dat") && + ((it->path().filename().string().substr(0,3) == "blk") || + (it->path().filename().string().substr(0,3) == "rev"))) + boost::filesystem::remove(it->path()); + } + } +} + void ThreadImport(std::vector<boost::filesystem::path> vImportFiles) { RenameThread("bitcoin-loadblk"); - // -reindex if (fReindex) { CImportingNow imp; @@ -674,6 +702,21 @@ bool AppInit2(boost::thread_group& threadGroup) if (nFD - MIN_CORE_FILEDESCRIPTORS < nMaxConnections) nMaxConnections = nFD - MIN_CORE_FILEDESCRIPTORS; + // if using block pruning, then disable txindex + // also disable the wallet (for now, until SPV support is implemented in wallet) + if (GetArg("-prune", 0)) { + if (GetBoolArg("-txindex", false)) + return InitError(_("Prune mode is incompatible with -txindex.")); +#ifdef ENABLE_WALLET + if (!GetBoolArg("-disablewallet", false)) { + if (SoftSetBoolArg("-disablewallet", true)) + LogPrintf("%s : parameter interaction: -prune -> setting -disablewallet=1\n", __func__); + else + return InitError(_("Can't run with a wallet in prune mode.")); + } +#endif + } + // ********************************************************* Step 3: parameter-to-internal-flags fDebug = !mapMultiArgs["-debug"].empty(); @@ -710,6 +753,21 @@ bool AppInit2(boost::thread_group& threadGroup) nScriptCheckThreads = MAX_SCRIPTCHECK_THREADS; fServer = GetBoolArg("-server", false); + + // block pruning; get the amount of disk space (in MB) to allot for block & undo files + int64_t nSignedPruneTarget = GetArg("-prune", 0) * 1024 * 1024; + if (nSignedPruneTarget < 0) { + return InitError(_("Prune cannot be configured with a negative value.")); + } + nPruneTarget = (uint64_t) nSignedPruneTarget; + if (nPruneTarget) { + if (nPruneTarget < MIN_DISK_SPACE_FOR_BLOCK_FILES) { + return InitError(strprintf(_("Prune configured below the minimum of %d MB. Please use a higher number."), MIN_DISK_SPACE_FOR_BLOCK_FILES / 1024 / 1024)); + } + LogPrintf("Prune configured to target %uMiB on disk for block and undo files.\n", nPruneTarget / 1024 / 1024); + fPruneMode = true; + } + #ifdef ENABLE_WALLET bool fDisableWallet = GetBoolArg("-disablewallet", false); #endif @@ -1030,8 +1088,12 @@ bool AppInit2(boost::thread_group& threadGroup) pcoinscatcher = new CCoinsViewErrorCatcher(pcoinsdbview); pcoinsTip = new CCoinsViewCache(pcoinscatcher); - if (fReindex) + if (fReindex) { pblocktree->WriteReindexing(true); + //If we're reindexing in prune mode, wipe away all our block and undo data files + if (fPruneMode) + DeleteAllBlockFiles(); + } if (!LoadBlockIndex()) { strLoadError = _("Error loading block database"); @@ -1055,7 +1117,18 @@ bool AppInit2(boost::thread_group& threadGroup) break; } + // Check for changed -prune state. What we are concerned about is a user who has pruned blocks + // in the past, but is now trying to run unpruned. + if (fHavePruned && !fPruneMode) { + strLoadError = _("You need to rebuild the database using -reindex to go back to unpruned mode. This will redownload the entire blockchain"); + break; + } + uiInterface.InitMessage(_("Verifying blocks...")); + if (fHavePruned && GetArg("-checkblocks", 288) > MIN_BLOCKS_TO_KEEP) { + LogPrintf("Prune: pruned datadir may not have more than %d blocks; -checkblocks=%d may fail\n", + MIN_BLOCKS_TO_KEEP, GetArg("-checkblocks", 288)); + } if (!CVerifyDB().VerifyDB(pcoinsdbview, GetArg("-checklevel", 3), GetArg("-checkblocks", 288))) { strLoadError = _("Corrupted block database detected"); @@ -1106,6 +1179,15 @@ bool AppInit2(boost::thread_group& threadGroup) mempool.ReadFeeEstimates(est_filein); fFeeEstimatesInitialized = true; + // if prune mode, unset NODE_NETWORK and prune block files + if (fPruneMode) { + LogPrintf("Unsetting NODE_NETWORK on prune mode\n"); + nLocalServices &= ~NODE_NETWORK; + if (!fReindex) { + PruneAndFlush(); + } + } + // ********************************************************* Step 8: load wallet #ifdef ENABLE_WALLET if (fDisableWallet) { diff --git a/src/main.cpp b/src/main.cpp index 4352c719a1..bf32ac91e7 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -52,9 +52,12 @@ int nScriptCheckThreads = 0; bool fImporting = false; bool fReindex = false; bool fTxIndex = false; +bool fHavePruned = false; +bool fPruneMode = false; bool fIsBareMultisigStd = true; bool fCheckBlockIndex = false; unsigned int nCoinCacheSize = 5000; +uint64_t nPruneTarget = 0; /** Fees smaller than this (in satoshi) are considered zero fee (for relaying and mining) */ CFeeRate minRelayTxFee = CFeeRate(1000); @@ -110,17 +113,25 @@ namespace { /** * The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself and all ancestors) and - * as good as our current tip or better. Entries may be failed, though. + * as good as our current tip or better. Entries may be failed, though, and pruning nodes may be + * missing the data for the block. */ set<CBlockIndex*, CBlockIndexWorkComparator> setBlockIndexCandidates; /** Number of nodes with fSyncStarted. */ int nSyncStarted = 0; - /** All pairs A->B, where A (or one if its ancestors) misses transactions, but B has transactions. */ + /** All pairs A->B, where A (or one if its ancestors) misses transactions, but B has transactions. + * Pruned nodes may have entries where B is missing data. + */ multimap<CBlockIndex*, CBlockIndex*> mapBlocksUnlinked; CCriticalSection cs_LastBlockFile; std::vector<CBlockFileInfo> vinfoBlockFile; int nLastBlockFile = 0; + /** Global flag to indicate we should check to see if there are + * block/undo files that should be deleted. Set on startup + * or if we allocate more file space when we're in prune mode + */ + bool fCheckForPruning = false; /** * Every received block is assigned a unique and increasing identifier, so we @@ -1849,6 +1860,7 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin } enum FlushStateMode { + FLUSH_STATE_NONE, FLUSH_STATE_IF_NEEDED, FLUSH_STATE_PERIODIC, FLUSH_STATE_ALWAYS @@ -1856,16 +1868,30 @@ enum FlushStateMode { /** * Update the on-disk chain state. - * The caches and indexes are flushed if either they're too large, forceWrite is set, or - * fast is not set and it's been a while since the last write. + * The caches and indexes are flushed depending on the mode we're called with + * if they're too large, if it's been a while since the last write, + * or always and in all cases if we're in prune mode and are deleting files. */ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { - LOCK(cs_main); + LOCK2(cs_main, cs_LastBlockFile); static int64_t nLastWrite = 0; + std::set<int> setFilesToPrune; + bool fFlushForPrune = false; try { + if (fPruneMode && fCheckForPruning) { + FindFilesToPrune(setFilesToPrune); + if (!setFilesToPrune.empty()) { + fFlushForPrune = true; + if (!fHavePruned) { + pblocktree->WriteFlag("prunedblockfiles", true); + fHavePruned = true; + } + } + } if ((mode == FLUSH_STATE_ALWAYS) || ((mode == FLUSH_STATE_PERIODIC || mode == FLUSH_STATE_IF_NEEDED) && pcoinsTip->GetCacheSize() > nCoinCacheSize) || - (mode == FLUSH_STATE_PERIODIC && GetTimeMicros() > nLastWrite + DATABASE_WRITE_INTERVAL * 1000000)) { + (mode == FLUSH_STATE_PERIODIC && GetTimeMicros() > nLastWrite + DATABASE_WRITE_INTERVAL * 1000000) || + fFlushForPrune) { // Typical CCoins structures on disk are around 100 bytes in size. // Pushing a new one to the database can cause it to be written // twice (once in the log, and once in the tables). This is already @@ -1893,9 +1919,16 @@ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { return state.Abort("Files to write to block index database"); } } - // Finally flush the chainstate (which may refer to block index entries). + // Flush the chainstate (which may refer to block index entries). if (!pcoinsTip->Flush()) return state.Abort("Failed to write to coin database"); + + // Finally remove any pruned files + if (fFlushForPrune) { + UnlinkPrunedFiles(setFilesToPrune); + fCheckForPruning = false; + } + // Update best block in wallet (so we can detect restored wallets). if (mode != FLUSH_STATE_IF_NEEDED) { GetMainSignals().SetBestChain(chainActive.GetLocator()); @@ -1913,6 +1946,12 @@ void FlushStateToDisk() { FlushStateToDisk(state, FLUSH_STATE_ALWAYS); } +void PruneAndFlush() { + CValidationState state; + fCheckForPruning = true; + FlushStateToDisk(state, FLUSH_STATE_NONE); +} + /** Update chainActive and related internal data structures. */ void static UpdateTip(CBlockIndex *pindexNew) { chainActive.SetTip(pindexNew); @@ -2083,15 +2122,29 @@ static CBlockIndex* FindMostWorkChain() { CBlockIndex *pindexTest = pindexNew; bool fInvalidAncestor = false; while (pindexTest && !chainActive.Contains(pindexTest)) { - assert(pindexTest->nStatus & BLOCK_HAVE_DATA); assert(pindexTest->nChainTx || pindexTest->nHeight == 0); - if (pindexTest->nStatus & BLOCK_FAILED_MASK) { - // Candidate has an invalid ancestor, remove entire chain from the set. - if (pindexBestInvalid == NULL || pindexNew->nChainWork > pindexBestInvalid->nChainWork) + + // Pruned nodes may have entries in setBlockIndexCandidates for + // which block files have been deleted. Remove those as candidates + // for the most work chain if we come across them; we can't switch + // to a chain unless we have all the non-active-chain parent blocks. + bool fFailedChain = pindexTest->nStatus & BLOCK_FAILED_MASK; + bool fMissingData = !(pindexTest->nStatus & BLOCK_HAVE_DATA); + if (fFailedChain || fMissingData) { + // Candidate chain is not usable (either invalid or missing data) + if (fFailedChain && (pindexBestInvalid == NULL || pindexNew->nChainWork > pindexBestInvalid->nChainWork)) pindexBestInvalid = pindexNew; CBlockIndex *pindexFailed = pindexNew; + // Remove the entire chain from the set. while (pindexTest != pindexFailed) { - pindexFailed->nStatus |= BLOCK_FAILED_CHILD; + if (fFailedChain) { + pindexFailed->nStatus |= BLOCK_FAILED_CHILD; + } else if (fMissingData) { + // If we're missing data, then add back to mapBlocksUnlinked, + // so that if the block arrives in the future we can try adding + // to setBlockIndexCandidates again. + mapBlocksUnlinked.insert(std::make_pair(pindexFailed->pprev, pindexFailed)); + } setBlockIndexCandidates.erase(pindexFailed); pindexFailed = pindexFailed->pprev; } @@ -2219,7 +2272,9 @@ bool ActivateBestChain(CValidationState &state, CBlock *pblock) { uint256 hashNewTip = pindexNewTip->GetBlockHash(); // Relay inventory, but don't relay old inventory during initial block download. int nBlockEstimate = Checkpoints::GetTotalBlocksEstimate(); - { + // Don't relay blocks if pruning -- could cause a peer to try to download, resulting + // in a stalled download if the block file is pruned before the request. + if (nLocalServices & NODE_NETWORK) { LOCK(cs_vNodes); BOOST_FOREACH(CNode* pnode, vNodes) if (chainActive.Height() > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : nBlockEstimate)) @@ -2419,6 +2474,8 @@ bool FindBlockPos(CValidationState &state, CDiskBlockPos &pos, unsigned int nAdd unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; unsigned int nNewChunks = (vinfoBlockFile[nFile].nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; if (nNewChunks > nOldChunks) { + if (fPruneMode) + fCheckForPruning = true; if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos)) { FILE *file = OpenBlockFile(pos); if (file) { @@ -2450,6 +2507,8 @@ bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigne unsigned int nOldChunks = (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; unsigned int nNewChunks = (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; if (nNewChunks > nOldChunks) { + if (fPruneMode) + fCheckForPruning = true; if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos)) { FILE *file = OpenUndoFile(pos); if (file) { @@ -2665,7 +2724,10 @@ bool AcceptBlock(CBlock& block, CValidationState& state, CBlockIndex** ppindex, if (!AcceptBlockHeader(block, state, &pindex)) return false; - if (pindex->nStatus & BLOCK_HAVE_DATA) { + // If we're pruning, ensure that we don't allow a peer to dump a copy + // of old blocks. But we might need blocks that are not on the main chain + // to handle a reorg, even if we've processed once. + if (pindex->nStatus & BLOCK_HAVE_DATA || chainActive.Contains(pindex)) { // TODO: deal better with duplicate blocks. // return state.DoS(20, error("AcceptBlock(): already have block %d %s", pindex->nHeight, pindex->GetBlockHash().ToString()), REJECT_DUPLICATE, "duplicate"); return true; @@ -2698,6 +2760,9 @@ bool AcceptBlock(CBlock& block, CValidationState& state, CBlockIndex** ppindex, return state.Abort(std::string("System error: ") + e.what()); } + if (fCheckForPruning) + FlushStateToDisk(state, FLUSH_STATE_NONE); // we just allocated more disk space for block files + return true; } @@ -2785,6 +2850,112 @@ bool AbortNode(const std::string &strMessage, const std::string &userMessage) { return false; } + +/** + * BLOCK PRUNING CODE + */ + +/* Calculate the amount of disk space the block & undo files currently use */ +uint64_t CalculateCurrentUsage() +{ + uint64_t retval = 0; + BOOST_FOREACH(const CBlockFileInfo &file, vinfoBlockFile) { + retval += file.nSize + file.nUndoSize; + } + return retval; +} + +/* Prune a block file (modify associated database entries)*/ +void PruneOneBlockFile(const int fileNumber) +{ + for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); ++it) { + CBlockIndex* pindex = it->second; + if (pindex->nFile == fileNumber) { + pindex->nStatus &= ~BLOCK_HAVE_DATA; + pindex->nStatus &= ~BLOCK_HAVE_UNDO; + pindex->nFile = 0; + pindex->nDataPos = 0; + pindex->nUndoPos = 0; + setDirtyBlockIndex.insert(pindex); + + // Prune from mapBlocksUnlinked -- any block we prune would have + // to be downloaded again in order to consider its chain, at which + // point it would be considered as a candidate for + // mapBlocksUnlinked or setBlockIndexCandidates. + std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> range = mapBlocksUnlinked.equal_range(pindex->pprev); + while (range.first != range.second) { + std::multimap<CBlockIndex *, CBlockIndex *>::iterator it = range.first; + range.first++; + if (it->second == pindex) { + mapBlocksUnlinked.erase(it); + } + } + } + } + + vinfoBlockFile[fileNumber].SetNull(); + setDirtyFileInfo.insert(fileNumber); +} + + +void UnlinkPrunedFiles(std::set<int>& setFilesToPrune) +{ + for (set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { + CDiskBlockPos pos(*it, 0); + boost::filesystem::remove(GetBlockPosFilename(pos, "blk")); + boost::filesystem::remove(GetBlockPosFilename(pos, "rev")); + LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); + } +} + +/* Calculate the block/rev files that should be deleted to remain under target*/ +void FindFilesToPrune(std::set<int>& setFilesToPrune) +{ + LOCK2(cs_main, cs_LastBlockFile); + if (chainActive.Tip() == NULL || nPruneTarget == 0) { + return; + } + if (chainActive.Tip()->nHeight <= Params().PruneAfterHeight()) { + return; + } + + unsigned int nLastBlockWeMustKeep = chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP; + uint64_t nCurrentUsage = CalculateCurrentUsage(); + // We don't check to prune until after we've allocated new space for files + // So we should leave a buffer under our target to account for another allocation + // before the next pruning. + uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE; + uint64_t nBytesToPrune; + int count=0; + + if (nCurrentUsage + nBuffer >= nPruneTarget) { + for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) { + nBytesToPrune = vinfoBlockFile[fileNumber].nSize + vinfoBlockFile[fileNumber].nUndoSize; + + if (vinfoBlockFile[fileNumber].nSize == 0) + continue; + + if (nCurrentUsage + nBuffer < nPruneTarget) // are we below our target? + break; + + // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip + if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeMustKeep) + break; + + PruneOneBlockFile(fileNumber); + // Queue up the files for removal + setFilesToPrune.insert(fileNumber); + nCurrentUsage -= nBytesToPrune; + count++; + } + } + + LogPrint("prune", "Prune: target=%dMiB actual=%dMiB diff=%dMiB min_must_keep=%d removed %d blk/rev pairs\n", + nPruneTarget/1024/1024, nCurrentUsage/1024/1024, + ((int64_t)nPruneTarget - (int64_t)nCurrentUsage)/1024/1024, + nLastBlockWeMustKeep, count); +} + bool CheckDiskSpace(uint64_t nAdditionalBytes) { uint64_t nFreeBytesAvailable = boost::filesystem::space(GetDataDir()).available; @@ -2872,7 +3043,9 @@ bool static LoadBlockIndexDB() { CBlockIndex* pindex = item.second; pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex); - if (pindex->nStatus & BLOCK_HAVE_DATA) { + // We can link the chain of blocks for which we've received transactions at some point. + // Pruned nodes may have deleted the block. + if (pindex->nTx > 0) { if (pindex->pprev) { if (pindex->pprev->nChainTx) { pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; @@ -2929,6 +3102,11 @@ bool static LoadBlockIndexDB() } } + // Check whether we have ever pruned block & undo files + pblocktree->ReadFlag("prunedblockfiles", fHavePruned); + if (fHavePruned) + LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n"); + // Check whether we need to continue reindexing bool fReindexing = false; pblocktree->ReadReindexing(fReindexing); @@ -3069,6 +3247,7 @@ void UnloadBlockIndex() delete entry.second; } mapBlockIndex.clear(); + fHavePruned = false; } bool LoadBlockIndex() @@ -3260,6 +3439,7 @@ void static CheckBlockIndex() int nHeight = 0; CBlockIndex* pindexFirstInvalid = NULL; // Oldest ancestor of pindex which is invalid. CBlockIndex* pindexFirstMissing = NULL; // Oldest ancestor of pindex which does not have BLOCK_HAVE_DATA. + CBlockIndex* pindexFirstNeverProcessed = NULL; // Oldest ancestor of pindex for which nTx == 0. CBlockIndex* pindexFirstNotTreeValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_TREE (regardless of being valid or not). CBlockIndex* pindexFirstNotTransactionsValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_TRANSACTIONS (regardless of being valid or not). CBlockIndex* pindexFirstNotChainValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_CHAIN (regardless of being valid or not). @@ -3268,6 +3448,7 @@ void static CheckBlockIndex() nNodes++; if (pindexFirstInvalid == NULL && pindex->nStatus & BLOCK_FAILED_VALID) pindexFirstInvalid = pindex; if (pindexFirstMissing == NULL && !(pindex->nStatus & BLOCK_HAVE_DATA)) pindexFirstMissing = pindex; + if (pindexFirstNeverProcessed == NULL && pindex->nTx == 0) pindexFirstNeverProcessed = pindex; if (pindex->pprev != NULL && pindexFirstNotTreeValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TREE) pindexFirstNotTreeValid = pindex; if (pindex->pprev != NULL && pindexFirstNotTransactionsValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) pindexFirstNotTransactionsValid = pindex; if (pindex->pprev != NULL && pindexFirstNotChainValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_CHAIN) pindexFirstNotChainValid = pindex; @@ -3279,12 +3460,21 @@ void static CheckBlockIndex() assert(pindex->GetBlockHash() == consensusParams.hashGenesisBlock); // Genesis block's hash must match. assert(pindex == chainActive.Genesis()); // The current active chain's genesis block must be this block. } - // HAVE_DATA is equivalent to VALID_TRANSACTIONS and equivalent to nTx > 0 (we stored the number of transactions in the block) - assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0)); - assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); if (pindex->nChainTx == 0) assert(pindex->nSequenceId == 0); // nSequenceId can't be set for blocks that aren't linked - // All parents having data is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to nChainTx being set. - assert((pindexFirstMissing != NULL) == (pindex->nChainTx == 0)); // nChainTx == 0 is used to signal that all parent block's transaction data is available. + // VALID_TRANSACTIONS is equivalent to nTx > 0 for all nodes (whether or not pruning has occurred). + // HAVE_DATA is only equivalent to nTx > 0 (or VALID_TRANSACTIONS) if no pruning has occurred. + if (!fHavePruned) { + // If we've never pruned, then HAVE_DATA should be equivalent to nTx > 0 + assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0)); + assert(pindexFirstMissing == pindexFirstNeverProcessed); + } else { + // If we have pruned, then we can only say that HAVE_DATA implies nTx > 0 + if (pindex->nStatus & BLOCK_HAVE_DATA) assert(pindex->nTx > 0); + } + if (pindex->nStatus & BLOCK_HAVE_UNDO) assert(pindex->nStatus & BLOCK_HAVE_DATA); + assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // This is pruning-independent. + // All parents having had data (at some point) is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to nChainTx being set. + assert((pindexFirstNeverProcessed != NULL) == (pindex->nChainTx == 0)); // nChainTx != 0 is used to signal that all parent blocks have been processed (but may have been pruned). assert((pindexFirstNotTransactionsValid != NULL) == (pindex->nChainTx == 0)); assert(pindex->nHeight == nHeight); // nHeight must be consistent. assert(pindex->pprev == NULL || pindex->nChainWork >= pindex->pprev->nChainWork); // For every block except the genesis block, the chainwork must be larger than the parent's. @@ -3297,11 +3487,20 @@ void static CheckBlockIndex() // Checks for not-invalid blocks. assert((pindex->nStatus & BLOCK_FAILED_MASK) == 0); // The failed mask cannot be set for blocks without invalid parents. } - if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && pindexFirstMissing == NULL) { - if (pindexFirstInvalid == NULL) { // If this block sorts at least as good as the current tip and is valid, it must be in setBlockIndexCandidates. - assert(setBlockIndexCandidates.count(pindex)); + if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && pindexFirstNeverProcessed == NULL) { + if (pindexFirstInvalid == NULL) { + // If this block sorts at least as good as the current tip and + // is valid and we have all data for its parents, it must be in + // setBlockIndexCandidates. chainActive.Tip() must also be there + // even if some data has been pruned. + if (pindexFirstMissing == NULL || pindex == chainActive.Tip()) { + assert(setBlockIndexCandidates.count(pindex)); + } + // If some parent is missing, then it could be that this block was in + // setBlockIndexCandidates but had to be removed because of the missing data. + // In this case it must be in mapBlocksUnlinked -- see test below. } - } else { // If this block sorts worse than the current tip, it cannot be in setBlockIndexCandidates. + } else { // If this block sorts worse than the current tip or some ancestor's block has never been seen, it cannot be in setBlockIndexCandidates. assert(setBlockIndexCandidates.count(pindex) == 0); } // Check whether this block is in mapBlocksUnlinked. @@ -3315,12 +3514,28 @@ void static CheckBlockIndex() } rangeUnlinked.first++; } - if (pindex->pprev && pindex->nStatus & BLOCK_HAVE_DATA && pindexFirstMissing != NULL) { - if (pindexFirstInvalid == NULL) { // If this block has block data available, some parent doesn't, and has no invalid parents, it must be in mapBlocksUnlinked. - assert(foundInUnlinked); + if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed != NULL && pindexFirstInvalid == NULL) { + // If this block has block data available, some parent was never received, and has no invalid parents, it must be in mapBlocksUnlinked. + assert(foundInUnlinked); + } + if (!(pindex->nStatus & BLOCK_HAVE_DATA)) assert(!foundInUnlinked); // Can't be in mapBlocksUnlinked if we don't HAVE_DATA + if (pindexFirstMissing == NULL) assert(!foundInUnlinked); // We aren't missing data for any parent -- cannot be in mapBlocksUnlinked. + if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed == NULL && pindexFirstMissing != NULL) { + // We HAVE_DATA for this block, have received data for all parents at some point, but we're currently missing data for some parent. + assert(fHavePruned); // We must have pruned. + // This block may have entered mapBlocksUnlinked if: + // - it has a descendant that at some point had more work than the + // tip, and + // - we tried switching to that descendant but were missing + // data for some intermediate block between chainActive and the + // tip. + // So if this block is itself better than chainActive.Tip() and it wasn't in + // setBlockIndexCandidates, then it must be in mapBlocksUnlinked. + if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && setBlockIndexCandidates.count(pindex) == 0) { + if (pindexFirstInvalid == NULL) { + assert(foundInUnlinked); + } } - } else { // If this block does not have block data available, or all parents do, it cannot be in mapBlocksUnlinked. - assert(!foundInUnlinked); } // assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow // End: actual consistency checks. @@ -3340,6 +3555,7 @@ void static CheckBlockIndex() // If pindex was the first with a certain property, unset the corresponding variable. if (pindex == pindexFirstInvalid) pindexFirstInvalid = NULL; if (pindex == pindexFirstMissing) pindexFirstMissing = NULL; + if (pindex == pindexFirstNeverProcessed) pindexFirstNeverProcessed = NULL; if (pindex == pindexFirstNotTreeValid) pindexFirstNotTreeValid = NULL; if (pindex == pindexFirstNotTransactionsValid) pindexFirstNotTransactionsValid = NULL; if (pindex == pindexFirstNotChainValid) pindexFirstNotChainValid = NULL; @@ -3497,7 +3713,9 @@ void static ProcessGetData(CNode* pfrom) } } } - if (send) + // Pruned nodes may have deleted the block, so check whether + // it's available before trying to send. + if (send && (mi->second->nStatus & BLOCK_HAVE_DATA)) { // Send block from disk CBlock block; diff --git a/src/main.h b/src/main.h index bb6fd6f204..d34f6c30d1 100644 --- a/src/main.h +++ b/src/main.h @@ -132,6 +132,26 @@ extern CBlockIndex *pindexBestHeader; /** Minimum disk space required - used in CheckDiskSpace() */ static const uint64_t nMinDiskSpace = 52428800; +/** Pruning-related variables and constants */ +/** True if any block files have ever been pruned. */ +extern bool fHavePruned; +/** True if we're running in -prune mode. */ +extern bool fPruneMode; +/** Number of MiB of block files that we're trying to stay below. */ +extern uint64_t nPruneTarget; +/** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of chainActive.Tip() will not be pruned. */ +static const signed int MIN_BLOCKS_TO_KEEP = 288; + +// Require that user allocate at least 550MB for block & undo files (blk???.dat and rev???.dat) +// At 1MB per block, 288 blocks = 288MB. +// Add 15% for Undo data = 331MB +// Add 20% for Orphan block rate = 397MB +// We want the low water mark after pruning to be at least 397 MB and since we prune in +// full block file chunks, we need the high water mark which triggers the prune to be +// one 128MB block file + added 15% undo data = 147MB greater for a total of 545MB +// Setting the target to > than 550MB will make it likely we can respect the target. +static const signed int MIN_DISK_SPACE_FOR_BLOCK_FILES = 550 * 1024 * 1024; + /** Register with a network node to receive its signals */ void RegisterNodeSignals(CNodeSignals& nodeSignals); /** Unregister a network node */ @@ -186,6 +206,28 @@ bool GetTransaction(const uint256 &hash, CTransaction &tx, uint256 &hashBlock, b bool ActivateBestChain(CValidationState &state, CBlock *pblock = NULL); CAmount GetBlockValue(int nHeight, const CAmount& nFees); +/** + * Prune block and undo files (blk???.dat and undo???.dat) so that the disk space used is less than a user-defined target. + * The user sets the target (in MB) on the command line or in config file. This will be run on startup and whenever new + * space is allocated in a block or undo file, staying below the target. Changing back to unpruned requires a reindex + * (which in this case means the blockchain must be re-downloaded.) + * + * Pruning functions are called from FlushStateToDisk when the global fCheckForPruning flag has been set. + * Block and undo files are deleted in lock-step (when blk00003.dat is deleted, so is rev00003.dat.) + * Pruning cannot take place until the longest chain is at least a certain length (100000 on mainnet, 1000 on testnet, 10 on regtest). + * Pruning will never delete a block within a defined distance (currently 288) from the active chain's tip. + * The block index is updated by unsetting HAVE_DATA and HAVE_UNDO for any blocks that were stored in the deleted files. + * A db flag records the fact that at least some block files have been pruned. + * + * @param[out] setFilesToPrune The set of file indices that can be unlinked will be returned + */ +void FindFilesToPrune(std::set<int>& setFilesToPrune); + +/** + * Actually unlink the specified files + */ +void UnlinkPrunedFiles(std::set<int>& setFilesToPrune); + /** Create a new block index entry for a given block hash */ CBlockIndex * InsertBlockIndex(uint256 hash); /** Abort with a message */ @@ -196,7 +238,8 @@ bool GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats); void Misbehaving(NodeId nodeid, int howmuch); /** Flush all state, indexes and buffers to disk. */ void FlushStateToDisk(); - +/** Prune block files and flush state to disk. */ +void PruneAndFlush(); /** (try to) add transaction to memory pool **/ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, |