diff options
author | W. J. van der Laan <laanwj@protonmail.com> | 2021-05-05 15:52:29 +0200 |
---|---|---|
committer | W. J. van der Laan <laanwj@protonmail.com> | 2021-05-05 16:03:03 +0200 |
commit | 3275c6e578b41a821a90ff33667b95d79aaf3642 (patch) | |
tree | 7d8cd2a375dc380d6a85cd588092a086eaeeb2ec | |
parent | 23109cc5485b9ab51cf1b6e655ead2e472ce0005 (diff) | |
parent | fa09a9eac8d8ab65ce4064c35a9f21349a644982 (diff) | |
download | bitcoin-3275c6e578b41a821a90ff33667b95d79aaf3642.tar.xz |
Merge bitcoin/bitcoin#21727: refactor: Move more stuff to blockstorage
fa09a9eac8d8ab65ce4064c35a9f21349a644982 style: Add { } to multi-line if (MarcoFalke)
fadafab83379ff10d86ada179c6f9641d19464fe move-only: Move functions to blockstorage (MarcoFalke)
fa7e64d58615fffea91cd64dc4a2790221ceff0a move-only: Move constants to blockstorage (MarcoFalke)
fa247a327fc7c7cea6bc8f93637b8babd3015ffa refactor: Move block storage globals to blockstorage (MarcoFalke)
fa81c30c6f1adac79517c958090db174eb6aeda2 refactor: Move pruning/reindex/importing globals to blockstorage (MarcoFalke)
Pull request description:
See #21575
ACKs for top commit:
Sjors:
ACK fa09a9eac8d8ab65ce4064c35a9f21349a644982
kiminuo:
ACK fa09a9e
laanwj:
Code review ACK fa09a9eac8d8ab65ce4064c35a9f21349a644982
promag:
Code review ACK fa09a9eac8d8ab65ce4064c35a9f21349a644982. Since last review
Tree-SHA512: 2eb6962ff44da6b77f3058fc02ec66ab742e25ae8dcc8ec62b062896571910d43ca7c4bb16fb3ccb5e5245195b8dec6384b6c8d442fa97ca28d93bdff347d677
-rw-r--r-- | src/index/txindex.cpp | 1 | ||||
-rw-r--r-- | src/init.cpp | 41 | ||||
-rw-r--r-- | src/node/blockstorage.cpp | 326 | ||||
-rw-r--r-- | src/node/blockstorage.h | 41 | ||||
-rw-r--r-- | src/validation.cpp | 285 | ||||
-rw-r--r-- | src/validation.h | 35 | ||||
-rw-r--r-- | src/wallet/test/wallet_tests.cpp | 1 | ||||
-rwxr-xr-x | test/lint/lint-locale-dependence.sh | 2 |
8 files changed, 383 insertions, 349 deletions
diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 06f5910238..3feefe8619 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -4,6 +4,7 @@ #include <index/disktxpos.h> #include <index/txindex.h> +#include <node/blockstorage.h> #include <node/ui_interface.h> #include <shutdown.h> #include <util/system.h> diff --git a/src/init.cpp b/src/init.cpp index 10404f9887..8aa94f123f 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -602,47 +602,6 @@ static void BlockNotifyGenesisWait(const CBlockIndex* pBlockIndex) } } -// If we're using -prune with -reindex, then delete block files that will be ignored by the -// reindex. Since reindexing works by starting at block file 0 and looping until a blockfile -// is missing, do the same here to delete any later block files after a gap. Also delete all -// rev files since they'll be rewritten by the reindex anyway. This ensures that vinfoBlockFile -// is in sync with what's actually on disk by the time we start downloading, so that pruning -// works correctly. -static void CleanupBlockRevFiles() -{ - std::map<std::string, fs::path> mapBlockFiles; - - // Glob all blk?????.dat and rev?????.dat files from the blocks directory. - // Remove the rev files immediately and insert the blk file paths into an - // ordered map keyed by block file index. - LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n"); - fs::path blocksdir = gArgs.GetBlocksDirPath(); - for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) { - if (fs::is_regular_file(*it) && - it->path().filename().string().length() == 12 && - it->path().filename().string().substr(8,4) == ".dat") - { - if (it->path().filename().string().substr(0,3) == "blk") - mapBlockFiles[it->path().filename().string().substr(3,5)] = it->path(); - else if (it->path().filename().string().substr(0,3) == "rev") - remove(it->path()); - } - } - - // Remove all block files that aren't part of a contiguous set starting at - // zero by walking the ordered map (keys are block file indices) by - // keeping a separate counter. Once we hit a gap (or if 0 doesn't exist) - // start removing block files. - int nContigCounter = 0; - for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) { - if (atoi(item.first) == nContigCounter) { - nContigCounter++; - continue; - } - remove(item.second); - } -} - #if HAVE_SYSTEM static void StartupNotify(const ArgsManager& args) { diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index daed6605e8..6c66c565ad 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -6,17 +6,310 @@ #include <chain.h> #include <chainparams.h> +#include <clientversion.h> +#include <consensus/validation.h> #include <flatfile.h> #include <fs.h> +#include <hash.h> #include <pow.h> #include <shutdown.h> #include <signet.h> #include <streams.h> +#include <undo.h> #include <util/system.h> #include <validation.h> -// From validation. TODO move here -bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown = false); +std::atomic_bool fImporting(false); +std::atomic_bool fReindex(false); +bool fHavePruned = false; +bool fPruneMode = false; +uint64_t nPruneTarget = 0; + +// TODO make namespace { +RecursiveMutex cs_LastBlockFile; +std::vector<CBlockFileInfo> vinfoBlockFile; +int nLastBlockFile = 0; +/** Global flag to indicate we should check to see if there are +* block/undo files that should be deleted. Set on startup +* or if we allocate more file space when we're in prune mode +*/ +bool fCheckForPruning = false; + +/** Dirty block index entries. */ +std::set<CBlockIndex*> setDirtyBlockIndex; + +/** Dirty block file entries. */ +std::set<int> setDirtyFileInfo; +// } // namespace + +static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false); +static FlatFileSeq BlockFileSeq(); +static FlatFileSeq UndoFileSeq(); + +bool IsBlockPruned(const CBlockIndex* pblockindex) +{ + return (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0); +} + +// If we're using -prune with -reindex, then delete block files that will be ignored by the +// reindex. Since reindexing works by starting at block file 0 and looping until a blockfile +// is missing, do the same here to delete any later block files after a gap. Also delete all +// rev files since they'll be rewritten by the reindex anyway. This ensures that vinfoBlockFile +// is in sync with what's actually on disk by the time we start downloading, so that pruning +// works correctly. +void CleanupBlockRevFiles() +{ + std::map<std::string, fs::path> mapBlockFiles; + + // Glob all blk?????.dat and rev?????.dat files from the blocks directory. + // Remove the rev files immediately and insert the blk file paths into an + // ordered map keyed by block file index. + LogPrintf("Removing unusable blk?????.dat and rev?????.dat files for -reindex with -prune\n"); + fs::path blocksdir = gArgs.GetBlocksDirPath(); + for (fs::directory_iterator it(blocksdir); it != fs::directory_iterator(); it++) { + if (fs::is_regular_file(*it) && + it->path().filename().string().length() == 12 && + it->path().filename().string().substr(8,4) == ".dat") + { + if (it->path().filename().string().substr(0, 3) == "blk") { + mapBlockFiles[it->path().filename().string().substr(3, 5)] = it->path(); + } else if (it->path().filename().string().substr(0, 3) == "rev") { + remove(it->path()); + } + } + } + + // Remove all block files that aren't part of a contiguous set starting at + // zero by walking the ordered map (keys are block file indices) by + // keeping a separate counter. Once we hit a gap (or if 0 doesn't exist) + // start removing block files. + int nContigCounter = 0; + for (const std::pair<const std::string, fs::path>& item : mapBlockFiles) { + if (atoi(item.first) == nContigCounter) { + nContigCounter++; + continue; + } + remove(item.second); + } +} + +std::string CBlockFileInfo::ToString() const +{ + return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); +} + +CBlockFileInfo* GetBlockFileInfo(size_t n) +{ + LOCK(cs_LastBlockFile); + + return &vinfoBlockFile.at(n); +} + +static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) +{ + // Open history file to append + CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); + if (fileout.IsNull()) { + return error("%s: OpenUndoFile failed", __func__); + } + + // Write index header + unsigned int nSize = GetSerializeSize(blockundo, fileout.GetVersion()); + fileout << messageStart << nSize; + + // Write undo data + long fileOutPos = ftell(fileout.Get()); + if (fileOutPos < 0) { + return error("%s: ftell failed", __func__); + } + pos.nPos = (unsigned int)fileOutPos; + fileout << blockundo; + + // calculate & write checksum + CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); + hasher << hashBlock; + hasher << blockundo; + fileout << hasher.GetHash(); + + return true; +} + +bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) +{ + FlatFilePos pos = pindex->GetUndoPos(); + if (pos.IsNull()) { + return error("%s: no undo data available", __func__); + } + + // Open history file to read + CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION); + if (filein.IsNull()) { + return error("%s: OpenUndoFile failed", __func__); + } + + // Read block + uint256 hashChecksum; + CHashVerifier<CAutoFile> verifier(&filein); // We need a CHashVerifier as reserializing may lose data + try { + verifier << pindex->pprev->GetBlockHash(); + verifier >> blockundo; + filein >> hashChecksum; + } catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } + + // Verify checksum + if (hashChecksum != verifier.GetHash()) { + return error("%s: Checksum mismatch", __func__); + } + + return true; +} + +static void FlushUndoFile(int block_file, bool finalize = false) +{ + FlatFilePos undo_pos_old(block_file, vinfoBlockFile[block_file].nUndoSize); + if (!UndoFileSeq().Flush(undo_pos_old, finalize)) { + AbortNode("Flushing undo file to disk failed. This is likely the result of an I/O error."); + } +} + +void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false) +{ + LOCK(cs_LastBlockFile); + FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize); + if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) { + AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); + } + // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks, + // e.g. during IBD or a sync after a node going offline + if (!fFinalize || finalize_undo) FlushUndoFile(nLastBlockFile, finalize_undo); +} + +uint64_t CalculateCurrentUsage() +{ + LOCK(cs_LastBlockFile); + + uint64_t retval = 0; + for (const CBlockFileInfo& file : vinfoBlockFile) { + retval += file.nSize + file.nUndoSize; + } + return retval; +} + +void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) +{ + for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { + FlatFilePos pos(*it, 0); + fs::remove(BlockFileSeq().FileName(pos)); + fs::remove(UndoFileSeq().FileName(pos)); + LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); + } +} + +static FlatFileSeq BlockFileSeq() +{ + return FlatFileSeq(gArgs.GetBlocksDirPath(), "blk", gArgs.GetBoolArg("-fastprune", false) ? 0x4000 /* 16kb */ : BLOCKFILE_CHUNK_SIZE); +} + +static FlatFileSeq UndoFileSeq() +{ + return FlatFileSeq(gArgs.GetBlocksDirPath(), "rev", UNDOFILE_CHUNK_SIZE); +} + +FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) +{ + return BlockFileSeq().Open(pos, fReadOnly); +} + +/** Open an undo file (rev?????.dat) */ +static FILE* OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) +{ + return UndoFileSeq().Open(pos, fReadOnly); +} + +fs::path GetBlockPosFilename(const FlatFilePos& pos) +{ + return BlockFileSeq().FileName(pos); +} + +bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown = false) +{ + LOCK(cs_LastBlockFile); + + unsigned int nFile = fKnown ? pos.nFile : nLastBlockFile; + if (vinfoBlockFile.size() <= nFile) { + vinfoBlockFile.resize(nFile + 1); + } + + bool finalize_undo = false; + if (!fKnown) { + while (vinfoBlockFile[nFile].nSize + nAddSize >= (gArgs.GetBoolArg("-fastprune", false) ? 0x10000 /* 64kb */ : MAX_BLOCKFILE_SIZE)) { + // when the undo file is keeping up with the block file, we want to flush it explicitly + // when it is lagging behind (more blocks arrive than are being connected), we let the + // undo block write case handle it + assert(std::addressof(::ChainActive()) == std::addressof(active_chain)); + finalize_undo = (vinfoBlockFile[nFile].nHeightLast == (unsigned int)active_chain.Tip()->nHeight); + nFile++; + if (vinfoBlockFile.size() <= nFile) { + vinfoBlockFile.resize(nFile + 1); + } + } + pos.nFile = nFile; + pos.nPos = vinfoBlockFile[nFile].nSize; + } + + if ((int)nFile != nLastBlockFile) { + if (!fKnown) { + LogPrint(BCLog::VALIDATION, "Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); + } + FlushBlockFile(!fKnown, finalize_undo); + nLastBlockFile = nFile; + } + + vinfoBlockFile[nFile].AddBlock(nHeight, nTime); + if (fKnown) { + vinfoBlockFile[nFile].nSize = std::max(pos.nPos + nAddSize, vinfoBlockFile[nFile].nSize); + } else { + vinfoBlockFile[nFile].nSize += nAddSize; + } + + if (!fKnown) { + bool out_of_space; + size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode("Disk space is too low!", _("Disk space is too low!")); + } + if (bytes_allocated != 0 && fPruneMode) { + fCheckForPruning = true; + } + } + + setDirtyFileInfo.insert(nFile); + return true; +} + +static bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize) +{ + pos.nFile = nFile; + + LOCK(cs_LastBlockFile); + + pos.nPos = vinfoBlockFile[nFile].nUndoSize; + vinfoBlockFile[nFile].nUndoSize += nAddSize; + setDirtyFileInfo.insert(nFile); + + bool out_of_space; + size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + return AbortNode(state, "Disk space is too low!", _("Disk space is too low!")); + } + if (bytes_allocated != 0 && fPruneMode) { + fCheckForPruning = true; + } + + return true; +} static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) { @@ -41,6 +334,35 @@ static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessa return true; } +bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) +{ + // Write undo information to disk + if (pindex->GetUndoPos().IsNull()) { + FlatFilePos _pos; + if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) { + return error("ConnectBlock(): FindUndoPos failed"); + } + if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) { + return AbortNode(state, "Failed to write undo data"); + } + // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order) + // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height + // in the block file info as below; note that this does not catch the case where the undo writes are keeping up + // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in + // the FindBlockPos function + if (_pos.nFile < nLastBlockFile && static_cast<uint32_t>(pindex->nHeight) == vinfoBlockFile[_pos.nFile].nHeightLast) { + FlushUndoFile(_pos.nFile, true); + } + + // update nUndoPos in block index + pindex->nUndoPos = _pos.nPos; + pindex->nStatus |= BLOCK_HAVE_UNDO; + setDirtyBlockIndex.insert(pindex); + } + + return true; +} + bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams) { block.SetNull(); diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index 3b546f0719..faf99dea81 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -12,7 +12,9 @@ #include <protocol.h> // For CMessageHeader::MessageStartChars class ArgsManager; +class BlockValidationState; class CBlock; +class CBlockFileInfo; class CBlockIndex; class CBlockUndo; class CChain; @@ -25,6 +27,44 @@ struct Params; static constexpr bool DEFAULT_STOPAFTERBLOCKIMPORT{false}; +/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ +static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB +/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ +static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB +/** The maximum size of a blk?????.dat file (since 0.8) */ +static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB + +extern std::atomic_bool fImporting; +extern std::atomic_bool fReindex; +/** Pruning-related variables and constants */ +/** True if any block files have ever been pruned. */ +extern bool fHavePruned; +/** True if we're running in -prune mode. */ +extern bool fPruneMode; +/** Number of MiB of block files that we're trying to stay below. */ +extern uint64_t nPruneTarget; + +//! Check whether the block associated with this index entry is pruned or not. +bool IsBlockPruned(const CBlockIndex* pblockindex); + +void CleanupBlockRevFiles(); + +/** Open a block file (blk?????.dat) */ +FILE* OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false); +/** Translation to a filesystem path */ +fs::path GetBlockPosFilename(const FlatFilePos& pos); + +/** Get block file info entry for one block file */ +CBlockFileInfo* GetBlockFileInfo(size_t n); + +/** Calculate the amount of disk space the block & undo files currently use */ +uint64_t CalculateCurrentUsage(); + +/** + * Actually unlink the specified files + */ +void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune); + /** Functions for disk access for blocks */ bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams); bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams); @@ -32,6 +72,7 @@ bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatFilePos& pos, c bool ReadRawBlockFromDisk(std::vector<uint8_t>& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start); bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); +bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams); FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp); diff --git a/src/validation.cpp b/src/validation.cpp index bf774a0791..54d062b63c 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -66,10 +66,6 @@ static const unsigned int EXTRA_DESCENDANT_TX_SIZE_LIMIT = 10000; /** Maximum kilobytes for transactions to store for processing during reorg */ static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20000; -/** The pre-allocation chunk size for blk?????.dat files (since 0.8) */ -static const unsigned int BLOCKFILE_CHUNK_SIZE = 0x1000000; // 16 MiB -/** The pre-allocation chunk size for rev?????.dat files (since 0.8) */ -static const unsigned int UNDOFILE_CHUNK_SIZE = 0x100000; // 1 MiB /** Time to wait between writing blocks/block index to disk. */ static constexpr std::chrono::hours DATABASE_WRITE_INTERVAL{1}; /** Time to wait between flushing chainstate to disk. */ @@ -135,14 +131,9 @@ Mutex g_best_block_mutex; std::condition_variable g_best_block_cv; uint256 g_best_block; bool g_parallel_script_checks{false}; -std::atomic_bool fImporting(false); -std::atomic_bool fReindex(false); -bool fHavePruned = false; -bool fPruneMode = false; bool fRequireStandard = true; bool fCheckBlockIndex = false; bool fCheckpointsEnabled = DEFAULT_CHECKPOINTS_ENABLED; -uint64_t nPruneTarget = 0; int64_t nMaxTipAge = DEFAULT_MAX_TIP_AGE; uint256 hashAssumeValid; @@ -153,22 +144,17 @@ CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); // Internal stuff namespace { CBlockIndex* pindexBestInvalid = nullptr; - - RecursiveMutex cs_LastBlockFile; - std::vector<CBlockFileInfo> vinfoBlockFile; - int nLastBlockFile = 0; - /** Global flag to indicate we should check to see if there are - * block/undo files that should be deleted. Set on startup - * or if we allocate more file space when we're in prune mode - */ - bool fCheckForPruning = false; - - /** Dirty block index entries. */ - std::set<CBlockIndex*> setDirtyBlockIndex; - - /** Dirty block file entries. */ - std::set<int> setDirtyFileInfo; -} // anon namespace +} // namespace + +// Internal stuff from blockstorage ... +extern RecursiveMutex cs_LastBlockFile; +extern std::vector<CBlockFileInfo> vinfoBlockFile; +extern int nLastBlockFile; +extern bool fCheckForPruning; +extern std::set<CBlockIndex*> setDirtyBlockIndex; +extern std::set<int> setDirtyFileInfo; +void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false); +// ... TODO move fully to blockstorage CBlockIndex* BlockManager::LookupBlockIndex(const uint256& hash) const { @@ -205,9 +191,6 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, bool cacheFullScriptStore, PrecomputedTransactionData& txdata, std::vector<CScriptCheck>* pvChecks = nullptr) EXCLUSIVE_LOCKS_REQUIRED(cs_main); -static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly = false); -static FlatFileSeq BlockFileSeq(); -static FlatFileSeq UndoFileSeq(); bool CheckFinalTx(const CBlockIndex* active_chain_tip, const CTransaction &tx, int flags) { @@ -1462,65 +1445,7 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, return true; } -static bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) -{ - // Open history file to append - CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); - if (fileout.IsNull()) - return error("%s: OpenUndoFile failed", __func__); - - // Write index header - unsigned int nSize = GetSerializeSize(blockundo, fileout.GetVersion()); - fileout << messageStart << nSize; - - // Write undo data - long fileOutPos = ftell(fileout.Get()); - if (fileOutPos < 0) - return error("%s: ftell failed", __func__); - pos.nPos = (unsigned int)fileOutPos; - fileout << blockundo; - - // calculate & write checksum - CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); - hasher << hashBlock; - hasher << blockundo; - fileout << hasher.GetHash(); - - return true; -} - -bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex) -{ - FlatFilePos pos = pindex->GetUndoPos(); - if (pos.IsNull()) { - return error("%s: no undo data available", __func__); - } - - // Open history file to read - CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION); - if (filein.IsNull()) - return error("%s: OpenUndoFile failed", __func__); - - // Read block - uint256 hashChecksum; - CHashVerifier<CAutoFile> verifier(&filein); // We need a CHashVerifier as reserializing may lose data - try { - verifier << pindex->pprev->GetBlockHash(); - verifier >> blockundo; - filein >> hashChecksum; - } - catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s", __func__, e.what()); - } - - // Verify checksum - if (hashChecksum != verifier.GetHash()) - return error("%s: Checksum mismatch", __func__); - - return true; -} - -static bool AbortNode(BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage = bilingual_str()) +bool AbortNode(BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage) { AbortNode(strMessage, userMessage); return state.Error(strMessage); @@ -1620,55 +1545,6 @@ DisconnectResult CChainState::DisconnectBlock(const CBlock& block, const CBlockI return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } -static void FlushUndoFile(int block_file, bool finalize = false) -{ - FlatFilePos undo_pos_old(block_file, vinfoBlockFile[block_file].nUndoSize); - if (!UndoFileSeq().Flush(undo_pos_old, finalize)) { - AbortNode("Flushing undo file to disk failed. This is likely the result of an I/O error."); - } -} - -static void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false) -{ - LOCK(cs_LastBlockFile); - FlatFilePos block_pos_old(nLastBlockFile, vinfoBlockFile[nLastBlockFile].nSize); - if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) { - AbortNode("Flushing block file to disk failed. This is likely the result of an I/O error."); - } - // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks, - // e.g. during IBD or a sync after a node going offline - if (!fFinalize || finalize_undo) FlushUndoFile(nLastBlockFile, finalize_undo); -} - -static bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize); - -static bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex* pindex, const CChainParams& chainparams) -{ - // Write undo information to disk - if (pindex->GetUndoPos().IsNull()) { - FlatFilePos _pos; - if (!FindUndoPos(state, pindex->nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) - return error("ConnectBlock(): FindUndoPos failed"); - if (!UndoWriteToDisk(blockundo, _pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) - return AbortNode(state, "Failed to write undo data"); - // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order) - // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height - // in the block file info as below; note that this does not catch the case where the undo writes are keeping up - // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in - // the FindBlockPos function - if (_pos.nFile < nLastBlockFile && static_cast<uint32_t>(pindex->nHeight) == vinfoBlockFile[_pos.nFile].nHeightLast) { - FlushUndoFile(_pos.nFile, true); - } - - // update nUndoPos in block index - pindex->nUndoPos = _pos.nPos; - pindex->nStatus |= BLOCK_HAVE_UNDO; - setDirtyBlockIndex.insert(pindex); - } - - return true; -} - static CCheckQueue<CScriptCheck> scriptcheckqueue(128); void StartScriptCheckWorkerThreads(int threads_num) @@ -3102,84 +2978,6 @@ void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pi } } -// TODO move to blockstorage -bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown = false) -{ - LOCK(cs_LastBlockFile); - - unsigned int nFile = fKnown ? pos.nFile : nLastBlockFile; - if (vinfoBlockFile.size() <= nFile) { - vinfoBlockFile.resize(nFile + 1); - } - - bool finalize_undo = false; - if (!fKnown) { - while (vinfoBlockFile[nFile].nSize + nAddSize >= (gArgs.GetBoolArg("-fastprune", false) ? 0x10000 /* 64kb */ : MAX_BLOCKFILE_SIZE)) { - // when the undo file is keeping up with the block file, we want to flush it explicitly - // when it is lagging behind (more blocks arrive than are being connected), we let the - // undo block write case handle it - assert(std::addressof(::ChainActive()) == std::addressof(active_chain)); - finalize_undo = (vinfoBlockFile[nFile].nHeightLast == (unsigned int)active_chain.Tip()->nHeight); - nFile++; - if (vinfoBlockFile.size() <= nFile) { - vinfoBlockFile.resize(nFile + 1); - } - } - pos.nFile = nFile; - pos.nPos = vinfoBlockFile[nFile].nSize; - } - - if ((int)nFile != nLastBlockFile) { - if (!fKnown) { - LogPrint(BCLog::VALIDATION, "Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); - } - FlushBlockFile(!fKnown, finalize_undo); - nLastBlockFile = nFile; - } - - vinfoBlockFile[nFile].AddBlock(nHeight, nTime); - if (fKnown) - vinfoBlockFile[nFile].nSize = std::max(pos.nPos + nAddSize, vinfoBlockFile[nFile].nSize); - else - vinfoBlockFile[nFile].nSize += nAddSize; - - if (!fKnown) { - bool out_of_space; - size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); - if (out_of_space) { - return AbortNode("Disk space is too low!", _("Disk space is too low!")); - } - if (bytes_allocated != 0 && fPruneMode) { - fCheckForPruning = true; - } - } - - setDirtyFileInfo.insert(nFile); - return true; -} - -static bool FindUndoPos(BlockValidationState &state, int nFile, FlatFilePos &pos, unsigned int nAddSize) -{ - pos.nFile = nFile; - - LOCK(cs_LastBlockFile); - - pos.nPos = vinfoBlockFile[nFile].nUndoSize; - vinfoBlockFile[nFile].nUndoSize += nAddSize; - setDirtyFileInfo.insert(nFile); - - bool out_of_space; - size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space); - if (out_of_space) { - return AbortNode(state, "Disk space is too low!", _("Disk space is too low!")); - } - if (bytes_allocated != 0 && fPruneMode) { - fCheckForPruning = true; - } - - return true; -} - static bool CheckBlockHeader(const CBlockHeader& block, BlockValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW = true) { // Check proof of work matches claimed amount @@ -3737,18 +3535,6 @@ bool TestBlockValidity(BlockValidationState& state, * BLOCK PRUNING CODE */ -/* Calculate the amount of disk space the block & undo files currently use */ -uint64_t CalculateCurrentUsage() -{ - LOCK(cs_LastBlockFile); - - uint64_t retval = 0; - for (const CBlockFileInfo &file : vinfoBlockFile) { - retval += file.nSize + file.nUndoSize; - } - return retval; -} - void BlockManager::PruneOneBlockFile(const int fileNumber) { AssertLockHeld(cs_main); @@ -3783,17 +3569,6 @@ void BlockManager::PruneOneBlockFile(const int fileNumber) setDirtyFileInfo.insert(fileNumber); } - -void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune) -{ - for (std::set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { - FlatFilePos pos(*it, 0); - fs::remove(BlockFileSeq().FileName(pos)); - fs::remove(UndoFileSeq().FileName(pos)); - LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); - } -} - void BlockManager::FindFilesToPruneManual(std::set<int>& setFilesToPrune, int nManualPruneHeight, int chain_tip_height) { assert(fPruneMode && nManualPruneHeight > 0); @@ -3888,30 +3663,6 @@ void BlockManager::FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPr nLastBlockWeCanPrune, count); } -static FlatFileSeq BlockFileSeq() -{ - return FlatFileSeq(gArgs.GetBlocksDirPath(), "blk", gArgs.GetBoolArg("-fastprune", false) ? 0x4000 /* 16kb */ : BLOCKFILE_CHUNK_SIZE); -} - -static FlatFileSeq UndoFileSeq() -{ - return FlatFileSeq(gArgs.GetBlocksDirPath(), "rev", UNDOFILE_CHUNK_SIZE); -} - -FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly) { - return BlockFileSeq().Open(pos, fReadOnly); -} - -/** Open an undo file (rev?????.dat) */ -static FILE* OpenUndoFile(const FlatFilePos &pos, bool fReadOnly) { - return UndoFileSeq().Open(pos, fReadOnly); -} - -fs::path GetBlockPosFilename(const FlatFilePos &pos) -{ - return BlockFileSeq().FileName(pos); -} - CBlockIndex * BlockManager::InsertBlockIndex(const uint256& hash) { AssertLockHeld(cs_main); @@ -4744,18 +4495,6 @@ bool CChainState::ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) return ret; } -std::string CBlockFileInfo::ToString() const -{ - return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, FormatISO8601Date(nTimeFirst), FormatISO8601Date(nTimeLast)); -} - -CBlockFileInfo* GetBlockFileInfo(size_t n) -{ - LOCK(cs_LastBlockFile); - - return &vinfoBlockFile.at(n); -} - static const uint64_t MEMPOOL_DUMP_VERSION = 1; bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mockable_fopen_function) diff --git a/src/validation.h b/src/validation.h index 63a8bdc096..8fa45c3f77 100644 --- a/src/validation.h +++ b/src/validation.h @@ -27,6 +27,7 @@ #include <serialize.h> #include <util/check.h> #include <util/hasher.h> +#include <util/translation.h> #include <atomic> #include <map> @@ -70,8 +71,6 @@ static const unsigned int DEFAULT_DESCENDANT_LIMIT = 25; static const unsigned int DEFAULT_DESCENDANT_SIZE_LIMIT = 101; /** Default for -mempoolexpiry, expiration time for mempool transactions in hours */ static const unsigned int DEFAULT_MEMPOOL_EXPIRY = 336; -/** The maximum size of a blk?????.dat file (since 0.8) */ -static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB /** Maximum number of dedicated script-checking threads allowed */ static const int MAX_SCRIPTCHECK_THREADS = 15; /** -par default (number of script-checking threads, 0 = auto) */ @@ -113,8 +112,6 @@ typedef std::unordered_map<uint256, CBlockIndex*, BlockHasher> BlockMap; extern Mutex g_best_block_mutex; extern std::condition_variable g_best_block_cv; extern uint256 g_best_block; -extern std::atomic_bool fImporting; -extern std::atomic_bool fReindex; /** Whether there are dedicated script-checking threads running. * False indicates all script checking is done on the main threadMessageHandler thread. */ @@ -136,20 +133,9 @@ extern arith_uint256 nMinimumChainWork; /** Best header we've seen so far (used for getheaders queries' starting points). */ extern CBlockIndex *pindexBestHeader; -/** Pruning-related variables and constants */ -/** True if any block files have ever been pruned. */ -extern bool fHavePruned; -/** True if we're running in -prune mode. */ -extern bool fPruneMode; -/** Number of MiB of block files that we're trying to stay below. */ -extern uint64_t nPruneTarget; /** Documentation for argument 'checklevel'. */ extern const std::vector<std::string> CHECKLEVEL_DOC; -/** Open a block file (blk?????.dat) */ -FILE* OpenBlockFile(const FlatFilePos &pos, bool fReadOnly = false); -/** Translation to a filesystem path */ -fs::path GetBlockPosFilename(const FlatFilePos &pos); /** Unload database information */ void UnloadBlockIndex(CTxMemPool* mempool, ChainstateManager& chainman); /** Run instances of script checking worker threads */ @@ -171,17 +157,11 @@ void StopScriptCheckWorkerThreads(); CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMemPool* const mempool, const uint256& hash, const Consensus::Params& consensusParams, uint256& hashBlock); CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams); +bool AbortNode(BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage = bilingual_str{}); + /** Guess verification progress (as a fraction between 0.0=genesis and 1.0=current tip). */ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex* pindex); -/** Calculate the amount of disk space the block & undo files currently use */ -uint64_t CalculateCurrentUsage(); - -/** - * Actually unlink the specified files - */ -void UnlinkPrunedFiles(const std::set<int>& setFilesToPrune); - /** Prune block files up to a given height */ void PruneBlockFilesManual(CChainState& active_chainstate, int nManualPruneHeight); @@ -1009,9 +989,6 @@ extern VersionBitsCache versionbitscache; */ int32_t ComputeBlockVersion(const CBlockIndex* pindexPrev, const Consensus::Params& params); -/** Get block file info entry for one block file */ -CBlockFileInfo* GetBlockFileInfo(size_t n); - using FopenFn = std::function<FILE*(const fs::path&, const char*)>; /** Dump the mempool to disk. */ @@ -1020,12 +997,6 @@ bool DumpMempool(const CTxMemPool& pool, FopenFn mockable_fopen_function = fsbri /** Load the mempool from disk. */ bool LoadMempool(CTxMemPool& pool, CChainState& active_chainstate, FopenFn mockable_fopen_function = fsbridge::fopen); -//! Check whether the block associated with this index entry is pruned or not. -inline bool IsBlockPruned(const CBlockIndex* pblockindex) -{ - return (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0); -} - /** * Return the expected assumeutxo value for a given height, if one exists. * diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index 011d59ecaf..c8e3c8f819 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -11,6 +11,7 @@ #include <vector> #include <interfaces/chain.h> +#include <node/blockstorage.h> #include <node/context.h> #include <policy/policy.h> #include <rpc/server.h> diff --git a/test/lint/lint-locale-dependence.sh b/test/lint/lint-locale-dependence.sh index e5657f7555..737d35a397 100755 --- a/test/lint/lint-locale-dependence.sh +++ b/test/lint/lint-locale-dependence.sh @@ -43,7 +43,7 @@ KNOWN_VIOLATIONS=( "src/dbwrapper.cpp.*stoul" "src/dbwrapper.cpp:.*vsnprintf" "src/httprpc.cpp.*trim" - "src/init.cpp:.*atoi" + "src/node/blockstorage.cpp:.*atoi" "src/qt/rpcconsole.cpp:.*atoi" "src/rest.cpp:.*strtol" "src/test/dbwrapper_tests.cpp:.*snprintf" |