From fa0c7d9ad24d3c9515d3f9c136af4071cbd79055 Mon Sep 17 00:00:00 2001 From: MarcoFalke Date: Fri, 2 Apr 2021 20:42:05 +0200 Subject: move-only: Move *Disk functions to blockstorage Can be reviewed with the git options --color-moved=dimmed-zebra --color-moved-ws=ignore-all-space --- src/index/base.cpp | 3 +- src/index/blockfilterindex.cpp | 2 +- src/net_processing.cpp | 1 + src/node/blockstorage.cpp | 142 +++++++++++++++++++++++++++++++- src/node/blockstorage.h | 21 +++++ src/node/interfaces.cpp | 4 +- src/rest.cpp | 1 + src/rpc/blockchain.cpp | 1 + src/rpc/rawtransaction.cpp | 1 + src/test/util/blockfilter.cpp | 1 + src/validation.cpp | 140 +------------------------------ src/validation.h | 9 -- src/zmq/zmqpublishnotifier.cpp | 3 +- test/lint/lint-circular-dependencies.sh | 4 +- 14 files changed, 181 insertions(+), 152 deletions(-) diff --git a/src/index/base.cpp b/src/index/base.cpp index 25644c3b41..9e637c9c6f 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -4,12 +4,13 @@ #include #include +#include #include #include #include #include #include -#include +#include // For g_chainman #include constexpr char DB_BEST_BLOCK = 'B'; diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index 32271fb7ab..154d7a7027 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -6,8 +6,8 @@ #include #include +#include #include -#include /* The index database stores three items for each block: the disk location of the encoded filter, * its dSHA256 hash, and the header. Those belonging to blocks on the active chain are indexed by diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 4108de2c8a..61faf7e1ee 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -16,6 +16,7 @@ #include #include #include +#include #include #include #include diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 235dbb1693..de54bf1f42 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -2,14 +2,154 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include +#include +#include #include +#include #include +#include #include +#include +#include #include #include +// From validation. TODO move here +bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown = false); + +static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) +{ + // Open history file to append + CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION); + if (fileout.IsNull()) + return error("WriteBlockToDisk: OpenBlockFile failed"); + + // Write index header + unsigned int nSize = GetSerializeSize(block, fileout.GetVersion()); + fileout << messageStart << nSize; + + // Write block + long fileOutPos = ftell(fileout.Get()); + if (fileOutPos < 0) + return error("WriteBlockToDisk: ftell failed"); + pos.nPos = (unsigned int)fileOutPos; + fileout << block; + + return true; +} + +bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams) +{ + block.SetNull(); + + // Open history file to read + CAutoFile filein(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION); + if (filein.IsNull()) + return error("ReadBlockFromDisk: OpenBlockFile failed for %s", pos.ToString()); + + // Read block + try { + filein >> block; + } + catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s at %s", __func__, e.what(), pos.ToString()); + } + + // Check the header + if (!CheckProofOfWork(block.GetHash(), block.nBits, consensusParams)) + return error("ReadBlockFromDisk: Errors in block header at %s", pos.ToString()); + + // Signet only: check block solution + if (consensusParams.signet_blocks && !CheckSignetBlockSolution(block, consensusParams)) { + return error("ReadBlockFromDisk: Errors in block solution at %s", pos.ToString()); + } + + return true; +} + +bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams) +{ + FlatFilePos blockPos; + { + LOCK(cs_main); + blockPos = pindex->GetBlockPos(); + } + + if (!ReadBlockFromDisk(block, blockPos, consensusParams)) + return false; + if (block.GetHash() != pindex->GetBlockHash()) + return error("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() doesn't match index for %s at %s", + pindex->ToString(), pindex->GetBlockPos().ToString()); + return true; +} + +bool ReadRawBlockFromDisk(std::vector& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start) +{ + FlatFilePos hpos = pos; + hpos.nPos -= 8; // Seek back 8 bytes for meta header + CAutoFile filein(OpenBlockFile(hpos, true), SER_DISK, CLIENT_VERSION); + if (filein.IsNull()) { + return error("%s: OpenBlockFile failed for %s", __func__, pos.ToString()); + } + + try { + CMessageHeader::MessageStartChars blk_start; + unsigned int blk_size; + + filein >> blk_start >> blk_size; + + if (memcmp(blk_start, message_start, CMessageHeader::MESSAGE_START_SIZE)) { + return error("%s: Block magic mismatch for %s: %s versus expected %s", __func__, pos.ToString(), + HexStr(blk_start), + HexStr(message_start)); + } + + if (blk_size > MAX_SIZE) { + return error("%s: Block data is larger than maximum deserialization size for %s: %s versus %s", __func__, pos.ToString(), + blk_size, MAX_SIZE); + } + + block.resize(blk_size); // Zeroing of memory is intentional here + filein.read((char*)block.data(), blk_size); + } catch (const std::exception& e) { + return error("%s: Read from block file failed: %s for %s", __func__, e.what(), pos.ToString()); + } + + return true; +} + +bool ReadRawBlockFromDisk(std::vector& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start) +{ + FlatFilePos block_pos; + { + LOCK(cs_main); + block_pos = pindex->GetBlockPos(); + } + + return ReadRawBlockFromDisk(block, block_pos, message_start); +} + +/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ +FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp) +{ + unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); + FlatFilePos blockPos; + if (dbp != nullptr) + blockPos = *dbp; + if (!FindBlockPos(blockPos, nBlockSize + 8, nHeight, active_chain, block.GetBlockTime(), dbp != nullptr)) { + error("%s: FindBlockPos failed", __func__); + return FlatFilePos(); + } + if (dbp == nullptr) { + if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) { + AbortNode("Failed to write block"); + return FlatFilePos(); + } + } + return blockPos; +} + struct CImportingNow { CImportingNow() { diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index d81b39f9f9..3b546f0719 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -5,15 +5,36 @@ #ifndef BITCOIN_NODE_BLOCKSTORAGE_H #define BITCOIN_NODE_BLOCKSTORAGE_H +#include #include #include +#include // For CMessageHeader::MessageStartChars class ArgsManager; +class CBlock; +class CBlockIndex; +class CBlockUndo; +class CChain; +class CChainParams; class ChainstateManager; +struct FlatFilePos; +namespace Consensus { +struct Params; +} static constexpr bool DEFAULT_STOPAFTERBLOCKIMPORT{false}; +/** Functions for disk access for blocks */ +bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams); +bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams); +bool ReadRawBlockFromDisk(std::vector& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start); +bool ReadRawBlockFromDisk(std::vector& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start); + +bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); + +FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp); + void ThreadImport(ChainstateManager& chainman, std::vector vImportFiles, const ArgsManager& args); #endif // BITCOIN_NODE_BLOCKSTORAGE_H diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 7ad02f81dc..264bec58f9 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -4,7 +4,6 @@ #include #include -#include #include #include #include @@ -17,6 +16,7 @@ #include #include #include +#include #include #include #include @@ -53,6 +53,8 @@ #include #include +#include + using interfaces::BlockTip; using interfaces::Chain; using interfaces::FoundBlock; diff --git a/src/rest.cpp b/src/rest.cpp index aa97470ca7..1398de19e3 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -8,6 +8,7 @@ #include #include #include +#include #include #include #include diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index e1501d7254..183fa50228 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 7932bd2915..527e9be87c 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include diff --git a/src/test/util/blockfilter.cpp b/src/test/util/blockfilter.cpp index bccff5e5a6..b8ab9d2344 100644 --- a/src/test/util/blockfilter.cpp +++ b/src/test/util/blockfilter.cpp @@ -5,6 +5,7 @@ #include #include +#include #include diff --git a/src/validation.cpp b/src/validation.cpp index 6f184780c8..652282920e 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -1148,123 +1149,6 @@ CTransactionRef GetTransaction(const CBlockIndex* const block_index, const CTxMe return nullptr; } -////////////////////////////////////////////////////////////////////////////// -// -// CBlock and CBlockIndex -// - -static bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos, const CMessageHeader::MessageStartChars& messageStart) -{ - // Open history file to append - CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION); - if (fileout.IsNull()) - return error("WriteBlockToDisk: OpenBlockFile failed"); - - // Write index header - unsigned int nSize = GetSerializeSize(block, fileout.GetVersion()); - fileout << messageStart << nSize; - - // Write block - long fileOutPos = ftell(fileout.Get()); - if (fileOutPos < 0) - return error("WriteBlockToDisk: ftell failed"); - pos.nPos = (unsigned int)fileOutPos; - fileout << block; - - return true; -} - -bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams) -{ - block.SetNull(); - - // Open history file to read - CAutoFile filein(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION); - if (filein.IsNull()) - return error("ReadBlockFromDisk: OpenBlockFile failed for %s", pos.ToString()); - - // Read block - try { - filein >> block; - } - catch (const std::exception& e) { - return error("%s: Deserialize or I/O error - %s at %s", __func__, e.what(), pos.ToString()); - } - - // Check the header - if (!CheckProofOfWork(block.GetHash(), block.nBits, consensusParams)) - return error("ReadBlockFromDisk: Errors in block header at %s", pos.ToString()); - - // Signet only: check block solution - if (consensusParams.signet_blocks && !CheckSignetBlockSolution(block, consensusParams)) { - return error("ReadBlockFromDisk: Errors in block solution at %s", pos.ToString()); - } - - return true; -} - -bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams) -{ - FlatFilePos blockPos; - { - LOCK(cs_main); - blockPos = pindex->GetBlockPos(); - } - - if (!ReadBlockFromDisk(block, blockPos, consensusParams)) - return false; - if (block.GetHash() != pindex->GetBlockHash()) - return error("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() doesn't match index for %s at %s", - pindex->ToString(), pindex->GetBlockPos().ToString()); - return true; -} - -bool ReadRawBlockFromDisk(std::vector& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start) -{ - FlatFilePos hpos = pos; - hpos.nPos -= 8; // Seek back 8 bytes for meta header - CAutoFile filein(OpenBlockFile(hpos, true), SER_DISK, CLIENT_VERSION); - if (filein.IsNull()) { - return error("%s: OpenBlockFile failed for %s", __func__, pos.ToString()); - } - - try { - CMessageHeader::MessageStartChars blk_start; - unsigned int blk_size; - - filein >> blk_start >> blk_size; - - if (memcmp(blk_start, message_start, CMessageHeader::MESSAGE_START_SIZE)) { - return error("%s: Block magic mismatch for %s: %s versus expected %s", __func__, pos.ToString(), - HexStr(blk_start), - HexStr(message_start)); - } - - if (blk_size > MAX_SIZE) { - return error("%s: Block data is larger than maximum deserialization size for %s: %s versus %s", __func__, pos.ToString(), - blk_size, MAX_SIZE); - } - - block.resize(blk_size); // Zeroing of memory is intentional here - filein.read((char*)block.data(), blk_size); - } catch(const std::exception& e) { - return error("%s: Read from block file failed: %s for %s", __func__, e.what(), pos.ToString()); - } - - return true; -} - -bool ReadRawBlockFromDisk(std::vector& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start) -{ - FlatFilePos block_pos; - { - LOCK(cs_main); - block_pos = pindex->GetBlockPos(); - } - - return ReadRawBlockFromDisk(block, block_pos, message_start); -} - CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams) { int halvings = nHeight / consensusParams.nSubsidyHalvingInterval; @@ -3218,7 +3102,8 @@ void CChainState::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pi } } -static bool FindBlockPos(FlatFilePos &pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown = false) +// TODO move to blockstorage +bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown = false) { LOCK(cs_LastBlockFile); @@ -3695,25 +3580,6 @@ bool ChainstateManager::ProcessNewBlockHeaders(const std::vector& return true; } -/** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ -static FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const CChainParams& chainparams, const FlatFilePos* dbp) { - unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); - FlatFilePos blockPos; - if (dbp != nullptr) - blockPos = *dbp; - if (!FindBlockPos(blockPos, nBlockSize+8, nHeight, active_chain, block.GetBlockTime(), dbp != nullptr)) { - error("%s: FindBlockPos failed", __func__); - return FlatFilePos(); - } - if (dbp == nullptr) { - if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) { - AbortNode("Failed to write block"); - return FlatFilePos(); - } - } - return blockPos; -} - /** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ bool CChainState::AcceptBlock(const std::shared_ptr& pblock, BlockValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock) { diff --git a/src/validation.h b/src/validation.h index 3f7e967ec7..1eb010889d 100644 --- a/src/validation.h +++ b/src/validation.h @@ -300,15 +300,6 @@ public: /** Initializes the script-execution cache */ void InitScriptExecutionCache(); - -/** Functions for disk access for blocks */ -bool ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos, const Consensus::Params& consensusParams); -bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams); -bool ReadRawBlockFromDisk(std::vector& block, const FlatFilePos& pos, const CMessageHeader::MessageStartChars& message_start); -bool ReadRawBlockFromDisk(std::vector& block, const CBlockIndex* pindex, const CMessageHeader::MessageStartChars& message_start); - -bool UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex* pindex); - /** Functions for validating blocks and updating the block tree */ /** Context-independent validity checks */ diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index 168ba841c8..25afa94d0f 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -6,10 +6,11 @@ #include #include +#include #include #include #include -#include +#include // For cs_main #include #include diff --git a/test/lint/lint-circular-dependencies.sh b/test/lint/lint-circular-dependencies.sh index 0b15f99448..ad2333a808 100755 --- a/test/lint/lint-circular-dependencies.sh +++ b/test/lint/lint-circular-dependencies.sh @@ -11,7 +11,9 @@ export LC_ALL=C EXPECTED_CIRCULAR_DEPENDENCIES=( "chainparamsbase -> util/system -> chainparamsbase" "index/txindex -> validation -> index/txindex" - "index/blockfilterindex -> validation -> index/blockfilterindex" + "node/blockstorage -> validation -> node/blockstorage" + "index/blockfilterindex -> node/blockstorage -> validation -> index/blockfilterindex" + "index/base -> validation -> index/blockfilterindex -> index/base" "policy/fees -> txmempool -> policy/fees" "qt/addresstablemodel -> qt/walletmodel -> qt/addresstablemodel" "qt/bitcoingui -> qt/walletframe -> qt/bitcoingui" -- cgit v1.2.3