diff options
Diffstat (limited to 'src/node')
-rw-r--r-- | src/node/abort.cpp | 9 | ||||
-rw-r--r-- | src/node/abort.h | 7 | ||||
-rw-r--r-- | src/node/blockstorage.cpp | 289 | ||||
-rw-r--r-- | src/node/blockstorage.h | 53 | ||||
-rw-r--r-- | src/node/chainstate.cpp | 20 | ||||
-rw-r--r-- | src/node/chainstate.h | 9 | ||||
-rw-r--r-- | src/node/context.cpp | 2 | ||||
-rw-r--r-- | src/node/context.h | 11 | ||||
-rw-r--r-- | src/node/interfaces.cpp | 20 | ||||
-rw-r--r-- | src/node/kernel_notifications.cpp | 13 | ||||
-rw-r--r-- | src/node/kernel_notifications.h | 5 | ||||
-rw-r--r-- | src/node/timeoffsets.cpp | 69 | ||||
-rw-r--r-- | src/node/timeoffsets.h | 39 | ||||
-rw-r--r-- | src/node/utxo_snapshot.h | 66 |
14 files changed, 415 insertions, 197 deletions
diff --git a/src/node/abort.cpp b/src/node/abort.cpp index 1bdc91670d..b727608384 100644 --- a/src/node/abort.cpp +++ b/src/node/abort.cpp @@ -16,14 +16,13 @@ namespace node { -void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const std::string& debug_message, const bilingual_str& user_message) +void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const bilingual_str& message) { - SetMiscWarning(Untranslated(debug_message)); - LogPrintf("*** %s\n", debug_message); - InitError(user_message.empty() ? _("A fatal internal error occurred, see debug.log for details") : user_message); + SetMiscWarning(message); + InitError(_("A fatal internal error occurred, see debug.log for details: ") + message); exit_status.store(EXIT_FAILURE); if (shutdown && !(*shutdown)()) { - LogPrintf("Error: failed to send shutdown signal\n"); + LogError("Failed to send shutdown signal\n"); }; } } // namespace node diff --git a/src/node/abort.h b/src/node/abort.h index 28d021cc78..1092279142 100644 --- a/src/node/abort.h +++ b/src/node/abort.h @@ -5,17 +5,16 @@ #ifndef BITCOIN_NODE_ABORT_H #define BITCOIN_NODE_ABORT_H -#include <util/translation.h> - #include <atomic> -#include <string> + +struct bilingual_str; namespace util { class SignalInterrupt; } // namespace util namespace node { -void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const std::string& debug_message, const bilingual_str& user_message = {}); +void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const bilingual_str& message); } // namespace node #endif // BITCOIN_NODE_ABORT_H diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index f78f33e371..fb62e78138 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -150,7 +150,6 @@ bool BlockTreeDB::LoadBlockIndexGuts(const Consensus::Params& consensusParams, s } // namespace kernel namespace node { -std::atomic_bool fReindex(false); bool CBlockIndexWorkComparator::operator()(const CBlockIndex* pa, const CBlockIndex* pb) const { @@ -404,7 +403,7 @@ bool BlockManager::LoadBlockIndex(const std::optional<uint256>& snapshot_blockha if (snapshot_blockhash) { const std::optional<AssumeutxoData> maybe_au_data = GetParams().AssumeutxoForBlockhash(*snapshot_blockhash); if (!maybe_au_data) { - m_opts.notifications.fatalError(strprintf("Assumeutxo data not found for the given blockhash '%s'.", snapshot_blockhash->ToString())); + m_opts.notifications.fatalError(strprintf(_("Assumeutxo data not found for the given blockhash '%s'."), snapshot_blockhash->ToString())); return false; } const AssumeutxoData& au_data = *Assert(maybe_au_data); @@ -552,7 +551,7 @@ bool BlockManager::LoadBlockIndexDB(const std::optional<uint256>& snapshot_block // Check whether we need to continue reindexing bool fReindexing = false; m_block_tree_db->ReadReindexing(fReindexing); - if (fReindexing) fReindex = true; + if (fReindexing) m_blockfiles_indexed = false; return true; } @@ -741,7 +740,7 @@ bool BlockManager::FlushUndoFile(int block_file, bool finalize) { FlatFilePos undo_pos_old(block_file, m_blockfile_info[block_file].nUndoSize); if (!UndoFileSeq().Flush(undo_pos_old, finalize)) { - m_opts.notifications.flushError("Flushing undo file to disk failed. This is likely the result of an I/O error."); + m_opts.notifications.flushError(_("Flushing undo file to disk failed. This is likely the result of an I/O error.")); return false; } return true; @@ -763,7 +762,7 @@ bool BlockManager::FlushBlockFile(int blockfile_num, bool fFinalize, bool finali FlatFilePos block_pos_old(blockfile_num, m_blockfile_info[blockfile_num].nSize); if (!BlockFileSeq().Flush(block_pos_old, fFinalize)) { - m_opts.notifications.flushError("Flushing block file to disk failed. This is likely the result of an I/O error."); + m_opts.notifications.flushError(_("Flushing block file to disk failed. This is likely the result of an I/O error.")); success = false; } // we do not always flush the undo file, as the chain tip may be lagging behind the incoming blocks, @@ -848,7 +847,7 @@ fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const return BlockFileSeq().FileName(pos); } -bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown) +FlatFilePos BlockManager::FindNextBlockPos(unsigned int nAddSize, unsigned int nHeight, uint64_t nTime) { LOCK(cs_LastBlockFile); @@ -863,88 +862,101 @@ bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigne } const int last_blockfile = m_blockfile_cursors[chain_type]->file_num; - int nFile = fKnown ? pos.nFile : last_blockfile; + int nFile = last_blockfile; if (static_cast<int>(m_blockfile_info.size()) <= nFile) { m_blockfile_info.resize(nFile + 1); } bool finalize_undo = false; - if (!fKnown) { - unsigned int max_blockfile_size{MAX_BLOCKFILE_SIZE}; - // Use smaller blockfiles in test-only -fastprune mode - but avoid - // the possibility of having a block not fit into the block file. - if (m_opts.fast_prune) { - max_blockfile_size = 0x10000; // 64kiB - if (nAddSize >= max_blockfile_size) { - // dynamically adjust the blockfile size to be larger than the added size - max_blockfile_size = nAddSize + 1; - } + unsigned int max_blockfile_size{MAX_BLOCKFILE_SIZE}; + // Use smaller blockfiles in test-only -fastprune mode - but avoid + // the possibility of having a block not fit into the block file. + if (m_opts.fast_prune) { + max_blockfile_size = 0x10000; // 64kiB + if (nAddSize >= max_blockfile_size) { + // dynamically adjust the blockfile size to be larger than the added size + max_blockfile_size = nAddSize + 1; } - assert(nAddSize < max_blockfile_size); - - while (m_blockfile_info[nFile].nSize + nAddSize >= max_blockfile_size) { - // when the undo file is keeping up with the block file, we want to flush it explicitly - // when it is lagging behind (more blocks arrive than are being connected), we let the - // undo block write case handle it - finalize_undo = (static_cast<int>(m_blockfile_info[nFile].nHeightLast) == - Assert(m_blockfile_cursors[chain_type])->undo_height); - - // Try the next unclaimed blockfile number - nFile = this->MaxBlockfileNum() + 1; - // Set to increment MaxBlockfileNum() for next iteration - m_blockfile_cursors[chain_type] = BlockfileCursor{nFile}; - - if (static_cast<int>(m_blockfile_info.size()) <= nFile) { - m_blockfile_info.resize(nFile + 1); - } + } + assert(nAddSize < max_blockfile_size); + + while (m_blockfile_info[nFile].nSize + nAddSize >= max_blockfile_size) { + // when the undo file is keeping up with the block file, we want to flush it explicitly + // when it is lagging behind (more blocks arrive than are being connected), we let the + // undo block write case handle it + finalize_undo = (static_cast<int>(m_blockfile_info[nFile].nHeightLast) == + Assert(m_blockfile_cursors[chain_type])->undo_height); + + // Try the next unclaimed blockfile number + nFile = this->MaxBlockfileNum() + 1; + // Set to increment MaxBlockfileNum() for next iteration + m_blockfile_cursors[chain_type] = BlockfileCursor{nFile}; + + if (static_cast<int>(m_blockfile_info.size()) <= nFile) { + m_blockfile_info.resize(nFile + 1); } - pos.nFile = nFile; - pos.nPos = m_blockfile_info[nFile].nSize; } + FlatFilePos pos; + pos.nFile = nFile; + pos.nPos = m_blockfile_info[nFile].nSize; if (nFile != last_blockfile) { - if (!fKnown) { - LogPrint(BCLog::BLOCKSTORAGE, "Leaving block file %i: %s (onto %i) (height %i)\n", - last_blockfile, m_blockfile_info[last_blockfile].ToString(), nFile, nHeight); - - // Do not propagate the return code. The flush concerns a previous block - // and undo file that has already been written to. If a flush fails - // here, and we crash, there is no expected additional block data - // inconsistency arising from the flush failure here. However, the undo - // data may be inconsistent after a crash if the flush is called during - // a reindex. A flush error might also leave some of the data files - // untrimmed. - if (!FlushBlockFile(last_blockfile, !fKnown, finalize_undo)) { - LogPrintLevel(BCLog::BLOCKSTORAGE, BCLog::Level::Warning, - "Failed to flush previous block file %05i (finalize=%i, finalize_undo=%i) before opening new block file %05i\n", - last_blockfile, !fKnown, finalize_undo, nFile); - } + LogPrint(BCLog::BLOCKSTORAGE, "Leaving block file %i: %s (onto %i) (height %i)\n", + last_blockfile, m_blockfile_info[last_blockfile].ToString(), nFile, nHeight); + + // Do not propagate the return code. The flush concerns a previous block + // and undo file that has already been written to. If a flush fails + // here, and we crash, there is no expected additional block data + // inconsistency arising from the flush failure here. However, the undo + // data may be inconsistent after a crash if the flush is called during + // a reindex. A flush error might also leave some of the data files + // untrimmed. + if (!FlushBlockFile(last_blockfile, /*fFinalize=*/true, finalize_undo)) { + LogPrintLevel(BCLog::BLOCKSTORAGE, BCLog::Level::Warning, + "Failed to flush previous block file %05i (finalize=1, finalize_undo=%i) before opening new block file %05i\n", + last_blockfile, finalize_undo, nFile); } // No undo data yet in the new file, so reset our undo-height tracking. m_blockfile_cursors[chain_type] = BlockfileCursor{nFile}; } m_blockfile_info[nFile].AddBlock(nHeight, nTime); - if (fKnown) { - m_blockfile_info[nFile].nSize = std::max(pos.nPos + nAddSize, m_blockfile_info[nFile].nSize); - } else { - m_blockfile_info[nFile].nSize += nAddSize; + m_blockfile_info[nFile].nSize += nAddSize; + + bool out_of_space; + size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); + if (out_of_space) { + m_opts.notifications.fatalError(_("Disk space is too low!")); + return {}; + } + if (bytes_allocated != 0 && IsPruneMode()) { + m_check_for_pruning = true; } - if (!fKnown) { - bool out_of_space; - size_t bytes_allocated = BlockFileSeq().Allocate(pos, nAddSize, out_of_space); - if (out_of_space) { - m_opts.notifications.fatalError("Disk space is too low!", _("Disk space is too low!")); - return false; - } - if (bytes_allocated != 0 && IsPruneMode()) { - m_check_for_pruning = true; - } + m_dirty_fileinfo.insert(nFile); + return pos; +} + +void BlockManager::UpdateBlockInfo(const CBlock& block, unsigned int nHeight, const FlatFilePos& pos) +{ + LOCK(cs_LastBlockFile); + + // Update the cursor so it points to the last file. + const BlockfileType chain_type{BlockfileTypeForHeight(nHeight)}; + auto& cursor{m_blockfile_cursors[chain_type]}; + if (!cursor || cursor->file_num < pos.nFile) { + m_blockfile_cursors[chain_type] = BlockfileCursor{pos.nFile}; } + // Update the file information with the current block. + const unsigned int added_size = ::GetSerializeSize(TX_WITH_WITNESS(block)); + const int nFile = pos.nFile; + if (static_cast<int>(m_blockfile_info.size()) <= nFile) { + m_blockfile_info.resize(nFile + 1); + } + m_blockfile_info[nFile].AddBlock(nHeight, block.GetBlockTime()); + m_blockfile_info[nFile].nSize = std::max(pos.nPos + added_size, m_blockfile_info[nFile].nSize); m_dirty_fileinfo.insert(nFile); - return true; } bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize) @@ -960,7 +972,7 @@ bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFileP bool out_of_space; size_t bytes_allocated = UndoFileSeq().Allocate(pos, nAddSize, out_of_space); if (out_of_space) { - return FatalError(m_opts.notifications, state, "Disk space is too low!", _("Disk space is too low!")); + return FatalError(m_opts.notifications, state, _("Disk space is too low!")); } if (bytes_allocated != 0 && IsPruneMode()) { m_check_for_pruning = true; @@ -1008,13 +1020,13 @@ bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValid return false; } if (!UndoWriteToDisk(blockundo, _pos, block.pprev->GetBlockHash())) { - return FatalError(m_opts.notifications, state, "Failed to write undo data"); + return FatalError(m_opts.notifications, state, _("Failed to write undo data.")); } // rev files are written in block height order, whereas blk files are written as blocks come in (often out of order) // we want to flush the rev (undo) file once we've written the last block, which is indicated by the last height // in the block file info as below; note that this does not catch the case where the undo writes are keeping up // with the block writes (usually when a synced up node is getting newly mined blocks) -- this case is caught in - // the FindBlockPos function + // the FindNextBlockPos function if (_pos.nFile < cursor.file_num && static_cast<uint32_t>(block.nHeight) == m_blockfile_info[_pos.nFile].nHeightLast) { // Do not propagate the return code, a failed flush here should not // be an indication for a failed write. If it were propagated here, @@ -1130,28 +1142,20 @@ bool BlockManager::ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatF return true; } -FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, const FlatFilePos* dbp) +FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight) { unsigned int nBlockSize = ::GetSerializeSize(TX_WITH_WITNESS(block)); - FlatFilePos blockPos; - const auto position_known {dbp != nullptr}; - if (position_known) { - blockPos = *dbp; - } else { - // when known, blockPos.nPos points at the offset of the block data in the blk file. that already accounts for - // the serialization header present in the file (the 4 magic message start bytes + the 4 length bytes = 8 bytes = BLOCK_SERIALIZATION_HEADER_SIZE). - // we add BLOCK_SERIALIZATION_HEADER_SIZE only for new blocks since they will have the serialization header added when written to disk. - nBlockSize += static_cast<unsigned int>(BLOCK_SERIALIZATION_HEADER_SIZE); - } - if (!FindBlockPos(blockPos, nBlockSize, nHeight, block.GetBlockTime(), position_known)) { - LogError("%s: FindBlockPos failed\n", __func__); + // Account for the 4 magic message start bytes + the 4 length bytes (8 bytes total, + // defined as BLOCK_SERIALIZATION_HEADER_SIZE) + nBlockSize += static_cast<unsigned int>(BLOCK_SERIALIZATION_HEADER_SIZE); + FlatFilePos blockPos{FindNextBlockPos(nBlockSize, nHeight, block.GetBlockTime())}; + if (blockPos.IsNull()) { + LogError("%s: FindNextBlockPos failed\n", __func__); return FlatFilePos(); } - if (!position_known) { - if (!WriteBlockToDisk(block, blockPos)) { - m_opts.notifications.fatalError("Failed to write block"); - return FlatFilePos(); - } + if (!WriteBlockToDisk(block, blockPos)) { + m_opts.notifications.fatalError(_("Failed to write block.")); + return FlatFilePos(); } return blockPos; } @@ -1175,69 +1179,66 @@ public: void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFiles) { - ScheduleBatchPriority(); - - { - ImportingNow imp{chainman.m_blockman.m_importing}; - - // -reindex - if (fReindex) { - int nFile = 0; - // Map of disk positions for blocks with unknown parent (only used for reindex); - // parent hash -> child disk position, multiple children can have the same parent. - std::multimap<uint256, FlatFilePos> blocks_with_unknown_parent; - while (true) { - FlatFilePos pos(nFile, 0); - if (!fs::exists(chainman.m_blockman.GetBlockPosFilename(pos))) { - break; // No block files left to reindex - } - AutoFile file{chainman.m_blockman.OpenBlockFile(pos, true)}; - if (file.IsNull()) { - break; // This error is logged in OpenBlockFile - } - LogPrintf("Reindexing block file blk%05u.dat...\n", (unsigned int)nFile); - chainman.LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent); - if (chainman.m_interrupt) { - LogPrintf("Interrupt requested. Exit %s\n", __func__); - return; - } - nFile++; + ImportingNow imp{chainman.m_blockman.m_importing}; + + // -reindex + if (!chainman.m_blockman.m_blockfiles_indexed) { + int nFile = 0; + // Map of disk positions for blocks with unknown parent (only used for reindex); + // parent hash -> child disk position, multiple children can have the same parent. + std::multimap<uint256, FlatFilePos> blocks_with_unknown_parent; + while (true) { + FlatFilePos pos(nFile, 0); + if (!fs::exists(chainman.m_blockman.GetBlockPosFilename(pos))) { + break; // No block files left to reindex } - WITH_LOCK(::cs_main, chainman.m_blockman.m_block_tree_db->WriteReindexing(false)); - fReindex = false; - LogPrintf("Reindexing finished\n"); - // To avoid ending up in a situation without genesis block, re-try initializing (no-op if reindexing worked): - chainman.ActiveChainstate().LoadGenesisBlock(); + AutoFile file{chainman.m_blockman.OpenBlockFile(pos, true)}; + if (file.IsNull()) { + break; // This error is logged in OpenBlockFile + } + LogPrintf("Reindexing block file blk%05u.dat...\n", (unsigned int)nFile); + chainman.LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent); + if (chainman.m_interrupt) { + LogPrintf("Interrupt requested. Exit %s\n", __func__); + return; + } + nFile++; } - - // -loadblock= - for (const fs::path& path : vImportFiles) { - AutoFile file{fsbridge::fopen(path, "rb")}; - if (!file.IsNull()) { - LogPrintf("Importing blocks file %s...\n", fs::PathToString(path)); - chainman.LoadExternalBlockFile(file); - if (chainman.m_interrupt) { - LogPrintf("Interrupt requested. Exit %s\n", __func__); - return; - } - } else { - LogPrintf("Warning: Could not open blocks file %s\n", fs::PathToString(path)); + WITH_LOCK(::cs_main, chainman.m_blockman.m_block_tree_db->WriteReindexing(false)); + chainman.m_blockman.m_blockfiles_indexed = true; + LogPrintf("Reindexing finished\n"); + // To avoid ending up in a situation without genesis block, re-try initializing (no-op if reindexing worked): + chainman.ActiveChainstate().LoadGenesisBlock(); + } + + // -loadblock= + for (const fs::path& path : vImportFiles) { + AutoFile file{fsbridge::fopen(path, "rb")}; + if (!file.IsNull()) { + LogPrintf("Importing blocks file %s...\n", fs::PathToString(path)); + chainman.LoadExternalBlockFile(file); + if (chainman.m_interrupt) { + LogPrintf("Interrupt requested. Exit %s\n", __func__); + return; } + } else { + LogPrintf("Warning: Could not open blocks file %s\n", fs::PathToString(path)); } + } - // scan for better chains in the block chain database, that are not yet connected in the active best chain + // scan for better chains in the block chain database, that are not yet connected in the active best chain - // We can't hold cs_main during ActivateBestChain even though we're accessing - // the chainman unique_ptrs since ABC requires us not to be holding cs_main, so retrieve - // the relevant pointers before the ABC call. - for (Chainstate* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) { - BlockValidationState state; - if (!chainstate->ActivateBestChain(state, nullptr)) { - chainman.GetNotifications().fatalError(strprintf("Failed to connect best block (%s)", state.ToString())); - return; - } + // We can't hold cs_main during ActivateBestChain even though we're accessing + // the chainman unique_ptrs since ABC requires us not to be holding cs_main, so retrieve + // the relevant pointers before the ABC call. + for (Chainstate* chainstate : WITH_LOCK(::cs_main, return chainman.GetAll())) { + BlockValidationState state; + if (!chainstate->ActivateBestChain(state, nullptr)) { + chainman.GetNotifications().fatalError(strprintf(_("Failed to connect best block (%s)."), state.ToString())); + return; } - } // End scope of ImportingNow + } + // End scope of ImportingNow } std::ostream& operator<<(std::ostream& os, const BlockfileType& type) { diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index ce514cc645..108a08a72b 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -76,8 +76,6 @@ static const unsigned int MAX_BLOCKFILE_SIZE = 0x8000000; // 128 MiB /** Size of header written by WriteBlockToDisk before a serialized CBlock */ static constexpr size_t BLOCK_SERIALIZATION_HEADER_SIZE = std::tuple_size_v<MessageStartChars> + sizeof(unsigned int); -extern std::atomic_bool fReindex; - // Because validation code takes pointers to the map's CBlockIndex objects, if // we ever switch to another associative container, we need to either use a // container that has stable addressing (true of all std associative @@ -155,7 +153,16 @@ private: /** Return false if undo file flushing fails. */ [[nodiscard]] bool FlushUndoFile(int block_file, bool finalize = false); - [[nodiscard]] bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown); + /** + * Helper function performing various preparations before a block can be saved to disk: + * Returns the correct position for the block to be saved, which may be in the current or a new + * block file depending on nAddSize. May flush the previous blockfile to disk if full, updates + * blockfile info, and checks if there is enough disk space to save the block. + * + * The nAddSize argument passed to this function should include not just the size of the serialized CBlock, but also the size of + * separator fields which are written before it by WriteBlockToDisk (BLOCK_SERIALIZATION_HEADER_SIZE). + */ + [[nodiscard]] FlatFilePos FindNextBlockPos(unsigned int nAddSize, unsigned int nHeight, uint64_t nTime); [[nodiscard]] bool FlushChainstateBlockFile(int tip_height); bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize); @@ -164,6 +171,12 @@ private: AutoFile OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const; + /** + * Write a block to disk. The pos argument passed to this function is modified by this call. Before this call, it should + * point to an unused file location where separator fields will be written, followed by the serialized CBlock data. + * After this call, it will point to the beginning of the serialized CBlock data, after the separator fields + * (BLOCK_SERIALIZATION_HEADER_SIZE) + */ bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const; bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock) const; @@ -206,7 +219,7 @@ private: //! effectively. //! //! This data structure maintains separate blockfile number cursors for each - //! BlockfileType. The ASSUMED state is initialized, when necessary, in FindBlockPos(). + //! BlockfileType. The ASSUMED state is initialized, when necessary, in FindNextBlockPos(). //! //! The first element is the NORMAL cursor, second is ASSUMED. std::array<std::optional<BlockfileCursor>, BlockfileType::NUM_TYPES> @@ -254,11 +267,19 @@ public: explicit BlockManager(const util::SignalInterrupt& interrupt, Options opts) : m_prune_mode{opts.prune_target > 0}, m_opts{std::move(opts)}, - m_interrupt{interrupt} {}; + m_interrupt{interrupt} {} const util::SignalInterrupt& m_interrupt; std::atomic<bool> m_importing{false}; + /** + * Whether all blockfiles have been added to the block tree database. + * Normally true, but set to false when a reindex is requested and the + * database is wiped. The value is persisted in the database across restarts + * and will be false until reindexing completes. + */ + std::atomic_bool m_blockfiles_indexed{true}; + BlockMap m_block_index GUARDED_BY(cs_main); /** @@ -312,8 +333,24 @@ public: bool WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValidationState& state, CBlockIndex& block) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - /** Store block on disk. If dbp is not nullptr, then it provides the known position of the block within a block file on disk. */ - FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, const FlatFilePos* dbp); + /** Store block on disk and update block file statistics. + * + * @param[in] block the block to be stored + * @param[in] nHeight the height of the block + * + * @returns in case of success, the position to which the block was written to + * in case of an error, an empty FlatFilePos + */ + FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight); + + /** Update blockfile info while processing a block during reindex. The block must be available on disk. + * + * @param[in] block the block being processed + * @param[in] nHeight the height of the block + * @param[in] pos the position of the serialized CBlock on disk. This is the position returned + * by WriteBlockToDisk pointing at the CBlock, not the separator fields before it + */ + void UpdateBlockInfo(const CBlock& block, unsigned int nHeight, const FlatFilePos& pos); /** Whether running in -prune mode. */ [[nodiscard]] bool IsPruneMode() const { return m_prune_mode; } @@ -322,7 +359,7 @@ public: [[nodiscard]] uint64_t GetPruneTarget() const { return m_opts.prune_target; } static constexpr auto PRUNE_TARGET_MANUAL{std::numeric_limits<uint64_t>::max()}; - [[nodiscard]] bool LoadingBlocks() const { return m_importing || fReindex; } + [[nodiscard]] bool LoadingBlocks() const { return m_importing || !m_blockfiles_indexed; } /** Calculate the amount of disk space the block & undo files currently use */ uint64_t CalculateCurrentUsage(); diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index bf1fc06b0b..d7e6176be1 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -45,11 +45,12 @@ static ChainstateLoadResult CompleteChainstateInitialization( .path = chainman.m_options.datadir / "blocks" / "index", .cache_bytes = static_cast<size_t>(cache_sizes.block_tree_db), .memory_only = options.block_tree_db_in_memory, - .wipe_data = options.reindex, + .wipe_data = options.wipe_block_tree_db, .options = chainman.m_options.block_tree_db}); - if (options.reindex) { + if (options.wipe_block_tree_db) { pblocktree->WriteReindexing(true); + chainman.m_blockman.m_blockfiles_indexed = false; //If we're reindexing in prune mode, wipe away unusable block files and all undo data files if (options.prune) { chainman.m_blockman.CleanupBlockRevFiles(); @@ -60,8 +61,7 @@ static ChainstateLoadResult CompleteChainstateInitialization( // LoadBlockIndex will load m_have_pruned if we've ever removed a // block file from disk. - // Note that it also sets fReindex global based on the disk flag! - // From here on, fReindex and options.reindex values may be different! + // Note that it also sets m_blockfiles_indexed based on the disk flag! if (!chainman.LoadBlockIndex()) { if (chainman.m_interrupt) return {ChainstateLoadStatus::INTERRUPTED, {}}; return {ChainstateLoadStatus::FAILURE, _("Error loading block database")}; @@ -84,12 +84,12 @@ static ChainstateLoadResult CompleteChainstateInitialization( // If we're not mid-reindex (based on disk + args), add a genesis block on disk // (otherwise we use the one already on disk). // This is called again in ImportBlocks after the reindex completes. - if (!fReindex && !chainman.ActiveChainstate().LoadGenesisBlock()) { + if (chainman.m_blockman.m_blockfiles_indexed && !chainman.ActiveChainstate().LoadGenesisBlock()) { return {ChainstateLoadStatus::FAILURE, _("Error initializing block database")}; } auto is_coinsview_empty = [&](Chainstate* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { - return options.reindex || options.reindex_chainstate || chainstate->CoinsTip().GetBestBlock().IsNull(); + return options.wipe_chainstate_db || chainstate->CoinsTip().GetBestBlock().IsNull(); }; assert(chainman.m_total_coinstip_cache > 0); @@ -110,7 +110,7 @@ static ChainstateLoadResult CompleteChainstateInitialization( chainstate->InitCoinsDB( /*cache_size_bytes=*/chainman.m_total_coinsdb_cache * init_cache_fraction, /*in_memory=*/options.coins_db_in_memory, - /*should_wipe=*/options.reindex || options.reindex_chainstate); + /*should_wipe=*/options.wipe_chainstate_db); if (options.coins_error_cb) { chainstate->CoinsErrorCatcher().AddReadErrCallback(options.coins_error_cb); @@ -142,7 +142,7 @@ static ChainstateLoadResult CompleteChainstateInitialization( } } - if (!options.reindex) { + if (!options.wipe_block_tree_db) { auto chainstates{chainman.GetAll()}; if (std::any_of(chainstates.begin(), chainstates.end(), [](const Chainstate* cs) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { return cs->NeedsRedownload(); })) { @@ -188,7 +188,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize // Load a chain created from a UTXO snapshot, if any exist. bool has_snapshot = chainman.DetectSnapshotChainstate(); - if (has_snapshot && (options.reindex || options.reindex_chainstate)) { + if (has_snapshot && options.wipe_chainstate_db) { LogPrintf("[snapshot] deleting snapshot chainstate due to reindexing\n"); if (!chainman.DeleteSnapshotChainstate()) { return {ChainstateLoadStatus::FAILURE_FATAL, Untranslated("Couldn't remove snapshot chainstate.")}; @@ -247,7 +247,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize ChainstateLoadResult VerifyLoadedChainstate(ChainstateManager& chainman, const ChainstateLoadOptions& options) { auto is_coinsview_empty = [&](Chainstate* chainstate) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { - return options.reindex || options.reindex_chainstate || chainstate->CoinsTip().GetBestBlock().IsNull(); + return options.wipe_chainstate_db || chainstate->CoinsTip().GetBestBlock().IsNull(); }; LOCK(cs_main); diff --git a/src/node/chainstate.h b/src/node/chainstate.h index a6e9a0331b..bb0c4f2b87 100644 --- a/src/node/chainstate.h +++ b/src/node/chainstate.h @@ -22,8 +22,13 @@ struct ChainstateLoadOptions { CTxMemPool* mempool{nullptr}; bool block_tree_db_in_memory{false}; bool coins_db_in_memory{false}; - bool reindex{false}; - bool reindex_chainstate{false}; + // Whether to wipe the block tree database when loading it. If set, this + // will also set a reindexing flag so any existing block data files will be + // scanned and added to the database. + bool wipe_block_tree_db{false}; + // Whether to wipe the chainstate database when loading it. If set, this + // will cause the chainstate database to be rebuilt starting from genesis. + bool wipe_chainstate_db{false}; bool prune{false}; //! Setting require_full_verification to true will require all checks at //! check_level (below) to succeed for loading to succeed. Setting it to diff --git a/src/node/context.cpp b/src/node/context.cpp index ca56fa0b86..e32d21b383 100644 --- a/src/node/context.cpp +++ b/src/node/context.cpp @@ -8,6 +8,7 @@ #include <banman.h> #include <interfaces/chain.h> #include <kernel/context.h> +#include <key.h> #include <net.h> #include <net_processing.h> #include <netgroup.h> @@ -16,6 +17,7 @@ #include <scheduler.h> #include <txmempool.h> #include <validation.h> +#include <validationinterface.h> namespace node { NodeContext::NodeContext() = default; diff --git a/src/node/context.h b/src/node/context.h index 245f230aec..a7d92989dd 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -5,10 +5,7 @@ #ifndef BITCOIN_NODE_CONTEXT_H #define BITCOIN_NODE_CONTEXT_H -#include <kernel/context.h> - #include <atomic> -#include <cassert> #include <cstdlib> #include <functional> #include <memory> @@ -24,6 +21,7 @@ class ValidationSignals; class CScheduler; class CTxMemPool; class ChainstateManager; +class ECC_Context; class NetGroupManager; class PeerManager; namespace interfaces { @@ -32,6 +30,12 @@ class ChainClient; class Init; class WalletLoader; } // namespace interfaces +namespace kernel { +struct Context; +} +namespace util { +class SignalInterrupt; +} namespace node { class KernelNotifications; @@ -49,6 +53,7 @@ class KernelNotifications; struct NodeContext { //! libbitcoin_kernel context std::unique_ptr<kernel::Context> kernel; + std::unique_ptr<ECC_Context> ecc_context; //! Init interface for initializing current process and connecting to other processes. interfaces::Init* init{nullptr}; //! Interrupt object used to track whether node shutdown was requested. diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index f9a372e3de..216f44ab9e 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -17,6 +17,7 @@ #include <interfaces/node.h> #include <interfaces/wallet.h> #include <kernel/chain.h> +#include <kernel/context.h> #include <kernel/mempool_entry.h> #include <logging.h> #include <mapport.h> @@ -47,14 +48,13 @@ #include <util/check.h> #include <util/result.h> #include <util/signalinterrupt.h> +#include <util/string.h> #include <util/translation.h> #include <validation.h> #include <validationinterface.h> #include <warnings.h> -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif +#include <config/bitcoin-config.h> // IWYU pragma: keep #include <any> #include <memory> @@ -91,7 +91,7 @@ public: explicit NodeImpl(NodeContext& context) { setContext(&context); } void initLogging() override { InitLogging(args()); } void initParameterInteraction() override { InitParameterInteraction(args()); } - bilingual_str getWarnings() override { return GetWarnings(true); } + bilingual_str getWarnings() override { return Join(GetWarnings(), Untranslated("<hr />")); } int getExitStatus() override { return Assert(m_context)->exit_status.load(); } uint32_t getLogCategories() override { return LogInstance().GetCategoryMask(); } bool baseInitialize() override @@ -100,6 +100,7 @@ public: if (!AppInitParameterInteraction(args())) return false; m_context->kernel = std::make_unique<kernel::Context>(); + m_context->ecc_context = std::make_unique<ECC_Context>(); if (!AppInitSanityChecks(*m_context->kernel)) return false; if (!AppInitLockDataDirectory()) return false; @@ -317,7 +318,7 @@ public: CFeeRate getDustRelayFee() override { if (!m_context->mempool) return CFeeRate{DUST_RELAY_TX_FEE}; - return m_context->mempool->m_dust_relay_feerate; + return m_context->mempool->m_opts.dust_relay_feerate; } UniValue executeRpc(const std::string& command, const UniValue& params, const std::string& uri) override { @@ -406,6 +407,7 @@ public: NodeContext* m_context{nullptr}; }; +// NOLINTNEXTLINE(misc-no-recursion) bool FillBlock(const CBlockIndex* index, const FoundBlock& block, UniqueLock<RecursiveMutex>& lock, const CChain& active, const BlockManager& blockman) { if (!index) return false; @@ -699,7 +701,7 @@ public: { const CTxMemPool::Limits default_limits{}; - const CTxMemPool::Limits& limits{m_node.mempool ? m_node.mempool->m_limits : default_limits}; + const CTxMemPool::Limits& limits{m_node.mempool ? m_node.mempool->m_opts.limits : default_limits}; limit_ancestor_count = limits.ancestor_count; limit_descendant_count = limits.descendant_count; @@ -730,17 +732,17 @@ public: CFeeRate relayMinFee() override { if (!m_node.mempool) return CFeeRate{DEFAULT_MIN_RELAY_TX_FEE}; - return m_node.mempool->m_min_relay_feerate; + return m_node.mempool->m_opts.min_relay_feerate; } CFeeRate relayIncrementalFee() override { if (!m_node.mempool) return CFeeRate{DEFAULT_INCREMENTAL_RELAY_FEE}; - return m_node.mempool->m_incremental_relay_feerate; + return m_node.mempool->m_opts.incremental_relay_feerate; } CFeeRate relayDustFee() override { if (!m_node.mempool) return CFeeRate{DUST_RELAY_TX_FEE}; - return m_node.mempool->m_dust_relay_feerate; + return m_node.mempool->m_opts.dust_relay_feerate; } bool havePruned() override { diff --git a/src/node/kernel_notifications.cpp b/src/node/kernel_notifications.cpp index 1fd3bad296..e326d4a1f2 100644 --- a/src/node/kernel_notifications.cpp +++ b/src/node/kernel_notifications.cpp @@ -4,9 +4,7 @@ #include <node/kernel_notifications.h> -#if defined(HAVE_CONFIG_H) -#include <config/bitcoin-config.h> -#endif +#include <config/bitcoin-config.h> // IWYU pragma: keep #include <chain.h> #include <common/args.h> @@ -16,6 +14,7 @@ #include <node/abort.h> #include <node/interface_ui.h> #include <util/check.h> +#include <util/signalinterrupt.h> #include <util/strencodings.h> #include <util/string.h> #include <util/translation.h> @@ -84,15 +83,15 @@ void KernelNotifications::warning(const bilingual_str& warning) DoWarning(warning); } -void KernelNotifications::flushError(const std::string& debug_message) +void KernelNotifications::flushError(const bilingual_str& message) { - AbortNode(&m_shutdown, m_exit_status, debug_message); + AbortNode(&m_shutdown, m_exit_status, message); } -void KernelNotifications::fatalError(const std::string& debug_message, const bilingual_str& user_message) +void KernelNotifications::fatalError(const bilingual_str& message) { node::AbortNode(m_shutdown_on_fatal_error ? &m_shutdown : nullptr, - m_exit_status, debug_message, user_message); + m_exit_status, message); } void ReadNotificationArgs(const ArgsManager& args, KernelNotifications& notifications) diff --git a/src/node/kernel_notifications.h b/src/node/kernel_notifications.h index 38d8600ac6..f4d97a0fff 100644 --- a/src/node/kernel_notifications.h +++ b/src/node/kernel_notifications.h @@ -9,7 +9,6 @@ #include <atomic> #include <cstdint> -#include <string> class ArgsManager; class CBlockIndex; @@ -37,9 +36,9 @@ public: void warning(const bilingual_str& warning) override; - void flushError(const std::string& debug_message) override; + void flushError(const bilingual_str& message) override; - void fatalError(const std::string& debug_message, const bilingual_str& user_message = {}) override; + void fatalError(const bilingual_str& message) override; //! Block height after which blockTip notification will return Interrupted{}, if >0. int m_stop_at_height{DEFAULT_STOPATHEIGHT}; diff --git a/src/node/timeoffsets.cpp b/src/node/timeoffsets.cpp new file mode 100644 index 0000000000..62f527be8a --- /dev/null +++ b/src/node/timeoffsets.cpp @@ -0,0 +1,69 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <logging.h> +#include <node/interface_ui.h> +#include <node/timeoffsets.h> +#include <sync.h> +#include <tinyformat.h> +#include <util/time.h> +#include <util/translation.h> +#include <warnings.h> + +#include <algorithm> +#include <chrono> +#include <cstdint> +#include <deque> +#include <limits> +#include <optional> + +using namespace std::chrono_literals; + +void TimeOffsets::Add(std::chrono::seconds offset) +{ + LOCK(m_mutex); + + if (m_offsets.size() >= MAX_SIZE) { + m_offsets.pop_front(); + } + m_offsets.push_back(offset); + LogDebug(BCLog::NET, "Added time offset %+ds, total samples %d\n", + Ticks<std::chrono::seconds>(offset), m_offsets.size()); +} + +std::chrono::seconds TimeOffsets::Median() const +{ + LOCK(m_mutex); + + // Only calculate the median if we have 5 or more offsets + if (m_offsets.size() < 5) return 0s; + + auto sorted_copy = m_offsets; + std::sort(sorted_copy.begin(), sorted_copy.end()); + return sorted_copy[sorted_copy.size() / 2]; // approximate median is good enough, keep it simple +} + +bool TimeOffsets::WarnIfOutOfSync() const +{ + // when median == std::numeric_limits<int64_t>::min(), calling std::chrono::abs is UB + auto median{std::max(Median(), std::chrono::seconds(std::numeric_limits<int64_t>::min() + 1))}; + if (std::chrono::abs(median) <= WARN_THRESHOLD) { + SetMedianTimeOffsetWarning(std::nullopt); + uiInterface.NotifyAlertChanged(); + return false; + } + + bilingual_str msg{strprintf(_( + "Your computer's date and time appear to be more than %d minutes out of sync with the network, " + "this may lead to consensus failure. After you've confirmed your computer's clock, this message " + "should no longer appear when you restart your node. Without a restart, it should stop showing " + "automatically after you've connected to a sufficient number of new outbound peers, which may " + "take some time. You can inspect the `timeoffset` field of the `getpeerinfo` and `getnetworkinfo` " + "RPC methods to get more info." + ), Ticks<std::chrono::minutes>(WARN_THRESHOLD))}; + LogWarning("%s\n", msg.original); + SetMedianTimeOffsetWarning(msg); + uiInterface.NotifyAlertChanged(); + return true; +} diff --git a/src/node/timeoffsets.h b/src/node/timeoffsets.h new file mode 100644 index 0000000000..2b12584e12 --- /dev/null +++ b/src/node/timeoffsets.h @@ -0,0 +1,39 @@ +// Copyright (c) 2024-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_NODE_TIMEOFFSETS_H +#define BITCOIN_NODE_TIMEOFFSETS_H + +#include <sync.h> + +#include <chrono> +#include <cstddef> +#include <deque> + +class TimeOffsets +{ + //! Maximum number of timeoffsets stored. + static constexpr size_t MAX_SIZE{50}; + //! Minimum difference between system and network time for a warning to be raised. + static constexpr std::chrono::minutes WARN_THRESHOLD{10}; + + mutable Mutex m_mutex; + /** The observed time differences between our local clock and those of our outbound peers. A + * positive offset means our peer's clock is ahead of our local clock. */ + std::deque<std::chrono::seconds> m_offsets GUARDED_BY(m_mutex){}; + +public: + /** Add a new time offset sample. */ + void Add(std::chrono::seconds offset) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); + + /** Compute and return the median of the collected time offset samples. The median is returned + * as 0 when there are less than 5 samples. */ + std::chrono::seconds Median() const EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); + + /** Raise warnings if the median time offset exceeds the warnings threshold. Returns true if + * warnings were raised. */ + bool WarnIfOutOfSync() const EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); +}; + +#endif // BITCOIN_NODE_TIMEOFFSETS_H diff --git a/src/node/utxo_snapshot.h b/src/node/utxo_snapshot.h index 1160bb55f0..a7c4135787 100644 --- a/src/node/utxo_snapshot.h +++ b/src/node/utxo_snapshot.h @@ -6,16 +6,23 @@ #ifndef BITCOIN_NODE_UTXO_SNAPSHOT_H #define BITCOIN_NODE_UTXO_SNAPSHOT_H +#include <chainparams.h> +#include <kernel/chainparams.h> #include <kernel/cs_main.h> #include <serialize.h> #include <sync.h> #include <uint256.h> +#include <util/chaintype.h> +#include <util/check.h> #include <util/fs.h> #include <cstdint> #include <optional> #include <string_view> +// UTXO set snapshot magic bytes +static constexpr std::array<uint8_t, 5> SNAPSHOT_MAGIC_BYTES = {'u', 't', 'x', 'o', 0xff}; + class Chainstate; namespace node { @@ -23,23 +30,78 @@ namespace node { //! assumeutxo Chainstate can be constructed. class SnapshotMetadata { + const uint16_t m_version{1}; + const std::set<uint16_t> m_supported_versions{1}; + const MessageStartChars m_network_magic; public: //! The hash of the block that reflects the tip of the chain for the //! UTXO set contained in this snapshot. uint256 m_base_blockhash; + uint32_t m_base_blockheight; + //! The number of coins in the UTXO set contained in this snapshot. Used //! during snapshot load to estimate progress of UTXO set reconstruction. uint64_t m_coins_count = 0; - SnapshotMetadata() { } SnapshotMetadata( + const MessageStartChars network_magic) : + m_network_magic(network_magic) { } + SnapshotMetadata( + const MessageStartChars network_magic, const uint256& base_blockhash, + const int base_blockheight, uint64_t coins_count) : + m_network_magic(network_magic), m_base_blockhash(base_blockhash), + m_base_blockheight(base_blockheight), m_coins_count(coins_count) { } - SERIALIZE_METHODS(SnapshotMetadata, obj) { READWRITE(obj.m_base_blockhash, obj.m_coins_count); } + template <typename Stream> + inline void Serialize(Stream& s) const { + s << SNAPSHOT_MAGIC_BYTES; + s << m_version; + s << m_network_magic; + s << m_base_blockheight; + s << m_base_blockhash; + s << m_coins_count; + } + + template <typename Stream> + inline void Unserialize(Stream& s) { + // Read the snapshot magic bytes + std::array<uint8_t, SNAPSHOT_MAGIC_BYTES.size()> snapshot_magic; + s >> snapshot_magic; + if (snapshot_magic != SNAPSHOT_MAGIC_BYTES) { + throw std::ios_base::failure("Invalid UTXO set snapshot magic bytes. Please check if this is indeed a snapshot file or if you are using an outdated snapshot format."); + } + + // Read the version + uint16_t version; + s >> version; + if (m_supported_versions.find(version) == m_supported_versions.end()) { + throw std::ios_base::failure(strprintf("Version of snapshot %s does not match any of the supported versions.", version)); + } + + // Read the network magic (pchMessageStart) + MessageStartChars message; + s >> message; + if (!std::equal(message.begin(), message.end(), m_network_magic.data())) { + auto metadata_network{GetNetworkForMagic(message)}; + if (metadata_network) { + std::string network_string{ChainTypeToString(metadata_network.value())}; + auto node_network{GetNetworkForMagic(m_network_magic)}; + std::string node_network_string{ChainTypeToString(node_network.value())}; + throw std::ios_base::failure(strprintf("The network of the snapshot (%s) does not match the network of this node (%s).", network_string, node_network_string)); + } else { + throw std::ios_base::failure("This snapshot has been created for an unrecognized network. This could be a custom signet, a new testnet or possibly caused by data corruption."); + } + } + + s >> m_base_blockheight; + s >> m_base_blockhash; + s >> m_coins_count; + } }; //! The file in the snapshot chainstate dir which stores the base blockhash. This is |