diff options
-rw-r--r-- | doc/design/assumeutxo.md | 5 | ||||
-rw-r--r-- | src/bench/load_external.cpp | 3 | ||||
-rw-r--r-- | src/chain.h | 20 | ||||
-rw-r--r-- | src/node/blockstorage.cpp | 16 | ||||
-rw-r--r-- | src/node/blockstorage.h | 18 | ||||
-rw-r--r-- | src/node/chainstate.cpp | 2 | ||||
-rw-r--r-- | src/test/blockmanager_tests.cpp | 7 | ||||
-rw-r--r-- | src/test/coinstatsindex_tests.cpp | 2 | ||||
-rw-r--r-- | src/test/fuzz/load_external_block_file.cpp | 4 | ||||
-rw-r--r-- | src/test/util/chainstate.h | 17 | ||||
-rw-r--r-- | src/test/validation_chainstate_tests.cpp | 24 | ||||
-rw-r--r-- | src/test/validation_chainstatemanager_tests.cpp | 75 | ||||
-rw-r--r-- | src/validation.cpp | 289 | ||||
-rw-r--r-- | src/validation.h | 150 |
14 files changed, 378 insertions, 254 deletions
diff --git a/doc/design/assumeutxo.md b/doc/design/assumeutxo.md index 469c551536..1492877e62 100644 --- a/doc/design/assumeutxo.md +++ b/doc/design/assumeutxo.md @@ -17,10 +17,9 @@ respectively generate and load UTXO snapshots. The utility script - A new block index `nStatus` flag is introduced, `BLOCK_ASSUMED_VALID`, to mark block index entries that are required to be assumed-valid by a chainstate created - from a UTXO snapshot. This flag is mostly used as a way to modify certain + from a UTXO snapshot. This flag is used as a way to modify certain CheckBlockIndex() logic to account for index entries that are pending validation by a - chainstate running asynchronously in the background. We also use this flag to control - which index entries are added to setBlockIndexCandidates during LoadBlockIndex(). + chainstate running asynchronously in the background. - The concept of UTXO snapshots is treated as an implementation detail that lives behind the ChainstateManager interface. The external presentation of the changes diff --git a/src/bench/load_external.cpp b/src/bench/load_external.cpp index 1378a7b20a..252cbb163b 100644 --- a/src/bench/load_external.cpp +++ b/src/bench/load_external.cpp @@ -49,14 +49,13 @@ static void LoadExternalBlockFile(benchmark::Bench& bench) fclose(file); } - Chainstate& chainstate{testing_setup->m_node.chainman->ActiveChainstate()}; std::multimap<uint256, FlatFilePos> blocks_with_unknown_parent; FlatFilePos pos; bench.run([&] { // "rb" is "binary, O_RDONLY", positioned to the start of the file. // The file will be closed by LoadExternalBlockFile(). FILE* file{fsbridge::fopen(blkfile, "rb")}; - chainstate.LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent); + testing_setup->m_node.chainman->LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent); }); fs::remove(blkfile); } diff --git a/src/chain.h b/src/chain.h index f5dd0fd315..2e1fb37bec 100644 --- a/src/chain.h +++ b/src/chain.h @@ -113,10 +113,10 @@ enum BlockStatus : uint32_t { BLOCK_VALID_TRANSACTIONS = 3, //! Outputs do not overspend inputs, no double spends, coinbase output ok, no immature coinbase spends, BIP30. - //! Implies all parents are also at least CHAIN. + //! Implies all parents are either at least VALID_CHAIN, or are ASSUMED_VALID BLOCK_VALID_CHAIN = 4, - //! Scripts & signatures ok. Implies all parents are also at least SCRIPTS. + //! Scripts & signatures ok. Implies all parents are either at least VALID_SCRIPTS, or are ASSUMED_VALID. BLOCK_VALID_SCRIPTS = 5, //! All validity bits. @@ -134,10 +134,18 @@ enum BlockStatus : uint32_t { BLOCK_OPT_WITNESS = 128, //!< block data in blk*.dat was received with a witness-enforcing client /** - * If set, this indicates that the block index entry is assumed-valid. - * Certain diagnostics will be skipped in e.g. CheckBlockIndex(). - * It almost certainly means that the block's full validation is pending - * on a background chainstate. See `doc/design/assumeutxo.md`. + * If ASSUMED_VALID is set, it means that this block has not been validated + * and has validity status less than VALID_SCRIPTS. Also that it may have + * descendant blocks with VALID_SCRIPTS set, because they can be validated + * based on an assumeutxo snapshot. + * + * When an assumeutxo snapshot is loaded, the ASSUMED_VALID flag is added to + * unvalidated blocks at the snapshot height and below. Then, as the background + * validation progresses, and these blocks are validated, the ASSUMED_VALID + * flags are removed. See `doc/design/assumeutxo.md` for details. + * + * This flag is only used to implement checks in CheckBlockIndex() and + * should not be used elsewhere. */ BLOCK_ASSUMED_VALID = 256, }; diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 78416ec576..0d25c798ce 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -618,7 +618,7 @@ fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const return BlockFileSeq().FileName(pos); } -bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown) +bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown) { LOCK(cs_LastBlockFile); @@ -644,7 +644,7 @@ bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigne // when the undo file is keeping up with the block file, we want to flush it explicitly // when it is lagging behind (more blocks arrive than are being connected), we let the // undo block write case handle it - finalize_undo = (m_blockfile_info[nFile].nHeightLast == (unsigned int)active_chain.Tip()->nHeight); + finalize_undo = (m_blockfile_info[nFile].nHeightLast == m_undo_height_in_last_blockfile); nFile++; if (m_blockfile_info.size() <= nFile) { m_blockfile_info.resize(nFile + 1); @@ -660,6 +660,7 @@ bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigne } FlushBlockFile(!fKnown, finalize_undo); m_last_blockfile = nFile; + m_undo_height_in_last_blockfile = 0; // No undo data yet in the new file, so reset our undo-height tracking. } m_blockfile_info[nFile].AddBlock(nHeight, nTime); @@ -749,8 +750,9 @@ bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValid // the FindBlockPos function if (_pos.nFile < m_last_blockfile && static_cast<uint32_t>(block.nHeight) == m_blockfile_info[_pos.nFile].nHeightLast) { FlushUndoFile(_pos.nFile, true); + } else if (_pos.nFile == m_last_blockfile && static_cast<uint32_t>(block.nHeight) > m_undo_height_in_last_blockfile) { + m_undo_height_in_last_blockfile = block.nHeight; } - // update nUndoPos in block index block.nUndoPos = _pos.nPos; block.nStatus |= BLOCK_HAVE_UNDO; @@ -839,7 +841,7 @@ bool BlockManager::ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatF return true; } -FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const FlatFilePos* dbp) +FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, const FlatFilePos* dbp) { unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); FlatFilePos blockPos; @@ -852,7 +854,7 @@ FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, CCha // we add BLOCK_SERIALIZATION_HEADER_SIZE only for new blocks since they will have the serialization header added when written to disk. nBlockSize += static_cast<unsigned int>(BLOCK_SERIALIZATION_HEADER_SIZE); } - if (!FindBlockPos(blockPos, nBlockSize, nHeight, active_chain, block.GetBlockTime(), position_known)) { + if (!FindBlockPos(blockPos, nBlockSize, nHeight, block.GetBlockTime(), position_known)) { error("%s: FindBlockPos failed", __func__); return FlatFilePos(); } @@ -905,7 +907,7 @@ void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFile break; // This error is logged in OpenBlockFile } LogPrintf("Reindexing block file blk%05u.dat...\n", (unsigned int)nFile); - chainman.ActiveChainstate().LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent); + chainman.LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent); if (chainman.m_interrupt) { LogPrintf("Interrupt requested. Exit %s\n", __func__); return; @@ -924,7 +926,7 @@ void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFile FILE* file = fsbridge::fopen(path, "rb"); if (file) { LogPrintf("Importing blocks file %s...\n", fs::PathToString(path)); - chainman.ActiveChainstate().LoadExternalBlockFile(file); + chainman.LoadExternalBlockFile(file); if (chainman.m_interrupt) { LogPrintf("Interrupt requested. Exit %s\n", __func__); return; diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index c2e903e470..eb40d45aba 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -24,7 +24,6 @@ class BlockValidationState; class CBlock; class CBlockFileInfo; class CBlockUndo; -class CChain; class CChainParams; class Chainstate; class ChainstateManager; @@ -94,7 +93,7 @@ private: EXCLUSIVE_LOCKS_REQUIRED(cs_main); void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false); void FlushUndoFile(int block_file, bool finalize = false); - bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown); + bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown); bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize); FlatFileSeq BlockFileSeq() const; @@ -128,6 +127,19 @@ private: RecursiveMutex cs_LastBlockFile; std::vector<CBlockFileInfo> m_blockfile_info; int m_last_blockfile = 0; + + // Track the height of the highest block in m_last_blockfile whose undo + // data has been written. Block data is written to block files in download + // order, but is written to undo files in validation order, which is + // usually in order by height. To avoid wasting disk space, undo files will + // be trimmed whenever the corresponding block file is finalized and + // the height of the highest block written to the block file equals the + // height of the highest block written to the undo file. This is a + // heuristic and can sometimes preemptively trim undo files that will write + // more data later, and sometimes fail to trim undo files that can't have + // more data written later. + unsigned int m_undo_height_in_last_blockfile = 0; + /** Global flag to indicate we should check to see if there are * block/undo files that should be deleted. Set on startup * or if we allocate more file space when we're in prune mode @@ -202,7 +214,7 @@ public: EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** Store block on disk. If dbp is not nullptr, then it provides the known position of the block within a block file on disk. */ - FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const FlatFilePos* dbp); + FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, const FlatFilePos* dbp); /** Whether running in -prune mode. */ [[nodiscard]] bool IsPruneMode() const { return m_prune_mode; } diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index 255d8be0ec..0828f64856 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -221,7 +221,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize // A reload of the block index is required to recompute setBlockIndexCandidates // for the fully validated chainstate. - chainman.ActiveChainstate().UnloadBlockIndex(); + chainman.ActiveChainstate().ClearBlockIndexCandidates(); auto [init_status, init_error] = CompleteChainstateInitialization(chainman, cache_sizes, options); if (init_status != ChainstateLoadStatus::SUCCESS) { diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp index e4ed861b12..f52c692649 100644 --- a/src/test/blockmanager_tests.cpp +++ b/src/test/blockmanager_tests.cpp @@ -30,22 +30,21 @@ BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos) .notifications = notifications, }; BlockManager blockman{m_node.kernel->interrupt, blockman_opts}; - CChain chain {}; // simulate adding a genesis block normally - BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, nullptr).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); + BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, nullptr).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); // simulate what happens during reindex // simulate a well-formed genesis block being found at offset 8 in the blk00000.dat file // the block is found at offset 8 because there is an 8 byte serialization header // consisting of 4 magic bytes + 4 length bytes before each block in a well-formed blk file. FlatFilePos pos{0, BLOCK_SERIALIZATION_HEADER_SIZE}; - BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, &pos).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); + BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, &pos).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); // now simulate what happens after reindex for the first new block processed // the actual block contents don't matter, just that it's a block. // verify that the write position is at offset 0x12d. // this is a check to make sure that https://github.com/bitcoin/bitcoin/issues/21379 does not recur // 8 bytes (for serialization header) + 285 (for serialized genesis block) = 293 // add another 8 bytes for the second block's serialization header and we get 293 + 8 = 301 - FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1, chain, nullptr)}; + FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1, nullptr)}; BOOST_CHECK_EQUAL(actual.nPos, BLOCK_SERIALIZATION_HEADER_SIZE + ::GetSerializeSize(params->GenesisBlock(), CLIENT_VERSION) + BLOCK_SERIALIZATION_HEADER_SIZE); } diff --git a/src/test/coinstatsindex_tests.cpp b/src/test/coinstatsindex_tests.cpp index 74d6d7231a..787a196a0c 100644 --- a/src/test/coinstatsindex_tests.cpp +++ b/src/test/coinstatsindex_tests.cpp @@ -98,7 +98,7 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_unclean_shutdown, TestChain100Setup) LOCK(cs_main); BlockValidationState state; BOOST_CHECK(CheckBlock(block, state, params.GetConsensus())); - BOOST_CHECK(chainstate.AcceptBlock(new_block, state, &new_block_index, true, nullptr, nullptr, true)); + BOOST_CHECK(m_node.chainman->AcceptBlock(new_block, state, &new_block_index, true, nullptr, nullptr, true)); CCoinsViewCache view(&chainstate.CoinsTip()); BOOST_CHECK(chainstate.ConnectBlock(block, state, new_block_index, view)); } diff --git a/src/test/fuzz/load_external_block_file.cpp b/src/test/fuzz/load_external_block_file.cpp index 502f7b897c..bdaa4ad1b8 100644 --- a/src/test/fuzz/load_external_block_file.cpp +++ b/src/test/fuzz/load_external_block_file.cpp @@ -35,9 +35,9 @@ FUZZ_TARGET(load_external_block_file, .init = initialize_load_external_block_fil // Corresponds to the -reindex case (track orphan blocks across files). FlatFilePos flat_file_pos; std::multimap<uint256, FlatFilePos> blocks_with_unknown_parent; - g_setup->m_node.chainman->ActiveChainstate().LoadExternalBlockFile(fuzzed_block_file, &flat_file_pos, &blocks_with_unknown_parent); + g_setup->m_node.chainman->LoadExternalBlockFile(fuzzed_block_file, &flat_file_pos, &blocks_with_unknown_parent); } else { // Corresponds to the -loadblock= case (orphan blocks aren't tracked across files). - g_setup->m_node.chainman->ActiveChainstate().LoadExternalBlockFile(fuzzed_block_file); + g_setup->m_node.chainman->LoadExternalBlockFile(fuzzed_block_file); } } diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h index bf8f8b5819..9ff2c08807 100644 --- a/src/test/util/chainstate.h +++ b/src/test/util/chainstate.h @@ -71,6 +71,7 @@ CreateAndActivateUTXOSnapshot( // This is a stripped-down version of node::LoadChainstate which // preserves the block index. LOCK(::cs_main); + CBlockIndex *orig_tip = node.chainman->ActiveChainstate().m_chain.Tip(); uint256 gen_hash = node.chainman->ActiveChainstate().m_chain[0]->GetBlockHash(); node.chainman->ResetChainstates(); node.chainman->InitializeChainstate(node.mempool.get()); @@ -83,6 +84,22 @@ CreateAndActivateUTXOSnapshot( chain.setBlockIndexCandidates.insert(node.chainman->m_blockman.LookupBlockIndex(gen_hash)); chain.LoadChainTip(); node.chainman->MaybeRebalanceCaches(); + + // Reset the HAVE_DATA flags below the snapshot height, simulating + // never-having-downloaded them in the first place. + // TODO: perhaps we could improve this by using pruning to delete + // these blocks instead + CBlockIndex *pindex = orig_tip; + while (pindex && pindex != chain.m_chain.Tip()) { + pindex->nStatus &= ~BLOCK_HAVE_DATA; + pindex->nStatus &= ~BLOCK_HAVE_UNDO; + // We have to set the ASSUMED_VALID flag, because otherwise it + // would not be possible to have a block index entry without HAVE_DATA + // and with nTx > 0 (since we aren't setting the pruned flag); + // see CheckBlockIndex(). + pindex->nStatus |= BLOCK_ASSUMED_VALID; + pindex = pindex->pprev; + } } BlockValidationState state; if (!node.chainman->ActiveChainstate().ActivateBestChain(state)) { diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 2078fcd8f8..fe2d2ba592 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -77,6 +77,13 @@ BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) // After adding some blocks to the tip, best block should have changed. BOOST_CHECK(::g_best_block != curr_tip); + // Grab block 1 from disk; we'll add it to the background chain later. + std::shared_ptr<CBlock> pblockone = std::make_shared<CBlock>(); + { + LOCK(::cs_main); + chainman.m_blockman.ReadBlockFromDisk(*pblockone, *chainman.ActiveChain()[1]); + } + BOOST_REQUIRE(CreateAndActivateUTXOSnapshot( this, NoMalleation, /*reset_chainstate=*/ true)); @@ -104,11 +111,7 @@ BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) assert(false); }()}; - // Create a block to append to the validation chain. - std::vector<CMutableTransaction> noTxns; - CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; - CBlock validation_block = this->CreateBlock(noTxns, scriptPubKey, background_cs); - auto pblock = std::make_shared<const CBlock>(validation_block); + // Append the first block to the background chain. BlockValidationState state; CBlockIndex* pindex = nullptr; const CChainParams& chainparams = Params(); @@ -118,17 +121,18 @@ BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) // once it is changed to support multiple chainstates. { LOCK(::cs_main); - bool checked = CheckBlock(*pblock, state, chainparams.GetConsensus()); + bool checked = CheckBlock(*pblockone, state, chainparams.GetConsensus()); BOOST_CHECK(checked); - bool accepted = background_cs.AcceptBlock( - pblock, state, &pindex, true, nullptr, &newblock, true); + bool accepted = chainman.AcceptBlock( + pblockone, state, &pindex, true, nullptr, &newblock, true); BOOST_CHECK(accepted); } + // UpdateTip is called here - bool block_added = background_cs.ActivateBestChain(state, pblock); + bool block_added = background_cs.ActivateBestChain(state, pblockone); // Ensure tip is as expected - BOOST_CHECK_EQUAL(background_cs.m_chain.Tip()->GetBlockHash(), validation_block.GetHash()); + BOOST_CHECK_EQUAL(background_cs.m_chain.Tip()->GetBlockHash(), pblockone->GetHash()); // g_best_block should be unchanged after adding a block to the background // validation chain. diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 99860961a2..160a807f69 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -49,6 +49,9 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) c1.InitCoinsDB( /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); + c1.LoadGenesisBlock(); + BlockValidationState val_state; + BOOST_CHECK(c1.ActivateBestChain(val_state, nullptr)); BOOST_CHECK(!manager.IsSnapshotActive()); BOOST_CHECK(WITH_LOCK(::cs_main, return !manager.IsSnapshotValidated())); @@ -58,7 +61,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) auto& active_chain = WITH_LOCK(manager.GetMutex(), return manager.ActiveChain()); BOOST_CHECK_EQUAL(&active_chain, &c1.m_chain); - BOOST_CHECK_EQUAL(WITH_LOCK(manager.GetMutex(), return manager.ActiveHeight()), -1); + BOOST_CHECK_EQUAL(WITH_LOCK(manager.GetMutex(), return manager.ActiveHeight()), 0); auto active_tip = WITH_LOCK(manager.GetMutex(), return manager.ActiveTip()); auto exp_tip = c1.m_chain.Tip(); @@ -68,7 +71,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // Create a snapshot-based chainstate. // - const uint256 snapshot_blockhash = GetRandHash(); + const uint256 snapshot_blockhash = active_tip->GetBlockHash(); Chainstate& c2 = WITH_LOCK(::cs_main, return manager.ActivateExistingSnapshot( &mempool, snapshot_blockhash)); chainstates.push_back(&c2); @@ -78,8 +81,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) c2.InitCoinsDB( /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); WITH_LOCK(::cs_main, c2.InitCoinsCache(1 << 23)); - // Unlike c1, which doesn't have any blocks. Gets us different tip, height. - c2.LoadGenesisBlock(); + c2.m_chain.SetTip(*active_tip); BlockValidationState _; BOOST_CHECK(c2.ActivateBestChain(_, nullptr)); @@ -99,16 +101,14 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) auto exp_tip2 = c2.m_chain.Tip(); BOOST_CHECK_EQUAL(active_tip2, exp_tip2); - // Ensure that these pointers actually correspond to different - // CCoinsViewCache instances. - BOOST_CHECK(exp_tip != exp_tip2); + BOOST_CHECK_EQUAL(exp_tip, exp_tip2); // Let scheduler events finish running to avoid accessing memory that is going to be unloaded SyncWithValidationInterfaceQueue(); } //! Test rebalancing the caches associated with each chainstate. -BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) +BOOST_FIXTURE_TEST_CASE(chainstatemanager_rebalance_caches, TestChain100Setup) { ChainstateManager& manager = *m_node.chainman; CTxMemPool& mempool = *m_node.mempool; @@ -121,7 +121,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) // Create a legacy (IBD) chainstate. // - Chainstate& c1 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(&mempool)); + Chainstate& c1 = manager.ActiveChainstate(); chainstates.push_back(&c1); c1.InitCoinsDB( /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); @@ -129,8 +129,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) { LOCK(::cs_main); c1.InitCoinsCache(1 << 23); - BOOST_REQUIRE(c1.LoadGenesisBlock()); - c1.CoinsTip().SetBestBlock(InsecureRand256()); manager.MaybeRebalanceCaches(); } @@ -139,7 +137,8 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) // Create a snapshot-based chainstate. // - Chainstate& c2 = WITH_LOCK(cs_main, return manager.ActivateExistingSnapshot(&mempool, GetRandHash())); + CBlockIndex* snapshot_base{WITH_LOCK(manager.GetMutex(), return manager.ActiveChain()[manager.ActiveChain().Height() / 2])}; + Chainstate& c2 = WITH_LOCK(cs_main, return manager.ActivateExistingSnapshot(&mempool, *snapshot_base->phashBlock)); chainstates.push_back(&c2); c2.InitCoinsDB( /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); @@ -147,8 +146,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) { LOCK(::cs_main); c2.InitCoinsCache(1 << 23); - BOOST_REQUIRE(c2.LoadGenesisBlock()); - c2.CoinsTip().SetBestBlock(InsecureRand256()); manager.MaybeRebalanceCaches(); } @@ -415,7 +412,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, SnapshotTestSetup) //! - Then mark a region of the chain BLOCK_ASSUMED_VALID and introduce a second chainstate //! that will tolerate assumed-valid blocks. Run LoadBlockIndex() and ensure that the first //! chainstate only contains fully validated blocks and the other chainstate contains all blocks, -//! even those assumed-valid. +//! except those marked assume-valid, because those entries don't HAVE_DATA. //! BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) { @@ -430,28 +427,34 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) const int assumed_valid_start_idx = last_assumed_valid_idx - expected_assumed_valid; CBlockIndex* validated_tip{nullptr}; + CBlockIndex* assumed_base{nullptr}; CBlockIndex* assumed_tip{WITH_LOCK(chainman.GetMutex(), return chainman.ActiveChain().Tip())}; auto reload_all_block_indexes = [&]() { + // For completeness, we also reset the block sequence counters to + // ensure that no state which affects the ranking of tip-candidates is + // retained (even though this isn't strictly necessary). + WITH_LOCK(::cs_main, return chainman.ResetBlockSequenceCounters()); for (Chainstate* cs : chainman.GetAll()) { LOCK(::cs_main); - cs->UnloadBlockIndex(); + cs->ClearBlockIndexCandidates(); BOOST_CHECK(cs->setBlockIndexCandidates.empty()); } WITH_LOCK(::cs_main, chainman.LoadBlockIndex()); }; - // Ensure that without any assumed-valid BlockIndex entries, all entries are considered - // tip candidates. + // Ensure that without any assumed-valid BlockIndex entries, only the current tip is + // considered as a candidate. reload_all_block_indexes(); - BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.size(), cs1.m_chain.Height() + 1); + BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.size(), 1); - // Mark some region of the chain assumed-valid. + // Mark some region of the chain assumed-valid, and remove the HAVE_DATA flag. for (int i = 0; i <= cs1.m_chain.Height(); ++i) { LOCK(::cs_main); auto index = cs1.m_chain[i]; + // Blocks with heights in range [20, 40) are marked ASSUMED_VALID if (i < last_assumed_valid_idx && i >= assumed_valid_start_idx) { index->nStatus = BlockStatus::BLOCK_VALID_TREE | BlockStatus::BLOCK_ASSUMED_VALID; } @@ -464,25 +467,41 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) validated_tip = index; BOOST_CHECK(!index->IsAssumedValid()); } + // Note the last assumed valid block as the snapshot base + if (i == last_assumed_valid_idx - 1) { + assumed_base = index; + BOOST_CHECK(index->IsAssumedValid()); + } else if (i == last_assumed_valid_idx) { + BOOST_CHECK(!index->IsAssumedValid()); + } } BOOST_CHECK_EQUAL(expected_assumed_valid, num_assumed_valid); + // Note: cs2's tip is not set when ActivateExistingSnapshot is called. Chainstate& cs2 = WITH_LOCK(::cs_main, - return chainman.ActivateExistingSnapshot(&mempool, GetRandHash())); + return chainman.ActivateExistingSnapshot(&mempool, *assumed_base->phashBlock)); + + // Set tip of the fully validated chain to be the validated tip + cs1.m_chain.SetTip(*validated_tip); + + // Set tip of the assume-valid-based chain to the assume-valid block + cs2.m_chain.SetTip(*assumed_base); reload_all_block_indexes(); - // The fully validated chain only has candidates up to the start of the assumed-valid - // blocks. + // The fully validated chain should have the current validated tip + // and the assumed valid base as candidates. + BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.size(), 2); BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.count(validated_tip), 1); - BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.count(assumed_tip), 0); - BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.size(), assumed_valid_start_idx); + BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.count(assumed_base), 1); - // The assumed-valid tolerant chain has all blocks as candidates. - BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.count(validated_tip), 1); + // The assumed-valid tolerant chain has the assumed valid base as a + // candidate, but otherwise has none of the assumed-valid (which do not + // HAVE_DATA) blocks as candidates. + BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.count(validated_tip), 0); BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.count(assumed_tip), 1); - BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.size(), num_indexes); + BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.size(), num_indexes - last_assumed_valid_idx + 1); } //! Ensure that snapshot chainstates initialize properly when found on disk. diff --git a/src/validation.cpp b/src/validation.cpp index e6def01db5..cd6654abe4 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1579,6 +1579,13 @@ Chainstate::Chainstate( m_chainman(chainman), m_from_snapshot_blockhash(from_snapshot_blockhash) {} +const CBlockIndex* Chainstate::SnapshotBase() +{ + if (!m_from_snapshot_blockhash) return nullptr; + if (!m_cached_snapshot_base) m_cached_snapshot_base = Assert(m_chainman.m_blockman.LookupBlockIndex(*m_from_snapshot_blockhash)); + return m_cached_snapshot_base; +} + void Chainstate::InitCoinsDB( size_t cache_size_bytes, bool in_memory, @@ -3193,7 +3200,8 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< // that the best block hash is non-null. if (m_chainman.m_interrupt) break; } while (pindexNewTip != pindexMostWork); - CheckBlockIndex(); + + m_chainman.CheckBlockIndex(); // Write changes periodically to disk, after relay. if (!FlushStateToDisk(state, FlushStateMode::PERIODIC)) { @@ -3213,17 +3221,17 @@ bool Chainstate::PreciousBlock(BlockValidationState& state, CBlockIndex* pindex) // Nothing to do, this block is not at the tip. return true; } - if (m_chain.Tip()->nChainWork > nLastPreciousChainwork) { + if (m_chain.Tip()->nChainWork > m_chainman.nLastPreciousChainwork) { // The chain has been extended since the last call, reset the counter. - nBlockReverseSequenceId = -1; + m_chainman.nBlockReverseSequenceId = -1; } - nLastPreciousChainwork = m_chain.Tip()->nChainWork; + m_chainman.nLastPreciousChainwork = m_chain.Tip()->nChainWork; setBlockIndexCandidates.erase(pindex); - pindex->nSequenceId = nBlockReverseSequenceId; - if (nBlockReverseSequenceId > std::numeric_limits<int32_t>::min()) { + pindex->nSequenceId = m_chainman.nBlockReverseSequenceId; + if (m_chainman.nBlockReverseSequenceId > std::numeric_limits<int32_t>::min()) { // We can't keep reducing the counter if somebody really wants to // call preciousblock 2**31-1 times on the same set of tips... - nBlockReverseSequenceId--; + m_chainman.nBlockReverseSequenceId--; } if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && pindex->HaveTxsDownloaded()) { setBlockIndexCandidates.insert(pindex); @@ -3339,7 +3347,7 @@ bool Chainstate::InvalidateBlock(BlockValidationState& state, CBlockIndex* pinde to_mark_failed = invalid_walk_tip; } - CheckBlockIndex(); + m_chainman.CheckBlockIndex(); { LOCK(cs_main); @@ -3416,8 +3424,32 @@ void Chainstate::ResetBlockFailureFlags(CBlockIndex *pindex) { } } +void Chainstate::TryAddBlockIndexCandidate(CBlockIndex* pindex) +{ + AssertLockHeld(cs_main); + // The block only is a candidate for the most-work-chain if it has more work than our current tip. + if (m_chain.Tip() != nullptr && setBlockIndexCandidates.value_comp()(pindex, m_chain.Tip())) { + return; + } + + bool is_active_chainstate = this == &m_chainman.ActiveChainstate(); + if (is_active_chainstate) { + // The active chainstate should always add entries that have more + // work than the tip. + setBlockIndexCandidates.insert(pindex); + } else if (!m_disabled) { + // For the background chainstate, we only consider connecting blocks + // towards the snapshot base (which can't be nullptr or else we'll + // never make progress). + const CBlockIndex* snapshot_base{Assert(m_chainman.GetSnapshotBaseBlock())}; + if (snapshot_base->GetAncestor(pindex->nHeight) == pindex) { + setBlockIndexCandidates.insert(pindex); + } + } +} + /** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ -void Chainstate::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos) +void ChainstateManager::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos) { AssertLockHeld(cs_main); pindexNew->nTx = block.vtx.size(); @@ -3426,7 +3458,7 @@ void Chainstate::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pin pindexNew->nDataPos = pos.nPos; pindexNew->nUndoPos = 0; pindexNew->nStatus |= BLOCK_HAVE_DATA; - if (DeploymentActiveAt(*pindexNew, m_chainman, Consensus::DEPLOYMENT_SEGWIT)) { + if (DeploymentActiveAt(*pindexNew, *this, Consensus::DEPLOYMENT_SEGWIT)) { pindexNew->nStatus |= BLOCK_OPT_WITNESS; } pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); @@ -3443,8 +3475,8 @@ void Chainstate::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pin queue.pop_front(); pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; pindex->nSequenceId = nBlockSequenceId++; - if (m_chain.Tip() == nullptr || !setBlockIndexCandidates.value_comp()(pindex, m_chain.Tip())) { - setBlockIndexCandidates.insert(pindex); + for (Chainstate *c : GetAll()) { + c->TryAddBlockIndexCandidate(pindex); } std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> range = m_blockman.m_blocks_unlinked.equal_range(pindex); while (range.first != range.second) { @@ -3858,7 +3890,7 @@ bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& for (const CBlockHeader& header : headers) { CBlockIndex *pindex = nullptr; // Use a temp pindex instead of ppindex to avoid a const_cast bool accepted{AcceptBlockHeader(header, state, &pindex, min_pow_checked)}; - ActiveChainstate().CheckBlockIndex(); + CheckBlockIndex(); if (!accepted) { return false; @@ -3905,7 +3937,7 @@ void ChainstateManager::ReportHeadersPresync(const arith_uint256& work, int64_t } /** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ -bool Chainstate::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock, bool min_pow_checked) +bool ChainstateManager::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock, bool min_pow_checked) { const CBlock& block = *pblock; @@ -3915,23 +3947,24 @@ bool Chainstate::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockV CBlockIndex *pindexDummy = nullptr; CBlockIndex *&pindex = ppindex ? *ppindex : pindexDummy; - bool accepted_header{m_chainman.AcceptBlockHeader(block, state, &pindex, min_pow_checked)}; + bool accepted_header{AcceptBlockHeader(block, state, &pindex, min_pow_checked)}; CheckBlockIndex(); if (!accepted_header) return false; - // Try to process all requested blocks that we don't have, but only - // process an unrequested block if it's new and has enough work to - // advance our tip, and isn't too many blocks ahead. + // Check all requested blocks that we do not already have for validity and + // save them to disk. Skip processing of unrequested blocks as an anti-DoS + // measure, unless the blocks have more work than the active chain tip, and + // aren't too far ahead of it, so are likely to be attached soon. bool fAlreadyHave = pindex->nStatus & BLOCK_HAVE_DATA; - bool fHasMoreOrSameWork = (m_chain.Tip() ? pindex->nChainWork >= m_chain.Tip()->nChainWork : true); + bool fHasMoreOrSameWork = (ActiveTip() ? pindex->nChainWork >= ActiveTip()->nChainWork : true); // Blocks that are too out-of-order needlessly limit the effectiveness of // pruning, because pruning will not delete block files that contain any // blocks which are too close in height to the tip. Apply this test // regardless of whether pruning is enabled; it should generally be safe to // not process unrequested blocks. - bool fTooFarAhead{pindex->nHeight > m_chain.Height() + int(MIN_BLOCKS_TO_KEEP)}; + bool fTooFarAhead{pindex->nHeight > ActiveHeight() + int(MIN_BLOCKS_TO_KEEP)}; // TODO: Decouple this function from the block download logic by removing fRequested // This requires some new chain data structure to efficiently look up if a @@ -3951,13 +3984,13 @@ bool Chainstate::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockV // If our tip is behind, a peer could try to send us // low-work blocks on a fake chain that we would never // request; don't process these. - if (pindex->nChainWork < m_chainman.MinimumChainWork()) return true; + if (pindex->nChainWork < MinimumChainWork()) return true; } - const CChainParams& params{m_chainman.GetParams()}; + const CChainParams& params{GetParams()}; if (!CheckBlock(block, state, params.GetConsensus()) || - !ContextualCheckBlock(block, state, m_chainman, pindex->pprev)) { + !ContextualCheckBlock(block, state, *this, pindex->pprev)) { if (state.IsInvalid() && state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { pindex->nStatus |= BLOCK_FAILED_VALID; m_blockman.m_dirty_blockindex.insert(pindex); @@ -3967,23 +4000,30 @@ bool Chainstate::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockV // Header is valid/has work, merkle tree and segwit merkle tree are good...RELAY NOW // (but if it does not build on our best tip, let the SendMessages loop relay it) - if (!IsInitialBlockDownload() && m_chain.Tip() == pindex->pprev) + if (!ActiveChainstate().IsInitialBlockDownload() && ActiveTip() == pindex->pprev) GetMainSignals().NewPoWValidBlock(pindex, pblock); // Write block to history file if (fNewBlock) *fNewBlock = true; try { - FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, pindex->nHeight, m_chain, dbp)}; + FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, pindex->nHeight, dbp)}; if (blockPos.IsNull()) { state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); return false; } ReceivedBlockTransactions(block, pindex, blockPos); } catch (const std::runtime_error& e) { - return FatalError(m_chainman.GetNotifications(), state, std::string("System error: ") + e.what()); + return FatalError(GetNotifications(), state, std::string("System error: ") + e.what()); } - FlushStateToDisk(state, FlushStateMode::NONE); + // TODO: FlushStateToDisk() handles flushing of both block and chainstate + // data, so we should move this to ChainstateManager so that we can be more + // intelligent about how we flush. + // For now, since FlushStateMode::NONE is used, all that can happen is that + // the block files may be pruned, so we can just call this on one + // chainstate (particularly if we haven't implemented pruning with + // background validation yet). + ActiveChainstate().FlushStateToDisk(state, FlushStateMode::NONE); CheckBlockIndex(); @@ -4011,7 +4051,7 @@ bool ChainstateManager::ProcessNewBlock(const std::shared_ptr<const CBlock>& blo bool ret = CheckBlock(*block, state, GetConsensus()); if (ret) { // Store to disk - ret = ActiveChainstate().AcceptBlock(block, state, &pindex, force_processing, nullptr, new_block, min_pow_checked); + ret = AcceptBlock(block, state, &pindex, force_processing, nullptr, new_block, min_pow_checked); } if (!ret) { GetMainSignals().BlockChecked(*block, state); @@ -4379,10 +4419,9 @@ bool Chainstate::NeedsRedownload() const return false; } -void Chainstate::UnloadBlockIndex() +void Chainstate::ClearBlockIndexCandidates() { AssertLockHeld(::cs_main); - nBlockSequenceId = 1; setBlockIndexCandidates.clear(); } @@ -4401,62 +4440,19 @@ bool ChainstateManager::LoadBlockIndex() std::sort(vSortedByHeight.begin(), vSortedByHeight.end(), CBlockIndexHeightOnlyComparator()); - // Find start of assumed-valid region. - int first_assumed_valid_height = std::numeric_limits<int>::max(); - - for (const CBlockIndex* block : vSortedByHeight) { - if (block->IsAssumedValid()) { - auto chainstates = GetAll(); - - // If we encounter an assumed-valid block index entry, ensure that we have - // one chainstate that tolerates assumed-valid entries and another that does - // not (i.e. the background validation chainstate), since assumed-valid - // entries should always be pending validation by a fully-validated chainstate. - auto any_chain = [&](auto fnc) { return std::any_of(chainstates.cbegin(), chainstates.cend(), fnc); }; - assert(any_chain([](auto chainstate) { return chainstate->reliesOnAssumedValid(); })); - assert(any_chain([](auto chainstate) { return !chainstate->reliesOnAssumedValid(); })); - - first_assumed_valid_height = block->nHeight; - LogPrintf("Saw first assumedvalid block at height %d (%s)\n", - first_assumed_valid_height, block->ToString()); - break; - } - } - for (CBlockIndex* pindex : vSortedByHeight) { if (m_interrupt) return false; - if (pindex->IsAssumedValid() || + // If we have an assumeutxo-based chainstate, then the snapshot + // block will be a candidate for the tip, but it may not be + // VALID_TRANSACTIONS (eg if we haven't yet downloaded the block), + // so we special-case the snapshot block as a potential candidate + // here. + if (pindex == GetSnapshotBaseBlock() || (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && (pindex->HaveTxsDownloaded() || pindex->pprev == nullptr))) { - // Fill each chainstate's block candidate set. Only add assumed-valid - // blocks to the tip candidate set if the chainstate is allowed to rely on - // assumed-valid blocks. - // - // If all setBlockIndexCandidates contained the assumed-valid blocks, the - // background chainstate's ActivateBestChain() call would add assumed-valid - // blocks to the chain (based on how FindMostWorkChain() works). Obviously - // we don't want this since the purpose of the background validation chain - // is to validate assued-valid blocks. - // - // Note: This is considering all blocks whose height is greater or equal to - // the first assumed-valid block to be assumed-valid blocks, and excluding - // them from the background chainstate's setBlockIndexCandidates set. This - // does mean that some blocks which are not technically assumed-valid - // (later blocks on a fork beginning before the first assumed-valid block) - // might not get added to the background chainstate, but this is ok, - // because they will still be attached to the active chainstate if they - // actually contain more work. - // - // Instead of this height-based approach, an earlier attempt was made at - // detecting "holistically" whether the block index under consideration - // relied on an assumed-valid ancestor, but this proved to be too slow to - // be practical. for (Chainstate* chainstate : GetAll()) { - if (chainstate->reliesOnAssumedValid() || - pindex->nHeight < first_assumed_valid_height) { - chainstate->setBlockIndexCandidates.insert(pindex); - } + chainstate->TryAddBlockIndexCandidate(pindex); } } if (pindex->nStatus & BLOCK_FAILED_MASK && (!m_best_invalid || pindex->nChainWork > m_best_invalid->nChainWork)) { @@ -4496,12 +4492,12 @@ bool Chainstate::LoadGenesisBlock() try { const CBlock& block = params.GenesisBlock(); - FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, 0, m_chain, nullptr)}; + FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, 0, nullptr)}; if (blockPos.IsNull()) { return error("%s: writing genesis block to disk failed", __func__); } CBlockIndex* pindex = m_blockman.AddToBlockIndex(block, m_chainman.m_best_header); - ReceivedBlockTransactions(block, pindex, blockPos); + m_chainman.ReceivedBlockTransactions(block, pindex, blockPos); } catch (const std::runtime_error& e) { return error("%s: failed to write genesis block: %s", __func__, e.what()); } @@ -4509,18 +4505,16 @@ bool Chainstate::LoadGenesisBlock() return true; } -void Chainstate::LoadExternalBlockFile( +void ChainstateManager::LoadExternalBlockFile( FILE* fileIn, FlatFilePos* dbp, std::multimap<uint256, FlatFilePos>* blocks_with_unknown_parent) { - AssertLockNotHeld(m_chainstate_mutex); - // Either both should be specified (-reindex), or neither (-loadblock). assert(!dbp == !blocks_with_unknown_parent); const auto start{SteadyClock::now()}; - const CChainParams& params{m_chainman.GetParams()}; + const CChainParams& params{GetParams()}; int nLoaded = 0; try { @@ -4530,7 +4524,7 @@ void Chainstate::LoadExternalBlockFile( // such as a block fails to deserialize. uint64_t nRewind = blkdat.GetPos(); while (!blkdat.eof()) { - if (m_chainman.m_interrupt) return; + if (m_interrupt) return; blkdat.SetPos(nRewind); nRewind++; // start one byte further next time, in case of failure @@ -4605,8 +4599,15 @@ void Chainstate::LoadExternalBlockFile( // Activate the genesis block so normal node progress can continue if (hash == params.GetConsensus().hashGenesisBlock) { - BlockValidationState state; - if (!ActivateBestChain(state, nullptr)) { + bool genesis_activation_failure = false; + for (auto c : GetAll()) { + BlockValidationState state; + if (!c->ActivateBestChain(state, nullptr)) { + genesis_activation_failure = true; + break; + } + } + if (genesis_activation_failure) { break; } } @@ -4619,14 +4620,21 @@ void Chainstate::LoadExternalBlockFile( // until after all of the block files are loaded. ActivateBestChain can be // called by concurrent network message processing. but, that is not // reliable for the purpose of pruning while importing. - BlockValidationState state; - if (!ActivateBestChain(state, pblock)) { - LogPrint(BCLog::REINDEX, "failed to activate chain (%s)\n", state.ToString()); + bool activation_failure = false; + for (auto c : GetAll()) { + BlockValidationState state; + if (!c->ActivateBestChain(state, pblock)) { + LogPrint(BCLog::REINDEX, "failed to activate chain (%s)\n", state.ToString()); + activation_failure = true; + break; + } + } + if (activation_failure) { break; } } - NotifyHeaderTip(*this); + NotifyHeaderTip(ActiveChainstate()); if (!blocks_with_unknown_parent) continue; @@ -4652,7 +4660,7 @@ void Chainstate::LoadExternalBlockFile( } range.first++; blocks_with_unknown_parent->erase(it); - NotifyHeaderTip(*this); + NotifyHeaderTip(ActiveChainstate()); } } } catch (const std::exception& e) { @@ -4671,14 +4679,14 @@ void Chainstate::LoadExternalBlockFile( } } } catch (const std::runtime_error& e) { - m_chainman.GetNotifications().fatalError(std::string("System error: ") + e.what()); + GetNotifications().fatalError(std::string("System error: ") + e.what()); } LogPrintf("Loaded %i blocks from external file in %dms\n", nLoaded, Ticks<std::chrono::milliseconds>(SteadyClock::now() - start)); } -void Chainstate::CheckBlockIndex() +void ChainstateManager::CheckBlockIndex() { - if (!m_chainman.ShouldCheckBlockIndex()) { + if (!ShouldCheckBlockIndex()) { return; } @@ -4687,7 +4695,7 @@ void Chainstate::CheckBlockIndex() // During a reindex, we read the genesis block and call CheckBlockIndex before ActivateBestChain, // so we have the genesis block in m_blockman.m_block_index but no active chain. (A few of the // tests when iterating the block tree require that m_chain has been initialized.) - if (m_chain.Height() < 0) { + if (ActiveChain().Height() < 0) { assert(m_blockman.m_block_index.size() <= 1); return; } @@ -4717,12 +4725,12 @@ void Chainstate::CheckBlockIndex() CBlockIndex* pindexFirstNotTransactionsValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_TRANSACTIONS (regardless of being valid or not). CBlockIndex* pindexFirstNotChainValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_CHAIN (regardless of being valid or not). CBlockIndex* pindexFirstNotScriptsValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_SCRIPTS (regardless of being valid or not). + CBlockIndex* pindexFirstAssumeValid = nullptr; // Oldest ancestor of pindex which has BLOCK_ASSUMED_VALID while (pindex != nullptr) { nNodes++; + if (pindexFirstAssumeValid == nullptr && pindex->nStatus & BLOCK_ASSUMED_VALID) pindexFirstAssumeValid = pindex; if (pindexFirstInvalid == nullptr && pindex->nStatus & BLOCK_FAILED_VALID) pindexFirstInvalid = pindex; - // Assumed-valid index entries will not have data since we haven't downloaded the - // full block yet. - if (pindexFirstMissing == nullptr && !(pindex->nStatus & BLOCK_HAVE_DATA) && !pindex->IsAssumedValid()) { + if (pindexFirstMissing == nullptr && !(pindex->nStatus & BLOCK_HAVE_DATA)) { pindexFirstMissing = pindex; } if (pindexFirstNeverProcessed == nullptr && pindex->nTx == 0) pindexFirstNeverProcessed = pindex; @@ -4751,8 +4759,12 @@ void Chainstate::CheckBlockIndex() // Begin: actual consistency checks. if (pindex->pprev == nullptr) { // Genesis block checks. - assert(pindex->GetBlockHash() == m_chainman.GetConsensus().hashGenesisBlock); // Genesis block's hash must match. - assert(pindex == m_chain.Genesis()); // The current active chain's genesis block must be this block. + assert(pindex->GetBlockHash() == GetConsensus().hashGenesisBlock); // Genesis block's hash must match. + for (auto c : GetAll()) { + if (c->m_chain.Genesis() != nullptr) { + assert(pindex == c->m_chain.Genesis()); // The chain's genesis block must be this block. + } + } } if (!pindex->HaveTxsDownloaded()) assert(pindex->nSequenceId <= 0); // nSequenceId can't be set positive for blocks that aren't linked (negative is used for preciousblock) // VALID_TRANSACTIONS is equivalent to nTx > 0 for all nodes (whether or not pruning has occurred). @@ -4762,7 +4774,13 @@ void Chainstate::CheckBlockIndex() if (!m_blockman.m_have_pruned && !pindex->IsAssumedValid()) { // If we've never pruned, then HAVE_DATA should be equivalent to nTx > 0 assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0)); - assert(pindexFirstMissing == pindexFirstNeverProcessed); + if (pindexFirstAssumeValid == nullptr) { + // If we've got some assume valid blocks, then we might have + // missing blocks (not HAVE_DATA) but still treat them as + // having been processed (with a fake nTx value). Otherwise, we + // can assert that these are the same. + assert(pindexFirstMissing == pindexFirstNeverProcessed); + } } else { // If we have pruned, then we can only say that HAVE_DATA implies nTx > 0 if (pindex->nStatus & BLOCK_HAVE_DATA) assert(pindex->nTx > 0); @@ -4792,27 +4810,32 @@ void Chainstate::CheckBlockIndex() // Checks for not-invalid blocks. assert((pindex->nStatus & BLOCK_FAILED_MASK) == 0); // The failed mask cannot be set for blocks without invalid parents. } - if (!CBlockIndexWorkComparator()(pindex, m_chain.Tip()) && pindexFirstNeverProcessed == nullptr) { - if (pindexFirstInvalid == nullptr) { - const bool is_active = this == &m_chainman.ActiveChainstate(); - - // If this block sorts at least as good as the current tip and - // is valid and we have all data for its parents, it must be in - // setBlockIndexCandidates. m_chain.Tip() must also be there - // even if some data has been pruned. - // - // Don't perform this check for the background chainstate since - // its setBlockIndexCandidates shouldn't have some entries (i.e. those past the - // snapshot block) which do exist in the block index for the active chainstate. - if (is_active && (pindexFirstMissing == nullptr || pindex == m_chain.Tip())) { - assert(setBlockIndexCandidates.count(pindex)); + // Chainstate-specific checks on setBlockIndexCandidates + for (auto c : GetAll()) { + if (c->m_chain.Tip() == nullptr) continue; + if (!CBlockIndexWorkComparator()(pindex, c->m_chain.Tip()) && pindexFirstNeverProcessed == nullptr) { + if (pindexFirstInvalid == nullptr) { + const bool is_active = c == &ActiveChainstate(); + // If this block sorts at least as good as the current tip and + // is valid and we have all data for its parents, it must be in + // setBlockIndexCandidates. m_chain.Tip() must also be there + // even if some data has been pruned. + // + if ((pindexFirstMissing == nullptr || pindex == c->m_chain.Tip())) { + // The active chainstate should always have this block + // as a candidate, but a background chainstate should + // only have it if it is an ancestor of the snapshot base. + if (is_active || GetSnapshotBaseBlock()->GetAncestor(pindex->nHeight) == pindex) { + assert(c->setBlockIndexCandidates.count(pindex)); + } + } + // If some parent is missing, then it could be that this block was in + // setBlockIndexCandidates but had to be removed because of the missing data. + // In this case it must be in m_blocks_unlinked -- see test below. } - // If some parent is missing, then it could be that this block was in - // setBlockIndexCandidates but had to be removed because of the missing data. - // In this case it must be in m_blocks_unlinked -- see test below. + } else { // If this block sorts worse than the current tip or some ancestor's block has never been seen, it cannot be in setBlockIndexCandidates. + assert(c->setBlockIndexCandidates.count(pindex) == 0); } - } else { // If this block sorts worse than the current tip or some ancestor's block has never been seen, it cannot be in setBlockIndexCandidates. - assert(setBlockIndexCandidates.count(pindex) == 0); } // Check whether this block is in m_blocks_unlinked. std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> rangeUnlinked = m_blockman.m_blocks_unlinked.equal_range(pindex->pprev); @@ -4833,18 +4856,23 @@ void Chainstate::CheckBlockIndex() if (pindexFirstMissing == nullptr) assert(!foundInUnlinked); // We aren't missing data for any parent -- cannot be in m_blocks_unlinked. if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed == nullptr && pindexFirstMissing != nullptr) { // We HAVE_DATA for this block, have received data for all parents at some point, but we're currently missing data for some parent. - assert(m_blockman.m_have_pruned); // We must have pruned. + assert(m_blockman.m_have_pruned || pindexFirstAssumeValid != nullptr); // We must have pruned, or else we're using a snapshot (causing us to have faked the received data for some parent(s)). // This block may have entered m_blocks_unlinked if: // - it has a descendant that at some point had more work than the // tip, and // - we tried switching to that descendant but were missing // data for some intermediate block between m_chain and the // tip. - // So if this block is itself better than m_chain.Tip() and it wasn't in + // So if this block is itself better than any m_chain.Tip() and it wasn't in // setBlockIndexCandidates, then it must be in m_blocks_unlinked. - if (!CBlockIndexWorkComparator()(pindex, m_chain.Tip()) && setBlockIndexCandidates.count(pindex) == 0) { - if (pindexFirstInvalid == nullptr) { - assert(foundInUnlinked); + for (auto c : GetAll()) { + const bool is_active = c == &ActiveChainstate(); + if (!CBlockIndexWorkComparator()(pindex, c->m_chain.Tip()) && c->setBlockIndexCandidates.count(pindex) == 0) { + if (pindexFirstInvalid == nullptr) { + if (is_active || GetSnapshotBaseBlock()->GetAncestor(pindex->nHeight) == pindex) { + assert(foundInUnlinked); + } + } } } } @@ -4871,6 +4899,7 @@ void Chainstate::CheckBlockIndex() if (pindex == pindexFirstNotTransactionsValid) pindexFirstNotTransactionsValid = nullptr; if (pindex == pindexFirstNotChainValid) pindexFirstNotChainValid = nullptr; if (pindex == pindexFirstNotScriptsValid) pindexFirstNotScriptsValid = nullptr; + if (pindex == pindexFirstAssumeValid) pindexFirstAssumeValid = nullptr; // Find our parent. CBlockIndex* pindexPar = pindex->pprev; // Find which child we just visited. @@ -5682,9 +5711,7 @@ util::Result<void> Chainstate::InvalidateCoinsDBOnDisk() const CBlockIndex* ChainstateManager::GetSnapshotBaseBlock() const { - const auto blockhash_op = this->SnapshotBlockhash(); - if (!blockhash_op) return nullptr; - return Assert(m_blockman.LookupBlockIndex(*blockhash_op)); + return m_active_chainstate ? m_active_chainstate->SnapshotBase() : nullptr; } std::optional<int> ChainstateManager::GetSnapshotBaseHeight() const diff --git a/src/validation.h b/src/validation.h index af8ceb5dfa..d7ad86a5e8 100644 --- a/src/validation.h +++ b/src/validation.h @@ -466,17 +466,6 @@ class Chainstate { protected: /** - * Every received block is assigned a unique and increasing identifier, so we - * know which one to give priority in case of a fork. - */ - /** Blocks loaded from disk are assigned id 0, so start the counter at 1. */ - int32_t nBlockSequenceId GUARDED_BY(::cs_main) = 1; - /** Decreasing counter (used by subsequent preciousblock calls). */ - int32_t nBlockReverseSequenceId = -1; - /** chainwork for the last block that preciousblock has been applied to. */ - arith_uint256 nLastPreciousChainwork = 0; - - /** * The ChainState Mutex * A lock that must be held when modifying this ChainState - held in ActivateBestChain() and * InvalidateBlock() @@ -511,6 +500,9 @@ protected: //! is set to true on the snapshot chainstate. bool m_disabled GUARDED_BY(::cs_main) {false}; + //! Cached result of LookupBlockIndex(*m_from_snapshot_blockhash) + const CBlockIndex* m_cached_snapshot_base GUARDED_BY(::cs_main) {nullptr}; + public: //! Reference to a BlockManager instance which itself is shared across all //! Chainstate instances. @@ -562,9 +554,12 @@ public: */ const std::optional<uint256> m_from_snapshot_blockhash; - //! Return true if this chainstate relies on blocks that are assumed-valid. In - //! practice this means it was created based on a UTXO snapshot. - bool reliesOnAssumedValid() { return m_from_snapshot_blockhash.has_value(); } + /** + * The base of the snapshot this chainstate was created from. + * + * nullptr if this chainstate was not created from a snapshot. + */ + const CBlockIndex* SnapshotBase() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** * The set of all CBlockIndex entries with either BLOCK_VALID_TRANSACTIONS (for @@ -620,37 +615,6 @@ public: bool ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - /** - * Import blocks from an external file - * - * During reindexing, this function is called for each block file (datadir/blocks/blk?????.dat). - * It reads all blocks contained in the given file and attempts to process them (add them to the - * block index). The blocks may be out of order within each file and across files. Often this - * function reads a block but finds that its parent hasn't been read yet, so the block can't be - * processed yet. The function will add an entry to the blocks_with_unknown_parent map (which is - * passed as an argument), so that when the block's parent is later read and processed, this - * function can re-read the child block from disk and process it. - * - * Because a block's parent may be in a later file, not just later in the same file, the - * blocks_with_unknown_parent map must be passed in and out with each call. It's a multimap, - * rather than just a map, because multiple blocks may have the same parent (when chain splits - * or stale blocks exist). It maps from parent-hash to child-disk-position. - * - * This function can also be used to read blocks from user-specified block files using the - * -loadblock= option. There's no unknown-parent tracking, so the last two arguments are omitted. - * - * - * @param[in] fileIn FILE handle to file containing blocks to read - * @param[in] dbp (optional) Disk block position (only for reindex) - * @param[in,out] blocks_with_unknown_parent (optional) Map of disk positions for blocks with - * unknown parent, key is parent block hash - * (only used for reindex) - * */ - void LoadExternalBlockFile( - FILE* fileIn, - FlatFilePos* dbp = nullptr, - std::multimap<uint256, FlatFilePos>* blocks_with_unknown_parent = nullptr) - EXCLUSIVE_LOCKS_REQUIRED(!m_chainstate_mutex); /** * Update the on-disk chain state. @@ -702,8 +666,6 @@ public: EXCLUSIVE_LOCKS_REQUIRED(!m_chainstate_mutex) LOCKS_EXCLUDED(::cs_main); - bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock, bool min_pow_checked) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - // Block (dis)connection on a given view: DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); @@ -738,9 +700,11 @@ public: /** Ensures we have a genesis block in the block tree, possibly writing one to disk. */ bool LoadGenesisBlock(); + void TryAddBlockIndexCandidate(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void PruneBlockIndexCandidates(); - void UnloadBlockIndex() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + void ClearBlockIndexCandidates() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** Check whether we are doing an initial block download (synchronizing from disk or network) */ bool IsInitialBlockDownload() const; @@ -748,13 +712,6 @@ public: /** Find the last common block of this chain and a locator. */ const CBlockIndex* FindForkInGlobalIndex(const CBlockLocator& locator) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); - /** - * Make various assertions about the state of the block index. - * - * By default this only executes fully when using the Regtest chain; see: m_options.check_block_index. - */ - void CheckBlockIndex(); - /** Load the persisted mempool from disk */ void LoadMempool(const fs::path& load_path, fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen); @@ -784,7 +741,6 @@ private: void InvalidBlockFound(CBlockIndex* pindex, const BlockValidationState& state) EXCLUSIVE_LOCKS_REQUIRED(cs_main); CBlockIndex* FindMostWorkChain() EXCLUSIVE_LOCKS_REQUIRED(cs_main); - void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos) EXCLUSIVE_LOCKS_REQUIRED(cs_main); bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -971,6 +927,13 @@ public: kernel::Notifications& GetNotifications() const { return m_options.notifications; }; /** + * Make various assertions about the state of the block index. + * + * By default this only executes fully when using the Regtest chain; see: m_options.check_block_index. + */ + void CheckBlockIndex(); + + /** * Alias for ::cs_main. * Should be used in new code to make it easier to make ::cs_main a member * of this class. @@ -991,6 +954,27 @@ public: node::BlockManager m_blockman; /** + * Every received block is assigned a unique and increasing identifier, so we + * know which one to give priority in case of a fork. + */ + /** Blocks loaded from disk are assigned id 0, so start the counter at 1. */ + int32_t nBlockSequenceId GUARDED_BY(::cs_main) = 1; + /** Decreasing counter (used by subsequent preciousblock calls). */ + int32_t nBlockReverseSequenceId = -1; + /** chainwork for the last block that preciousblock has been applied to. */ + arith_uint256 nLastPreciousChainwork = 0; + + // Reset the memory-only sequence counters we use to track block arrival + // (used by tests to reset state) + void ResetBlockSequenceCounters() EXCLUSIVE_LOCKS_REQUIRED(::cs_main) + { + AssertLockHeld(::cs_main); + nBlockSequenceId = 1; + nBlockReverseSequenceId = -1; + } + + + /** * In order to efficiently track invalidity of headers, we keep the set of * blocks which we tried to connect and found to be invalid here (ie which * were set to BLOCK_FAILED_VALID since the last restart). We can then @@ -1086,6 +1070,37 @@ public: } /** + * Import blocks from an external file + * + * During reindexing, this function is called for each block file (datadir/blocks/blk?????.dat). + * It reads all blocks contained in the given file and attempts to process them (add them to the + * block index). The blocks may be out of order within each file and across files. Often this + * function reads a block but finds that its parent hasn't been read yet, so the block can't be + * processed yet. The function will add an entry to the blocks_with_unknown_parent map (which is + * passed as an argument), so that when the block's parent is later read and processed, this + * function can re-read the child block from disk and process it. + * + * Because a block's parent may be in a later file, not just later in the same file, the + * blocks_with_unknown_parent map must be passed in and out with each call. It's a multimap, + * rather than just a map, because multiple blocks may have the same parent (when chain splits + * or stale blocks exist). It maps from parent-hash to child-disk-position. + * + * This function can also be used to read blocks from user-specified block files using the + * -loadblock= option. There's no unknown-parent tracking, so the last two arguments are omitted. + * + * + * @param[in] fileIn FILE handle to file containing blocks to read + * @param[in] dbp (optional) Disk block position (only for reindex) + * @param[in,out] blocks_with_unknown_parent (optional) Map of disk positions for blocks with + * unknown parent, key is parent block hash + * (only used for reindex) + * */ + void LoadExternalBlockFile( + FILE* fileIn, + FlatFilePos* dbp = nullptr, + std::multimap<uint256, FlatFilePos>* blocks_with_unknown_parent = nullptr); + + /** * Process an incoming block. This only returns after the best known valid * block is made active. Note that it does not, however, guarantee that the * specific block passed to it has been checked for validity! @@ -1125,6 +1140,29 @@ public: bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, bool min_pow_checked, BlockValidationState& state, const CBlockIndex** ppindex = nullptr) LOCKS_EXCLUDED(cs_main); /** + * Sufficiently validate a block for disk storage (and store on disk). + * + * @param[in] pblock The block we want to process. + * @param[in] fRequested Whether we requested this block from a + * peer. + * @param[in] dbp The location on disk, if we are importing + * this block from prior storage. + * @param[in] min_pow_checked True if proof-of-work anti-DoS checks have + * been done by caller for headers chain + * + * @param[out] state The state of the block validation. + * @param[out] ppindex Optional return parameter to get the + * CBlockIndex pointer for this block. + * @param[out] fNewBlock Optional return parameter to indicate if the + * block is new to our storage. + * + * @returns False if the block or header is invalid, or if saving to disk fails (likely a fatal error); true otherwise. + */ + bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock, bool min_pow_checked) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + /** * Try to add a transaction to the memory pool. * * @param[in] tx The transaction to submit for mempool acceptance. |