From 7fea48467442079cd0b4021b580761d7e33fa8a1 Mon Sep 17 00:00:00 2001 From: Pieter Wuille Date: Sun, 21 Oct 2012 21:23:13 +0200 Subject: Add -reindex, to perform in-place reindexing of block chain files Flushes the blktree/ and coins/ databases, and reindexes the block chain files, as if their contents was loaded via -loadblock. Based on earlier work by Jeff Garzik. --- src/main.cpp | 132 ++++++++++++++++++++++++++++++++++++++++------------------- 1 file changed, 89 insertions(+), 43 deletions(-) (limited to 'src/main.cpp') diff --git a/src/main.cpp b/src/main.cpp index 64adae8edd..0940260f9e 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -41,6 +41,7 @@ CBlockIndex* pindexBest = NULL; set setBlockIndexValid; // may contain all CBlockIndex*'s that have validness >=BLOCK_VALID_TRANSACTIONS, and must contain those who aren't failed int64 nTimeBestReceived = 0; bool fImporting = false; +bool fReindex = false; unsigned int nCoinCacheSize = 5000; CMedianFilter cPeerBlockCounts(5, 0); // Amount of blocks that other nodes claim to have @@ -1145,7 +1146,7 @@ int GetNumBlocksOfPeers() bool IsInitialBlockDownload() { - if (pindexBest == NULL || nBestHeight < Checkpoints::GetTotalBlocksEstimate()) + if (pindexBest == NULL || nBestHeight < Checkpoints::GetTotalBlocksEstimate() || fReindex || fImporting) return true; static int64 nLastUpdate; static CBlockIndex* pindexLastBest; @@ -1862,35 +1863,45 @@ bool CBlock::AddToBlockIndex(const CDiskBlockPos &pos) } -bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64 nTime) +bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64 nTime, bool fKnown = false) { bool fUpdatedLast = false; LOCK(cs_LastBlockFile); - while (infoLastBlockFile.nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { - printf("Leaving block file %i: %s\n", nLastBlockFile, infoLastBlockFile.ToString().c_str()); - FlushBlockFile(); - nLastBlockFile++; - infoLastBlockFile.SetNull(); - pblocktree->ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile); // check whether data for the new file somehow already exist; can fail just fine - fUpdatedLast = true; + if (fKnown) { + if (nLastBlockFile != pos.nFile) { + nLastBlockFile = pos.nFile; + infoLastBlockFile.SetNull(); + pblocktree->ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile); + } + } else { + while (infoLastBlockFile.nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { + printf("Leaving block file %i: %s\n", nLastBlockFile, infoLastBlockFile.ToString().c_str()); + FlushBlockFile(); + nLastBlockFile++; + infoLastBlockFile.SetNull(); + pblocktree->ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile); // check whether data for the new file somehow already exist; can fail just fine + fUpdatedLast = true; + } + pos.nFile = nLastBlockFile; + pos.nPos = infoLastBlockFile.nSize; } - pos.nFile = nLastBlockFile; - pos.nPos = infoLastBlockFile.nSize; infoLastBlockFile.nSize += nAddSize; infoLastBlockFile.AddBlock(nHeight, nTime); - unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; - unsigned int nNewChunks = (infoLastBlockFile.nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; - if (nNewChunks > nOldChunks) { - FILE *file = OpenBlockFile(pos); - if (file) { - printf("Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile); - AllocateFileRange(file, pos.nPos, nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos); + if (!fKnown) { + unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; + unsigned int nNewChunks = (infoLastBlockFile.nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; + if (nNewChunks > nOldChunks) { + FILE *file = OpenBlockFile(pos); + if (file) { + printf("Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile); + AllocateFileRange(file, pos.nPos, nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos); + } + fclose(file); } - fclose(file); } if (!pblocktree->WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) @@ -1996,7 +2007,7 @@ bool CBlock::CheckBlock(bool fCheckPOW, bool fCheckMerkleRoot) const return true; } -bool CBlock::AcceptBlock() +bool CBlock::AcceptBlock(CDiskBlockPos *dbp) { // Check for duplicate uint256 hash = GetHash(); @@ -2004,11 +2015,15 @@ bool CBlock::AcceptBlock() return error("AcceptBlock() : block already in mapBlockIndex"); // Get prev block index + CBlockIndex* pindexPrev = NULL; + int nHeight = 0; + if (hash != hashGenesisBlock) { + map::iterator mi = mapBlockIndex.find(hashPrevBlock); if (mi == mapBlockIndex.end()) return DoS(10, error("AcceptBlock() : prev block not found")); - CBlockIndex* pindexPrev = (*mi).second; - int nHeight = pindexPrev->nHeight+1; + pindexPrev = (*mi).second; + nHeight = pindexPrev->nHeight+1; // Check proof of work if (nBits != GetNextWorkRequired(pindexPrev, this)) @@ -2048,16 +2063,22 @@ bool CBlock::AcceptBlock() return DoS(100, error("AcceptBlock() : block height mismatch in coinbase")); } } + } // Write block to history file unsigned int nBlockSize = ::GetSerializeSize(*this, SER_DISK, CLIENT_VERSION); - if (!CheckDiskSpace(::GetSerializeSize(*this, SER_DISK, CLIENT_VERSION))) - return error("AcceptBlock() : out of disk space"); CDiskBlockPos blockPos; - if (!FindBlockPos(blockPos, nBlockSize+8, nHeight, nTime)) + if (dbp == NULL) { + if (!CheckDiskSpace(::GetSerializeSize(*this, SER_DISK, CLIENT_VERSION))) + return error("AcceptBlock() : out of disk space"); + } else { + blockPos = *dbp; + } + if (!FindBlockPos(blockPos, nBlockSize+8, nHeight, nTime, dbp != NULL)) return error("AcceptBlock() : FindBlockPos failed"); - if (!WriteToDisk(blockPos)) - return error("AcceptBlock() : WriteToDisk failed"); + if (dbp == NULL) + if (!WriteToDisk(blockPos)) + return error("AcceptBlock() : WriteToDisk failed"); if (!AddToBlockIndex(blockPos)) return error("AcceptBlock() : AddToBlockIndex failed"); @@ -2086,7 +2107,7 @@ bool CBlockIndex::IsSuperMajority(int minVersion, const CBlockIndex* pstart, uns return (nFound >= nRequired); } -bool ProcessBlock(CNode* pfrom, CBlock* pblock) +bool ProcessBlock(CNode* pfrom, CBlock* pblock, CDiskBlockPos *dbp) { // Check for duplicate uint256 hash = pblock->GetHash(); @@ -2124,7 +2145,7 @@ bool ProcessBlock(CNode* pfrom, CBlock* pblock) // If we don't already have its previous block, shunt it off to holding area until we get it - if (!mapBlockIndex.count(pblock->hashPrevBlock)) + if (pblock->hashPrevBlock != 0 && !mapBlockIndex.count(pblock->hashPrevBlock)) { printf("ProcessBlock: ORPHAN BLOCK, prev=%s\n", pblock->hashPrevBlock.ToString().substr(0,20).c_str()); @@ -2141,7 +2162,7 @@ bool ProcessBlock(CNode* pfrom, CBlock* pblock) } // Store to disk - if (!pblock->AcceptBlock()) + if (!pblock->AcceptBlock(dbp)) return error("ProcessBlock() : AcceptBlock FAILED"); // Recursively process any orphan blocks that depended on this one @@ -2304,6 +2325,11 @@ bool static LoadBlockIndexDB() // Load bnBestInvalidWork, OK if it doesn't exist pblocktree->ReadBestInvalidWork(bnBestInvalidWork); + // Check whether we need to continue reindexing + bool fReindexing = false; + pblocktree->ReadReindexing(fReindexing); + fReindex |= fReindexing; + // Verify blocks in the best chain int nCheckLevel = GetArg("-checklevel", 1); int nCheckDepth = GetArg( "-checkblocks", 2500); @@ -2337,7 +2363,7 @@ bool static LoadBlockIndexDB() return true; } -bool LoadBlockIndex(bool fAllowNew) +bool LoadBlockIndex() { if (fTestNet) { @@ -2348,6 +2374,9 @@ bool LoadBlockIndex(bool fAllowNew) hashGenesisBlock = uint256("000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943"); } + if (fReindex) + return true; + // // Load block index from databases // @@ -2359,9 +2388,6 @@ bool LoadBlockIndex(bool fAllowNew) // if (mapBlockIndex.empty()) { - if (!fAllowNew) - return false; - // Genesis Block: // CBlock(hash=000000000019d6, ver=1, hashPrevBlock=00000000000000, hashMerkleRoot=4a5e1e, nTime=1231006505, nBits=1d00ffff, nNonce=2083236893, vtx=1) // CTransaction(hash=4a5e1e, ver=1, vin.size=1, vout.size=1, nLockTime=0) @@ -2487,18 +2513,28 @@ void PrintBlockTree() } } -bool LoadExternalBlockFile(FILE* fileIn) +bool LoadExternalBlockFile(FILE* fileIn, CDiskBlockPos *dbp) { int64 nStart = GetTimeMillis(); int nLoaded = 0; { CBufferedFile blkdat(fileIn, 2*MAX_BLOCK_SIZE, MAX_BLOCK_SIZE+8, SER_DISK, CLIENT_VERSION); + uint64 nStartByte = 0; + if (dbp) { + // (try to) skip already indexed part + CBlockFileInfo info; + if (pblocktree->ReadBlockFileInfo(dbp->nFile, info)) { + nStartByte = info.nSize; + blkdat.Seek(info.nSize); + } + } uint64 nRewind = blkdat.GetPos(); while (blkdat.good() && !blkdat.eof() && !fShutdown) { blkdat.SetPos(nRewind); nRewind++; // start one byte further next time, in case of failure blkdat.SetLimit(); // remove former limit + unsigned int nSize = 0; try { // locate a header unsigned char buf[4]; @@ -2508,18 +2544,27 @@ bool LoadExternalBlockFile(FILE* fileIn) if (memcmp(buf, pchMessageStart, 4)) continue; // read size - unsigned int nSize; blkdat >> nSize; if (nSize < 80 || nSize > MAX_BLOCK_SIZE) continue; + } catch (std::exception &e) { + // no valid block header found; don't complain + break; + } + try { // read block - blkdat.SetLimit(blkdat.GetPos() + nSize); + uint64 nBlockPos = blkdat.GetPos(); + blkdat.SetLimit(nBlockPos + nSize); CBlock block; blkdat >> block; nRewind = blkdat.GetPos(); - { + + // process block + if (nBlockPos >= nStartByte) { LOCK(cs_main); - if (ProcessBlock(NULL,&block)) + if (dbp) + dbp->nPos = nBlockPos; + if (ProcessBlock(NULL, &block, dbp)) nLoaded++; } } catch (std::exception &e) { @@ -2528,7 +2573,8 @@ bool LoadExternalBlockFile(FILE* fileIn) } fclose(fileIn); } - printf("Loaded %i blocks from external file in %"PRI64d"ms\n", nLoaded, GetTimeMillis() - nStart); + if (nLoaded > 0) + printf("Loaded %i blocks from external file in %"PRI64d"ms\n", nLoaded, GetTimeMillis() - nStart); return nLoaded > 0; } @@ -2741,7 +2787,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) // Ask the first connected node for block updates static int nAskedForBlocks = 0; - if (!pfrom->fClient && !pfrom->fOneShot && !fImporting && + if (!pfrom->fClient && !pfrom->fOneShot && !fImporting && !fReindex && (pfrom->nStartingHeight > (nBestHeight - 144)) && (pfrom->nVersion < NOBLKS_VERSION_START || pfrom->nVersion >= NOBLKS_VERSION_END) && @@ -2878,7 +2924,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) printf(" got inventory: %s %s\n", inv.ToString().c_str(), fAlreadyHave ? "have" : "new"); if (!fAlreadyHave) { - if (!fImporting) + if (!fImporting && !fReindex) pfrom->AskFor(inv); } else if (inv.type == MSG_BLOCK && mapOrphanBlocks.count(inv.hash)) { pfrom->PushGetBlocks(pindexBest, GetOrphanRoot(mapOrphanBlocks[inv.hash])); @@ -3108,7 +3154,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) } - else if (strCommand == "block") + else if (strCommand == "block" && !fImporting && !fReindex) // Ignore blocks received while importing { CBlock block; vRecv >> block; -- cgit v1.2.3