diff options
author | Pieter Wuille <pieter.wuille@gmail.com> | 2012-11-09 14:50:30 -0800 |
---|---|---|
committer | Pieter Wuille <pieter.wuille@gmail.com> | 2012-11-09 14:50:30 -0800 |
commit | 485cf044baf562d549e214c4836437dd698d3099 (patch) | |
tree | 70997acd33477bdb882447117571dcfa650c1fa7 /src/main.cpp | |
parent | 8146591a5305a737557aadec56002af4d11dc4b3 (diff) | |
parent | 7fea48467442079cd0b4021b580761d7e33fa8a1 (diff) | |
download | bitcoin-485cf044baf562d549e214c4836437dd698d3099.tar.xz |
Merge pull request #1943 from sipa/reindex2
Add -reindex, to perform in-place reindexing of block chain files
Diffstat (limited to 'src/main.cpp')
-rw-r--r-- | src/main.cpp | 239 |
1 files changed, 113 insertions, 126 deletions
diff --git a/src/main.cpp b/src/main.cpp index a08e393caa..0940260f9e 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -41,6 +41,7 @@ CBlockIndex* pindexBest = NULL; set<CBlockIndex*, CBlockIndexWorkComparator> setBlockIndexValid; // may contain all CBlockIndex*'s that have validness >=BLOCK_VALID_TRANSACTIONS, and must contain those who aren't failed int64 nTimeBestReceived = 0; bool fImporting = false; +bool fReindex = false; unsigned int nCoinCacheSize = 5000; CMedianFilter<int> cPeerBlockCounts(5, 0); // Amount of blocks that other nodes claim to have @@ -1145,7 +1146,7 @@ int GetNumBlocksOfPeers() bool IsInitialBlockDownload() { - if (pindexBest == NULL || nBestHeight < Checkpoints::GetTotalBlocksEstimate()) + if (pindexBest == NULL || nBestHeight < Checkpoints::GetTotalBlocksEstimate() || fReindex || fImporting) return true; static int64 nLastUpdate; static CBlockIndex* pindexLastBest; @@ -1862,35 +1863,45 @@ bool CBlock::AddToBlockIndex(const CDiskBlockPos &pos) } -bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64 nTime) +bool FindBlockPos(CDiskBlockPos &pos, unsigned int nAddSize, unsigned int nHeight, uint64 nTime, bool fKnown = false) { bool fUpdatedLast = false; LOCK(cs_LastBlockFile); - while (infoLastBlockFile.nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { - printf("Leaving block file %i: %s\n", nLastBlockFile, infoLastBlockFile.ToString().c_str()); - FlushBlockFile(); - nLastBlockFile++; - infoLastBlockFile.SetNull(); - pblocktree->ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile); // check whether data for the new file somehow already exist; can fail just fine - fUpdatedLast = true; + if (fKnown) { + if (nLastBlockFile != pos.nFile) { + nLastBlockFile = pos.nFile; + infoLastBlockFile.SetNull(); + pblocktree->ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile); + } + } else { + while (infoLastBlockFile.nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { + printf("Leaving block file %i: %s\n", nLastBlockFile, infoLastBlockFile.ToString().c_str()); + FlushBlockFile(); + nLastBlockFile++; + infoLastBlockFile.SetNull(); + pblocktree->ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile); // check whether data for the new file somehow already exist; can fail just fine + fUpdatedLast = true; + } + pos.nFile = nLastBlockFile; + pos.nPos = infoLastBlockFile.nSize; } - pos.nFile = nLastBlockFile; - pos.nPos = infoLastBlockFile.nSize; infoLastBlockFile.nSize += nAddSize; infoLastBlockFile.AddBlock(nHeight, nTime); - unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; - unsigned int nNewChunks = (infoLastBlockFile.nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; - if (nNewChunks > nOldChunks) { - FILE *file = OpenBlockFile(pos); - if (file) { - printf("Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile); - AllocateFileRange(file, pos.nPos, nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos); + if (!fKnown) { + unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; + unsigned int nNewChunks = (infoLastBlockFile.nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; + if (nNewChunks > nOldChunks) { + FILE *file = OpenBlockFile(pos); + if (file) { + printf("Pre-allocating up to position 0x%x in blk%05u.dat\n", nNewChunks * BLOCKFILE_CHUNK_SIZE, pos.nFile); + AllocateFileRange(file, pos.nPos, nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos); + } + fclose(file); } - fclose(file); } if (!pblocktree->WriteBlockFileInfo(nLastBlockFile, infoLastBlockFile)) @@ -1996,7 +2007,7 @@ bool CBlock::CheckBlock(bool fCheckPOW, bool fCheckMerkleRoot) const return true; } -bool CBlock::AcceptBlock() +bool CBlock::AcceptBlock(CDiskBlockPos *dbp) { // Check for duplicate uint256 hash = GetHash(); @@ -2004,11 +2015,15 @@ bool CBlock::AcceptBlock() return error("AcceptBlock() : block already in mapBlockIndex"); // Get prev block index + CBlockIndex* pindexPrev = NULL; + int nHeight = 0; + if (hash != hashGenesisBlock) { + map<uint256, CBlockIndex*>::iterator mi = mapBlockIndex.find(hashPrevBlock); if (mi == mapBlockIndex.end()) return DoS(10, error("AcceptBlock() : prev block not found")); - CBlockIndex* pindexPrev = (*mi).second; - int nHeight = pindexPrev->nHeight+1; + pindexPrev = (*mi).second; + nHeight = pindexPrev->nHeight+1; // Check proof of work if (nBits != GetNextWorkRequired(pindexPrev, this)) @@ -2048,16 +2063,22 @@ bool CBlock::AcceptBlock() return DoS(100, error("AcceptBlock() : block height mismatch in coinbase")); } } + } // Write block to history file unsigned int nBlockSize = ::GetSerializeSize(*this, SER_DISK, CLIENT_VERSION); - if (!CheckDiskSpace(::GetSerializeSize(*this, SER_DISK, CLIENT_VERSION))) - return error("AcceptBlock() : out of disk space"); CDiskBlockPos blockPos; - if (!FindBlockPos(blockPos, nBlockSize+8, nHeight, nTime)) + if (dbp == NULL) { + if (!CheckDiskSpace(::GetSerializeSize(*this, SER_DISK, CLIENT_VERSION))) + return error("AcceptBlock() : out of disk space"); + } else { + blockPos = *dbp; + } + if (!FindBlockPos(blockPos, nBlockSize+8, nHeight, nTime, dbp != NULL)) return error("AcceptBlock() : FindBlockPos failed"); - if (!WriteToDisk(blockPos)) - return error("AcceptBlock() : WriteToDisk failed"); + if (dbp == NULL) + if (!WriteToDisk(blockPos)) + return error("AcceptBlock() : WriteToDisk failed"); if (!AddToBlockIndex(blockPos)) return error("AcceptBlock() : AddToBlockIndex failed"); @@ -2086,7 +2107,7 @@ bool CBlockIndex::IsSuperMajority(int minVersion, const CBlockIndex* pstart, uns return (nFound >= nRequired); } -bool ProcessBlock(CNode* pfrom, CBlock* pblock) +bool ProcessBlock(CNode* pfrom, CBlock* pblock, CDiskBlockPos *dbp) { // Check for duplicate uint256 hash = pblock->GetHash(); @@ -2124,7 +2145,7 @@ bool ProcessBlock(CNode* pfrom, CBlock* pblock) // If we don't already have its previous block, shunt it off to holding area until we get it - if (!mapBlockIndex.count(pblock->hashPrevBlock)) + if (pblock->hashPrevBlock != 0 && !mapBlockIndex.count(pblock->hashPrevBlock)) { printf("ProcessBlock: ORPHAN BLOCK, prev=%s\n", pblock->hashPrevBlock.ToString().substr(0,20).c_str()); @@ -2141,7 +2162,7 @@ bool ProcessBlock(CNode* pfrom, CBlock* pblock) } // Store to disk - if (!pblock->AcceptBlock()) + if (!pblock->AcceptBlock(dbp)) return error("ProcessBlock() : AcceptBlock FAILED"); // Recursively process any orphan blocks that depended on this one @@ -2304,6 +2325,11 @@ bool static LoadBlockIndexDB() // Load bnBestInvalidWork, OK if it doesn't exist pblocktree->ReadBestInvalidWork(bnBestInvalidWork); + // Check whether we need to continue reindexing + bool fReindexing = false; + pblocktree->ReadReindexing(fReindexing); + fReindex |= fReindexing; + // Verify blocks in the best chain int nCheckLevel = GetArg("-checklevel", 1); int nCheckDepth = GetArg( "-checkblocks", 2500); @@ -2337,7 +2363,7 @@ bool static LoadBlockIndexDB() return true; } -bool LoadBlockIndex(bool fAllowNew) +bool LoadBlockIndex() { if (fTestNet) { @@ -2348,6 +2374,9 @@ bool LoadBlockIndex(bool fAllowNew) hashGenesisBlock = uint256("000000000933ea01ad0ee984209779baaec3ced90fa3f408719526f8d77f4943"); } + if (fReindex) + return true; + // // Load block index from databases // @@ -2359,9 +2388,6 @@ bool LoadBlockIndex(bool fAllowNew) // if (mapBlockIndex.empty()) { - if (!fAllowNew) - return false; - // Genesis Block: // CBlock(hash=000000000019d6, ver=1, hashPrevBlock=00000000000000, hashMerkleRoot=4a5e1e, nTime=1231006505, nBits=1d00ffff, nNonce=2083236893, vtx=1) // CTransaction(hash=4a5e1e, ver=1, vin.size=1, vout.size=1, nLockTime=0) @@ -2487,110 +2513,71 @@ void PrintBlockTree() } } -bool LoadExternalBlockFile(FILE* fileIn) +bool LoadExternalBlockFile(FILE* fileIn, CDiskBlockPos *dbp) { int64 nStart = GetTimeMillis(); int nLoaded = 0; { - try { - CAutoFile blkdat(fileIn, SER_DISK, CLIENT_VERSION); - unsigned int nPos = 0; - while (nPos != (unsigned int)-1 && blkdat.good() && !fRequestShutdown) - { - unsigned char pchData[65536]; - do { - fseek(blkdat, nPos, SEEK_SET); - int nRead = fread(pchData, 1, sizeof(pchData), blkdat); - if (nRead <= 8) - { - nPos = (unsigned int)-1; - break; - } - void* nFind = memchr(pchData, pchMessageStart[0], nRead+1-sizeof(pchMessageStart)); - if (nFind) - { - if (memcmp(nFind, pchMessageStart, sizeof(pchMessageStart))==0) - { - nPos += ((unsigned char*)nFind - pchData) + sizeof(pchMessageStart); - break; - } - nPos += ((unsigned char*)nFind - pchData) + 1; - } - else - nPos += sizeof(pchData) - sizeof(pchMessageStart) + 1; - } while(!fRequestShutdown); - if (nPos == (unsigned int)-1) - break; - fseek(blkdat, nPos, SEEK_SET); - unsigned int nSize; + CBufferedFile blkdat(fileIn, 2*MAX_BLOCK_SIZE, MAX_BLOCK_SIZE+8, SER_DISK, CLIENT_VERSION); + uint64 nStartByte = 0; + if (dbp) { + // (try to) skip already indexed part + CBlockFileInfo info; + if (pblocktree->ReadBlockFileInfo(dbp->nFile, info)) { + nStartByte = info.nSize; + blkdat.Seek(info.nSize); + } + } + uint64 nRewind = blkdat.GetPos(); + while (blkdat.good() && !blkdat.eof() && !fShutdown) { + blkdat.SetPos(nRewind); + nRewind++; // start one byte further next time, in case of failure + blkdat.SetLimit(); // remove former limit + unsigned int nSize = 0; + try { + // locate a header + unsigned char buf[4]; + blkdat.FindByte(pchMessageStart[0]); + nRewind = blkdat.GetPos()+1; + blkdat >> FLATDATA(buf); + if (memcmp(buf, pchMessageStart, 4)) + continue; + // read size blkdat >> nSize; - if (nSize > 0 && nSize <= MAX_BLOCK_SIZE) - { - CBlock block; - blkdat >> block; + if (nSize < 80 || nSize > MAX_BLOCK_SIZE) + continue; + } catch (std::exception &e) { + // no valid block header found; don't complain + break; + } + try { + // read block + uint64 nBlockPos = blkdat.GetPos(); + blkdat.SetLimit(nBlockPos + nSize); + CBlock block; + blkdat >> block; + nRewind = blkdat.GetPos(); + + // process block + if (nBlockPos >= nStartByte) { LOCK(cs_main); - if (ProcessBlock(NULL,&block)) - { + if (dbp) + dbp->nPos = nBlockPos; + if (ProcessBlock(NULL, &block, dbp)) nLoaded++; - nPos += 4 + nSize; - } } + } catch (std::exception &e) { + printf("%s() : Deserialize or I/O error caught during load\n", __PRETTY_FUNCTION__); } } - catch (std::exception &e) { - printf("%s() : Deserialize or I/O error caught during load\n", - __PRETTY_FUNCTION__); - } + fclose(fileIn); } - printf("Loaded %i blocks from external file in %"PRI64d"ms\n", nLoaded, GetTimeMillis() - nStart); + if (nLoaded > 0) + printf("Loaded %i blocks from external file in %"PRI64d"ms\n", nLoaded, GetTimeMillis() - nStart); return nLoaded > 0; } -struct CImportingNow -{ - CImportingNow() { - assert(fImporting == false); - fImporting = true; - } - - ~CImportingNow() { - assert(fImporting == true); - fImporting = false; - } -}; - -void ThreadImport(void *data) { - std::vector<boost::filesystem::path> *vFiles = reinterpret_cast<std::vector<boost::filesystem::path>*>(data); - - RenameThread("bitcoin-loadblk"); - - CImportingNow imp; - vnThreadsRunning[THREAD_IMPORT]++; - - // -loadblock= - BOOST_FOREACH(boost::filesystem::path &path, *vFiles) { - FILE *file = fopen(path.string().c_str(), "rb"); - if (file) - LoadExternalBlockFile(file); - } - - // hardcoded $DATADIR/bootstrap.dat - filesystem::path pathBootstrap = GetDataDir() / "bootstrap.dat"; - if (filesystem::exists(pathBootstrap)) { - FILE *file = fopen(pathBootstrap.string().c_str(), "rb"); - if (file) { - filesystem::path pathBootstrapOld = GetDataDir() / "bootstrap.dat.old"; - LoadExternalBlockFile(file); - RenameOver(pathBootstrap, pathBootstrapOld); - } - } - - delete vFiles; - - vnThreadsRunning[THREAD_IMPORT]--; -} - @@ -2800,7 +2787,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) // Ask the first connected node for block updates static int nAskedForBlocks = 0; - if (!pfrom->fClient && !pfrom->fOneShot && !fImporting && + if (!pfrom->fClient && !pfrom->fOneShot && !fImporting && !fReindex && (pfrom->nStartingHeight > (nBestHeight - 144)) && (pfrom->nVersion < NOBLKS_VERSION_START || pfrom->nVersion >= NOBLKS_VERSION_END) && @@ -2937,7 +2924,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) printf(" got inventory: %s %s\n", inv.ToString().c_str(), fAlreadyHave ? "have" : "new"); if (!fAlreadyHave) { - if (!fImporting) + if (!fImporting && !fReindex) pfrom->AskFor(inv); } else if (inv.type == MSG_BLOCK && mapOrphanBlocks.count(inv.hash)) { pfrom->PushGetBlocks(pindexBest, GetOrphanRoot(mapOrphanBlocks[inv.hash])); @@ -3167,7 +3154,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv) } - else if (strCommand == "block") + else if (strCommand == "block" && !fImporting && !fReindex) // Ignore blocks received while importing { CBlock block; vRecv >> block; |