// Copyright (c) 2009-2010 Satoshi Nakamoto // Copyright (c) 2009-2012 The Bitcoin developers // Distributed under the MIT/X11 software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "db.h" #include "util.h" #include "main.h" #include #include #include #ifndef WIN32 #include "sys/stat.h" #endif using namespace std; using namespace boost; unsigned int nWalletDBUpdated; // // CDB // CDBEnv bitdb; void CDBEnv::EnvShutdown() { if (!fDbEnvInit) return; fDbEnvInit = false; int ret = dbenv.close(0); if (ret != 0) printf("EnvShutdown exception: %s (%d)\n", DbEnv::strerror(ret), ret); if (!fMockDb) DbEnv(0).remove(GetDataDir().string().c_str(), 0); } CDBEnv::CDBEnv() : dbenv(DB_CXX_NO_EXCEPTIONS) { } CDBEnv::~CDBEnv() { EnvShutdown(); } void CDBEnv::Close() { EnvShutdown(); } bool CDBEnv::Open(boost::filesystem::path pathEnv_) { if (fDbEnvInit) return true; if (fShutdown) return false; pathEnv = pathEnv_; filesystem::path pathDataDir = pathEnv; filesystem::path pathLogDir = pathDataDir / "database"; filesystem::create_directory(pathLogDir); filesystem::path pathErrorFile = pathDataDir / "db.log"; printf("dbenv.open LogDir=%s ErrorFile=%s\n", pathLogDir.string().c_str(), pathErrorFile.string().c_str()); unsigned int nEnvFlags = 0; if (GetBoolArg("-privdb", true)) nEnvFlags |= DB_PRIVATE; int nDbCache = GetArg("-dbcache", 25); dbenv.set_lg_dir(pathLogDir.string().c_str()); dbenv.set_cachesize(nDbCache / 1024, (nDbCache % 1024)*1048576, 1); dbenv.set_lg_bsize(1048576); dbenv.set_lg_max(10485760); dbenv.set_lk_max_locks(40000); dbenv.set_lk_max_objects(40000); dbenv.set_errfile(fopen(pathErrorFile.string().c_str(), "a")); /// debug dbenv.set_flags(DB_AUTO_COMMIT, 1); dbenv.set_flags(DB_TXN_WRITE_NOSYNC, 1); dbenv.log_set_config(DB_LOG_AUTO_REMOVE, 1); int ret = dbenv.open(pathDataDir.string().c_str(), DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_THREAD | DB_RECOVER | nEnvFlags, S_IRUSR | S_IWUSR); if (ret != 0) return error("CDB() : error %s (%d) opening database environment", DbEnv::strerror(ret), ret); fDbEnvInit = true; fMockDb = false; return true; } void CDBEnv::MakeMock() { if (fDbEnvInit) throw runtime_error("CDBEnv::MakeMock(): already initialized"); if (fShutdown) throw runtime_error("CDBEnv::MakeMock(): during shutdown"); printf("CDBEnv::MakeMock()\n"); dbenv.set_cachesize(1, 0, 1); dbenv.set_lg_bsize(10485760*4); dbenv.set_lg_max(10485760); dbenv.set_lk_max_locks(10000); dbenv.set_lk_max_objects(10000); dbenv.set_flags(DB_AUTO_COMMIT, 1); dbenv.log_set_config(DB_LOG_IN_MEMORY, 1); int ret = dbenv.open(NULL, DB_CREATE | DB_INIT_LOCK | DB_INIT_LOG | DB_INIT_MPOOL | DB_INIT_TXN | DB_THREAD | DB_PRIVATE, S_IRUSR | S_IWUSR); if (ret > 0) throw runtime_error(strprintf("CDBEnv::MakeMock(): error %d opening database environment", ret)); fDbEnvInit = true; fMockDb = true; } CDBEnv::VerifyResult CDBEnv::Verify(std::string strFile, bool (*recoverFunc)(CDBEnv& dbenv, std::string strFile)) { LOCK(cs_db); assert(mapFileUseCount.count(strFile) == 0); Db db(&dbenv, 0); int result = db.verify(strFile.c_str(), NULL, NULL, 0); if (result == 0) return VERIFY_OK; else if (recoverFunc == NULL) return RECOVER_FAIL; // Try to recover: bool fRecovered = (*recoverFunc)(*this, strFile); return (fRecovered ? RECOVER_OK : RECOVER_FAIL); } bool CDBEnv::Salvage(std::string strFile, bool fAggressive, std::vector& vResult) { LOCK(cs_db); assert(mapFileUseCount.count(strFile) == 0); u_int32_t flags = DB_SALVAGE; if (fAggressive) flags |= DB_AGGRESSIVE; stringstream strDump; Db db(&dbenv, 0); int result = db.verify(strFile.c_str(), NULL, &strDump, flags); if (result != 0) { printf("ERROR: db salvage failed\n"); return false; } // Format of bdb dump is ascii lines: // header lines... // HEADER=END // hexadecimal key // hexadecimal value // ... repeated // DATA=END string strLine; while (!strDump.eof() && strLine != "HEADER=END") getline(strDump, strLine); // Skip past header std::string keyHex, valueHex; while (!strDump.eof() && keyHex != "DATA=END") { getline(strDump, keyHex); if (keyHex != "DATA_END") { getline(strDump, valueHex); vResult.push_back(make_pair(ParseHex(keyHex),ParseHex(valueHex))); } } return (result == 0); } void CDBEnv::CheckpointLSN(std::string strFile) { dbenv.txn_checkpoint(0, 0, 0); if (fMockDb) return; dbenv.lsn_reset(strFile.c_str(), 0); } CDB::CDB(const char *pszFile, const char* pszMode) : pdb(NULL), activeTxn(NULL) { int ret; if (pszFile == NULL) return; fReadOnly = (!strchr(pszMode, '+') && !strchr(pszMode, 'w')); bool fCreate = strchr(pszMode, 'c'); unsigned int nFlags = DB_THREAD; if (fCreate) nFlags |= DB_CREATE; { LOCK(bitdb.cs_db); if (!bitdb.Open(GetDataDir())) throw runtime_error("env open failed"); strFile = pszFile; ++bitdb.mapFileUseCount[strFile]; pdb = bitdb.mapDb[strFile]; if (pdb == NULL) { pdb = new Db(&bitdb.dbenv, 0); bool fMockDb = bitdb.IsMock(); if (fMockDb) { DbMpoolFile*mpf = pdb->get_mpf(); ret = mpf->set_flags(DB_MPOOL_NOFILE, 1); if (ret != 0) throw runtime_error(strprintf("CDB() : failed to configure for no temp file backing for database %s", pszFile)); } ret = pdb->open(NULL, // Txn pointer fMockDb ? NULL : pszFile, // Filename fMockDb ? pszFile : "main", // Logical db name DB_BTREE, // Database type nFlags, // Flags 0); if (ret != 0) { delete pdb; pdb = NULL; --bitdb.mapFileUseCount[strFile]; strFile = ""; throw runtime_error(strprintf("CDB() : can't open database file %s, error %d", pszFile, ret)); } if (fCreate && !Exists(string("version"))) { bool fTmp = fReadOnly; fReadOnly = false; WriteVersion(CLIENT_VERSION); fReadOnly = fTmp; } bitdb.mapDb[strFile] = pdb; } } } static bool IsChainFile(std::string strFile) { if (strFile == "coins.dat" || strFile == "blktree.dat") return true; return false; } void CDB::Flush() { if (activeTxn) return; // Flush database activity from memory pool to disk log unsigned int nMinutes = 0; if (fReadOnly) nMinutes = 1; if (IsChainFile(strFile)) nMinutes = 2; if (IsChainFile(strFile) && IsInitialBlockDownload()) nMinutes = 5; bitdb.dbenv.txn_checkpoint(nMinutes ? GetArg("-dblogsize", 100)*1024 : 0, nMinutes, 0); } void CDB::Close() { if (!pdb) return; if (activeTxn) activeTxn->abort(); activeTxn = NULL; pdb = NULL; Flush(); { LOCK(bitdb.cs_db); --bitdb.mapFileUseCount[strFile]; } } void CDBEnv::CloseDb(const string& strFile) { { LOCK(cs_db); if (mapDb[strFile] != NULL) { // Close the database handle Db* pdb = mapDb[strFile]; pdb->close(0); delete pdb; mapDb[strFile] = NULL; } } } bool CDBEnv::RemoveDb(const string& strFile) { this->CloseDb(strFile); LOCK(cs_db); int rc = dbenv.dbremove(NULL, strFile.c_str(), NULL, DB_AUTO_COMMIT); return (rc == 0); } bool CDB::Rewrite(const string& strFile, const char* pszSkip) { while (!fShutdown) { { LOCK(bitdb.cs_db); if (!bitdb.mapFileUseCount.count(strFile) || bitdb.mapFileUseCount[strFile] == 0) { // Flush log data to the dat file bitdb.CloseDb(strFile); bitdb.CheckpointLSN(strFile); bitdb.mapFileUseCount.erase(strFile); bool fSuccess = true; printf("Rewriting %s...\n", strFile.c_str()); string strFileRes = strFile + ".rewrite"; { // surround usage of db with extra {} CDB db(strFile.c_str(), "r"); Db* pdbCopy = new Db(&bitdb.dbenv, 0); int ret = pdbCopy->open(NULL, // Txn pointer strFileRes.c_str(), // Filename "main", // Logical db name DB_BTREE, // Database type DB_CREATE, // Flags 0); if (ret > 0) { printf("Cannot create database file %s\n", strFileRes.c_str()); fSuccess = false; } Dbc* pcursor = db.GetCursor(); if (pcursor) while (fSuccess) { CDataStream ssKey(SER_DISK, CLIENT_VERSION); CDataStream ssValue(SER_DISK, CLIENT_VERSION); int ret = db.ReadAtCursor(pcursor, ssKey, ssValue, DB_NEXT); if (ret == DB_NOTFOUND) { pcursor->close(); break; } else if (ret != 0) { pcursor->close(); fSuccess = false; break; } if (pszSkip && strncmp(&ssKey[0], pszSkip, std::min(ssKey.size(), strlen(pszSkip))) == 0) continue; if (strncmp(&ssKey[0], "\x07version", 8) == 0) { // Update version: ssValue.clear(); ssValue << CLIENT_VERSION; } Dbt datKey(&ssKey[0], ssKey.size()); Dbt datValue(&ssValue[0], ssValue.size()); int ret2 = pdbCopy->put(NULL, &datKey, &datValue, DB_NOOVERWRITE); if (ret2 > 0) fSuccess = false; } if (fSuccess) { db.Close(); bitdb.CloseDb(strFile); if (pdbCopy->close(0)) fSuccess = false; delete pdbCopy; } } if (fSuccess) { Db dbA(&bitdb.dbenv, 0); if (dbA.remove(strFile.c_str(), NULL, 0)) fSuccess = false; Db dbB(&bitdb.dbenv, 0); if (dbB.rename(strFileRes.c_str(), NULL, strFile.c_str(), 0)) fSuccess = false; } if (!fSuccess) printf("Rewriting of %s FAILED!\n", strFileRes.c_str()); return fSuccess; } } Sleep(100); } return false; } void CDBEnv::Flush(bool fShutdown) { int64 nStart = GetTimeMillis(); // Flush log data to the actual data file // on all files that are not in use printf("Flush(%s)%s\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " db not started"); if (!fDbEnvInit) return; { LOCK(cs_db); map::iterator mi = mapFileUseCount.begin(); while (mi != mapFileUseCount.end()) { string strFile = (*mi).first; int nRefCount = (*mi).second; printf("%s refcount=%d\n", strFile.c_str(), nRefCount); if (nRefCount == 0) { // Move log data to the dat file CloseDb(strFile); printf("%s checkpoint\n", strFile.c_str()); dbenv.txn_checkpoint(0, 0, 0); if (!IsChainFile(strFile) || fDetachDB) { printf("%s detach\n", strFile.c_str()); if (!fMockDb) dbenv.lsn_reset(strFile.c_str(), 0); } printf("%s closed\n", strFile.c_str()); mapFileUseCount.erase(mi++); } else mi++; } printf("DBFlush(%s)%s ended %15"PRI64d"ms\n", fShutdown ? "true" : "false", fDbEnvInit ? "" : " db not started", GetTimeMillis() - nStart); if (fShutdown) { char** listp; if (mapFileUseCount.empty()) { dbenv.log_archive(&listp, DB_ARCH_REMOVE); Close(); } } } } // // CBlockTreeDB and CCoinsDB // bool CCoinsDB::HaveCoins(uint256 hash) { assert(!fClient); return Exists(make_pair('c', hash)); } bool CCoinsDB::ReadCoins(uint256 hash, CCoins &coins) { assert(!fClient); return Read(make_pair('c', hash), coins); } bool CCoinsDB::WriteCoins(uint256 hash, const CCoins &coins) { assert(!fClient); if (coins.IsPruned()) return Erase(make_pair('c', hash)); else return Write(make_pair('c', hash), coins); } bool CBlockTreeDB::WriteBlockIndex(const CDiskBlockIndex& blockindex) { return Write(make_pair('b', blockindex.GetBlockHash()), blockindex); } bool CCoinsDB::ReadHashBestChain(uint256& hashBestChain) { return Read('B', hashBestChain); } bool CCoinsDB::WriteHashBestChain(uint256 hashBestChain) { return Write('B', hashBestChain); } bool CBlockTreeDB::ReadBestInvalidWork(CBigNum& bnBestInvalidWork) { return Read('I', bnBestInvalidWork); } bool CBlockTreeDB::WriteBestInvalidWork(CBigNum bnBestInvalidWork) { return Write('I', bnBestInvalidWork); } bool CBlockTreeDB::WriteBlockFileInfo(int nFile, const CBlockFileInfo &info) { return Write(make_pair('f', nFile), info); } bool CBlockTreeDB::ReadBlockFileInfo(int nFile, CBlockFileInfo &info) { return Read(make_pair('f', nFile), info); } bool CBlockTreeDB::WriteLastBlockFile(int nFile) { return Write('l', nFile); } bool CBlockTreeDB::ReadLastBlockFile(int &nFile) { return Read('l', nFile); } CCoinsViewDB::CCoinsViewDB() : db("cr+") {} bool CCoinsViewDB::GetCoins(uint256 txid, CCoins &coins) { return db.ReadCoins(txid, coins); } bool CCoinsViewDB::SetCoins(uint256 txid, const CCoins &coins) { return db.WriteCoins(txid, coins); } bool CCoinsViewDB::HaveCoins(uint256 txid) { return db.HaveCoins(txid); } CBlockIndex *CCoinsViewDB::GetBestBlock() { uint256 hashBestChain; if (!db.ReadHashBestChain(hashBestChain)) return NULL; std::map::iterator it = mapBlockIndex.find(hashBestChain); if (it == mapBlockIndex.end()) return NULL; return it->second; } bool CCoinsViewDB::SetBestBlock(CBlockIndex *pindex) { return db.WriteHashBestChain(pindex->GetBlockHash()); } bool CCoinsViewDB::BatchWrite(const std::map &mapCoins, CBlockIndex *pindex) { printf("Committing %u changed transactions to coin database...\n", (unsigned int)mapCoins.size()); if (!db.TxnBegin()) return false; bool fOk = true; for (std::map::const_iterator it = mapCoins.begin(); it != mapCoins.end(); it++) { fOk = db.WriteCoins(it->first, it->second); if (!fOk) break; } if (fOk) fOk = db.WriteHashBestChain(pindex->GetBlockHash()); if (!fOk) db.TxnAbort(); else fOk = db.TxnCommit(); return fOk; } CBlockIndex static * InsertBlockIndex(uint256 hash) { if (hash == 0) return NULL; // Return existing map::iterator mi = mapBlockIndex.find(hash); if (mi != mapBlockIndex.end()) return (*mi).second; // Create new CBlockIndex* pindexNew = new CBlockIndex(); if (!pindexNew) throw runtime_error("LoadBlockIndex() : new CBlockIndex failed"); mi = mapBlockIndex.insert(make_pair(hash, pindexNew)).first; pindexNew->phashBlock = &((*mi).first); return pindexNew; } bool LoadBlockIndexDB() { if (!pblocktree->LoadBlockIndexGuts()) return false; if (fRequestShutdown) return true; // Calculate bnChainWork vector > vSortedByHeight; vSortedByHeight.reserve(mapBlockIndex.size()); BOOST_FOREACH(const PAIRTYPE(uint256, CBlockIndex*)& item, mapBlockIndex) { CBlockIndex* pindex = item.second; vSortedByHeight.push_back(make_pair(pindex->nHeight, pindex)); } sort(vSortedByHeight.begin(), vSortedByHeight.end()); BOOST_FOREACH(const PAIRTYPE(int, CBlockIndex*)& item, vSortedByHeight) { CBlockIndex* pindex = item.second; pindex->bnChainWork = (pindex->pprev ? pindex->pprev->bnChainWork : 0) + pindex->GetBlockWork(); pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS && !(pindex->nStatus & BLOCK_FAILED_MASK)) setBlockIndexValid.insert(pindex); } // Load block file info pblocktree->ReadLastBlockFile(nLastBlockFile); printf("LoadBlockIndex(): last block file = %i\n", nLastBlockFile); if (pblocktree->ReadBlockFileInfo(nLastBlockFile, infoLastBlockFile)) printf("LoadBlockIndex(): last block file: %s\n", infoLastBlockFile.ToString().c_str()); // Load hashBestChain pointer to end of best chain pindexBest = pcoinsTip->GetBestBlock(); if (pindexBest == NULL) { if (pindexGenesisBlock == NULL) return true; } hashBestChain = pindexBest->GetBlockHash(); nBestHeight = pindexBest->nHeight; bnBestChainWork = pindexBest->bnChainWork; // set 'next' pointers in best chain CBlockIndex *pindex = pindexBest; while(pindex != NULL && pindex->pprev != NULL) { CBlockIndex *pindexPrev = pindex->pprev; pindexPrev->pnext = pindex; pindex = pindexPrev; } printf("LoadBlockIndex(): hashBestChain=%s height=%d date=%s\n", hashBestChain.ToString().substr(0,20).c_str(), nBestHeight, DateTimeStrFormat("%x %H:%M:%S", pindexBest->GetBlockTime()).c_str()); // Load bnBestInvalidWork, OK if it doesn't exist pblocktree->ReadBestInvalidWork(bnBestInvalidWork); // Verify blocks in the best chain int nCheckLevel = GetArg("-checklevel", 1); int nCheckDepth = GetArg( "-checkblocks", 2500); if (nCheckDepth == 0) nCheckDepth = 1000000000; // suffices until the year 19000 if (nCheckDepth > nBestHeight) nCheckDepth = nBestHeight; printf("Verifying last %i blocks at level %i\n", nCheckDepth, nCheckLevel); CBlockIndex* pindexFork = NULL; for (CBlockIndex* pindex = pindexBest; pindex && pindex->pprev; pindex = pindex->pprev) { if (fRequestShutdown || pindex->nHeight < nBestHeight-nCheckDepth) break; CBlock block; if (!block.ReadFromDisk(pindex)) return error("LoadBlockIndex() : block.ReadFromDisk failed"); // check level 1: verify block validity if (nCheckLevel>0 && !block.CheckBlock()) { printf("LoadBlockIndex() : *** found bad block at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString().c_str()); pindexFork = pindex->pprev; } // TODO: stronger verifications } if (pindexFork && !fRequestShutdown) { // TODO: reorg back return error("LoadBlockIndex(): chain database corrupted"); } return true; } bool CBlockTreeDB::LoadBlockIndexGuts() { // Get database cursor Dbc* pcursor = GetCursor(); if (!pcursor) return false; // Load mapBlockIndex unsigned int fFlags = DB_SET_RANGE; loop { // Read next record CDataStream ssKey(SER_DISK, CLIENT_VERSION); if (fFlags == DB_SET_RANGE) ssKey << make_pair('b', uint256(0)); CDataStream ssValue(SER_DISK, CLIENT_VERSION); int ret = ReadAtCursor(pcursor, ssKey, ssValue, fFlags); fFlags = DB_NEXT; if (ret == DB_NOTFOUND) break; else if (ret != 0) return false; // Unserialize try { char chType; ssKey >> chType; if (chType == 'b' && !fRequestShutdown) { CDiskBlockIndex diskindex; ssValue >> diskindex; // Construct block index object CBlockIndex* pindexNew = InsertBlockIndex(diskindex.GetBlockHash()); pindexNew->pprev = InsertBlockIndex(diskindex.hashPrev); pindexNew->nHeight = diskindex.nHeight; pindexNew->nFile = diskindex.nFile; pindexNew->nDataPos = diskindex.nDataPos; pindexNew->nUndoPos = diskindex.nUndoPos; pindexNew->nVersion = diskindex.nVersion; pindexNew->hashMerkleRoot = diskindex.hashMerkleRoot; pindexNew->nTime = diskindex.nTime; pindexNew->nBits = diskindex.nBits; pindexNew->nNonce = diskindex.nNonce; pindexNew->nStatus = diskindex.nStatus; pindexNew->nTx = diskindex.nTx; // Watch for genesis block if (pindexGenesisBlock == NULL && diskindex.GetBlockHash() == hashGenesisBlock) pindexGenesisBlock = pindexNew; if (!pindexNew->CheckIndex()) return error("LoadBlockIndex() : CheckIndex failed: %s", pindexNew->ToString().c_str()); } else { break; // if shutdown requested or finished loading block index } } // try catch (std::exception &e) { return error("%s() : deserialize error", __PRETTY_FUNCTION__); } } pcursor->close(); return true; } // // CAddrDB // CAddrDB::CAddrDB() { pathAddr = GetDataDir() / "peers.dat"; } bool CAddrDB::Write(const CAddrMan& addr) { // Generate random temporary filename unsigned short randv = 0; RAND_bytes((unsigned char *)&randv, sizeof(randv)); std::string tmpfn = strprintf("peers.dat.%04x", randv); // serialize addresses, checksum data up to that point, then append csum CDataStream ssPeers(SER_DISK, CLIENT_VERSION); ssPeers << FLATDATA(pchMessageStart); ssPeers << addr; uint256 hash = Hash(ssPeers.begin(), ssPeers.end()); ssPeers << hash; // open temp output file, and associate with CAutoFile boost::filesystem::path pathTmp = GetDataDir() / tmpfn; FILE *file = fopen(pathTmp.string().c_str(), "wb"); CAutoFile fileout = CAutoFile(file, SER_DISK, CLIENT_VERSION); if (!fileout) return error("CAddrman::Write() : open failed"); // Write and commit header, data try { fileout << ssPeers; } catch (std::exception &e) { return error("CAddrman::Write() : I/O error"); } FileCommit(fileout); fileout.fclose(); // replace existing peers.dat, if any, with new peers.dat.XXXX if (!RenameOver(pathTmp, pathAddr)) return error("CAddrman::Write() : Rename-into-place failed"); return true; } bool CAddrDB::Read(CAddrMan& addr) { // open input file, and associate with CAutoFile FILE *file = fopen(pathAddr.string().c_str(), "rb"); CAutoFile filein = CAutoFile(file, SER_DISK, CLIENT_VERSION); if (!filein) return error("CAddrman::Read() : open failed"); // use file size to size memory buffer int fileSize = GetFilesize(filein); int dataSize = fileSize - sizeof(uint256); vector vchData; vchData.resize(dataSize); uint256 hashIn; // read data and checksum from file try { filein.read((char *)&vchData[0], dataSize); filein >> hashIn; } catch (std::exception &e) { return error("CAddrman::Read() 2 : I/O error or stream data corrupted"); } filein.fclose(); CDataStream ssPeers(vchData, SER_DISK, CLIENT_VERSION); // verify stored checksum matches input data uint256 hashTmp = Hash(ssPeers.begin(), ssPeers.end()); if (hashIn != hashTmp) return error("CAddrman::Read() : checksum mismatch; data corrupted"); // de-serialize address data unsigned char pchMsgTmp[4]; try { ssPeers >> FLATDATA(pchMsgTmp); ssPeers >> addr; } catch (std::exception &e) { return error("CAddrman::Read() : I/O error or stream data corrupted"); } // finally, verify the network matches ours if (memcmp(pchMsgTmp, pchMessageStart, sizeof(pchMsgTmp))) return error("CAddrman::Read() : invalid network magic number"); return true; }