diff options
| -rw-r--r-- | src/index/base.cpp | 20 | ||||
| -rw-r--r-- | src/index/base.h | 18 | ||||
| -rw-r--r-- | src/index/txindex.cpp | 219 | ||||
| -rw-r--r-- | src/index/txindex.h | 12 | ||||
| -rw-r--r-- | src/txdb.cpp | 176 | ||||
| -rw-r--r-- | src/txdb.h | 64 |
6 files changed, 264 insertions, 245 deletions
diff --git a/src/index/base.cpp b/src/index/base.cpp index f381681a64..738166dc94 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -11,6 +11,8 @@ #include <validation.h> #include <warnings.h> +constexpr char DB_BEST_BLOCK = 'B'; + constexpr int64_t SYNC_LOG_INTERVAL = 30; // seconds constexpr int64_t SYNC_LOCATOR_WRITE_INTERVAL = 30; // seconds @@ -26,6 +28,24 @@ static void FatalError(const char* fmt, const Args&... args) StartShutdown(); } +BaseIndex::DB::DB(const fs::path& path, size_t n_cache_size, bool f_memory, bool f_wipe, bool f_obfuscate) : + CDBWrapper(path, n_cache_size, f_memory, f_wipe, f_obfuscate) +{} + +bool BaseIndex::DB::ReadBestBlock(CBlockLocator& locator) const +{ + bool success = Read(DB_BEST_BLOCK, locator); + if (!success) { + locator.SetNull(); + } + return success; +} + +bool BaseIndex::DB::WriteBestBlock(const CBlockLocator& locator) +{ + return Write(DB_BEST_BLOCK, locator); +} + BaseIndex::~BaseIndex() { Interrupt(); diff --git a/src/index/base.h b/src/index/base.h index 68efaaaf26..04ee6e6cc2 100644 --- a/src/index/base.h +++ b/src/index/base.h @@ -5,10 +5,10 @@ #ifndef BITCOIN_INDEX_BASE_H #define BITCOIN_INDEX_BASE_H +#include <dbwrapper.h> #include <primitives/block.h> #include <primitives/transaction.h> #include <threadinterrupt.h> -#include <txdb.h> #include <uint256.h> #include <validationinterface.h> @@ -21,6 +21,20 @@ class CBlockIndex; */ class BaseIndex : public CValidationInterface { +protected: + class DB : public CDBWrapper + { + public: + DB(const fs::path& path, size_t n_cache_size, + bool f_memory = false, bool f_wipe = false, bool f_obfuscate = false); + + /// Read block locator of the chain that the txindex is in sync with. + bool ReadBestBlock(CBlockLocator& locator) const; + + /// Write block locator of the chain that the txindex is in sync with. + bool WriteBestBlock(const CBlockLocator& locator); + }; + private: /// Whether the index is in sync with the main chain. The flag is flipped /// from false to true once, after which point this starts processing @@ -55,7 +69,7 @@ protected: /// Write update index entries for a newly connected block. virtual bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) { return true; } - virtual BaseIndexDB& GetDB() const = 0; + virtual DB& GetDB() const = 0; /// Get the name of the index for display in logs. virtual const char* GetName() const = 0; diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index 328039977f..e106b9b420 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -3,15 +3,232 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <index/txindex.h> +#include <init.h> +#include <ui_interface.h> #include <util.h> #include <validation.h> +#include <boost/thread.hpp> + +constexpr char DB_BEST_BLOCK = 'B'; +constexpr char DB_TXINDEX = 't'; +constexpr char DB_TXINDEX_BLOCK = 'T'; + std::unique_ptr<TxIndex> g_txindex; +struct CDiskTxPos : public CDiskBlockPos +{ + unsigned int nTxOffset; // after header + + ADD_SERIALIZE_METHODS; + + template <typename Stream, typename Operation> + inline void SerializationOp(Stream& s, Operation ser_action) { + READWRITEAS(CDiskBlockPos, *this); + READWRITE(VARINT(nTxOffset)); + } + + CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { + } + + CDiskTxPos() { + SetNull(); + } + + void SetNull() { + CDiskBlockPos::SetNull(); + nTxOffset = 0; + } +}; + +/** + * Access to the txindex database (indexes/txindex/) + * + * The database stores a block locator of the chain the database is synced to + * so that the TxIndex can efficiently determine the point it last stopped at. + * A locator is used instead of a simple hash of the chain tip because blocks + * and block index entries may not be flushed to disk until after this database + * is updated. + */ +class TxIndex::DB : public BaseIndex::DB +{ +public: + explicit DB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + + /// Read the disk location of the transaction data with the given hash. Returns false if the + /// transaction hash is not indexed. + bool ReadTxPos(const uint256& txid, CDiskTxPos& pos) const; + + /// Write a batch of transaction positions to the DB. + bool WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos); + + /// Migrate txindex data from the block tree DB, where it may be for older nodes that have not + /// been upgraded yet to the new database. + bool MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator); +}; + +TxIndex::DB::DB(size_t n_cache_size, bool f_memory, bool f_wipe) : + BaseIndex::DB(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe) +{} + +bool TxIndex::DB::ReadTxPos(const uint256 &txid, CDiskTxPos& pos) const +{ + return Read(std::make_pair(DB_TXINDEX, txid), pos); +} + +bool TxIndex::DB::WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos) +{ + CDBBatch batch(*this); + for (const auto& tuple : v_pos) { + batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second); + } + return WriteBatch(batch); +} + +/* + * Safely persist a transfer of data from the old txindex database to the new one, and compact the + * range of keys updated. This is used internally by MigrateData. + */ +static void WriteTxIndexMigrationBatches(CDBWrapper& newdb, CDBWrapper& olddb, + CDBBatch& batch_newdb, CDBBatch& batch_olddb, + const std::pair<unsigned char, uint256>& begin_key, + const std::pair<unsigned char, uint256>& end_key) +{ + // Sync new DB changes to disk before deleting from old DB. + newdb.WriteBatch(batch_newdb, /*fSync=*/ true); + olddb.WriteBatch(batch_olddb); + olddb.CompactRange(begin_key, end_key); + + batch_newdb.Clear(); + batch_olddb.Clear(); +} + +bool TxIndex::DB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator) +{ + // The prior implementation of txindex was always in sync with block index + // and presence was indicated with a boolean DB flag. If the flag is set, + // this means the txindex from a previous version is valid and in sync with + // the chain tip. The first step of the migration is to unset the flag and + // write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the + // index entries are copied over in batches to the new database. Finally, + // DB_TXINDEX_BLOCK is erased from the old database and the block hash is + // written to the new database. + // + // Unsetting the boolean flag ensures that if the node is downgraded to a + // previous version, it will not see a corrupted, partially migrated index + // -- it will see that the txindex is disabled. When the node is upgraded + // again, the migration will pick up where it left off and sync to the block + // with hash DB_TXINDEX_BLOCK. + bool f_legacy_flag = false; + block_tree_db.ReadFlag("txindex", f_legacy_flag); + if (f_legacy_flag) { + if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) { + return error("%s: cannot write block indicator", __func__); + } + if (!block_tree_db.WriteFlag("txindex", false)) { + return error("%s: cannot write block index db flag", __func__); + } + } + + CBlockLocator locator; + if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) { + return true; + } + + int64_t count = 0; + LogPrintf("Upgrading txindex database... [0%%]\n"); + uiInterface.ShowProgress(_("Upgrading txindex database"), 0, true); + int report_done = 0; + const size_t batch_size = 1 << 24; // 16 MiB + + CDBBatch batch_newdb(*this); + CDBBatch batch_olddb(block_tree_db); + + std::pair<unsigned char, uint256> key; + std::pair<unsigned char, uint256> begin_key{DB_TXINDEX, uint256()}; + std::pair<unsigned char, uint256> prev_key = begin_key; + + bool interrupted = false; + std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator()); + for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) { + boost::this_thread::interruption_point(); + if (ShutdownRequested()) { + interrupted = true; + break; + } + + if (!cursor->GetKey(key)) { + return error("%s: cannot get key from valid cursor", __func__); + } + if (key.first != DB_TXINDEX) { + break; + } + + // Log progress every 10%. + if (++count % 256 == 0) { + // Since txids are uniformly random and traversed in increasing order, the high 16 bits + // of the hash can be used to estimate the current progress. + const uint256& txid = key.second; + uint32_t high_nibble = + (static_cast<uint32_t>(*(txid.begin() + 0)) << 8) + + (static_cast<uint32_t>(*(txid.begin() + 1)) << 0); + int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5); + + uiInterface.ShowProgress(_("Upgrading txindex database"), percentage_done, true); + if (report_done < percentage_done/10) { + LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done); + report_done = percentage_done/10; + } + } + + CDiskTxPos value; + if (!cursor->GetValue(value)) { + return error("%s: cannot parse txindex record", __func__); + } + batch_newdb.Write(key, value); + batch_olddb.Erase(key); + + if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) { + // NOTE: it's OK to delete the key pointed at by the current DB cursor while iterating + // because LevelDB iterators are guaranteed to provide a consistent view of the + // underlying data, like a lightweight snapshot. + WriteTxIndexMigrationBatches(*this, block_tree_db, + batch_newdb, batch_olddb, + prev_key, key); + prev_key = key; + } + } + + // If these final DB batches complete the migration, write the best block + // hash marker to the new database and delete from the old one. This signals + // that the former is fully caught up to that point in the blockchain and + // that all txindex entries have been removed from the latter. + if (!interrupted) { + batch_olddb.Erase(DB_TXINDEX_BLOCK); + batch_newdb.Write(DB_BEST_BLOCK, locator); + } + + WriteTxIndexMigrationBatches(*this, block_tree_db, + batch_newdb, batch_olddb, + begin_key, key); + + if (interrupted) { + LogPrintf("[CANCELLED].\n"); + return false; + } + + uiInterface.ShowProgress("", 100, false); + + LogPrintf("[DONE].\n"); + return true; +} + TxIndex::TxIndex(size_t n_cache_size, bool f_memory, bool f_wipe) : m_db(MakeUnique<TxIndex::DB>(n_cache_size, f_memory, f_wipe)) {} +TxIndex::~TxIndex() {} + bool TxIndex::Init() { LOCK(cs_main); @@ -38,7 +255,7 @@ bool TxIndex::WriteBlock(const CBlock& block, const CBlockIndex* pindex) return m_db->WriteTxs(vPos); } -BaseIndexDB& TxIndex::GetDB() const { return *m_db; } +BaseIndex::DB& TxIndex::GetDB() const { return *m_db; } bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRef& tx) const { diff --git a/src/index/txindex.h b/src/index/txindex.h index 2a0c70e9d1..8202c3c951 100644 --- a/src/index/txindex.h +++ b/src/index/txindex.h @@ -5,7 +5,9 @@ #ifndef BITCOIN_INDEX_TXINDEX_H #define BITCOIN_INDEX_TXINDEX_H +#include <chain.h> #include <index/base.h> +#include <txdb.h> /** * TxIndex is used to look up transactions included in the blockchain by hash. @@ -14,8 +16,11 @@ */ class TxIndex final : public BaseIndex { +protected: + class DB; + private: - const std::unique_ptr<TxIndexDB> m_db; + const std::unique_ptr<DB> m_db; protected: /// Override base class init to migrate from old database. @@ -23,7 +28,7 @@ protected: bool WriteBlock(const CBlock& block, const CBlockIndex* pindex) override; - BaseIndexDB& GetDB() const override; + BaseIndex::DB& GetDB() const override; const char* GetName() const override { return "txindex"; } @@ -31,6 +36,9 @@ public: /// Constructs the index, which becomes available to be queried. explicit TxIndex(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); + // Destructor is declared because this class contains a unique_ptr to an incomplete type. + virtual ~TxIndex() override; + /// Look up a transaction by hash. /// /// @param[in] tx_hash The hash of the transaction to be returned. diff --git a/src/txdb.cpp b/src/txdb.cpp index 624b23962a..b1d5879c83 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -21,8 +21,6 @@ static const char DB_COIN = 'C'; static const char DB_COINS = 'c'; static const char DB_BLOCK_FILES = 'f'; -static const char DB_TXINDEX = 't'; -static const char DB_TXINDEX_BLOCK = 'T'; static const char DB_BLOCK_INDEX = 'b'; static const char DB_BEST_BLOCK = 'B'; @@ -414,177 +412,3 @@ bool CCoinsViewDB::Upgrade() { LogPrintf("[%s].\n", ShutdownRequested() ? "CANCELLED" : "DONE"); return !ShutdownRequested(); } - -BaseIndexDB::BaseIndexDB(const fs::path& path, size_t n_cache_size, bool f_memory, bool f_wipe, bool f_obfuscate) : - CDBWrapper(path, n_cache_size, f_memory, f_wipe, f_obfuscate) -{} - -TxIndexDB::TxIndexDB(size_t n_cache_size, bool f_memory, bool f_wipe) : - BaseIndexDB(GetDataDir() / "indexes" / "txindex", n_cache_size, f_memory, f_wipe) -{} - -bool TxIndexDB::ReadTxPos(const uint256 &txid, CDiskTxPos& pos) const -{ - return Read(std::make_pair(DB_TXINDEX, txid), pos); -} - -bool TxIndexDB::WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos) -{ - CDBBatch batch(*this); - for (const auto& tuple : v_pos) { - batch.Write(std::make_pair(DB_TXINDEX, tuple.first), tuple.second); - } - return WriteBatch(batch); -} - -bool BaseIndexDB::ReadBestBlock(CBlockLocator& locator) const -{ - bool success = Read(DB_BEST_BLOCK, locator); - if (!success) { - locator.SetNull(); - } - return success; -} - -bool BaseIndexDB::WriteBestBlock(const CBlockLocator& locator) -{ - return Write(DB_BEST_BLOCK, locator); -} - -/* - * Safely persist a transfer of data from the old txindex database to the new one, and compact the - * range of keys updated. This is used internally by MigrateData. - */ -static void WriteTxIndexMigrationBatches(TxIndexDB& newdb, CBlockTreeDB& olddb, - CDBBatch& batch_newdb, CDBBatch& batch_olddb, - const std::pair<unsigned char, uint256>& begin_key, - const std::pair<unsigned char, uint256>& end_key) -{ - // Sync new DB changes to disk before deleting from old DB. - newdb.WriteBatch(batch_newdb, /*fSync=*/ true); - olddb.WriteBatch(batch_olddb); - olddb.CompactRange(begin_key, end_key); - - batch_newdb.Clear(); - batch_olddb.Clear(); -} - -bool TxIndexDB::MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator) -{ - // The prior implementation of txindex was always in sync with block index - // and presence was indicated with a boolean DB flag. If the flag is set, - // this means the txindex from a previous version is valid and in sync with - // the chain tip. The first step of the migration is to unset the flag and - // write the chain hash to a separate key, DB_TXINDEX_BLOCK. After that, the - // index entries are copied over in batches to the new database. Finally, - // DB_TXINDEX_BLOCK is erased from the old database and the block hash is - // written to the new database. - // - // Unsetting the boolean flag ensures that if the node is downgraded to a - // previous version, it will not see a corrupted, partially migrated index - // -- it will see that the txindex is disabled. When the node is upgraded - // again, the migration will pick up where it left off and sync to the block - // with hash DB_TXINDEX_BLOCK. - bool f_legacy_flag = false; - block_tree_db.ReadFlag("txindex", f_legacy_flag); - if (f_legacy_flag) { - if (!block_tree_db.Write(DB_TXINDEX_BLOCK, best_locator)) { - return error("%s: cannot write block indicator", __func__); - } - if (!block_tree_db.WriteFlag("txindex", false)) { - return error("%s: cannot write block index db flag", __func__); - } - } - - CBlockLocator locator; - if (!block_tree_db.Read(DB_TXINDEX_BLOCK, locator)) { - return true; - } - - int64_t count = 0; - LogPrintf("Upgrading txindex database... [0%%]\n"); - uiInterface.ShowProgress(_("Upgrading txindex database"), 0, true); - int report_done = 0; - const size_t batch_size = 1 << 24; // 16 MiB - - CDBBatch batch_newdb(*this); - CDBBatch batch_olddb(block_tree_db); - - std::pair<unsigned char, uint256> key; - std::pair<unsigned char, uint256> begin_key{DB_TXINDEX, uint256()}; - std::pair<unsigned char, uint256> prev_key = begin_key; - - bool interrupted = false; - std::unique_ptr<CDBIterator> cursor(block_tree_db.NewIterator()); - for (cursor->Seek(begin_key); cursor->Valid(); cursor->Next()) { - boost::this_thread::interruption_point(); - if (ShutdownRequested()) { - interrupted = true; - break; - } - - if (!cursor->GetKey(key)) { - return error("%s: cannot get key from valid cursor", __func__); - } - if (key.first != DB_TXINDEX) { - break; - } - - // Log progress every 10%. - if (++count % 256 == 0) { - // Since txids are uniformly random and traversed in increasing order, the high 16 bits - // of the hash can be used to estimate the current progress. - const uint256& txid = key.second; - uint32_t high_nibble = - (static_cast<uint32_t>(*(txid.begin() + 0)) << 8) + - (static_cast<uint32_t>(*(txid.begin() + 1)) << 0); - int percentage_done = (int)(high_nibble * 100.0 / 65536.0 + 0.5); - - uiInterface.ShowProgress(_("Upgrading txindex database"), percentage_done, true); - if (report_done < percentage_done/10) { - LogPrintf("Upgrading txindex database... [%d%%]\n", percentage_done); - report_done = percentage_done/10; - } - } - - CDiskTxPos value; - if (!cursor->GetValue(value)) { - return error("%s: cannot parse txindex record", __func__); - } - batch_newdb.Write(key, value); - batch_olddb.Erase(key); - - if (batch_newdb.SizeEstimate() > batch_size || batch_olddb.SizeEstimate() > batch_size) { - // NOTE: it's OK to delete the key pointed at by the current DB cursor while iterating - // because LevelDB iterators are guaranteed to provide a consistent view of the - // underlying data, like a lightweight snapshot. - WriteTxIndexMigrationBatches(*this, block_tree_db, - batch_newdb, batch_olddb, - prev_key, key); - prev_key = key; - } - } - - // If these final DB batches complete the migration, write the best block - // hash marker to the new database and delete from the old one. This signals - // that the former is fully caught up to that point in the blockchain and - // that all txindex entries have been removed from the latter. - if (!interrupted) { - batch_olddb.Erase(DB_TXINDEX_BLOCK); - batch_newdb.Write(DB_BEST_BLOCK, locator); - } - - WriteTxIndexMigrationBatches(*this, block_tree_db, - batch_newdb, batch_olddb, - begin_key, key); - - if (interrupted) { - LogPrintf("[CANCELLED].\n"); - return false; - } - - uiInterface.ShowProgress("", 100, false); - - LogPrintf("[DONE].\n"); - return true; -} diff --git a/src/txdb.h b/src/txdb.h index f9d9e4246c..100adb428d 100644 --- a/src/txdb.h +++ b/src/txdb.h @@ -40,31 +40,6 @@ static const int64_t nMaxTxIndexCache = 1024; //! Max memory allocated to coin DB specific cache (MiB) static const int64_t nMaxCoinsDBCache = 8; -struct CDiskTxPos : public CDiskBlockPos -{ - unsigned int nTxOffset; // after header - - ADD_SERIALIZE_METHODS; - - template <typename Stream, typename Operation> - inline void SerializationOp(Stream& s, Operation ser_action) { - READWRITEAS(CDiskBlockPos, *this); - READWRITE(VARINT(nTxOffset)); - } - - CDiskTxPos(const CDiskBlockPos &blockIn, unsigned int nTxOffsetIn) : CDiskBlockPos(blockIn.nFile, blockIn.nPos), nTxOffset(nTxOffsetIn) { - } - - CDiskTxPos() { - SetNull(); - } - - void SetNull() { - CDiskBlockPos::SetNull(); - nTxOffset = 0; - } -}; - /** CCoinsView backed by the coin database (chainstate/) */ class CCoinsViewDB final : public CCoinsView { @@ -123,43 +98,4 @@ public: bool LoadBlockIndexGuts(const Consensus::Params& consensusParams, std::function<CBlockIndex*(const uint256&)> insertBlockIndex); }; -class BaseIndexDB : public CDBWrapper -{ -public: - BaseIndexDB(const fs::path& path, size_t n_cache_size, - bool f_memory = false, bool f_wipe = false, bool f_obfuscate = false); - - /// Read block locator of the chain that the index is in sync with. - bool ReadBestBlock(CBlockLocator& locator) const; - - /// Write block locator of the chain that the index is in sync with. - bool WriteBestBlock(const CBlockLocator& locator); -}; - -/** - * Access to the txindex database (indexes/txindex/) - * - * The database stores a block locator of the chain the database is synced to - * so that the TxIndex can efficiently determine the point it last stopped at. - * A locator is used instead of a simple hash of the chain tip because blocks - * and block index entries may not be flushed to disk until after this database - * is updated. - */ -class TxIndexDB : public BaseIndexDB -{ -public: - explicit TxIndexDB(size_t n_cache_size, bool f_memory = false, bool f_wipe = false); - - /// Read the disk location of the transaction data with the given hash. Returns false if the - /// transaction hash is not indexed. - bool ReadTxPos(const uint256& txid, CDiskTxPos& pos) const; - - /// Write a batch of transaction positions to the DB. - bool WriteTxs(const std::vector<std::pair<uint256, CDiskTxPos>>& v_pos); - - /// Migrate txindex data from the block tree DB, where it may be for older nodes that have not - /// been upgraded yet to the new database. - bool MigrateData(CBlockTreeDB& block_tree_db, const CBlockLocator& best_locator); -}; - #endif // BITCOIN_TXDB_H |