diff options
Diffstat (limited to 'src')
-rw-r--r-- | src/chain.h | 13 | ||||
-rw-r--r-- | src/chainparams.cpp | 26 | ||||
-rw-r--r-- | src/chainparams.h | 26 | ||||
-rw-r--r-- | src/coins.cpp | 8 | ||||
-rw-r--r-- | src/coins.h | 12 | ||||
-rw-r--r-- | src/logging.cpp | 6 | ||||
-rw-r--r-- | src/net.cpp | 42 | ||||
-rw-r--r-- | src/net.h | 29 | ||||
-rw-r--r-- | src/net_processing.cpp | 163 | ||||
-rw-r--r-- | src/net_processing.h | 4 | ||||
-rw-r--r-- | src/node/coinstats.cpp | 12 | ||||
-rw-r--r-- | src/qt/rpcconsole.cpp | 2 | ||||
-rw-r--r-- | src/rpc/blockchain.cpp | 24 | ||||
-rw-r--r-- | src/rpc/blockchain.h | 8 | ||||
-rw-r--r-- | src/rpc/net.cpp | 11 | ||||
-rw-r--r-- | src/test/fuzz/crypto.cpp | 13 | ||||
-rw-r--r-- | src/test/fuzz/muhash.cpp | 10 | ||||
-rw-r--r-- | src/test/miner_tests.cpp | 3 | ||||
-rw-r--r-- | src/test/net_tests.cpp | 6 | ||||
-rw-r--r-- | src/test/sock_tests.cpp | 2 | ||||
-rw-r--r-- | src/test/util/setup_common.cpp | 87 | ||||
-rw-r--r-- | src/test/util/setup_common.h | 29 | ||||
-rw-r--r-- | src/test/validation_chainstatemanager_tests.cpp | 187 | ||||
-rw-r--r-- | src/test/validation_tests.cpp | 24 | ||||
-rw-r--r-- | src/txdb.cpp | 14 | ||||
-rw-r--r-- | src/util/time.cpp | 9 | ||||
-rw-r--r-- | src/util/time.h | 13 | ||||
-rw-r--r-- | src/validation.cpp | 293 | ||||
-rw-r--r-- | src/validation.h | 46 |
29 files changed, 958 insertions, 164 deletions
diff --git a/src/chain.h b/src/chain.h index 43e8a39f36..04a5db5a17 100644 --- a/src/chain.h +++ b/src/chain.h @@ -163,14 +163,27 @@ public: //! Number of transactions in this block. //! Note: in a potential headers-first mode, this number cannot be relied upon + //! Note: this value is faked during UTXO snapshot load to ensure that + //! LoadBlockIndex() will load index entries for blocks that we lack data for. + //! @sa ActivateSnapshot unsigned int nTx{0}; //! (memory only) Number of transactions in the chain up to and including this block. //! This value will be non-zero only if and only if transactions for this block and all its parents are available. //! Change to 64-bit type when necessary; won't happen before 2030 + //! + //! Note: this value is faked during use of a UTXO snapshot because we don't + //! have the underlying block data available during snapshot load. + //! @sa AssumeutxoData + //! @sa ActivateSnapshot unsigned int nChainTx{0}; //! Verification status of this block. See enum BlockStatus + //! + //! Note: this value is modified to show BLOCK_OPT_WITNESS during UTXO snapshot + //! load to avoid the block index being spuriously rewound. + //! @sa RewindBlockIndex + //! @sa ActivateSnapshot uint32_t nStatus{0}; //! block header diff --git a/src/chainparams.cpp b/src/chainparams.cpp index 88cf5ef0a8..97280c0d16 100644 --- a/src/chainparams.cpp +++ b/src/chainparams.cpp @@ -8,7 +8,6 @@ #include <chainparamsseeds.h> #include <consensus/merkle.h> #include <hash.h> // for signet block challenge hash -#include <tinyformat.h> #include <util/system.h> #include <util/strencodings.h> #include <versionbitsinfo.h> @@ -161,6 +160,10 @@ public: } }; + m_assumeutxo_data = MapAssumeutxo{ + // TODO to be specified in a future patch. + }; + chainTxData = ChainTxData{ // Data from RPC: getchaintxstats 4096 0000000000000000000b9d2ec5a352ecba0592946514a92f14319dc2b367fc72 /* nTime */ 1603995752, @@ -250,6 +253,10 @@ public: } }; + m_assumeutxo_data = MapAssumeutxo{ + // TODO to be specified in a future patch. + }; + chainTxData = ChainTxData{ // Data from RPC: getchaintxstats 4096 000000000000006433d1efec504c53ca332b64963c425395515b01977bd7b3b0 /* nTime */ 1603359686, @@ -431,6 +438,17 @@ public: } }; + m_assumeutxo_data = MapAssumeutxo{ + { + 110, + {uint256S("0x76fd7334ac7c1baf57ddc0c626f073a655a35d98a4258cd1382c8cc2b8392e10"), 110}, + }, + { + 210, + {uint256S("0x9c5ed99ef98544b34f8920b6d1802f72ac28ae6e2bd2bd4c316ff10c230df3f2"), 210}, + }, + }; + chainTxData = ChainTxData{ 0, 0, @@ -526,3 +544,9 @@ void SelectParams(const std::string& network) SelectBaseParams(network); globalChainParams = CreateChainParams(gArgs, network); } + +std::ostream& operator<<(std::ostream& o, const AssumeutxoData& aud) +{ + o << strprintf("AssumeutxoData(%s, %s)", aud.hash_serialized.ToString(), aud.nChainTx); + return o; +} diff --git a/src/chainparams.h b/src/chainparams.h index d8b25c7220..4d24dcdb7c 100644 --- a/src/chainparams.h +++ b/src/chainparams.h @@ -31,6 +31,26 @@ struct CCheckpointData { }; /** + * Holds configuration for use during UTXO snapshot load and validation. The contents + * here are security critical, since they dictate which UTXO snapshots are recognized + * as valid. + */ +struct AssumeutxoData { + //! The expected hash of the deserialized UTXO set. + const uint256 hash_serialized; + + //! Used to populate the nChainTx value, which is used during BlockManager::LoadBlockIndex(). + //! + //! We need to hardcode the value here because this is computed cumulatively using block data, + //! which we do not necessarily have at the time of snapshot load. + const unsigned int nChainTx; +}; + +std::ostream& operator<<(std::ostream& o, const AssumeutxoData& aud); + +using MapAssumeutxo = std::map<int, const AssumeutxoData>; + +/** * Holds various statistics on transactions within a chain. Used to estimate * verification progress during chain sync. * @@ -90,6 +110,11 @@ public: const std::string& Bech32HRP() const { return bech32_hrp; } const std::vector<SeedSpec6>& FixedSeeds() const { return vFixedSeeds; } const CCheckpointData& Checkpoints() const { return checkpointData; } + + //! Get allowed assumeutxo configuration. + //! @see ChainstateManager + const MapAssumeutxo& Assumeutxo() const { return m_assumeutxo_data; } + const ChainTxData& TxData() const { return chainTxData; } protected: CChainParams() {} @@ -111,6 +136,7 @@ protected: bool m_is_test_chain; bool m_is_mockable_chain; CCheckpointData checkpointData; + MapAssumeutxo m_assumeutxo_data; ChainTxData chainTxData; }; diff --git a/src/coins.cpp b/src/coins.cpp index dd84e720e7..d52851cadd 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -97,6 +97,14 @@ void CCoinsViewCache::AddCoin(const COutPoint &outpoint, Coin&& coin, bool possi cachedCoinsUsage += it->second.coin.DynamicMemoryUsage(); } +void CCoinsViewCache::EmplaceCoinInternalDANGER(COutPoint&& outpoint, Coin&& coin) { + cachedCoinsUsage += coin.DynamicMemoryUsage(); + cacheCoins.emplace( + std::piecewise_construct, + std::forward_as_tuple(std::move(outpoint)), + std::forward_as_tuple(std::move(coin), CCoinsCacheEntry::DIRTY)); +} + void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight, bool check_for_overwrite) { bool fCoinbase = tx.IsCoinBase(); const uint256& txid = tx.GetHash(); diff --git a/src/coins.h b/src/coins.h index d2eb42d8cf..feb441fd6a 100644 --- a/src/coins.h +++ b/src/coins.h @@ -20,6 +20,8 @@ #include <functional> #include <unordered_map> +class ChainstateManager; + /** * A UTXO entry. * @@ -125,6 +127,7 @@ struct CCoinsCacheEntry CCoinsCacheEntry() : flags(0) {} explicit CCoinsCacheEntry(Coin&& coin_) : coin(std::move(coin_)), flags(0) {} + CCoinsCacheEntry(Coin&& coin_, unsigned char flag) : coin(std::move(coin_)), flags(flag) {} }; typedef std::unordered_map<COutPoint, CCoinsCacheEntry, SaltedOutpointHasher> CCoinsMap; @@ -263,6 +266,15 @@ public: void AddCoin(const COutPoint& outpoint, Coin&& coin, bool possible_overwrite); /** + * Emplace a coin into cacheCoins without performing any checks, marking + * the emplaced coin as dirty. + * + * NOT FOR GENERAL USE. Used only when loading coins from a UTXO snapshot. + * @sa ChainstateManager::PopulateAndValidateSnapshot() + */ + void EmplaceCoinInternalDANGER(COutPoint&& outpoint, Coin&& coin); + + /** * Spend a coin. Pass moveto in order to get the deleted data. * If no unspent output exists for the passed outpoint, this call * has no effect. diff --git a/src/logging.cpp b/src/logging.cpp index 4ddcf1d930..529bb11d5b 100644 --- a/src/logging.cpp +++ b/src/logging.cpp @@ -203,9 +203,9 @@ std::string BCLog::Logger::LogTimestampStr(const std::string& str) strStamped.pop_back(); strStamped += strprintf(".%06dZ", nTimeMicros%1000000); } - int64_t mocktime = GetMockTime(); - if (mocktime) { - strStamped += " (mocktime: " + FormatISO8601DateTime(mocktime) + ")"; + std::chrono::seconds mocktime = GetMockTime(); + if (mocktime > 0s) { + strStamped += " (mocktime: " + FormatISO8601DateTime(count_seconds(mocktime)) + ")"; } strStamped += ' ' + str; } else diff --git a/src/net.cpp b/src/net.cpp index 68388da094..81176785a2 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -601,21 +601,8 @@ void CNode::copyStats(CNodeStats &stats, const std::vector<bool> &m_asmap) stats.minFeeFilter = 0; } - // It is common for nodes with good ping times to suddenly become lagged, - // due to a new block arriving or other large transfer. - // Merely reporting pingtime might fool the caller into thinking the node was still responsive, - // since pingtime does not update until the ping is complete, which might take a while. - // So, if a ping is taking an unusually long time in flight, - // the caller can immediately detect that this is happening. - std::chrono::microseconds ping_wait{0}; - if ((0 != nPingNonceSent) && (0 != m_ping_start.load().count())) { - ping_wait = GetTime<std::chrono::microseconds>() - m_ping_start.load(); - } - - // Raw ping time is in microseconds, but show it to user as whole seconds (Bitcoin users should be well used to small numbers with many decimal places by now :) - stats.m_ping_usec = nPingUsecTime; - stats.m_min_ping_usec = nMinPingUsecTime; - stats.m_ping_wait_usec = count_microseconds(ping_wait); + stats.m_ping_usec = m_last_ping_time; + stats.m_min_ping_usec = m_min_ping_time; // Leave string empty if addrLocal invalid (not filled in yet) CService addrLocalUnlocked = GetAddrLocal(); @@ -837,7 +824,7 @@ size_t CConnman::SocketSendData(CNode& node) const static bool ReverseCompareNodeMinPingTime(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) { - return a.nMinPingUsecTime > b.nMinPingUsecTime; + return a.m_min_ping_time > b.m_min_ping_time; } static bool ReverseCompareNodeTimeConnected(const NodeEvictionCandidate &a, const NodeEvictionCandidate &b) @@ -992,7 +979,7 @@ bool CConnman::AttemptToEvictConnection() peer_relay_txes = node->m_tx_relay->fRelayTxes; peer_filter_not_null = node->m_tx_relay->pfilter != nullptr; } - NodeEvictionCandidate candidate = {node->GetId(), node->nTimeConnected, node->nMinPingUsecTime, + NodeEvictionCandidate candidate = {node->GetId(), node->nTimeConnected, node->m_min_ping_time, node->nLastBlockTime, node->nLastTXTime, HasAllDesirableServiceFlags(node->nServices), peer_relay_txes, peer_filter_not_null, node->nKeyedNetGroup, @@ -1221,18 +1208,17 @@ void CConnman::NotifyNumConnectionsChanged() } } +bool CConnman::RunInactivityChecks(const CNode& node) const +{ + return GetSystemTimeInSeconds() > node.nTimeConnected + m_peer_connect_timeout; +} + bool CConnman::InactivityCheck(const CNode& node) const { // Use non-mockable system time (otherwise these timers will pop when we // use setmocktime in the tests). int64_t now = GetSystemTimeInSeconds(); - if (now <= node.nTimeConnected + m_peer_connect_timeout) { - // Only run inactivity checks if the peer has been connected longer - // than m_peer_connect_timeout. - return false; - } - if (node.nLastRecv == 0 || node.nLastSend == 0) { LogPrint(BCLog::NET, "socket no message in first %i seconds, %d %d peer=%d\n", m_peer_connect_timeout, node.nLastRecv != 0, node.nLastSend != 0, node.GetId()); return true; @@ -1248,14 +1234,6 @@ bool CConnman::InactivityCheck(const CNode& node) const return true; } - if (node.nPingNonceSent && node.m_ping_start.load() + std::chrono::seconds{TIMEOUT_INTERVAL} < GetTime<std::chrono::microseconds>()) { - // We use mockable time for ping timeouts. This means that setmocktime - // may cause pings to time out for peers that have been connected for - // longer than m_peer_connect_timeout. - LogPrint(BCLog::NET, "ping timeout: %fs peer=%d\n", 0.000001 * count_microseconds(GetTime<std::chrono::microseconds>() - node.m_ping_start.load()), node.GetId()); - return true; - } - if (!node.fSuccessfullyConnected) { LogPrint(BCLog::NET, "version handshake timeout peer=%d\n", node.GetId()); return true; @@ -1537,7 +1515,7 @@ void CConnman::SocketHandler() if (bytes_sent) RecordBytesSent(bytes_sent); } - if (InactivityCheck(*pnode)) pnode->fDisconnect = true; + if (RunInactivityChecks(*pnode) && InactivityCheck(*pnode)) pnode->fDisconnect = true; } { LOCK(cs_vNodes); @@ -260,7 +260,6 @@ public: mapMsgCmdSize mapRecvBytesPerMsgCmd; NetPermissionFlags m_permissionFlags; int64_t m_ping_usec; - int64_t m_ping_wait_usec; int64_t m_min_ping_usec; CAmount minFeeFilter; // Our address, as reported by the peer @@ -591,17 +590,12 @@ public: * in CConnman::AttemptToEvictConnection. */ std::atomic<int64_t> nLastTXTime{0}; - // Ping time measurement: - // The pong reply we're expecting, or 0 if no pong expected. - std::atomic<uint64_t> nPingNonceSent{0}; - /** When the last ping was sent, or 0 if no ping was ever sent */ - std::atomic<std::chrono::microseconds> m_ping_start{0us}; - // Last measured round-trip time. - std::atomic<int64_t> nPingUsecTime{0}; - // Best measured round-trip time. - std::atomic<int64_t> nMinPingUsecTime{std::numeric_limits<int64_t>::max()}; - // Whether a ping is requested. - std::atomic<bool> fPingQueued{false}; + /** Last measured round-trip time. Used only for RPC/GUI stats/debugging.*/ + std::atomic<int64_t> m_last_ping_time{0}; + + /** Lowest measured round-trip time. Used as an inbound peer eviction + * criterium in CConnman::AttemptToEvictConnection. */ + std::atomic<int64_t> m_min_ping_time{std::numeric_limits<int64_t>::max()}; CNode(NodeId id, ServiceFlags nLocalServicesIn, SOCKET hSocketIn, const CAddress& addrIn, uint64_t nKeyedNetGroupIn, uint64_t nLocalHostNonceIn, const CAddress& addrBindIn, const std::string& addrNameIn, ConnectionType conn_type_in, bool inbound_onion); ~CNode(); @@ -721,6 +715,12 @@ public: std::string ConnectionTypeAsString() const { return ::ConnectionTypeAsString(m_conn_type); } + /** A ping-pong round trip has completed successfully. Update latest and minimum ping times. */ + void PongReceived(std::chrono::microseconds ping_time) { + m_last_ping_time = count_microseconds(ping_time); + m_min_ping_time = std::min(m_min_ping_time.load(), count_microseconds(ping_time)); + } + private: const NodeId id; const uint64_t nLocalHostNonce; @@ -1022,6 +1022,9 @@ public: void SetAsmap(std::vector<bool> asmap) { addrman.m_asmap = std::move(asmap); } + /** Return true if the peer has been connected for long enough to do inactivity checks. */ + bool RunInactivityChecks(const CNode& node) const; + private: struct ListenSocket { public: @@ -1250,7 +1253,7 @@ struct NodeEvictionCandidate { NodeId id; int64_t nTimeConnected; - int64_t nMinPingUsecTime; + int64_t m_min_ping_time; int64_t nLastBlockTime; int64_t nLastTXTime; bool fRelevantServices; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index c782498e14..ae45d3d807 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -219,6 +219,13 @@ struct Peer { /** This peer's reported block height when we connected */ std::atomic<int> m_starting_height{-1}; + /** The pong reply we're expecting, or 0 if no pong expected. */ + std::atomic<uint64_t> m_ping_nonce_sent{0}; + /** When the last ping was sent, or 0 if no ping was ever sent */ + std::atomic<std::chrono::microseconds> m_ping_start{0us}; + /** Whether a ping has been requested by the user */ + std::atomic<bool> m_ping_queued{false}; + /** Set of txids to reconsider once their parent transactions have been accepted **/ std::set<uint256> m_orphan_work_set GUARDED_BY(g_cs_orphans); @@ -256,6 +263,7 @@ public: void CheckForStaleTipAndEvictPeers() override; bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) override; bool IgnoresIncomingTxs() override { return m_ignore_incoming_txs; } + void SendPings() override; void SetBestHeight(int height) override { m_best_height = height; }; void Misbehaving(const NodeId pnode, const int howmuch, const std::string& message) override; void ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, @@ -302,9 +310,10 @@ private: /** Maybe disconnect a peer and discourage future connections from its address. * * @param[in] pnode The node to check. + * @param[in] peer The peer object to check. * @return True if the peer was marked for disconnection in this function */ - bool MaybeDiscourageAndDisconnect(CNode& pnode); + bool MaybeDiscourageAndDisconnect(CNode& pnode, Peer& peer); void ProcessOrphanTx(std::set<uint256>& orphan_work_set) EXCLUSIVE_LOCKS_REQUIRED(cs_main, g_cs_orphans); /** Process a single headers message from a peer. */ @@ -323,6 +332,10 @@ private: /** Send a version message to a peer */ void PushNodeVersion(CNode& pnode, int64_t nTime); + /** Send a ping message every PING_INTERVAL or if requested via RPC. May + * mark the peer to be disconnected if a ping has timed out. */ + void MaybeSendPing(CNode& node_to, Peer& peer); + const CChainParams& m_chainparams; CConnman& m_connman; /** Pointer to this node's banman. May be nullptr - check existence before dereferencing. */ @@ -646,7 +659,7 @@ static void UpdatePreferredDownload(const CNode& node, CNodeState* state) EXCLUS nPreferredDownload += state->fPreferredDownload; } -bool PeerManagerImpl::MarkBlockAsReceived(const uint256& hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +bool PeerManagerImpl::MarkBlockAsReceived(const uint256& hash) { std::map<uint256, std::pair<NodeId, std::list<QueuedBlock>::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); if (itInFlight != mapBlocksInFlight.end()) { @@ -670,7 +683,7 @@ bool PeerManagerImpl::MarkBlockAsReceived(const uint256& hash) EXCLUSIVE_LOCKS_R return false; } -bool PeerManagerImpl::MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const CBlockIndex* pindex, std::list<QueuedBlock>::iterator** pit) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +bool PeerManagerImpl::MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const CBlockIndex* pindex, std::list<QueuedBlock>::iterator** pit) { CNodeState *state = State(nodeid); assert(state != nullptr); @@ -739,7 +752,7 @@ static void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) EXCLUSIV } } -void PeerManagerImpl::MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +void PeerManagerImpl::MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid) { AssertLockHeld(cs_main); CNodeState* nodestate = State(nodeid); @@ -778,7 +791,7 @@ void PeerManagerImpl::MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid) EXCLUS } } -bool PeerManagerImpl::TipMayBeStale() EXCLUSIVE_LOCKS_REQUIRED(cs_main) +bool PeerManagerImpl::TipMayBeStale() { AssertLockHeld(cs_main); const Consensus::Params& consensusParams = m_chainparams.GetConsensus(); @@ -802,7 +815,7 @@ static bool PeerHasHeader(CNodeState *state, const CBlockIndex *pindex) EXCLUSIV return false; } -void PeerManagerImpl::FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vector<const CBlockIndex*>& vBlocks, NodeId& nodeStaller) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +void PeerManagerImpl::FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vector<const CBlockIndex*>& vBlocks, NodeId& nodeStaller) { if (count == 0) return; @@ -1088,6 +1101,18 @@ bool PeerManagerImpl::GetNodeStateStats(NodeId nodeid, CNodeStateStats &stats) PeerRef peer = GetPeerRef(nodeid); if (peer == nullptr) return false; stats.m_starting_height = peer->m_starting_height; + // It is common for nodes with good ping times to suddenly become lagged, + // due to a new block arriving or other large transfer. + // Merely reporting pingtime might fool the caller into thinking the node was still responsive, + // since pingtime does not update until the ping is complete, which might take a while. + // So, if a ping is taking an unusually long time in flight, + // the caller can immediately detect that this is happening. + std::chrono::microseconds ping_wait{0}; + if ((0 != peer->m_ping_nonce_sent) && (0 != peer->m_ping_start.load().count())) { + ping_wait = GetTime<std::chrono::microseconds>() - peer->m_ping_start.load(); + } + + stats.m_ping_wait_usec = count_microseconds(ping_wait); return true; } @@ -1591,7 +1616,7 @@ void PeerManagerImpl::BlockChecked(const CBlock& block, const BlockValidationSta // -bool PeerManagerImpl::AlreadyHaveTx(const GenTxid& gtxid) EXCLUSIVE_LOCKS_REQUIRED(cs_main) +bool PeerManagerImpl::AlreadyHaveTx(const GenTxid& gtxid) { assert(recentRejects); if (::ChainActive().Tip()->GetBlockHash() != hashRecentRejectsChainTip) { @@ -1627,6 +1652,12 @@ bool static AlreadyHaveBlock(const uint256& block_hash) EXCLUSIVE_LOCKS_REQUIRED return g_chainman.m_blockman.LookupBlockIndex(block_hash) != nullptr; } +void PeerManagerImpl::SendPings() +{ + LOCK(m_peer_mutex); + for(auto& it : m_peer_map) it.second->m_ping_queued = true; +} + void RelayTransaction(const uint256& txid, const uint256& wtxid, const CConnman& connman) { connman.ForEachNode([&txid, &wtxid](CNode* pnode) EXCLUSIVE_LOCKS_REQUIRED(::cs_main) { @@ -1853,7 +1884,7 @@ void static ProcessGetBlockData(CNode& pfrom, Peer& peer, const CChainParams& ch } } -CTransactionRef PeerManagerImpl::FindTxForGetData(const CNode& peer, const GenTxid& gtxid, const std::chrono::seconds mempool_req, const std::chrono::seconds now) LOCKS_EXCLUDED(cs_main) +CTransactionRef PeerManagerImpl::FindTxForGetData(const CNode& peer, const GenTxid& gtxid, const std::chrono::seconds mempool_req, const std::chrono::seconds now) { auto txinfo = m_mempool.info(gtxid); if (txinfo.tx) { @@ -1880,7 +1911,7 @@ CTransactionRef PeerManagerImpl::FindTxForGetData(const CNode& peer, const GenTx return {}; } -void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic<bool>& interruptMsgProc) EXCLUSIVE_LOCKS_REQUIRED(!cs_main, peer.m_getdata_requests_mutex) +void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic<bool>& interruptMsgProc) { AssertLockNotHeld(cs_main); @@ -3837,15 +3868,14 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, vRecv >> nonce; // Only process pong message if there is an outstanding ping (old ping without nonce should never pong) - if (pfrom.nPingNonceSent != 0) { - if (nonce == pfrom.nPingNonceSent) { + if (peer->m_ping_nonce_sent != 0) { + if (nonce == peer->m_ping_nonce_sent) { // Matching pong received, this ping is no longer outstanding bPingFinished = true; - const auto ping_time = ping_end - pfrom.m_ping_start.load(); + const auto ping_time = ping_end - peer->m_ping_start.load(); if (ping_time.count() >= 0) { - // Successful ping time measurement, replace previous - pfrom.nPingUsecTime = count_microseconds(ping_time); - pfrom.nMinPingUsecTime = std::min(pfrom.nMinPingUsecTime.load(), count_microseconds(ping_time)); + // Let connman know about this successful ping-pong + pfrom.PongReceived(ping_time); } else { // This should never happen sProblem = "Timing mishap"; @@ -3872,12 +3902,12 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, LogPrint(BCLog::NET, "pong peer=%d: %s, %x expected, %x received, %u bytes\n", pfrom.GetId(), sProblem, - pfrom.nPingNonceSent, + peer->m_ping_nonce_sent, nonce, nAvail); } if (bPingFinished) { - pfrom.nPingNonceSent = 0; + peer->m_ping_nonce_sent = 0; } return; } @@ -3996,43 +4026,39 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } -bool PeerManagerImpl::MaybeDiscourageAndDisconnect(CNode& pnode) +bool PeerManagerImpl::MaybeDiscourageAndDisconnect(CNode& pnode, Peer& peer) { - const NodeId peer_id{pnode.GetId()}; - PeerRef peer = GetPeerRef(peer_id); - if (peer == nullptr) return false; - { - LOCK(peer->m_misbehavior_mutex); + LOCK(peer.m_misbehavior_mutex); // There's nothing to do if the m_should_discourage flag isn't set - if (!peer->m_should_discourage) return false; + if (!peer.m_should_discourage) return false; - peer->m_should_discourage = false; + peer.m_should_discourage = false; } // peer.m_misbehavior_mutex if (pnode.HasPermission(PF_NOBAN)) { // We never disconnect or discourage peers for bad behavior if they have the NOBAN permission flag - LogPrintf("Warning: not punishing noban peer %d!\n", peer_id); + LogPrintf("Warning: not punishing noban peer %d!\n", peer.m_id); return false; } if (pnode.IsManualConn()) { // We never disconnect or discourage manual peers for bad behavior - LogPrintf("Warning: not punishing manually connected peer %d!\n", peer_id); + LogPrintf("Warning: not punishing manually connected peer %d!\n", peer.m_id); return false; } if (pnode.addr.IsLocal()) { // We disconnect local peers for bad behavior but don't discourage (since that would discourage // all peers on the same local address) - LogPrintf("Warning: disconnecting but not discouraging local peer %d!\n", peer_id); + LogPrintf("Warning: disconnecting but not discouraging local peer %d!\n", peer.m_id); pnode.fDisconnect = true; return true; } // Normal case: Disconnect the peer and discourage all nodes sharing the address - LogPrint(BCLog::NET, "Disconnecting and discouraging peer %d!\n", peer_id); + LogPrint(BCLog::NET, "Disconnecting and discouraging peer %d!\n", peer.m_id); if (m_banman) m_banman->Discourage(pnode.addr); m_connman.DisconnectNode(pnode.addr); return true; @@ -4295,6 +4321,50 @@ void PeerManagerImpl::CheckForStaleTipAndEvictPeers() } } +void PeerManagerImpl::MaybeSendPing(CNode& node_to, Peer& peer) +{ + // Use mockable time for ping timeouts. + // This means that setmocktime may cause pings to time out. + auto now = GetTime<std::chrono::microseconds>(); + + if (m_connman.RunInactivityChecks(node_to) && peer.m_ping_nonce_sent && + now > peer.m_ping_start.load() + std::chrono::seconds{TIMEOUT_INTERVAL}) { + LogPrint(BCLog::NET, "ping timeout: %fs peer=%d\n", 0.000001 * count_microseconds(now - peer.m_ping_start.load()), peer.m_id); + node_to.fDisconnect = true; + return; + } + + const CNetMsgMaker msgMaker(node_to.GetCommonVersion()); + bool pingSend = false; + + if (peer.m_ping_queued) { + // RPC ping request by user + pingSend = true; + } + + if (peer.m_ping_nonce_sent == 0 && now > peer.m_ping_start.load() + PING_INTERVAL) { + // Ping automatically sent as a latency probe & keepalive. + pingSend = true; + } + + if (pingSend) { + uint64_t nonce = 0; + while (nonce == 0) { + GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); + } + peer.m_ping_queued = false; + peer.m_ping_start = now; + if (node_to.GetCommonVersion() > BIP0031_VERSION) { + peer.m_ping_nonce_sent = nonce; + m_connman.PushMessage(&node_to, msgMaker.Make(NetMsgType::PING, nonce)); + } else { + // Peer is too old to support ping command with nonce, pong will never arrive. + peer.m_ping_nonce_sent = 0; + m_connman.PushMessage(&node_to, msgMaker.Make(NetMsgType::PING)); + } + } +} + namespace { class CompareInvMempoolOrder { @@ -4319,11 +4389,12 @@ public: bool PeerManagerImpl::SendMessages(CNode* pto) { PeerRef peer = GetPeerRef(pto->GetId()); + if (!peer) return false; const Consensus::Params& consensusParams = m_chainparams.GetConsensus(); // We must call MaybeDiscourageAndDisconnect first, to ensure that we'll // disconnect misbehaving peers even before the version handshake is complete. - if (MaybeDiscourageAndDisconnect(*pto)) return true; + if (MaybeDiscourageAndDisconnect(*pto, *peer)) return true; // Don't send anything until the version handshake is complete if (!pto->fSuccessfullyConnected || pto->fDisconnect) @@ -4332,34 +4403,10 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // If we get here, the outgoing message serialization version is set and can't change. const CNetMsgMaker msgMaker(pto->GetCommonVersion()); - // - // Message: ping - // - bool pingSend = false; - if (pto->fPingQueued) { - // RPC ping request by user - pingSend = true; - } - if (pto->nPingNonceSent == 0 && pto->m_ping_start.load() + PING_INTERVAL < GetTime<std::chrono::microseconds>()) { - // Ping automatically sent as a latency probe & keepalive. - pingSend = true; - } - if (pingSend) { - uint64_t nonce = 0; - while (nonce == 0) { - GetRandBytes((unsigned char*)&nonce, sizeof(nonce)); - } - pto->fPingQueued = false; - pto->m_ping_start = GetTime<std::chrono::microseconds>(); - if (pto->GetCommonVersion() > BIP0031_VERSION) { - pto->nPingNonceSent = nonce; - m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::PING, nonce)); - } else { - // Peer is too old to support ping command with nonce, pong will never arrive. - pto->nPingNonceSent = 0; - m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::PING)); - } - } + MaybeSendPing(*pto, *peer); + + // MaybeSendPing may have marked peer for disconnection + if (pto->fDisconnect) return true; { LOCK(cs_main); diff --git a/src/net_processing.h b/src/net_processing.h index eaa3b142a8..d7be453df5 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -30,6 +30,7 @@ struct CNodeStateStats { int nSyncHeight = -1; int nCommonHeight = -1; int m_starting_height = -1; + int64_t m_ping_wait_usec; std::vector<int> vHeightInFlight; }; @@ -47,6 +48,9 @@ public: /** Whether this node ignores txs received over p2p. */ virtual bool IgnoresIncomingTxs() = 0; + /** Send ping message to all peers */ + virtual void SendPings() = 0; + /** Set the best height */ virtual void SetBestHeight(int height) = 0; diff --git a/src/node/coinstats.cpp b/src/node/coinstats.cpp index b994e79391..06fcc33725 100644 --- a/src/node/coinstats.cpp +++ b/src/node/coinstats.cpp @@ -55,6 +55,18 @@ static void ApplyHash(CCoinsStats& stats, MuHash3072& muhash, const uint256& has muhash.Insert(MakeUCharSpan(ss)); } +//! Warning: be very careful when changing this! assumeutxo and UTXO snapshot +//! validation commitments are reliant on the hash constructed by this +//! function. +//! +//! If the construction of this hash is changed, it will invalidate +//! existing UTXO snapshots. This will not result in any kind of consensus +//! failure, but it will force clients that were expecting to make use of +//! assumeutxo to do traditional IBD instead. +//! +//! It is also possible, though very unlikely, that a change in this +//! construction could cause a previously invalid (and potentially malicious) +//! UTXO snapshot to be considered valid. template <typename T> static void ApplyStats(CCoinsStats& stats, T& hash_obj, const uint256& hash, const std::map<uint32_t, Coin>& outputs) { diff --git a/src/qt/rpcconsole.cpp b/src/qt/rpcconsole.cpp index 2758ee351a..ea12ce1583 100644 --- a/src/qt/rpcconsole.cpp +++ b/src/qt/rpcconsole.cpp @@ -1115,7 +1115,6 @@ void RPCConsole::updateDetailWidget() ui->peerBytesRecv->setText(GUIUtil::formatBytes(stats->nodeStats.nRecvBytes)); ui->peerConnTime->setText(GUIUtil::formatDurationStr(GetSystemTimeInSeconds() - stats->nodeStats.nTimeConnected)); ui->peerPingTime->setText(GUIUtil::formatPingTime(stats->nodeStats.m_ping_usec)); - ui->peerPingWait->setText(GUIUtil::formatPingTime(stats->nodeStats.m_ping_wait_usec)); ui->peerMinPing->setText(GUIUtil::formatPingTime(stats->nodeStats.m_min_ping_usec)); ui->timeoffset->setText(GUIUtil::formatTimeOffset(stats->nodeStats.nTimeOffset)); ui->peerVersion->setText(QString::number(stats->nodeStats.nVersion)); @@ -1149,6 +1148,7 @@ void RPCConsole::updateDetailWidget() ui->peerCommonHeight->setText(tr("Unknown")); ui->peerHeight->setText(QString::number(stats->nodeStateStats.m_starting_height)); + ui->peerPingWait->setText(GUIUtil::formatPingTime(stats->nodeStateStats.m_ping_wait_usec)); } ui->detailWidget->show(); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 5be02b1e4e..5dc33d7a98 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -2411,10 +2411,21 @@ static RPCHelpMan dumptxoutset() FILE* file{fsbridge::fopen(temppath, "wb")}; CAutoFile afile{file, SER_DISK, CLIENT_VERSION}; + NodeContext& node = EnsureNodeContext(request.context); + UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), afile); + fs::rename(temppath, path); + + result.pushKV("path", path.string()); + return result; +}, + }; +} + +UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFile& afile) +{ std::unique_ptr<CCoinsViewCursor> pcursor; CCoinsStats stats; CBlockIndex* tip; - NodeContext& node = EnsureNodeContext(request.context); { // We need to lock cs_main to ensure that the coinsdb isn't written to @@ -2431,13 +2442,13 @@ static RPCHelpMan dumptxoutset() // LOCK(::cs_main); - ::ChainstateActive().ForceFlushStateToDisk(); + chainstate.ForceFlushStateToDisk(); - if (!GetUTXOStats(&::ChainstateActive().CoinsDB(), stats, CoinStatsHashType::NONE, node.rpc_interruption_point)) { + if (!GetUTXOStats(&chainstate.CoinsDB(), stats, CoinStatsHashType::NONE, node.rpc_interruption_point)) { throw JSONRPCError(RPC_INTERNAL_ERROR, "Unable to read UTXO set"); } - pcursor = std::unique_ptr<CCoinsViewCursor>(::ChainstateActive().CoinsDB().Cursor()); + pcursor = std::unique_ptr<CCoinsViewCursor>(chainstate.CoinsDB().Cursor()); tip = g_chainman.m_blockman.LookupBlockIndex(stats.hashBlock); CHECK_NONFATAL(tip); } @@ -2462,16 +2473,13 @@ static RPCHelpMan dumptxoutset() } afile.fclose(); - fs::rename(temppath, path); UniValue result(UniValue::VOBJ); result.pushKV("coins_written", stats.coins_count); result.pushKV("base_hash", tip->GetBlockHash().ToString()); result.pushKV("base_height", tip->nHeight); - result.pushKV("path", path.string()); + return result; -}, - }; } void RegisterBlockchainRPCCommands(CRPCTable &t) diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index e4ce80400e..d8cae4dd24 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -6,6 +6,7 @@ #define BITCOIN_RPC_BLOCKCHAIN_H #include <amount.h> +#include <streams.h> #include <sync.h> #include <stdint.h> @@ -16,6 +17,7 @@ extern RecursiveMutex cs_main; class CBlock; class CBlockIndex; class CBlockPolicyEstimator; +class CChainState; class CTxMemPool; class ChainstateManager; class UniValue; @@ -57,4 +59,10 @@ CTxMemPool& EnsureMemPool(const util::Ref& context); ChainstateManager& EnsureChainman(const util::Ref& context); CBlockPolicyEstimator& EnsureFeeEstimator(const util::Ref& context); +/** + * Helper to create UTXO snapshots given a chainstate and a file handle. + * @return a UniValue map containing metadata about the snapshot. + */ +UniValue CreateUTXOSnapshot(NodeContext& node, CChainState& chainstate, CAutoFile& afile); + #endif diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index e83f66dd12..0224ee697a 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -77,13 +77,12 @@ static RPCHelpMan ping() [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { NodeContext& node = EnsureNodeContext(request.context); - if(!node.connman) + if (!node.peerman) { throw JSONRPCError(RPC_CLIENT_P2P_DISABLED, "Error: Peer-to-peer functionality missing or disabled"); + } // Request that each node send a ping during next message processing pass - node.connman->ForEachNode([](CNode* pnode) { - pnode->fPingQueued = true; - }); + node.peerman->SendPings(); return NullUniValue; }, }; @@ -209,8 +208,8 @@ static RPCHelpMan getpeerinfo() if (stats.m_min_ping_usec < std::numeric_limits<int64_t>::max()) { obj.pushKV("minping", ((double)stats.m_min_ping_usec) / 1e6); } - if (stats.m_ping_wait_usec > 0) { - obj.pushKV("pingwait", ((double)stats.m_ping_wait_usec) / 1e6); + if (fStateStats && statestats.m_ping_wait_usec > 0) { + obj.pushKV("pingwait", ((double)statestats.m_ping_wait_usec) / 1e6); } obj.pushKV("version", stats.nVersion); // Use the sanitized form of subver here, to avoid tricksy remote peers from diff --git a/src/test/fuzz/crypto.cpp b/src/test/fuzz/crypto.cpp index c2bb3a1a4e..17ac48fca7 100644 --- a/src/test/fuzz/crypto.cpp +++ b/src/test/fuzz/crypto.cpp @@ -4,7 +4,6 @@ #include <crypto/hmac_sha256.h> #include <crypto/hmac_sha512.h> -#include <crypto/muhash.h> #include <crypto/ripemd160.h> #include <crypto/sha1.h> #include <crypto/sha256.h> @@ -36,7 +35,6 @@ FUZZ_TARGET(crypto) CSHA512 sha512; SHA3_256 sha3; CSipHasher sip_hasher{fuzzed_data_provider.ConsumeIntegral<uint64_t>(), fuzzed_data_provider.ConsumeIntegral<uint64_t>()}; - MuHash3072 muhash; while (fuzzed_data_provider.ConsumeBool()) { CallOneOf( @@ -63,12 +61,6 @@ FUZZ_TARGET(crypto) (void)Hash(data); (void)Hash160(data); (void)sha512.Size(); - - if (fuzzed_data_provider.ConsumeBool()) { - muhash *= MuHash3072(data); - } else { - muhash /= MuHash3072(data); - } }, [&] { (void)hash160.Reset(); @@ -78,7 +70,6 @@ FUZZ_TARGET(crypto) (void)sha256.Reset(); (void)sha3.Reset(); (void)sha512.Reset(); - muhash = MuHash3072(); }, [&] { CallOneOf( @@ -122,10 +113,6 @@ FUZZ_TARGET(crypto) [&] { data.resize(SHA3_256::OUTPUT_SIZE); sha3.Finalize(data); - }, - [&] { - uint256 out; - muhash.Finalize(out); }); }); } diff --git a/src/test/fuzz/muhash.cpp b/src/test/fuzz/muhash.cpp index 8f843ca773..2d761cef15 100644 --- a/src/test/fuzz/muhash.cpp +++ b/src/test/fuzz/muhash.cpp @@ -41,6 +41,11 @@ FUZZ_TARGET(muhash) muhash.Finalize(out2); assert(out == out2); + MuHash3072 muhash3; + muhash3 *= muhash; + uint256 out3; + muhash3.Finalize(out3); + assert(out == out3); // Test that removing all added elements brings the object back to it's initial state muhash /= muhash; @@ -50,4 +55,9 @@ FUZZ_TARGET(muhash) muhash2.Finalize(out2); assert(out == out2); + + muhash3.Remove(data); + muhash3.Remove(data2); + muhash3.Finalize(out3); + assert(out == out3); } diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index e967273636..9c06283a58 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -123,6 +123,7 @@ void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, co m_node.mempool->addUnchecked(entry.Fee(50000).Time(GetTime()).SpendsCoinbase(false).FromTx(tx)); std::unique_ptr<CBlockTemplate> pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); + BOOST_REQUIRE_EQUAL(pblocktemplate->block.vtx.size(), 4); BOOST_CHECK(pblocktemplate->block.vtx[1]->GetHash() == hashParentTx); BOOST_CHECK(pblocktemplate->block.vtx[2]->GetHash() == hashHighFeeTx); BOOST_CHECK(pblocktemplate->block.vtx[3]->GetHash() == hashMediumFeeTx); @@ -157,6 +158,7 @@ void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, co hashLowFeeTx = tx.GetHash(); m_node.mempool->addUnchecked(entry.Fee(feeToUse+2).FromTx(tx)); pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); + BOOST_REQUIRE_EQUAL(pblocktemplate->block.vtx.size(), 6); BOOST_CHECK(pblocktemplate->block.vtx[4]->GetHash() == hashFreeTx); BOOST_CHECK(pblocktemplate->block.vtx[5]->GetHash() == hashLowFeeTx); @@ -191,6 +193,7 @@ void MinerTestingSetup::TestPackageSelection(const CChainParams& chainparams, co tx.vout[0].nValue = 100000000 - 10000; // 10k satoshi fee m_node.mempool->addUnchecked(entry.Fee(10000).FromTx(tx)); pblocktemplate = AssemblerForTest(chainparams).CreateNewBlock(scriptPubKey); + BOOST_REQUIRE_EQUAL(pblocktemplate->block.vtx.size(), 9); BOOST_CHECK(pblocktemplate->block.vtx[8]->GetHash() == hashLowFeeTx2); } diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 010a3375cf..f52905e1ef 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -794,7 +794,7 @@ std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(const int n_c candidates.push_back({ /* id */ id, /* nTimeConnected */ static_cast<int64_t>(random_context.randrange(100)), - /* nMinPingUsecTime */ static_cast<int64_t>(random_context.randrange(100)), + /* m_min_ping_time */ static_cast<int64_t>(random_context.randrange(100)), /* nLastBlockTime */ static_cast<int64_t>(random_context.randrange(100)), /* nLastTXTime */ static_cast<int64_t>(random_context.randrange(100)), /* fRelevantServices */ random_context.randbool(), @@ -854,7 +854,7 @@ BOOST_AUTO_TEST_CASE(node_eviction_test) // from eviction. BOOST_CHECK(!IsEvicted( number_of_nodes, [](NodeEvictionCandidate& candidate) { - candidate.nMinPingUsecTime = candidate.id; + candidate.m_min_ping_time = candidate.id; }, {0, 1, 2, 3, 4, 5, 6, 7}, random_context)); @@ -901,7 +901,7 @@ BOOST_AUTO_TEST_CASE(node_eviction_test) BOOST_CHECK(!IsEvicted( number_of_nodes, [number_of_nodes](NodeEvictionCandidate& candidate) { candidate.nKeyedNetGroup = number_of_nodes - candidate.id; // 4 protected - candidate.nMinPingUsecTime = candidate.id; // 8 protected + candidate.m_min_ping_time = candidate.id; // 8 protected candidate.nLastTXTime = number_of_nodes - candidate.id; // 4 protected candidate.nLastBlockTime = number_of_nodes - candidate.id; // 4 protected }, diff --git a/src/test/sock_tests.cpp b/src/test/sock_tests.cpp index cc0e6e7057..ed9780dfb5 100644 --- a/src/test/sock_tests.cpp +++ b/src/test/sock_tests.cpp @@ -95,7 +95,7 @@ static void CreateSocketPair(int s[2]) static void SendAndRecvMessage(const Sock& sender, const Sock& receiver) { const char* msg = "abcd"; - constexpr size_t msg_len = 4; + constexpr ssize_t msg_len = 4; char recv_buf[10]; BOOST_CHECK_EQUAL(sender.Send(msg, msg_len, 0), msg_len); diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index b9f3f8c955..03ab3f523d 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -199,14 +199,43 @@ TestingSetup::TestingSetup(const std::string& chainName, const std::vector<const } } -TestChain100Setup::TestChain100Setup() +TestChain100Setup::TestChain100Setup(bool deterministic) { + m_deterministic = deterministic; + + if (m_deterministic) { + SetMockTime(1598887952); + constexpr std::array<unsigned char, 32> vchKey = { + { + 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1 + } + }; + coinbaseKey.Set(vchKey.begin(), vchKey.end(), false); + } else { + coinbaseKey.MakeNewKey(true); + } + // Generate a 100-block chain: - coinbaseKey.MakeNewKey(true); + this->mineBlocks(COINBASE_MATURITY); + + if (m_deterministic) { + LOCK(::cs_main); + assert( + m_node.chainman->ActiveChain().Tip()->GetBlockHash().ToString() == + "49c95db1e470fed04496d801c9d8fbb78155d2c7f855232c918823d2c17d0cf6"); + } +} + +void TestChain100Setup::mineBlocks(int num_blocks) +{ CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; - for (int i = 0; i < COINBASE_MATURITY; i++) { + for (int i = 0; i < num_blocks; i++) + { std::vector<CMutableTransaction> noTxns; CBlock b = CreateAndProcessBlock(noTxns, scriptPubKey); + if (m_deterministic) { + SetMockTime(GetTime() + 1); + } m_coinbase_txns.push_back(b.vtx[0]); } } @@ -231,9 +260,61 @@ CBlock TestChain100Setup::CreateAndProcessBlock(const std::vector<CMutableTransa return block; } + +CMutableTransaction TestChain100Setup::CreateValidMempoolTransaction(CTransactionRef input_transaction, + int input_vout, + int input_height, + CKey input_signing_key, + CScript output_destination, + CAmount output_amount) +{ + // Transaction we will submit to the mempool + CMutableTransaction mempool_txn; + + // Create an input + COutPoint outpoint_to_spend(input_transaction->GetHash(), input_vout); + CTxIn input(outpoint_to_spend); + mempool_txn.vin.push_back(input); + + // Create an output + CTxOut output(output_amount, output_destination); + mempool_txn.vout.push_back(output); + + // Sign the transaction + // - Add the signing key to a keystore + FillableSigningProvider keystore; + keystore.AddKey(input_signing_key); + // - Populate a CoinsViewCache with the unspent output + CCoinsView coins_view; + CCoinsViewCache coins_cache(&coins_view); + AddCoins(coins_cache, *input_transaction.get(), input_height); + // - Use GetCoin to properly populate utxo_to_spend, + Coin utxo_to_spend; + assert(coins_cache.GetCoin(outpoint_to_spend, utxo_to_spend)); + // - Then add it to a map to pass in to SignTransaction + std::map<COutPoint, Coin> input_coins; + input_coins.insert({outpoint_to_spend, utxo_to_spend}); + // - Default signature hashing type + int nHashType = SIGHASH_ALL; + std::map<int, std::string> input_errors; + assert(SignTransaction(mempool_txn, &keystore, input_coins, nHashType, input_errors)); + + // Add transaction to the mempool + { + LOCK(cs_main); + const MempoolAcceptResult result = AcceptToMemoryPool(*m_node.mempool.get(), MakeTransactionRef(mempool_txn), /* bypass_limits */ false); + assert(result.m_result_type == MempoolAcceptResult::ResultType::VALID); + } + + return mempool_txn; +} + TestChain100Setup::~TestChain100Setup() { gArgs.ForceSetArg("-segwitheight", "0"); + if (m_deterministic) { + SetMockTime(0); + } } CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction& tx) const diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 331c1235cb..33f24e7c44 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -78,7 +78,6 @@ struct BasicTestingSetup { explicit BasicTestingSetup(const std::string& chainName = CBaseChainParams::MAIN, const std::vector<const char*>& extra_args = {}); ~BasicTestingSetup(); -private: const fs::path m_path_root; }; @@ -112,7 +111,7 @@ class CScript; * Testing fixture that pre-creates a 100-block REGTEST-mode block chain */ struct TestChain100Setup : public RegTestingSetup { - TestChain100Setup(); + TestChain100Setup(bool deterministic = false); /** * Create a new block with just given transactions, coinbase paying to @@ -121,12 +120,38 @@ struct TestChain100Setup : public RegTestingSetup { CBlock CreateAndProcessBlock(const std::vector<CMutableTransaction>& txns, const CScript& scriptPubKey); + //! Mine a series of new blocks on the active chain. + void mineBlocks(int num_blocks); + + /** + * Create a transaction and submit to the mempool. + * + * @param input_transaction The transaction to spend + * @param input_vout The vout to spend from the input_transaction + * @param input_height The height of the block that included the input_transaction + * @param input_signing_key The key to spend the input_transaction + * @param output_destination Where to send the output + * @param output_amount How much to send + */ + CMutableTransaction CreateValidMempoolTransaction(CTransactionRef input_transaction, + int input_vout, + int input_height, + CKey input_signing_key, + CScript output_destination, + CAmount output_amount = CAmount(1 * COIN)); + ~TestChain100Setup(); + bool m_deterministic; std::vector<CTransactionRef> m_coinbase_txns; // For convenience, coinbase transactions CKey coinbaseKey; // private/public key needed to spend coinbase transactions }; + +struct TestChain100DeterministicSetup : public TestChain100Setup { + TestChain100DeterministicSetup() : TestChain100Setup(true) { } +}; + class CTxMemPoolEntry; struct TestMemPoolEntryHelper diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 3d8570e27c..94d4277019 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -4,13 +4,18 @@ // #include <chainparams.h> #include <consensus/validation.h> +#include <node/utxo_snapshot.h> #include <random.h> +#include <rpc/blockchain.h> #include <sync.h> #include <test/util/setup_common.h> #include <uint256.h> #include <validation.h> #include <validationinterface.h> +#include <tinyformat.h> +#include <univalue.h> + #include <vector> #include <boost/test/unit_test.hpp> @@ -28,6 +33,8 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) std::vector<CChainState*> chainstates; const CChainParams& chainparams = Params(); + BOOST_CHECK(!manager.SnapshotBlockhash().has_value()); + // Create a legacy (IBD) chainstate. // CChainState& c1 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(mempool)); @@ -54,10 +61,17 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) auto& validated_cs = manager.ValidatedChainstate(); BOOST_CHECK_EQUAL(&validated_cs, &c1); + BOOST_CHECK(!manager.SnapshotBlockhash().has_value()); + // Create a snapshot-based chainstate. // - CChainState& c2 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(mempool, GetRandHash())); + const uint256 snapshot_blockhash = GetRandHash(); + CChainState& c2 = WITH_LOCK(::cs_main, return manager.InitializeChainstate( + mempool, snapshot_blockhash)); chainstates.push_back(&c2); + + BOOST_CHECK_EQUAL(manager.SnapshotBlockhash().value(), snapshot_blockhash); + c2.InitCoinsDB( /* cache_size_bytes */ 1 << 23, /* in_memory */ true, /* should_wipe */ false); WITH_LOCK(::cs_main, c2.InitCoinsCache(1 << 23)); @@ -155,4 +169,175 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) BOOST_CHECK_CLOSE(c2.m_coinsdb_cache_size_bytes, max_cache * 0.95, 1); } +auto NoMalleation = [](CAutoFile& file, SnapshotMetadata& meta){}; + +template<typename F = decltype(NoMalleation)> +static bool +CreateAndActivateUTXOSnapshot(NodeContext& node, const fs::path root, F malleation = NoMalleation) +{ + // Write out a snapshot to the test's tempdir. + // + int height; + WITH_LOCK(::cs_main, height = node.chainman->ActiveHeight()); + fs::path snapshot_path = root / tfm::format("test_snapshot.%d.dat", height); + FILE* outfile{fsbridge::fopen(snapshot_path, "wb")}; + CAutoFile auto_outfile{outfile, SER_DISK, CLIENT_VERSION}; + + UniValue result = CreateUTXOSnapshot(node, node.chainman->ActiveChainstate(), auto_outfile); + BOOST_TEST_MESSAGE( + "Wrote UTXO snapshot to " << snapshot_path.make_preferred().string() << ": " << result.write()); + + // Read the written snapshot in and then activate it. + // + FILE* infile{fsbridge::fopen(snapshot_path, "rb")}; + CAutoFile auto_infile{infile, SER_DISK, CLIENT_VERSION}; + SnapshotMetadata metadata; + auto_infile >> metadata; + + malleation(auto_infile, metadata); + + return node.chainman->ActivateSnapshot(auto_infile, metadata, /*in_memory*/ true); +} + +//! Test basic snapshot activation. +BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, TestChain100DeterministicSetup) +{ + ChainstateManager& chainman = *Assert(m_node.chainman); + + size_t initial_size; + size_t initial_total_coins{100}; + + // Make some initial assertions about the contents of the chainstate. + { + LOCK(::cs_main); + CCoinsViewCache& ibd_coinscache = chainman.ActiveChainstate().CoinsTip(); + initial_size = ibd_coinscache.GetCacheSize(); + size_t total_coins{0}; + + for (CTransactionRef& txn : m_coinbase_txns) { + COutPoint op{txn->GetHash(), 0}; + BOOST_CHECK(ibd_coinscache.HaveCoin(op)); + total_coins++; + } + + BOOST_CHECK_EQUAL(total_coins, initial_total_coins); + BOOST_CHECK_EQUAL(initial_size, initial_total_coins); + } + + // Snapshot should refuse to load at this height. + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(m_node, m_path_root)); + BOOST_CHECK(chainman.ActiveChainstate().m_from_snapshot_blockhash.IsNull()); + BOOST_CHECK_EQUAL( + chainman.ActiveChainstate().m_from_snapshot_blockhash, + chainman.SnapshotBlockhash().value_or(uint256())); + + // Mine 10 more blocks, putting at us height 110 where a valid assumeutxo value can + // be found. + mineBlocks(10); + initial_size += 10; + initial_total_coins += 10; + + // Should not load malleated snapshots + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + // A UTXO is missing but count is correct + metadata.m_coins_count -= 1; + + COutPoint outpoint; + Coin coin; + + auto_infile >> outpoint; + auto_infile >> coin; + })); + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + // Coins count is larger than coins in file + metadata.m_coins_count += 1; + })); + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot( + m_node, m_path_root, [](CAutoFile& auto_infile, SnapshotMetadata& metadata) { + // Coins count is smaller than coins in file + metadata.m_coins_count -= 1; + })); + + BOOST_REQUIRE(CreateAndActivateUTXOSnapshot(m_node, m_path_root)); + + // Ensure our active chain is the snapshot chainstate. + BOOST_CHECK(!chainman.ActiveChainstate().m_from_snapshot_blockhash.IsNull()); + BOOST_CHECK_EQUAL( + chainman.ActiveChainstate().m_from_snapshot_blockhash, + *chainman.SnapshotBlockhash()); + + // To be checked against later when we try loading a subsequent snapshot. + uint256 loaded_snapshot_blockhash{*chainman.SnapshotBlockhash()}; + + // Make some assertions about the both chainstates. These checks ensure the + // legacy chainstate hasn't changed and that the newly created chainstate + // reflects the expected content. + { + LOCK(::cs_main); + int chains_tested{0}; + + for (CChainState* chainstate : chainman.GetAll()) { + BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); + CCoinsViewCache& coinscache = chainstate->CoinsTip(); + + // Both caches will be empty initially. + BOOST_CHECK_EQUAL((unsigned int)0, coinscache.GetCacheSize()); + + size_t total_coins{0}; + + for (CTransactionRef& txn : m_coinbase_txns) { + COutPoint op{txn->GetHash(), 0}; + BOOST_CHECK(coinscache.HaveCoin(op)); + total_coins++; + } + + BOOST_CHECK_EQUAL(initial_size , coinscache.GetCacheSize()); + BOOST_CHECK_EQUAL(total_coins, initial_total_coins); + chains_tested++; + } + + BOOST_CHECK_EQUAL(chains_tested, 2); + } + + // Mine some new blocks on top of the activated snapshot chainstate. + constexpr size_t new_coins{100}; + mineBlocks(new_coins); // Defined in TestChain100Setup. + + { + LOCK(::cs_main); + size_t coins_in_active{0}; + size_t coins_in_ibd{0}; + size_t coins_missing_ibd{0}; + + for (CChainState* chainstate : chainman.GetAll()) { + BOOST_TEST_MESSAGE("Checking coins in " << chainstate->ToString()); + CCoinsViewCache& coinscache = chainstate->CoinsTip(); + bool is_ibd = chainman.IsBackgroundIBD(chainstate); + + for (CTransactionRef& txn : m_coinbase_txns) { + COutPoint op{txn->GetHash(), 0}; + if (coinscache.HaveCoin(op)) { + (is_ibd ? coins_in_ibd : coins_in_active)++; + } else if (is_ibd) { + coins_missing_ibd++; + } + } + } + + BOOST_CHECK_EQUAL(coins_in_active, initial_total_coins + new_coins); + BOOST_CHECK_EQUAL(coins_in_ibd, initial_total_coins); + BOOST_CHECK_EQUAL(coins_missing_ibd, new_coins); + } + + // Snapshot should refuse to load after one has already loaded. + BOOST_REQUIRE(!CreateAndActivateUTXOSnapshot(m_node, m_path_root)); + + // Snapshot blockhash should be unchanged. + BOOST_CHECK_EQUAL( + chainman.ActiveChainstate().m_from_snapshot_blockhash, + loaded_snapshot_blockhash); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/validation_tests.cpp b/src/test/validation_tests.cpp index 9e37f14921..ecf9453094 100644 --- a/src/test/validation_tests.cpp +++ b/src/test/validation_tests.cpp @@ -5,6 +5,7 @@ #include <chainparams.h> #include <net.h> #include <signet.h> +#include <uint256.h> #include <validation.h> #include <test/util/setup_common.h> @@ -119,4 +120,27 @@ BOOST_AUTO_TEST_CASE(signet_parse_tests) BOOST_CHECK(!CheckSignetBlockSolution(block, signet_params->GetConsensus())); } +//! Test retrieval of valid assumeutxo values. +BOOST_AUTO_TEST_CASE(test_assumeutxo) +{ + const auto params = CreateChainParams(*m_node.args, CBaseChainParams::REGTEST); + + // These heights don't have assumeutxo configurations associated, per the contents + // of chainparams.cpp. + std::vector<int> bad_heights{0, 100, 111, 115, 209, 211}; + + for (auto empty : bad_heights) { + const auto out = ExpectedAssumeutxo(empty, *params); + BOOST_CHECK(!out); + } + + const auto out110 = *ExpectedAssumeutxo(110, *params); + BOOST_CHECK_EQUAL(out110.hash_serialized, uint256S("76fd7334ac7c1baf57ddc0c626f073a655a35d98a4258cd1382c8cc2b8392e10")); + BOOST_CHECK_EQUAL(out110.nChainTx, (unsigned int)110); + + const auto out210 = *ExpectedAssumeutxo(210, *params); + BOOST_CHECK_EQUAL(out210.hash_serialized, uint256S("9c5ed99ef98544b34f8920b6d1802f72ac28ae6e2bd2bd4c316ff10c230df3f2")); + BOOST_CHECK_EQUAL(out210.nChainTx, (unsigned int)210); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/txdb.cpp b/src/txdb.cpp index 72460e7c69..4b4766e1ba 100644 --- a/src/txdb.cpp +++ b/src/txdb.cpp @@ -47,11 +47,15 @@ CCoinsViewDB::CCoinsViewDB(fs::path ldb_path, size_t nCacheSize, bool fMemory, b void CCoinsViewDB::ResizeCache(size_t new_cache_size) { - // Have to do a reset first to get the original `m_db` state to release its - // filesystem lock. - m_db.reset(); - m_db = MakeUnique<CDBWrapper>( - m_ldb_path, new_cache_size, m_is_memory, /*fWipe*/ false, /*obfuscate*/ true); + // We can't do this operation with an in-memory DB since we'll lose all the coins upon + // reset. + if (!m_is_memory) { + // Have to do a reset first to get the original `m_db` state to release its + // filesystem lock. + m_db.reset(); + m_db = MakeUnique<CDBWrapper>( + m_ldb_path, new_cache_size, m_is_memory, /*fWipe*/ false, /*obfuscate*/ true); + } } bool CCoinsViewDB::GetCoin(const COutPoint &outpoint, Coin &coin) const { diff --git a/src/util/time.cpp b/src/util/time.cpp index 295806c54a..7b0eb8fd8e 100644 --- a/src/util/time.cpp +++ b/src/util/time.cpp @@ -53,9 +53,14 @@ void SetMockTime(int64_t nMockTimeIn) nMockTime.store(nMockTimeIn, std::memory_order_relaxed); } -int64_t GetMockTime() +void SetMockTime(std::chrono::seconds mock_time_in) { - return nMockTime.load(std::memory_order_relaxed); + nMockTime.store(mock_time_in.count(), std::memory_order_relaxed); +} + +std::chrono::seconds GetMockTime() +{ + return std::chrono::seconds(nMockTime.load(std::memory_order_relaxed)); } int64_t GetTimeMillis() diff --git a/src/util/time.h b/src/util/time.h index 03b75b5be5..c4f930781a 100644 --- a/src/util/time.h +++ b/src/util/time.h @@ -43,10 +43,19 @@ int64_t GetTimeMicros(); /** Returns the system time (not mockable) */ int64_t GetSystemTimeInSeconds(); // Like GetTime(), but not mockable -/** For testing. Set e.g. with the setmocktime rpc, or -mocktime argument */ +/** + * DEPRECATED + * Use SetMockTime with chrono type + * + * @param[in] nMockTimeIn Time in seconds. + */ void SetMockTime(int64_t nMockTimeIn); + +/** For testing. Set e.g. with the setmocktime rpc, or -mocktime argument */ +void SetMockTime(std::chrono::seconds mock_time_in); + /** For testing */ -int64_t GetMockTime(); +std::chrono::seconds GetMockTime(); /** Return system time (or mocked time, if set) */ template <typename T> diff --git a/src/validation.cpp b/src/validation.cpp index 778d75ce18..31609ea3e5 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -20,6 +20,7 @@ #include <index/txindex.h> #include <logging.h> #include <logging/timer.h> +#include <node/coinstats.h> #include <node/ui_interface.h> #include <optional.h> #include <policy/policy.h> @@ -5147,7 +5148,8 @@ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex *pin Optional<uint256> ChainstateManager::SnapshotBlockhash() const { LOCK(::cs_main); - if (m_active_chainstate != nullptr) { + if (m_active_chainstate != nullptr && + !m_active_chainstate->m_from_snapshot_blockhash.IsNull()) { // If a snapshot chainstate exists, it will always be our active. return m_active_chainstate->m_from_snapshot_blockhash; } @@ -5192,6 +5194,295 @@ CChainState& ChainstateManager::InitializeChainstate(CTxMemPool& mempool, const return *to_modify; } +const AssumeutxoData* ExpectedAssumeutxo( + const int height, const CChainParams& chainparams) +{ + const MapAssumeutxo& valid_assumeutxos_map = chainparams.Assumeutxo(); + const auto assumeutxo_found = valid_assumeutxos_map.find(height); + + if (assumeutxo_found != valid_assumeutxos_map.end()) { + return &assumeutxo_found->second; + } + return nullptr; +} + +bool ChainstateManager::ActivateSnapshot( + CAutoFile& coins_file, + const SnapshotMetadata& metadata, + bool in_memory) +{ + uint256 base_blockhash = metadata.m_base_blockhash; + + if (this->SnapshotBlockhash()) { + LogPrintf("[snapshot] can't activate a snapshot-based chainstate more than once\n"); + return false; + } + + int64_t current_coinsdb_cache_size{0}; + int64_t current_coinstip_cache_size{0}; + + // Cache percentages to allocate to each chainstate. + // + // These particular percentages don't matter so much since they will only be + // relevant during snapshot activation; caches are rebalanced at the conclusion of + // this function. We want to give (essentially) all available cache capacity to the + // snapshot to aid the bulk load later in this function. + static constexpr double IBD_CACHE_PERC = 0.01; + static constexpr double SNAPSHOT_CACHE_PERC = 0.99; + + { + LOCK(::cs_main); + // Resize the coins caches to ensure we're not exceeding memory limits. + // + // Allocate the majority of the cache to the incoming snapshot chainstate, since + // (optimistically) getting to its tip will be the top priority. We'll need to call + // `MaybeRebalanceCaches()` once we're done with this function to ensure + // the right allocation (including the possibility that no snapshot was activated + // and that we should restore the active chainstate caches to their original size). + // + current_coinsdb_cache_size = this->ActiveChainstate().m_coinsdb_cache_size_bytes; + current_coinstip_cache_size = this->ActiveChainstate().m_coinstip_cache_size_bytes; + + // Temporarily resize the active coins cache to make room for the newly-created + // snapshot chain. + this->ActiveChainstate().ResizeCoinsCaches( + static_cast<size_t>(current_coinstip_cache_size * IBD_CACHE_PERC), + static_cast<size_t>(current_coinsdb_cache_size * IBD_CACHE_PERC)); + } + + auto snapshot_chainstate = WITH_LOCK(::cs_main, return MakeUnique<CChainState>( + this->ActiveChainstate().m_mempool, m_blockman, base_blockhash)); + + { + LOCK(::cs_main); + snapshot_chainstate->InitCoinsDB( + static_cast<size_t>(current_coinsdb_cache_size * SNAPSHOT_CACHE_PERC), + in_memory, false, "chainstate"); + snapshot_chainstate->InitCoinsCache( + static_cast<size_t>(current_coinstip_cache_size * SNAPSHOT_CACHE_PERC)); + } + + const bool snapshot_ok = this->PopulateAndValidateSnapshot( + *snapshot_chainstate, coins_file, metadata); + + if (!snapshot_ok) { + WITH_LOCK(::cs_main, this->MaybeRebalanceCaches()); + return false; + } + + { + LOCK(::cs_main); + assert(!m_snapshot_chainstate); + m_snapshot_chainstate.swap(snapshot_chainstate); + const bool chaintip_loaded = m_snapshot_chainstate->LoadChainTip(::Params()); + assert(chaintip_loaded); + + m_active_chainstate = m_snapshot_chainstate.get(); + + LogPrintf("[snapshot] successfully activated snapshot %s\n", base_blockhash.ToString()); + LogPrintf("[snapshot] (%.2f MB)\n", + m_snapshot_chainstate->CoinsTip().DynamicMemoryUsage() / (1000 * 1000)); + + this->MaybeRebalanceCaches(); + } + return true; +} + +bool ChainstateManager::PopulateAndValidateSnapshot( + CChainState& snapshot_chainstate, + CAutoFile& coins_file, + const SnapshotMetadata& metadata) +{ + // It's okay to release cs_main before we're done using `coins_cache` because we know + // that nothing else will be referencing the newly created snapshot_chainstate yet. + CCoinsViewCache& coins_cache = *WITH_LOCK(::cs_main, return &snapshot_chainstate.CoinsTip()); + + uint256 base_blockhash = metadata.m_base_blockhash; + + COutPoint outpoint; + Coin coin; + const uint64_t coins_count = metadata.m_coins_count; + uint64_t coins_left = metadata.m_coins_count; + + LogPrintf("[snapshot] loading coins from snapshot %s\n", base_blockhash.ToString()); + int64_t flush_now{0}; + int64_t coins_processed{0}; + + while (coins_left > 0) { + try { + coins_file >> outpoint; + } catch (const std::ios_base::failure&) { + LogPrintf("[snapshot] bad snapshot - no coins left after deserializing %d coins\n", + coins_count - coins_left); + return false; + } + coins_file >> coin; + coins_cache.EmplaceCoinInternalDANGER(std::move(outpoint), std::move(coin)); + + --coins_left; + ++coins_processed; + + if (coins_processed % 1000000 == 0) { + LogPrintf("[snapshot] %d coins loaded (%.2f%%, %.2f MB)\n", + coins_processed, + static_cast<float>(coins_processed) * 100 / static_cast<float>(coins_count), + coins_cache.DynamicMemoryUsage() / (1000 * 1000)); + } + + // Batch write and flush (if we need to) every so often. + // + // If our average Coin size is roughly 41 bytes, checking every 120,000 coins + // means <5MB of memory imprecision. + if (coins_processed % 120000 == 0) { + if (ShutdownRequested()) { + return false; + } + + const auto snapshot_cache_state = WITH_LOCK(::cs_main, + return snapshot_chainstate.GetCoinsCacheSizeState(&snapshot_chainstate.m_mempool)); + + if (snapshot_cache_state >= + CoinsCacheSizeState::CRITICAL) { + LogPrintf("[snapshot] flushing coins cache (%.2f MB)... ", /* Continued */ + coins_cache.DynamicMemoryUsage() / (1000 * 1000)); + flush_now = GetTimeMillis(); + + // This is a hack - we don't know what the actual best block is, but that + // doesn't matter for the purposes of flushing the cache here. We'll set this + // to its correct value (`base_blockhash`) below after the coins are loaded. + coins_cache.SetBestBlock(GetRandHash()); + + coins_cache.Flush(); + LogPrintf("done (%.2fms)\n", GetTimeMillis() - flush_now); + } + } + } + + // Important that we set this. This and the coins_cache accesses above are + // sort of a layer violation, but either we reach into the innards of + // CCoinsViewCache here or we have to invert some of the CChainState to + // embed them in a snapshot-activation-specific CCoinsViewCache bulk load + // method. + coins_cache.SetBestBlock(base_blockhash); + + bool out_of_coins{false}; + try { + coins_file >> outpoint; + } catch (const std::ios_base::failure&) { + // We expect an exception since we should be out of coins. + out_of_coins = true; + } + if (!out_of_coins) { + LogPrintf("[snapshot] bad snapshot - coins left over after deserializing %d coins\n", + coins_count); + return false; + } + + LogPrintf("[snapshot] loaded %d (%.2f MB) coins from snapshot %s\n", + coins_count, + coins_cache.DynamicMemoryUsage() / (1000 * 1000), + base_blockhash.ToString()); + + LogPrintf("[snapshot] flushing snapshot chainstate to disk\n"); + // No need to acquire cs_main since this chainstate isn't being used yet. + coins_cache.Flush(); // TODO: if #17487 is merged, add erase=false here for better performance. + + assert(coins_cache.GetBestBlock() == base_blockhash); + + CCoinsStats stats; + auto breakpoint_fnc = [] { /* TODO insert breakpoint here? */ }; + + // As above, okay to immediately release cs_main here since no other context knows + // about the snapshot_chainstate. + CCoinsViewDB* snapshot_coinsdb = WITH_LOCK(::cs_main, return &snapshot_chainstate.CoinsDB()); + + if (!GetUTXOStats(snapshot_coinsdb, stats, CoinStatsHashType::HASH_SERIALIZED, breakpoint_fnc)) { + LogPrintf("[snapshot] failed to generate coins stats\n"); + return false; + } + + // Ensure that the base blockhash appears in the known chain of valid headers. We're willing to + // wait a bit here because the snapshot may have been loaded on startup, before we've + // received headers from the network. + + int max_secs_to_wait_for_headers = 60 * 10; + CBlockIndex* snapshot_start_block = nullptr; + + while (max_secs_to_wait_for_headers > 0) { + snapshot_start_block = WITH_LOCK(::cs_main, + return m_blockman.LookupBlockIndex(base_blockhash)); + --max_secs_to_wait_for_headers; + + if (!snapshot_start_block) { + std::this_thread::sleep_for(std::chrono::seconds(1)); + } else { + break; + } + } + + if (snapshot_start_block == nullptr) { + LogPrintf("[snapshot] timed out waiting for snapshot start blockheader %s\n", + base_blockhash.ToString()); + return false; + } + + // Assert that the deserialized chainstate contents match the expected assumeutxo value. + + int base_height = snapshot_start_block->nHeight; + auto maybe_au_data = ExpectedAssumeutxo(base_height, ::Params()); + + if (!maybe_au_data) { + LogPrintf("[snapshot] assumeutxo height in snapshot metadata not recognized " /* Continued */ + "(%d) - refusing to load snapshot\n", base_height); + return false; + } + + const AssumeutxoData& au_data = *maybe_au_data; + + if (stats.hashSerialized != au_data.hash_serialized) { + LogPrintf("[snapshot] bad snapshot content hash: expected %s, got %s\n", + au_data.hash_serialized.ToString(), stats.hashSerialized.ToString()); + return false; + } + + snapshot_chainstate.m_chain.SetTip(snapshot_start_block); + + // The remainder of this function requires modifying data protected by cs_main. + LOCK(::cs_main); + + // Fake various pieces of CBlockIndex state: + // + // - nChainTx: so that we accurately report IBD-to-tip progress + // - nTx: so that LoadBlockIndex() loads assumed-valid CBlockIndex entries + // (among other things) + // - nStatus & BLOCK_OPT_WITNESS: so that RewindBlockIndex() doesn't zealously + // unwind the assumed-valid chain. + // + CBlockIndex* index = nullptr; + for (int i = 0; i <= snapshot_chainstate.m_chain.Height(); ++i) { + index = snapshot_chainstate.m_chain[i]; + + if (!index->nTx) { + index->nTx = 1; + } + index->nChainTx = index->pprev ? index->pprev->nChainTx + index->nTx : 1; + + // We need to fake this flag so that CChainState::RewindBlockIndex() + // won't try to rewind the entire assumed-valid chain on startup. + if (index->pprev && ::IsWitnessEnabled(index->pprev, ::Params().GetConsensus())) { + index->nStatus |= BLOCK_OPT_WITNESS; + } + } + + assert(index); + index->nChainTx = metadata.m_nchaintx; + snapshot_chainstate.setBlockIndexCandidates.insert(snapshot_start_block); + + LogPrintf("[snapshot] validated snapshot (%.2f MB)\n", + coins_cache.DynamicMemoryUsage() / (1000 * 1000)); + return true; +} + CChainState& ChainstateManager::ActiveChainstate() const { LOCK(::cs_main); diff --git a/src/validation.h b/src/validation.h index 238d6009b4..d6058e7dac 100644 --- a/src/validation.h +++ b/src/validation.h @@ -11,10 +11,12 @@ #endif #include <amount.h> +#include <attributes.h> #include <coins.h> #include <consensus/validation.h> #include <crypto/common.h> // for ReadLE64 #include <fs.h> +#include <node/utxo_snapshot.h> #include <optional.h> #include <policy/feerate.h> #include <protocol.h> // For CMessageHeader::MessageStartChars @@ -53,6 +55,7 @@ struct ChainTxData; struct DisconnectedBlockTransactions; struct PrecomputedTransactionData; struct LockPoints; +struct AssumeutxoData; /** Default for -minrelaytxfee, minimum relay fee for transactions */ static const unsigned int DEFAULT_MIN_RELAY_TX_FEE = 1000; @@ -830,9 +833,7 @@ private: //! using this pointer (e.g. net_processing). //! //! Once this pointer is set to a corresponding chainstate, it will not - //! be reset until init.cpp:Shutdown(). This means it is safe to acquire - //! the contents of this pointer with ::cs_main held, release the lock, - //! and then use the reference without concern of it being deconstructed. + //! be reset until init.cpp:Shutdown(). //! //! This is especially important when, e.g., calling ActivateBestChain() //! on all chainstates because we are not able to hold ::cs_main going into @@ -843,9 +844,7 @@ private: //! non-null, it is always our active chainstate. //! //! Once this pointer is set to a corresponding chainstate, it will not - //! be reset until init.cpp:Shutdown(). This means it is safe to acquire - //! the contents of this pointer with ::cs_main held, release the lock, - //! and then use the reference without concern of it being deconstructed. + //! be reset until init.cpp:Shutdown(). //! //! This is especially important when, e.g., calling ActivateBestChain() //! on all chainstates because we are not able to hold ::cs_main going into @@ -856,9 +855,7 @@ private: //! most-work chain. //! //! Once this pointer is set to a corresponding chainstate, it will not - //! be reset until init.cpp:Shutdown(). This means it is safe to acquire - //! the contents of this pointer with ::cs_main held, release the lock, - //! and then use the reference without concern of it being deconstructed. + //! be reset until init.cpp:Shutdown(). //! //! This is especially important when, e.g., calling ActivateBestChain() //! on all chainstates because we are not able to hold ::cs_main going into @@ -869,6 +866,12 @@ private: //! by the background validation chainstate. bool m_snapshot_validated{false}; + //! Internal helper for ActivateSnapshot(). + [[nodiscard]] bool PopulateAndValidateSnapshot( + CChainState& snapshot_chainstate, + CAutoFile& coins_file, + const SnapshotMetadata& metadata); + // For access to m_active_chainstate. friend CChainState& ChainstateActive(); friend CChain& ChainActive(); @@ -899,6 +902,22 @@ public: //! Get all chainstates currently being used. std::vector<CChainState*> GetAll(); + //! Construct and activate a Chainstate on the basis of UTXO snapshot data. + //! + //! Steps: + //! + //! - Initialize an unused CChainState. + //! - Load its `CoinsViews` contents from `coins_file`. + //! - Verify that the hash of the resulting coinsdb matches the expected hash + //! per assumeutxo chain parameters. + //! - Wait for our headers chain to include the base block of the snapshot. + //! - "Fast forward" the tip of the new chainstate to the base of the snapshot, + //! faking nTx* block index data along the way. + //! - Move the new chainstate to `m_snapshot_chainstate` and make it our + //! ChainstateActive(). + [[nodiscard]] bool ActivateSnapshot( + CAutoFile& coins_file, const SnapshotMetadata& metadata, bool in_memory); + //! The most-work chain. CChainState& ActiveChainstate() const; CChain& ActiveChain() const { return ActiveChainstate().m_chain; } @@ -1013,4 +1032,13 @@ inline bool IsBlockPruned(const CBlockIndex* pblockindex) return (fHavePruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0); } +/** + * Return the expected assumeutxo value for a given height, if one exists. + * + * @param height[in] Get the assumeutxo value for this height. + * + * @returns empty if no assumeutxo configuration exists for the given height. + */ +const AssumeutxoData* ExpectedAssumeutxo(const int height, const CChainParams& params); + #endif // BITCOIN_VALIDATION_H |