diff options
Diffstat (limited to 'src/main.cpp')
-rw-r--r-- | src/main.cpp | 4914 |
1 files changed, 3569 insertions, 1345 deletions
diff --git a/src/main.cpp b/src/main.cpp index 9e1c41ada7..73fbe53afb 100644 --- a/src/main.cpp +++ b/src/main.cpp @@ -1,33 +1,53 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2014 The Bitcoin developers +// Copyright (c) 2009-2015 The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include "main.h" #include "addrman.h" -#include "alert.h" +#include "arith_uint256.h" +#include "blockencodings.h" #include "chainparams.h" #include "checkpoints.h" #include "checkqueue.h" +#include "consensus/consensus.h" +#include "consensus/merkle.h" +#include "consensus/validation.h" +#include "hash.h" #include "init.h" #include "merkleblock.h" #include "net.h" +#include "policy/fees.h" +#include "policy/policy.h" #include "pow.h" +#include "primitives/block.h" +#include "primitives/transaction.h" +#include "random.h" +#include "script/script.h" +#include "script/sigcache.h" +#include "script/standard.h" +#include "tinyformat.h" #include "txdb.h" #include "txmempool.h" #include "ui_interface.h" +#include "undo.h" #include "util.h" #include "utilmoneystr.h" +#include "utilstrencodings.h" +#include "validationinterface.h" +#include "versionbits.h" +#include <atomic> #include <sstream> #include <boost/algorithm/string/replace.hpp> +#include <boost/algorithm/string/join.hpp> #include <boost/filesystem.hpp> #include <boost/filesystem/fstream.hpp> +#include <boost/math/distributions/poisson.hpp> #include <boost/thread.hpp> -using namespace boost; using namespace std; #if defined(NDEBUG) @@ -50,22 +70,49 @@ int nScriptCheckThreads = 0; bool fImporting = false; bool fReindex = false; bool fTxIndex = false; -bool fIsBareMultisigStd = true; -unsigned int nCoinCacheSize = 5000; - - -/** Fees smaller than this (in satoshi) are considered zero fee (for relaying and mining) */ -CFeeRate minRelayTxFee = CFeeRate(1000); +bool fHavePruned = false; +bool fPruneMode = false; +bool fIsBareMultisigStd = DEFAULT_PERMIT_BAREMULTISIG; +bool fRequireStandard = true; +unsigned int nBytesPerSigOp = DEFAULT_BYTES_PER_SIGOP; +bool fCheckBlockIndex = false; +bool fCheckpointsEnabled = DEFAULT_CHECKPOINTS_ENABLED; +size_t nCoinCacheUsage = 5000 * 300; +uint64_t nPruneTarget = 0; +int64_t nMaxTipAge = DEFAULT_MAX_TIP_AGE; +bool fEnableReplacement = DEFAULT_ENABLE_REPLACEMENT; + + +CFeeRate minRelayTxFee = CFeeRate(DEFAULT_MIN_RELAY_TX_FEE); +CAmount maxTxFee = DEFAULT_TRANSACTION_MAXFEE; CTxMemPool mempool(::minRelayTxFee); +FeeFilterRounder filterRounder(::minRelayTxFee); + +struct IteratorComparator +{ + template<typename I> + bool operator()(const I& a, const I& b) + { + return &(*a) < &(*b); + } +}; struct COrphanTx { CTransaction tx; NodeId fromPeer; + int64_t nTimeExpire; }; -map<uint256, COrphanTx> mapOrphanTransactions; -map<uint256, set<uint256> > mapOrphanTransactionsByPrev; -void EraseOrphansFor(NodeId peer); +map<uint256, COrphanTx> mapOrphanTransactions GUARDED_BY(cs_main); +map<COutPoint, set<map<uint256, COrphanTx>::iterator, IteratorComparator>> mapOrphanTransactionsByPrev GUARDED_BY(cs_main); +void EraseOrphansFor(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + +/** + * Returns true if there are nRequired or more blocks of minVersion or above + * in the last Consensus::Params::nMajorityWindow blocks, starting at pstart and going backwards. + */ +static bool IsSuperMajority(int minVersion, const CBlockIndex* pstart, unsigned nRequired, const Consensus::Params& consensusParams); +static void CheckBlockIndex(const Consensus::Params& consensusParams); /** Constant stuff for coinbase transactions we create: */ CScript COINBASE_FLAGS; @@ -77,7 +124,7 @@ namespace { struct CBlockIndexWorkComparator { - bool operator()(CBlockIndex *pa, CBlockIndex *pb) { + bool operator()(CBlockIndex *pa, CBlockIndex *pb) const { // First sort by most total work, ... if (pa->nChainWork > pb->nChainWork) return false; if (pa->nChainWork < pb->nChainWork) return true; @@ -99,18 +146,26 @@ namespace { CBlockIndex *pindexBestInvalid; /** - * The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS or better that are at least - * as good as our current tip. Entries may be failed, though. + * The set of all CBlockIndex entries with BLOCK_VALID_TRANSACTIONS (for itself and all ancestors) and + * as good as our current tip or better. Entries may be failed, though, and pruning nodes may be + * missing the data for the block. */ set<CBlockIndex*, CBlockIndexWorkComparator> setBlockIndexCandidates; /** Number of nodes with fSyncStarted. */ int nSyncStarted = 0; - /** All pairs A->B, where A (or one if its ancestors) misses transactions, but B has transactions. */ + /** All pairs A->B, where A (or one of its ancestors) misses transactions, but B has transactions. + * Pruned nodes may have entries where B is missing data. + */ multimap<CBlockIndex*, CBlockIndex*> mapBlocksUnlinked; CCriticalSection cs_LastBlockFile; std::vector<CBlockFileInfo> vinfoBlockFile; int nLastBlockFile = 0; + /** Global flag to indicate we should check to see if there are + * block/undo files that should be deleted. Set on startup + * or if we allocate more file space when we're in prune mode + */ + bool fCheckForPruning = false; /** * Every received block is assigned a unique and increasing identifier, so we @@ -121,19 +176,47 @@ namespace { uint32_t nBlockSequenceId = 1; /** - * Sources of received blocks, to be able to send them reject messages or ban - * them, if processing happens afterwards. Protected by cs_main. + * Sources of received blocks, saved to be able to send them reject + * messages or ban them when processing happens afterwards. Protected by + * cs_main. */ map<uint256, NodeId> mapBlockSource; + /** + * Filter for transactions that were recently rejected by + * AcceptToMemoryPool. These are not rerequested until the chain tip + * changes, at which point the entire filter is reset. Protected by + * cs_main. + * + * Without this filter we'd be re-requesting txs from each of our peers, + * increasing bandwidth consumption considerably. For instance, with 100 + * peers, half of which relay a tx we don't accept, that might be a 50x + * bandwidth increase. A flooding attacker attempting to roll-over the + * filter using minimum-sized, 60byte, transactions might manage to send + * 1000/sec if we have fast peers, so we pick 120,000 to give our peers a + * two minute window to send invs to us. + * + * Decreasing the false positive rate is fairly cheap, so we pick one in a + * million to make it highly unlikely for users to have issues with this + * filter. + * + * Memory used: 1.3 MB + */ + boost::scoped_ptr<CRollingBloomFilter> recentRejects; + uint256 hashRecentRejectsChainTip; + /** Blocks that are in flight, and that are in the queue to be downloaded. Protected by cs_main. */ struct QueuedBlock { uint256 hash; - CBlockIndex *pindex; //! Optional. - int64_t nTime; //! Time of "getdata" request in microseconds. + CBlockIndex* pindex; //!< Optional. + bool fValidatedHeaders; //!< Whether this block has validated headers at the time of request. + std::unique_ptr<PartiallyDownloadedBlock> partialBlock; //!< Optional, used for CMPCTBLOCK downloads }; map<uint256, pair<NodeId, list<QueuedBlock>::iterator> > mapBlocksInFlight; + /** Stack of nodes which we have set to announce using compact blocks */ + list<NodeId> lNodesAnnouncingHeaderAndIDs; + /** Number of preferable block download peers. */ int nPreferredDownload = 0; @@ -142,70 +225,17 @@ namespace { /** Dirty block file entries. */ set<int> setDirtyFileInfo; -} // anon namespace - -////////////////////////////////////////////////////////////////////////////// -// -// dispatching functions -// - -// These functions dispatch to one or all registered wallets - -namespace { -struct CMainSignals { - /** Notifies listeners of updated transaction data (transaction, and optionally the block it is found in. */ - boost::signals2::signal<void (const CTransaction &, const CBlock *)> SyncTransaction; - /** Notifies listeners of an erased transaction (currently disabled, requires transaction replacement). */ - boost::signals2::signal<void (const uint256 &)> EraseTransaction; - /** Notifies listeners of an updated transaction without new data (for now: a coinbase potentially becoming visible). */ - boost::signals2::signal<void (const uint256 &)> UpdatedTransaction; - /** Notifies listeners of a new active block chain. */ - boost::signals2::signal<void (const CBlockLocator &)> SetBestChain; - /** Notifies listeners about an inventory item being seen on the network. */ - boost::signals2::signal<void (const uint256 &)> Inventory; - /** Tells listeners to broadcast their data. */ - boost::signals2::signal<void ()> Broadcast; - /** Notifies listeners of a block validation result */ - boost::signals2::signal<void (const CBlock&, const CValidationState&)> BlockChecked; -} g_signals; + /** Number of peers from which we're downloading blocks. */ + int nPeersWithValidatedDownloads = 0; + /** Relay map, protected by cs_main. */ + typedef std::map<uint256, std::shared_ptr<const CTransaction>> MapRelay; + MapRelay mapRelay; + /** Expiration-time ordered list of (expire time, relay map entry) pairs, protected by cs_main). */ + std::deque<std::pair<int64_t, MapRelay::iterator>> vRelayExpiration; } // anon namespace -void RegisterValidationInterface(CValidationInterface* pwalletIn) { - g_signals.SyncTransaction.connect(boost::bind(&CValidationInterface::SyncTransaction, pwalletIn, _1, _2)); - g_signals.EraseTransaction.connect(boost::bind(&CValidationInterface::EraseFromWallet, pwalletIn, _1)); - g_signals.UpdatedTransaction.connect(boost::bind(&CValidationInterface::UpdatedTransaction, pwalletIn, _1)); - g_signals.SetBestChain.connect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); - g_signals.Inventory.connect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); - g_signals.Broadcast.connect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn)); - g_signals.BlockChecked.connect(boost::bind(&CValidationInterface::BlockChecked, pwalletIn, _1, _2)); -} - -void UnregisterValidationInterface(CValidationInterface* pwalletIn) { - g_signals.BlockChecked.disconnect(boost::bind(&CValidationInterface::BlockChecked, pwalletIn, _1, _2)); - g_signals.Broadcast.disconnect(boost::bind(&CValidationInterface::ResendWalletTransactions, pwalletIn)); - g_signals.Inventory.disconnect(boost::bind(&CValidationInterface::Inventory, pwalletIn, _1)); - g_signals.SetBestChain.disconnect(boost::bind(&CValidationInterface::SetBestChain, pwalletIn, _1)); - g_signals.UpdatedTransaction.disconnect(boost::bind(&CValidationInterface::UpdatedTransaction, pwalletIn, _1)); - g_signals.EraseTransaction.disconnect(boost::bind(&CValidationInterface::EraseFromWallet, pwalletIn, _1)); - g_signals.SyncTransaction.disconnect(boost::bind(&CValidationInterface::SyncTransaction, pwalletIn, _1, _2)); -} - -void UnregisterAllValidationInterfaces() { - g_signals.BlockChecked.disconnect_all_slots(); - g_signals.Broadcast.disconnect_all_slots(); - g_signals.Inventory.disconnect_all_slots(); - g_signals.SetBestChain.disconnect_all_slots(); - g_signals.UpdatedTransaction.disconnect_all_slots(); - g_signals.EraseTransaction.disconnect_all_slots(); - g_signals.SyncTransaction.disconnect_all_slots(); -} - -void SyncWithWallets(const CTransaction &tx, const CBlock *pblock) { - g_signals.SyncTransaction(tx, pblock); -} - ////////////////////////////////////////////////////////////////////////////// // // Registration of network node signals. @@ -226,6 +256,10 @@ struct CBlockReject { * and we're no longer holding the node's locks. */ struct CNodeState { + //! The peer's address + CService address; + //! Whether we have a fully established connection. + bool fCurrentlyConnected; //! Accumulated misbehaviour score for this peer. int nMisbehavior; //! Whether this peer should be disconnected and banned (unless whitelisted). @@ -240,25 +274,49 @@ struct CNodeState { uint256 hashLastUnknownBlock; //! The last full block we both have. CBlockIndex *pindexLastCommonBlock; + //! The best header we have sent our peer. + CBlockIndex *pindexBestHeaderSent; + //! Length of current-streak of unconnecting headers announcements + int nUnconnectingHeaders; //! Whether we've started headers synchronization with this peer. bool fSyncStarted; //! Since when we're stalling block download progress (in microseconds), or 0. int64_t nStallingSince; list<QueuedBlock> vBlocksInFlight; + //! When the first entry in vBlocksInFlight started downloading. Don't care when vBlocksInFlight is empty. + int64_t nDownloadingSince; int nBlocksInFlight; + int nBlocksInFlightValidHeaders; //! Whether we consider this a preferred download peer. bool fPreferredDownload; + //! Whether this peer wants invs or headers (when possible) for block announcements. + bool fPreferHeaders; + //! Whether this peer wants invs or cmpctblocks (when possible) for block announcements. + bool fPreferHeaderAndIDs; + //! Whether this peer will send us cmpctblocks if we request them + bool fProvidesHeaderAndIDs; + //! Whether this peer can give us witnesses + bool fHaveWitness; CNodeState() { + fCurrentlyConnected = false; nMisbehavior = 0; fShouldBan = false; pindexBestKnownBlock = NULL; - hashLastUnknownBlock = uint256(0); + hashLastUnknownBlock.SetNull(); pindexLastCommonBlock = NULL; + pindexBestHeaderSent = NULL; + nUnconnectingHeaders = 0; fSyncStarted = false; nStallingSince = 0; + nDownloadingSince = 0; nBlocksInFlight = 0; + nBlocksInFlightValidHeaders = 0; fPreferredDownload = false; + fPreferHeaders = false; + fPreferHeaderAndIDs = false; + fProvidesHeaderAndIDs = false; + fHaveWitness = false; } }; @@ -293,6 +351,7 @@ void InitializeNode(NodeId nodeid, const CNode *pnode) { LOCK(cs_main); CNodeState &state = mapNodeState.insert(std::make_pair(nodeid, CNodeState())).first->second; state.name = pnode->addrName; + state.address = pnode->addr; } void FinalizeNode(NodeId nodeid) { @@ -302,51 +361,98 @@ void FinalizeNode(NodeId nodeid) { if (state->fSyncStarted) nSyncStarted--; - BOOST_FOREACH(const QueuedBlock& entry, state->vBlocksInFlight) + if (state->nMisbehavior == 0 && state->fCurrentlyConnected) { + AddressCurrentlyConnected(state->address); + } + + BOOST_FOREACH(const QueuedBlock& entry, state->vBlocksInFlight) { mapBlocksInFlight.erase(entry.hash); + } EraseOrphansFor(nodeid); nPreferredDownload -= state->fPreferredDownload; + nPeersWithValidatedDownloads -= (state->nBlocksInFlightValidHeaders != 0); + assert(nPeersWithValidatedDownloads >= 0); mapNodeState.erase(nodeid); + + if (mapNodeState.empty()) { + // Do a consistency check after the last peer is removed. + assert(mapBlocksInFlight.empty()); + assert(nPreferredDownload == 0); + assert(nPeersWithValidatedDownloads == 0); + } } // Requires cs_main. -void MarkBlockAsReceived(const uint256& hash) { +// Returns a bool indicating whether we requested this block. +// Also used if a block was /not/ received and timed out or started with another peer +bool MarkBlockAsReceived(const uint256& hash) { map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); if (itInFlight != mapBlocksInFlight.end()) { CNodeState *state = State(itInFlight->second.first); + state->nBlocksInFlightValidHeaders -= itInFlight->second.second->fValidatedHeaders; + if (state->nBlocksInFlightValidHeaders == 0 && itInFlight->second.second->fValidatedHeaders) { + // Last validated block on the queue was received. + nPeersWithValidatedDownloads--; + } + if (state->vBlocksInFlight.begin() == itInFlight->second.second) { + // First block on the queue was received, update the start download time for the next one + state->nDownloadingSince = std::max(state->nDownloadingSince, GetTimeMicros()); + } state->vBlocksInFlight.erase(itInFlight->second.second); state->nBlocksInFlight--; state->nStallingSince = 0; mapBlocksInFlight.erase(itInFlight); + return true; } + return false; } // Requires cs_main. -void MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, CBlockIndex *pindex = NULL) { +// returns false, still setting pit, if the block was already in flight from the same peer +// pit will only be valid as long as the same cs_main lock is being held +bool MarkBlockAsInFlight(NodeId nodeid, const uint256& hash, const Consensus::Params& consensusParams, CBlockIndex *pindex = NULL, list<QueuedBlock>::iterator **pit = NULL) { CNodeState *state = State(nodeid); assert(state != NULL); + // Short-circuit most stuff in case its from the same node + map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator itInFlight = mapBlocksInFlight.find(hash); + if (itInFlight != mapBlocksInFlight.end() && itInFlight->second.first == nodeid) { + *pit = &itInFlight->second.second; + return false; + } + // Make sure it's not listed somewhere already. MarkBlockAsReceived(hash); - QueuedBlock newentry = {hash, pindex, GetTimeMicros()}; - list<QueuedBlock>::iterator it = state->vBlocksInFlight.insert(state->vBlocksInFlight.end(), newentry); + list<QueuedBlock>::iterator it = state->vBlocksInFlight.insert(state->vBlocksInFlight.end(), + {hash, pindex, pindex != NULL, std::unique_ptr<PartiallyDownloadedBlock>(pit ? new PartiallyDownloadedBlock(&mempool) : NULL)}); state->nBlocksInFlight++; - mapBlocksInFlight[hash] = std::make_pair(nodeid, it); + state->nBlocksInFlightValidHeaders += it->fValidatedHeaders; + if (state->nBlocksInFlight == 1) { + // We're starting a block download (batch) from this peer. + state->nDownloadingSince = GetTimeMicros(); + } + if (state->nBlocksInFlightValidHeaders == 1 && pindex != NULL) { + nPeersWithValidatedDownloads++; + } + itInFlight = mapBlocksInFlight.insert(std::make_pair(hash, std::make_pair(nodeid, it))).first; + if (pit) + *pit = &itInFlight->second.second; + return true; } -/** Check whether the last unknown block a peer advertized is not yet known. */ +/** Check whether the last unknown block a peer advertised is not yet known. */ void ProcessBlockAvailability(NodeId nodeid) { CNodeState *state = State(nodeid); assert(state != NULL); - if (state->hashLastUnknownBlock != 0) { + if (!state->hashLastUnknownBlock.IsNull()) { BlockMap::iterator itOld = mapBlockIndex.find(state->hashLastUnknownBlock); if (itOld != mapBlockIndex.end() && itOld->second->nChainWork > 0) { if (state->pindexBestKnownBlock == NULL || itOld->second->nChainWork >= state->pindexBestKnownBlock->nChainWork) state->pindexBestKnownBlock = itOld->second; - state->hashLastUnknownBlock = uint256(0); + state->hashLastUnknownBlock.SetNull(); } } } @@ -369,6 +475,48 @@ void UpdateBlockAvailability(NodeId nodeid, const uint256 &hash) { } } +void MaybeSetPeerAsAnnouncingHeaderAndIDs(const CNodeState* nodestate, CNode* pfrom) { + if (nLocalServices & NODE_WITNESS) { + // Don't ever request compact blocks when segwit is enabled. + return; + } + if (nodestate->fProvidesHeaderAndIDs) { + BOOST_FOREACH(const NodeId nodeid, lNodesAnnouncingHeaderAndIDs) + if (nodeid == pfrom->GetId()) + return; + bool fAnnounceUsingCMPCTBLOCK = false; + uint64_t nCMPCTBLOCKVersion = 1; + if (lNodesAnnouncingHeaderAndIDs.size() >= 3) { + // As per BIP152, we only get 3 of our peers to announce + // blocks using compact encodings. + CNode* pnodeStop = FindNode(lNodesAnnouncingHeaderAndIDs.front()); + if (pnodeStop) { + pnodeStop->PushMessage(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); + lNodesAnnouncingHeaderAndIDs.pop_front(); + } + } + fAnnounceUsingCMPCTBLOCK = true; + pfrom->PushMessage(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); + lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); + } +} + +// Requires cs_main +bool CanDirectFetch(const Consensus::Params &consensusParams) +{ + return chainActive.Tip()->GetBlockTime() > GetAdjustedTime() - consensusParams.nPowTargetSpacing * 20; +} + +// Requires cs_main +bool PeerHasHeader(CNodeState *state, CBlockIndex *pindex) +{ + if (state->pindexBestKnownBlock && pindex == state->pindexBestKnownBlock->GetAncestor(pindex->nHeight)) + return true; + if (state->pindexBestHeaderSent && pindex == state->pindexBestHeaderSent->GetAncestor(pindex->nHeight)) + return true; + return false; +} + /** Find the last common ancestor two blocks have. * Both pa and pb must be non-NULL. */ CBlockIndex* LastCommonAncestor(CBlockIndex* pa, CBlockIndex* pb) { @@ -413,7 +561,7 @@ void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vector<CBl } // If the peer reorganized, our previous pindexLastCommonBlock may not be an ancestor - // of their current tip anymore. Go back enough to fix that. + // of its current tip anymore. Go back enough to fix that. state->pindexLastCommonBlock = LastCommonAncestor(state->pindexLastCommonBlock, state->pindexBestKnownBlock); if (state->pindexLastCommonBlock == state->pindexBestKnownBlock) return; @@ -440,9 +588,14 @@ void FindNextBlocksToDownload(NodeId nodeid, unsigned int count, std::vector<CBl // Iterate over those blocks in vToFetch (in forward direction), adding the ones that // are not yet downloaded and not in flight to vBlocks. In the mean time, update - // pindexLastCommonBlock as long as all ancestors are already downloaded. + // pindexLastCommonBlock as long as all ancestors are already downloaded, or if it's + // already part of our chain (and therefore don't need it even if pruned). BOOST_FOREACH(CBlockIndex* pindex, vToFetch) { - if (pindex->nStatus & BLOCK_HAVE_DATA) { + if (!pindex->IsValid(BLOCK_VALID_TREE)) { + // We consider the chain that this peer is on invalid. + return; + } + if (pindex->nStatus & BLOCK_HAVE_DATA || chainActive.Contains(pindex)) { if (pindex->nChainTx) state->pindexLastCommonBlock = pindex; } else if (mapBlocksInFlight.count(pindex->GetBlockHash()) == 0) { @@ -512,6 +665,9 @@ CBlockIndex* FindForkInGlobalIndex(const CChain& chain, const CBlockLocator& loc CBlockIndex* pindex = (*mi).second; if (chain.Contains(pindex)) return pindex; + if (pindex->GetAncestor(chain.Height()) == chain.Tip()) { + return chain.Tip(); + } } } return chain.Genesis(); @@ -525,7 +681,7 @@ CBlockTreeDB *pblocktree = NULL; // mapOrphanTransactions // -bool AddOrphanTx(const CTransaction& tx, NodeId peer) +bool AddOrphanTx(const CTransaction& tx, NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { uint256 hash = tx.GetHash(); if (mapOrphanTransactions.count(hash)) @@ -536,40 +692,42 @@ bool AddOrphanTx(const CTransaction& tx, NodeId peer) // large transaction with a missing parent then we assume // it will rebroadcast it later, after the parent transaction(s) // have been mined or received. - // 10,000 orphans, each of which is at most 5,000 bytes big is - // at most 500 megabytes of orphans: - unsigned int sz = tx.GetSerializeSize(SER_NETWORK, CTransaction::CURRENT_VERSION); - if (sz > 5000) + // 100 orphans, each of which is at most 99,999 bytes big is + // at most 10 megabytes of orphans and somewhat more byprev index (in the worst case): + unsigned int sz = GetTransactionCost(tx); + if (sz >= MAX_STANDARD_TX_COST) { LogPrint("mempool", "ignoring large orphan tx (size: %u, hash: %s)\n", sz, hash.ToString()); return false; } - mapOrphanTransactions[hash].tx = tx; - mapOrphanTransactions[hash].fromPeer = peer; - BOOST_FOREACH(const CTxIn& txin, tx.vin) - mapOrphanTransactionsByPrev[txin.prevout.hash].insert(hash); + auto ret = mapOrphanTransactions.emplace(hash, COrphanTx{tx, peer, GetTime() + ORPHAN_TX_EXPIRE_TIME}); + assert(ret.second); + BOOST_FOREACH(const CTxIn& txin, tx.vin) { + mapOrphanTransactionsByPrev[txin.prevout].insert(ret.first); + } - LogPrint("mempool", "stored orphan tx %s (mapsz %u prevsz %u)\n", hash.ToString(), + LogPrint("mempool", "stored orphan tx %s (mapsz %u outsz %u)\n", hash.ToString(), mapOrphanTransactions.size(), mapOrphanTransactionsByPrev.size()); return true; } -void static EraseOrphanTx(uint256 hash) +int static EraseOrphanTx(uint256 hash) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { map<uint256, COrphanTx>::iterator it = mapOrphanTransactions.find(hash); if (it == mapOrphanTransactions.end()) - return; + return 0; BOOST_FOREACH(const CTxIn& txin, it->second.tx.vin) { - map<uint256, set<uint256> >::iterator itPrev = mapOrphanTransactionsByPrev.find(txin.prevout.hash); + auto itPrev = mapOrphanTransactionsByPrev.find(txin.prevout); if (itPrev == mapOrphanTransactionsByPrev.end()) continue; - itPrev->second.erase(hash); + itPrev->second.erase(it); if (itPrev->second.empty()) mapOrphanTransactionsByPrev.erase(itPrev); } mapOrphanTransactions.erase(it); + return 1; } void EraseOrphansFor(NodeId peer) @@ -581,17 +739,36 @@ void EraseOrphansFor(NodeId peer) map<uint256, COrphanTx>::iterator maybeErase = iter++; // increment to avoid iterator becoming invalid if (maybeErase->second.fromPeer == peer) { - EraseOrphanTx(maybeErase->second.tx.GetHash()); - ++nErased; + nErased += EraseOrphanTx(maybeErase->second.tx.GetHash()); } } if (nErased > 0) LogPrint("mempool", "Erased %d orphan tx from peer %d\n", nErased, peer); } -unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) +unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { unsigned int nEvicted = 0; + static int64_t nNextSweep; + int64_t nNow = GetTime(); + if (nNextSweep <= nNow) { + // Sweep out expired orphan pool entries: + int nErased = 0; + int64_t nMinExpTime = nNow + ORPHAN_TX_EXPIRE_TIME - ORPHAN_TX_EXPIRE_INTERVAL; + map<uint256, COrphanTx>::iterator iter = mapOrphanTransactions.begin(); + while (iter != mapOrphanTransactions.end()) + { + map<uint256, COrphanTx>::iterator maybeErase = iter++; + if (maybeErase->second.nTimeExpire <= nNow) { + nErased += EraseOrphanTx(maybeErase->second.tx.GetHash()); + } else { + nMinExpTime = std::min(maybeErase->second.nTimeExpire, nMinExpTime); + } + } + // Sweep again 5 minutes after the next entry that expires in order to batch the linear scan. + nNextSweep = nMinExpTime + ORPHAN_TX_EXPIRE_INTERVAL; + if (nErased > 0) LogPrint("mempool", "Erased %d orphan tx due to expiration\n", nErased); + } while (mapOrphanTransactions.size() > nMaxOrphans) { // Evict a random orphan: @@ -605,185 +782,223 @@ unsigned int LimitOrphanTxSize(unsigned int nMaxOrphans) return nEvicted; } - - - - - - -bool IsStandardTx(const CTransaction& tx, string& reason) +bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime) { - AssertLockHeld(cs_main); - if (tx.nVersion > CTransaction::CURRENT_VERSION || tx.nVersion < 1) { - reason = "version"; - return false; - } - - // Treat non-final transactions as non-standard to prevent a specific type - // of double-spend attack, as well as DoS attacks. (if the transaction - // can't be mined, the attacker isn't expending resources broadcasting it) - // Basically we don't want to propagate transactions that can't be included in - // the next block. - // - // However, IsFinalTx() is confusing... Without arguments, it uses - // chainActive.Height() to evaluate nLockTime; when a block is accepted, chainActive.Height() - // is set to the value of nHeight in the block. However, when IsFinalTx() - // is called within CBlock::AcceptBlock(), the height of the block *being* - // evaluated is what is used. Thus if we want to know if a transaction can - // be part of the *next* block, we need to call IsFinalTx() with one more - // than chainActive.Height(). - // - // Timestamps on the other hand don't get any special treatment, because we - // can't know what timestamp the next block will have, and there aren't - // timestamp applications where it matters. - if (!IsFinalTx(tx, chainActive.Height() + 1)) { - reason = "non-final"; - return false; + if (tx.nLockTime == 0) + return true; + if ((int64_t)tx.nLockTime < ((int64_t)tx.nLockTime < LOCKTIME_THRESHOLD ? (int64_t)nBlockHeight : nBlockTime)) + return true; + BOOST_FOREACH(const CTxIn& txin, tx.vin) { + if (!(txin.nSequence == CTxIn::SEQUENCE_FINAL)) + return false; } + return true; +} - // Extremely large transactions with lots of inputs can cost the network - // almost as much to process as they cost the sender in fees, because - // computing signature hashes is O(ninputs*txsize). Limiting transactions - // to MAX_STANDARD_TX_SIZE mitigates CPU exhaustion attacks. - unsigned int sz = tx.GetSerializeSize(SER_NETWORK, CTransaction::CURRENT_VERSION); - if (sz >= MAX_STANDARD_TX_SIZE) { - reason = "tx-size"; - return false; - } +bool CheckFinalTx(const CTransaction &tx, int flags) +{ + AssertLockHeld(cs_main); - BOOST_FOREACH(const CTxIn& txin, tx.vin) - { - // Biggest 'standard' txin is a 15-of-15 P2SH multisig with compressed - // keys. (remember the 520 byte limit on redeemScript size) That works - // out to a (15*(33+1))+3=513 byte redeemScript, 513+1+15*(73+1)+3=1627 - // bytes of scriptSig, which we round off to 1650 bytes for some minor - // future-proofing. That's also enough to spend a 20-of-20 - // CHECKMULTISIG scriptPubKey, though such a scriptPubKey is not - // considered standard) - if (txin.scriptSig.size() > 1650) { - reason = "scriptsig-size"; - return false; - } - if (!txin.scriptSig.IsPushOnly()) { - reason = "scriptsig-not-pushonly"; - return false; - } - } + // By convention a negative value for flags indicates that the + // current network-enforced consensus rules should be used. In + // a future soft-fork scenario that would mean checking which + // rules would be enforced for the next block and setting the + // appropriate flags. At the present time no soft-forks are + // scheduled, so no flags are set. + flags = std::max(flags, 0); + + // CheckFinalTx() uses chainActive.Height()+1 to evaluate + // nLockTime because when IsFinalTx() is called within + // CBlock::AcceptBlock(), the height of the block *being* + // evaluated is what is used. Thus if we want to know if a + // transaction can be part of the *next* block, we need to call + // IsFinalTx() with one more than chainActive.Height(). + const int nBlockHeight = chainActive.Height() + 1; + + // BIP113 will require that time-locked transactions have nLockTime set to + // less than the median time of the previous block they're contained in. + // When the next block is created its previous block will be the current + // chain tip, so we use that to calculate the median time passed to + // IsFinalTx() if LOCKTIME_MEDIAN_TIME_PAST is set. + const int64_t nBlockTime = (flags & LOCKTIME_MEDIAN_TIME_PAST) + ? chainActive.Tip()->GetMedianTimePast() + : GetAdjustedTime(); + + return IsFinalTx(tx, nBlockHeight, nBlockTime); +} - unsigned int nDataOut = 0; - txnouttype whichType; - BOOST_FOREACH(const CTxOut& txout, tx.vout) { - if (!::IsStandard(txout.scriptPubKey, whichType)) { - reason = "scriptpubkey"; - return false; +/** + * Calculates the block height and previous block's median time past at + * which the transaction will be considered final in the context of BIP 68. + * Also removes from the vector of input heights any entries which did not + * correspond to sequence locked inputs as they do not affect the calculation. + */ +static std::pair<int, int64_t> CalculateSequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block) +{ + assert(prevHeights->size() == tx.vin.size()); + + // Will be set to the equivalent height- and time-based nLockTime + // values that would be necessary to satisfy all relative lock- + // time constraints given our view of block chain history. + // The semantics of nLockTime are the last invalid height/time, so + // use -1 to have the effect of any height or time being valid. + int nMinHeight = -1; + int64_t nMinTime = -1; + + // tx.nVersion is signed integer so requires cast to unsigned otherwise + // we would be doing a signed comparison and half the range of nVersion + // wouldn't support BIP 68. + bool fEnforceBIP68 = static_cast<uint32_t>(tx.nVersion) >= 2 + && flags & LOCKTIME_VERIFY_SEQUENCE; + + // Do not enforce sequence numbers as a relative lock time + // unless we have been instructed to + if (!fEnforceBIP68) { + return std::make_pair(nMinHeight, nMinTime); + } + + for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { + const CTxIn& txin = tx.vin[txinIndex]; + + // Sequence numbers with the most significant bit set are not + // treated as relative lock-times, nor are they given any + // consensus-enforced meaning at this point. + if (txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_DISABLE_FLAG) { + // The height of this input is not relevant for sequence locks + (*prevHeights)[txinIndex] = 0; + continue; } - if (whichType == TX_NULL_DATA) - nDataOut++; - else if ((whichType == TX_MULTISIG) && (!fIsBareMultisigStd)) { - reason = "bare-multisig"; - return false; - } else if (txout.IsDust(::minRelayTxFee)) { - reason = "dust"; - return false; + int nCoinHeight = (*prevHeights)[txinIndex]; + + if (txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_TYPE_FLAG) { + int64_t nCoinTime = block.GetAncestor(std::max(nCoinHeight-1, 0))->GetMedianTimePast(); + // NOTE: Subtract 1 to maintain nLockTime semantics + // BIP 68 relative lock times have the semantics of calculating + // the first block or time at which the transaction would be + // valid. When calculating the effective block time or height + // for the entire transaction, we switch to using the + // semantics of nLockTime which is the last invalid block + // time or height. Thus we subtract 1 from the calculated + // time or height. + + // Time-based relative lock-times are measured from the + // smallest allowed timestamp of the block containing the + // txout being spent, which is the median time past of the + // block prior. + nMinTime = std::max(nMinTime, nCoinTime + (int64_t)((txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_MASK) << CTxIn::SEQUENCE_LOCKTIME_GRANULARITY) - 1); + } else { + nMinHeight = std::max(nMinHeight, nCoinHeight + (int)(txin.nSequence & CTxIn::SEQUENCE_LOCKTIME_MASK) - 1); } } - // only one OP_RETURN txout is permitted - if (nDataOut > 1) { - reason = "multi-op-return"; + return std::make_pair(nMinHeight, nMinTime); +} + +static bool EvaluateSequenceLocks(const CBlockIndex& block, std::pair<int, int64_t> lockPair) +{ + assert(block.pprev); + int64_t nBlockTime = block.pprev->GetMedianTimePast(); + if (lockPair.first >= block.nHeight || lockPair.second >= nBlockTime) return false; - } return true; } -bool IsFinalTx(const CTransaction &tx, int nBlockHeight, int64_t nBlockTime) +bool SequenceLocks(const CTransaction &tx, int flags, std::vector<int>* prevHeights, const CBlockIndex& block) { - AssertLockHeld(cs_main); - // Time based nLockTime implemented in 0.1.6 - if (tx.nLockTime == 0) - return true; - if (nBlockHeight == 0) - nBlockHeight = chainActive.Height(); - if (nBlockTime == 0) - nBlockTime = GetAdjustedTime(); - if ((int64_t)tx.nLockTime < ((int64_t)tx.nLockTime < LOCKTIME_THRESHOLD ? (int64_t)nBlockHeight : nBlockTime)) - return true; - BOOST_FOREACH(const CTxIn& txin, tx.vin) - if (!txin.IsFinal()) - return false; - return true; + return EvaluateSequenceLocks(block, CalculateSequenceLocks(tx, flags, prevHeights, block)); } -/** - * Check transaction inputs to mitigate two - * potential denial-of-service attacks: - * - * 1. scriptSigs with extra data stuffed into them, - * not consumed by scriptPubKey (or P2SH script) - * 2. P2SH scripts with a crazy number of expensive - * CHECKSIG/CHECKMULTISIG operations - */ -bool AreInputsStandard(const CTransaction& tx, const CCoinsViewCache& mapInputs) +bool TestLockPointValidity(const LockPoints* lp) { - if (tx.IsCoinBase()) - return true; // Coinbases don't use vin normally - - for (unsigned int i = 0; i < tx.vin.size(); i++) - { - const CTxOut& prev = mapInputs.GetOutputFor(tx.vin[i]); - - vector<vector<unsigned char> > vSolutions; - txnouttype whichType; - // get the scriptPubKey corresponding to this input: - const CScript& prevScript = prev.scriptPubKey; - if (!Solver(prevScript, whichType, vSolutions)) - return false; - int nArgsExpected = ScriptSigArgsExpected(whichType, vSolutions); - if (nArgsExpected < 0) + AssertLockHeld(cs_main); + assert(lp); + // If there are relative lock times then the maxInputBlock will be set + // If there are no relative lock times, the LockPoints don't depend on the chain + if (lp->maxInputBlock) { + // Check whether chainActive is an extension of the block at which the LockPoints + // calculation was valid. If not LockPoints are no longer valid + if (!chainActive.Contains(lp->maxInputBlock)) { return false; + } + } - // Transactions with extra stuff in their scriptSigs are - // non-standard. Note that this EvalScript() call will - // be quick, because if there are any operations - // beside "push data" in the scriptSig - // IsStandard() will have already returned false - // and this method isn't called. - vector<vector<unsigned char> > stack; - if (!EvalScript(stack, tx.vin[i].scriptSig, false, BaseSignatureChecker())) - return false; + // LockPoints still valid + return true; +} - if (whichType == TX_SCRIPTHASH) - { - if (stack.empty()) - return false; - CScript subscript(stack.back().begin(), stack.back().end()); - vector<vector<unsigned char> > vSolutions2; - txnouttype whichType2; - if (Solver(subscript, whichType2, vSolutions2)) - { - int tmpExpected = ScriptSigArgsExpected(whichType2, vSolutions2); - if (tmpExpected < 0) - return false; - nArgsExpected += tmpExpected; +bool CheckSequenceLocks(const CTransaction &tx, int flags, LockPoints* lp, bool useExistingLockPoints) +{ + AssertLockHeld(cs_main); + AssertLockHeld(mempool.cs); + + CBlockIndex* tip = chainActive.Tip(); + CBlockIndex index; + index.pprev = tip; + // CheckSequenceLocks() uses chainActive.Height()+1 to evaluate + // height based locks because when SequenceLocks() is called within + // ConnectBlock(), the height of the block *being* + // evaluated is what is used. + // Thus if we want to know if a transaction can be part of the + // *next* block, we need to use one more than chainActive.Height() + index.nHeight = tip->nHeight + 1; + + std::pair<int, int64_t> lockPair; + if (useExistingLockPoints) { + assert(lp); + lockPair.first = lp->height; + lockPair.second = lp->time; + } + else { + // pcoinsTip contains the UTXO set for chainActive.Tip() + CCoinsViewMemPool viewMemPool(pcoinsTip, mempool); + std::vector<int> prevheights; + prevheights.resize(tx.vin.size()); + for (size_t txinIndex = 0; txinIndex < tx.vin.size(); txinIndex++) { + const CTxIn& txin = tx.vin[txinIndex]; + CCoins coins; + if (!viewMemPool.GetCoins(txin.prevout.hash, coins)) { + return error("%s: Missing input", __func__); } - else - { - // Any other Script with less than 15 sigops OK: - unsigned int sigops = subscript.GetSigOpCount(true); - // ... extra data left on the stack after execution is OK, too: - return (sigops <= MAX_P2SH_SIGOPS); + if (coins.nHeight == MEMPOOL_HEIGHT) { + // Assume all mempool transaction confirm in the next block + prevheights[txinIndex] = tip->nHeight + 1; + } else { + prevheights[txinIndex] = coins.nHeight; } } - - if (stack.size() != (unsigned int)nArgsExpected) - return false; + lockPair = CalculateSequenceLocks(tx, flags, &prevheights, index); + if (lp) { + lp->height = lockPair.first; + lp->time = lockPair.second; + // Also store the hash of the block with the highest height of + // all the blocks which have sequence locked prevouts. + // This hash needs to still be on the chain + // for these LockPoint calculations to be valid + // Note: It is impossible to correctly calculate a maxInputBlock + // if any of the sequence locked inputs depend on unconfirmed txs, + // except in the special case where the relative lock time/height + // is 0, which is equivalent to no sequence lock. Since we assume + // input height of tip+1 for mempool txs and test the resulting + // lockPair from CalculateSequenceLocks against tip+1. We know + // EvaluateSequenceLocks will fail if there was a non-zero sequence + // lock on a mempool input, so we can use the return value of + // CheckSequenceLocks to indicate the LockPoints validity + int maxInputHeight = 0; + BOOST_FOREACH(int height, prevheights) { + // Can ignore mempool inputs since we'll fail if they had non-zero locks + if (height != tip->nHeight+1) { + maxInputHeight = std::max(maxInputHeight, height); + } + } + lp->maxInputBlock = tip->GetAncestor(maxInputHeight); + } } - - return true; + return EvaluateSequenceLocks(index, lockPair); } + unsigned int GetLegacySigOpCount(const CTransaction& tx) { unsigned int nSigOps = 0; @@ -813,8 +1028,24 @@ unsigned int GetP2SHSigOpCount(const CTransaction& tx, const CCoinsViewCache& in return nSigOps; } +int64_t GetTransactionSigOpCost(const CTransaction& tx, const CCoinsViewCache& inputs, int flags) +{ + int64_t nSigOps = GetLegacySigOpCount(tx) * WITNESS_SCALE_FACTOR; + if (tx.IsCoinBase()) + return nSigOps; + if (flags & SCRIPT_VERIFY_P2SH) { + nSigOps += GetP2SHSigOpCount(tx, inputs) * WITNESS_SCALE_FACTOR; + } + + for (unsigned int i = 0; i < tx.vin.size(); i++) + { + const CTxOut &prevout = inputs.GetOutputFor(tx.vin[i]); + nSigOps += CountWitnessSigOps(tx.vin[i].scriptSig, prevout.scriptPubKey, i < tx.wit.vtxinwit.size() ? &tx.wit.vtxinwit[i].scriptWitness : NULL, flags); + } + return nSigOps; +} @@ -824,30 +1055,24 @@ bool CheckTransaction(const CTransaction& tx, CValidationState &state) { // Basic checks that don't depend on any context if (tx.vin.empty()) - return state.DoS(10, error("CheckTransaction() : vin empty"), - REJECT_INVALID, "bad-txns-vin-empty"); + return state.DoS(10, false, REJECT_INVALID, "bad-txns-vin-empty"); if (tx.vout.empty()) - return state.DoS(10, error("CheckTransaction() : vout empty"), - REJECT_INVALID, "bad-txns-vout-empty"); - // Size limits - if (::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION) > MAX_BLOCK_SIZE) - return state.DoS(100, error("CheckTransaction() : size limits failed"), - REJECT_INVALID, "bad-txns-oversize"); + return state.DoS(10, false, REJECT_INVALID, "bad-txns-vout-empty"); + // Size limits (this doesn't take the witness into account, as that hasn't been checked for malleability) + if (::GetSerializeSize(tx, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) > MAX_BLOCK_BASE_SIZE) + return state.DoS(100, false, REJECT_INVALID, "bad-txns-oversize"); // Check for negative or overflow output values CAmount nValueOut = 0; BOOST_FOREACH(const CTxOut& txout, tx.vout) { if (txout.nValue < 0) - return state.DoS(100, error("CheckTransaction() : txout.nValue negative"), - REJECT_INVALID, "bad-txns-vout-negative"); + return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-negative"); if (txout.nValue > MAX_MONEY) - return state.DoS(100, error("CheckTransaction() : txout.nValue too high"), - REJECT_INVALID, "bad-txns-vout-toolarge"); + return state.DoS(100, false, REJECT_INVALID, "bad-txns-vout-toolarge"); nValueOut += txout.nValue; if (!MoneyRange(nValueOut)) - return state.DoS(100, error("CheckTransaction() : txout total out of range"), - REJECT_INVALID, "bad-txns-txouttotal-toolarge"); + return state.DoS(100, false, REJECT_INVALID, "bad-txns-txouttotal-toolarge"); } // Check for duplicate inputs @@ -855,95 +1080,130 @@ bool CheckTransaction(const CTransaction& tx, CValidationState &state) BOOST_FOREACH(const CTxIn& txin, tx.vin) { if (vInOutPoints.count(txin.prevout)) - return state.DoS(100, error("CheckTransaction() : duplicate inputs"), - REJECT_INVALID, "bad-txns-inputs-duplicate"); + return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputs-duplicate"); vInOutPoints.insert(txin.prevout); } if (tx.IsCoinBase()) { if (tx.vin[0].scriptSig.size() < 2 || tx.vin[0].scriptSig.size() > 100) - return state.DoS(100, error("CheckTransaction() : coinbase script size"), - REJECT_INVALID, "bad-cb-length"); + return state.DoS(100, false, REJECT_INVALID, "bad-cb-length"); } else { BOOST_FOREACH(const CTxIn& txin, tx.vin) if (txin.prevout.IsNull()) - return state.DoS(10, error("CheckTransaction() : prevout is null"), - REJECT_INVALID, "bad-txns-prevout-null"); + return state.DoS(10, false, REJECT_INVALID, "bad-txns-prevout-null"); } return true; } -CAmount GetMinRelayFee(const CTransaction& tx, unsigned int nBytes, bool fAllowFree) -{ - { - LOCK(mempool.cs); - uint256 hash = tx.GetHash(); - double dPriorityDelta = 0; - CAmount nFeeDelta = 0; - mempool.ApplyDeltas(hash, dPriorityDelta, nFeeDelta); - if (dPriorityDelta > 0 || nFeeDelta > 0) - return 0; - } - - CAmount nMinFee = ::minRelayTxFee.GetFee(nBytes); - - if (fAllowFree) - { - // There is a free transaction area in blocks created by most miners, - // * If we are relaying we allow transactions up to DEFAULT_BLOCK_PRIORITY_SIZE - 1000 - // to be considered to fall into this category. We don't want to encourage sending - // multiple transactions instead of one big transaction to avoid fees. - if (nBytes < (DEFAULT_BLOCK_PRIORITY_SIZE - 1000)) - nMinFee = 0; - } +void LimitMempoolSize(CTxMemPool& pool, size_t limit, unsigned long age) { + int expired = pool.Expire(GetTime() - age); + if (expired != 0) + LogPrint("mempool", "Expired %i transactions from the memory pool\n", expired); - if (!MoneyRange(nMinFee)) - nMinFee = MAX_MONEY; - return nMinFee; + std::vector<uint256> vNoSpendsRemaining; + pool.TrimToSize(limit, &vNoSpendsRemaining); + BOOST_FOREACH(const uint256& removed, vNoSpendsRemaining) + pcoinsTip->Uncache(removed); } +/** Convert CValidationState to a human-readable message for logging */ +std::string FormatStateMessage(const CValidationState &state) +{ + return strprintf("%s%s (code %i)", + state.GetRejectReason(), + state.GetDebugMessage().empty() ? "" : ", "+state.GetDebugMessage(), + state.GetRejectCode()); +} -bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, - bool* pfMissingInputs, bool fRejectInsaneFee) +bool AcceptToMemoryPoolWorker(CTxMemPool& pool, CValidationState& state, const CTransaction& tx, bool fLimitFree, + bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount& nAbsurdFee, + std::vector<uint256>& vHashTxnToUncache) { + const uint256 hash = tx.GetHash(); AssertLockHeld(cs_main); if (pfMissingInputs) *pfMissingInputs = false; if (!CheckTransaction(tx, state)) - return error("AcceptToMemoryPool: : CheckTransaction failed"); + return false; // state filled in by CheckTransaction // Coinbase is only valid in a block, not as a loose transaction if (tx.IsCoinBase()) - return state.DoS(100, error("AcceptToMemoryPool: : coinbase as individual tx"), - REJECT_INVALID, "coinbase"); + return state.DoS(100, false, REJECT_INVALID, "coinbase"); + + // Don't relay version 2 transactions until CSV is active, and we can be + // sure that such transactions will be mined (unless we're on + // -testnet/-regtest). + const CChainParams& chainparams = Params(); + if (fRequireStandard && tx.nVersion >= 2 && VersionBitsTipState(chainparams.GetConsensus(), Consensus::DEPLOYMENT_CSV) != THRESHOLD_ACTIVE) { + return state.DoS(0, false, REJECT_NONSTANDARD, "premature-version2-tx"); + } + + // Reject transactions with witness before segregated witness activates (override with -prematurewitness) + if (!GetBoolArg("-prematurewitness",false) && !tx.wit.IsNull() && !IsWitnessEnabled(chainActive.Tip(), Params().GetConsensus())) { + return state.DoS(0, false, REJECT_NONSTANDARD, "no-witness-yet", true); + } // Rather not work on nonstandard transactions (unless -testnet/-regtest) string reason; - if (Params().RequireStandard() && !IsStandardTx(tx, reason)) - return state.DoS(0, - error("AcceptToMemoryPool : nonstandard transaction: %s", reason), - REJECT_NONSTANDARD, reason); + if (fRequireStandard && !IsStandardTx(tx, reason)) + return state.DoS(0, false, REJECT_NONSTANDARD, reason); + + // Only accept nLockTime-using transactions that can be mined in the next + // block; we don't want our mempool filled up with transactions that can't + // be mined yet. + if (!CheckFinalTx(tx, STANDARD_LOCKTIME_VERIFY_FLAGS)) + return state.DoS(0, false, REJECT_NONSTANDARD, "non-final"); // is it already in the memory pool? - uint256 hash = tx.GetHash(); if (pool.exists(hash)) - return false; + return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-in-mempool"); // Check for conflicts with in-memory transactions + set<uint256> setConflicts; { LOCK(pool.cs); // protect pool.mapNextTx - for (unsigned int i = 0; i < tx.vin.size(); i++) + BOOST_FOREACH(const CTxIn &txin, tx.vin) { - COutPoint outpoint = tx.vin[i].prevout; - if (pool.mapNextTx.count(outpoint)) + auto itConflicting = pool.mapNextTx.find(txin.prevout); + if (itConflicting != pool.mapNextTx.end()) { - // Disable replacement feature for now - return false; + const CTransaction *ptxConflicting = itConflicting->second; + if (!setConflicts.count(ptxConflicting->GetHash())) + { + // Allow opt-out of transaction replacement by setting + // nSequence >= maxint-1 on all inputs. + // + // maxint-1 is picked to still allow use of nLockTime by + // non-replacable transactions. All inputs rather than just one + // is for the sake of multi-party protocols, where we don't + // want a single party to be able to disable replacement. + // + // The opt-out ignores descendants as anyone relying on + // first-seen mempool behavior should be checking all + // unconfirmed ancestors anyway; doing otherwise is hopelessly + // insecure. + bool fReplacementOptOut = true; + if (fEnableReplacement) + { + BOOST_FOREACH(const CTxIn &txin, ptxConflicting->vin) + { + if (txin.nSequence < std::numeric_limits<unsigned int>::max()-1) + { + fReplacementOptOut = false; + break; + } + } + } + if (fReplacementOptOut) + return state.Invalid(false, REJECT_CONFLICT, "txn-mempool-conflict"); + + setConflicts.insert(ptxConflicting->GetHash()); + } } } } @@ -953,30 +1213,36 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa CCoinsViewCache view(&dummy); CAmount nValueIn = 0; + LockPoints lp; { LOCK(pool.cs); CCoinsViewMemPool viewMemPool(pcoinsTip, pool); view.SetBackend(viewMemPool); // do we already have it? - if (view.HaveCoins(hash)) - return false; + bool fHadTxInCache = pcoinsTip->HaveCoinsInCache(hash); + if (view.HaveCoins(hash)) { + if (!fHadTxInCache) + vHashTxnToUncache.push_back(hash); + return state.Invalid(false, REJECT_ALREADY_KNOWN, "txn-already-known"); + } // do all inputs exist? // Note that this does not check for the presence of actual outputs (see the next check for that), - // only helps filling in pfMissingInputs (to determine missing vs spent). + // and only helps with filling in pfMissingInputs (to determine missing vs spent). BOOST_FOREACH(const CTxIn txin, tx.vin) { + if (!pcoinsTip->HaveCoinsInCache(txin.prevout.hash)) + vHashTxnToUncache.push_back(txin.prevout.hash); if (!view.HaveCoins(txin.prevout.hash)) { if (pfMissingInputs) *pfMissingInputs = true; - return false; + return false; // fMissingInputs and !state.IsInvalid() is used to detect this condition, don't set state.Invalid() } } // are the actual inputs available? if (!view.HaveInputs(tx)) - return state.Invalid(error("AcceptToMemoryPool : inputs already spent"), - REJECT_DUPLICATE, "bad-txns-inputs-spent"); + return state.Invalid(false, REJECT_DUPLICATE, "bad-txns-inputs-spent"); // Bring the best block into scope view.GetBestBlock(); @@ -985,43 +1251,67 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa // we have all inputs cached now, so switch back to dummy, so we don't need to keep lock on mempool view.SetBackend(dummy); + + // Only accept BIP68 sequence locked transactions that can be mined in the next + // block; we don't want our mempool filled up with transactions that can't + // be mined yet. + // Must keep pool.cs for this unless we change CheckSequenceLocks to take a + // CoinsViewCache instead of create its own + if (!CheckSequenceLocks(tx, STANDARD_LOCKTIME_VERIFY_FLAGS, &lp)) + return state.DoS(0, false, REJECT_NONSTANDARD, "non-BIP68-final"); } // Check for non-standard pay-to-script-hash in inputs - if (Params().RequireStandard() && !AreInputsStandard(tx, view)) - return error("AcceptToMemoryPool: : nonstandard transaction input"); + if (fRequireStandard && !AreInputsStandard(tx, view)) + return state.Invalid(false, REJECT_NONSTANDARD, "bad-txns-nonstandard-inputs"); - // Check that the transaction doesn't have an excessive number of - // sigops, making it impossible to mine. Since the coinbase transaction - // itself can contain sigops MAX_TX_SIGOPS is less than - // MAX_BLOCK_SIGOPS; we still consider this an invalid rather than - // merely non-standard transaction. - unsigned int nSigOps = GetLegacySigOpCount(tx); - nSigOps += GetP2SHSigOpCount(tx, view); - if (nSigOps > MAX_TX_SIGOPS) - return state.DoS(0, - error("AcceptToMemoryPool : too many sigops %s, %d > %d", - hash.ToString(), nSigOps, MAX_TX_SIGOPS), - REJECT_NONSTANDARD, "bad-txns-too-many-sigops"); + int64_t nSigOpsCost = GetTransactionSigOpCost(tx, view, STANDARD_SCRIPT_VERIFY_FLAGS); CAmount nValueOut = tx.GetValueOut(); CAmount nFees = nValueIn-nValueOut; - double dPriority = view.GetPriority(tx, chainActive.Height()); + // nModifiedFees includes any fee deltas from PrioritiseTransaction + CAmount nModifiedFees = nFees; + double nPriorityDummy = 0; + pool.ApplyDeltas(hash, nPriorityDummy, nModifiedFees); + + CAmount inChainInputValue; + double dPriority = view.GetPriority(tx, chainActive.Height(), inChainInputValue); + + // Keep track of transactions that spend a coinbase, which we re-scan + // during reorgs to ensure COINBASE_MATURITY is still met. + bool fSpendsCoinbase = false; + BOOST_FOREACH(const CTxIn &txin, tx.vin) { + const CCoins *coins = view.AccessCoins(txin.prevout.hash); + if (coins->IsCoinBase()) { + fSpendsCoinbase = true; + break; + } + } - CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height()); + CTxMemPoolEntry entry(tx, nFees, GetTime(), dPriority, chainActive.Height(), pool.HasNoInputsOf(tx), inChainInputValue, fSpendsCoinbase, nSigOpsCost, lp); unsigned int nSize = entry.GetTxSize(); - // Don't accept it if it can't get into a block - CAmount txMinFee = GetMinRelayFee(tx, nSize, true); - if (fLimitFree && nFees < txMinFee) - return state.DoS(0, error("AcceptToMemoryPool : not enough fees %s, %d < %d", - hash.ToString(), nFees, txMinFee), - REJECT_INSUFFICIENTFEE, "insufficient fee"); + // Check that the transaction doesn't have an excessive number of + // sigops, making it impossible to mine. Since the coinbase transaction + // itself can contain sigops MAX_STANDARD_TX_SIGOPS is less than + // MAX_BLOCK_SIGOPS; we still consider this an invalid rather than + // merely non-standard transaction. + if ((nSigOpsCost > MAX_STANDARD_TX_SIGOPS_COST) || (nBytesPerSigOp && nSigOpsCost > nSize * WITNESS_SCALE_FACTOR / nBytesPerSigOp)) + return state.DoS(0, false, REJECT_NONSTANDARD, "bad-txns-too-many-sigops", false, + strprintf("%d", nSigOpsCost)); + + CAmount mempoolRejectFee = pool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFee(nSize); + if (mempoolRejectFee > 0 && nModifiedFees < mempoolRejectFee) { + return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "mempool min fee not met", false, strprintf("%d < %d", nFees, mempoolRejectFee)); + } else if (GetBoolArg("-relaypriority", DEFAULT_RELAYPRIORITY) && nModifiedFees < ::minRelayTxFee.GetFee(nSize) && !AllowFree(entry.GetPriority(chainActive.Height() + 1))) { + // Require that free transactions have sufficient priority to be mined in the next block. + return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "insufficient priority"); + } - // Continuously rate-limit free (really, very-low-fee)transactions + // Continuously rate-limit free (really, very-low-fee) transactions // This mitigates 'penny-flooding' -- sending thousands of free transactions just to // be annoying or make others' transactions take longer to confirm. - if (fLimitFree && nFees < ::minRelayTxFee.GetFee(nSize)) + if (fLimitFree && nModifiedFees < ::minRelayTxFee.GetFee(nSize)) { static CCriticalSection csFreeLimiter; static double dFreeCount; @@ -1035,81 +1325,294 @@ bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransa nLastTime = nNow; // -limitfreerelay unit is thousand-bytes-per-minute // At default rate it would take over a month to fill 1GB - if (dFreeCount >= GetArg("-limitfreerelay", 15)*10*1000) - return state.DoS(0, error("AcceptToMemoryPool : free transaction rejected by rate limiter"), - REJECT_INSUFFICIENTFEE, "insufficient priority"); + if (dFreeCount + nSize >= GetArg("-limitfreerelay", DEFAULT_LIMITFREERELAY) * 10 * 1000) + return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "rate limited free transaction"); LogPrint("mempool", "Rate limit dFreeCount: %g => %g\n", dFreeCount, dFreeCount+nSize); dFreeCount += nSize; } - if (fRejectInsaneFee && nFees > ::minRelayTxFee.GetFee(nSize) * 10000) - return error("AcceptToMemoryPool: : insane fees %s, %d > %d", - hash.ToString(), - nFees, ::minRelayTxFee.GetFee(nSize) * 10000); + if (nAbsurdFee && nFees > nAbsurdFee) + return state.Invalid(false, + REJECT_HIGHFEE, "absurdly-high-fee", + strprintf("%d > %d", nFees, nAbsurdFee)); + + // Calculate in-mempool ancestors, up to a limit. + CTxMemPool::setEntries setAncestors; + size_t nLimitAncestors = GetArg("-limitancestorcount", DEFAULT_ANCESTOR_LIMIT); + size_t nLimitAncestorSize = GetArg("-limitancestorsize", DEFAULT_ANCESTOR_SIZE_LIMIT)*1000; + size_t nLimitDescendants = GetArg("-limitdescendantcount", DEFAULT_DESCENDANT_LIMIT); + size_t nLimitDescendantSize = GetArg("-limitdescendantsize", DEFAULT_DESCENDANT_SIZE_LIMIT)*1000; + std::string errString; + if (!pool.CalculateMemPoolAncestors(entry, setAncestors, nLimitAncestors, nLimitAncestorSize, nLimitDescendants, nLimitDescendantSize, errString)) { + return state.DoS(0, false, REJECT_NONSTANDARD, "too-long-mempool-chain", false, errString); + } + + // A transaction that spends outputs that would be replaced by it is invalid. Now + // that we have the set of all ancestors we can detect this + // pathological case by making sure setConflicts and setAncestors don't + // intersect. + BOOST_FOREACH(CTxMemPool::txiter ancestorIt, setAncestors) + { + const uint256 &hashAncestor = ancestorIt->GetTx().GetHash(); + if (setConflicts.count(hashAncestor)) + { + return state.DoS(10, false, + REJECT_INVALID, "bad-txns-spends-conflicting-tx", false, + strprintf("%s spends conflicting transaction %s", + hash.ToString(), + hashAncestor.ToString())); + } + } + + // Check if it's economically rational to mine this transaction rather + // than the ones it replaces. + CAmount nConflictingFees = 0; + size_t nConflictingSize = 0; + uint64_t nConflictingCount = 0; + CTxMemPool::setEntries allConflicting; + + // If we don't hold the lock allConflicting might be incomplete; the + // subsequent RemoveStaged() and addUnchecked() calls don't guarantee + // mempool consistency for us. + LOCK(pool.cs); + if (setConflicts.size()) + { + CFeeRate newFeeRate(nModifiedFees, nSize); + set<uint256> setConflictsParents; + const int maxDescendantsToVisit = 100; + CTxMemPool::setEntries setIterConflicting; + BOOST_FOREACH(const uint256 &hashConflicting, setConflicts) + { + CTxMemPool::txiter mi = pool.mapTx.find(hashConflicting); + if (mi == pool.mapTx.end()) + continue; + + // Save these to avoid repeated lookups + setIterConflicting.insert(mi); + + // Don't allow the replacement to reduce the feerate of the + // mempool. + // + // We usually don't want to accept replacements with lower + // feerates than what they replaced as that would lower the + // feerate of the next block. Requiring that the feerate always + // be increased is also an easy-to-reason about way to prevent + // DoS attacks via replacements. + // + // The mining code doesn't (currently) take children into + // account (CPFP) so we only consider the feerates of + // transactions being directly replaced, not their indirect + // descendants. While that does mean high feerate children are + // ignored when deciding whether or not to replace, we do + // require the replacement to pay more overall fees too, + // mitigating most cases. + CFeeRate oldFeeRate(mi->GetModifiedFee(), mi->GetTxSize()); + if (newFeeRate <= oldFeeRate) + { + return state.DoS(0, false, + REJECT_INSUFFICIENTFEE, "insufficient fee", false, + strprintf("rejecting replacement %s; new feerate %s <= old feerate %s", + hash.ToString(), + newFeeRate.ToString(), + oldFeeRate.ToString())); + } + + BOOST_FOREACH(const CTxIn &txin, mi->GetTx().vin) + { + setConflictsParents.insert(txin.prevout.hash); + } + + nConflictingCount += mi->GetCountWithDescendants(); + } + // This potentially overestimates the number of actual descendants + // but we just want to be conservative to avoid doing too much + // work. + if (nConflictingCount <= maxDescendantsToVisit) { + // If not too many to replace, then calculate the set of + // transactions that would have to be evicted + BOOST_FOREACH(CTxMemPool::txiter it, setIterConflicting) { + pool.CalculateDescendants(it, allConflicting); + } + BOOST_FOREACH(CTxMemPool::txiter it, allConflicting) { + nConflictingFees += it->GetModifiedFee(); + nConflictingSize += it->GetTxSize(); + } + } else { + return state.DoS(0, false, + REJECT_NONSTANDARD, "too many potential replacements", false, + strprintf("rejecting replacement %s; too many potential replacements (%d > %d)\n", + hash.ToString(), + nConflictingCount, + maxDescendantsToVisit)); + } + + for (unsigned int j = 0; j < tx.vin.size(); j++) + { + // We don't want to accept replacements that require low + // feerate junk to be mined first. Ideally we'd keep track of + // the ancestor feerates and make the decision based on that, + // but for now requiring all new inputs to be confirmed works. + if (!setConflictsParents.count(tx.vin[j].prevout.hash)) + { + // Rather than check the UTXO set - potentially expensive - + // it's cheaper to just check if the new input refers to a + // tx that's in the mempool. + if (pool.mapTx.find(tx.vin[j].prevout.hash) != pool.mapTx.end()) + return state.DoS(0, false, + REJECT_NONSTANDARD, "replacement-adds-unconfirmed", false, + strprintf("replacement %s adds unconfirmed input, idx %d", + hash.ToString(), j)); + } + } + + // The replacement must pay greater fees than the transactions it + // replaces - if we did the bandwidth used by those conflicting + // transactions would not be paid for. + if (nModifiedFees < nConflictingFees) + { + return state.DoS(0, false, + REJECT_INSUFFICIENTFEE, "insufficient fee", false, + strprintf("rejecting replacement %s, less fees than conflicting txs; %s < %s", + hash.ToString(), FormatMoney(nModifiedFees), FormatMoney(nConflictingFees))); + } + + // Finally in addition to paying more fees than the conflicts the + // new transaction must pay for its own bandwidth. + CAmount nDeltaFees = nModifiedFees - nConflictingFees; + if (nDeltaFees < ::minRelayTxFee.GetFee(nSize)) + { + return state.DoS(0, false, + REJECT_INSUFFICIENTFEE, "insufficient fee", false, + strprintf("rejecting replacement %s, not enough additional fees to relay; %s < %s", + hash.ToString(), + FormatMoney(nDeltaFees), + FormatMoney(::minRelayTxFee.GetFee(nSize)))); + } + } + + unsigned int scriptVerifyFlags = STANDARD_SCRIPT_VERIFY_FLAGS; + if (!Params().RequireStandard()) { + scriptVerifyFlags = GetArg("-promiscuousmempoolflags", scriptVerifyFlags); + } // Check against previous transactions // This is done last to help prevent CPU exhaustion denial-of-service attacks. - if (!CheckInputs(tx, state, view, true, STANDARD_SCRIPT_VERIFY_FLAGS, true)) + if (!CheckInputs(tx, state, view, true, scriptVerifyFlags, true)) { + // SCRIPT_VERIFY_CLEANSTACK requires SCRIPT_VERIFY_WITNESS, so we + // need to turn both off, and compare against just turning off CLEANSTACK + // to see if the failure is specifically due to witness validation. + if (CheckInputs(tx, state, view, true, scriptVerifyFlags & ~(SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_CLEANSTACK), true) && + !CheckInputs(tx, state, view, true, scriptVerifyFlags & ~SCRIPT_VERIFY_CLEANSTACK, true)) { + // Only the witness is wrong, so the transaction itself may be fine. + state.SetCorruptionPossible(); + } + return false; + } + + // Check again against just the consensus-critical mandatory script + // verification flags, in case of bugs in the standard flags that cause + // transactions to pass as valid when they're actually invalid. For + // instance the STRICTENC flag was incorrectly allowing certain + // CHECKSIG NOT scripts to pass, even though they were invalid. + // + // There is a similar check in CreateNewBlock() to prevent creating + // invalid blocks, however allowing such transactions into the mempool + // can be exploited as a DoS attack. + if (!CheckInputs(tx, state, view, true, MANDATORY_SCRIPT_VERIFY_FLAGS, true)) + { + return error("%s: BUG! PLEASE REPORT THIS! ConnectInputs failed against MANDATORY but not STANDARD flags %s, %s", + __func__, hash.ToString(), FormatStateMessage(state)); + } + + // Remove conflicting transactions from the mempool + BOOST_FOREACH(const CTxMemPool::txiter it, allConflicting) { - return error("AcceptToMemoryPool: : ConnectInputs failed %s", hash.ToString()); + LogPrint("mempool", "replacing tx %s with %s for %s BTC additional fees, %d delta bytes\n", + it->GetTx().GetHash().ToString(), + hash.ToString(), + FormatMoney(nModifiedFees - nConflictingFees), + (int)nSize - (int)nConflictingSize); } + pool.RemoveStaged(allConflicting, false); + // Store transaction in memory - pool.addUnchecked(hash, entry); + pool.addUnchecked(hash, entry, setAncestors, !IsInitialBlockDownload()); + + // trim mempool and check if tx was trimmed + if (!fOverrideMempoolLimit) { + LimitMempoolSize(pool, GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); + if (!pool.exists(hash)) + return state.DoS(0, false, REJECT_INSUFFICIENTFEE, "mempool full"); + } } - SyncWithWallets(tx, NULL); + SyncWithWallets(tx, NULL, NULL); return true; } +bool AcceptToMemoryPool(CTxMemPool& pool, CValidationState &state, const CTransaction &tx, bool fLimitFree, + bool* pfMissingInputs, bool fOverrideMempoolLimit, const CAmount nAbsurdFee) +{ + std::vector<uint256> vHashTxToUncache; + bool res = AcceptToMemoryPoolWorker(pool, state, tx, fLimitFree, pfMissingInputs, fOverrideMempoolLimit, nAbsurdFee, vHashTxToUncache); + if (!res) { + BOOST_FOREACH(const uint256& hashTx, vHashTxToUncache) + pcoinsTip->Uncache(hashTx); + } + return res; +} + /** Return transaction in tx, and if it was found inside a block, its hash is placed in hashBlock */ -bool GetTransaction(const uint256 &hash, CTransaction &txOut, uint256 &hashBlock, bool fAllowSlow) +bool GetTransaction(const uint256 &hash, CTransaction &txOut, const Consensus::Params& consensusParams, uint256 &hashBlock, bool fAllowSlow) { CBlockIndex *pindexSlow = NULL; + + LOCK(cs_main); + + std::shared_ptr<const CTransaction> ptx = mempool.get(hash); + if (ptx) { - LOCK(cs_main); - { - if (mempool.lookup(hash, txOut)) - { - return true; - } - } + txOut = *ptx; + return true; + } - if (fTxIndex) { - CDiskTxPos postx; - if (pblocktree->ReadTxIndex(hash, postx)) { - CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); - CBlockHeader header; - try { - file >> header; - fseek(file.Get(), postx.nTxOffset, SEEK_CUR); - file >> txOut; - } catch (std::exception &e) { - return error("%s : Deserialize or I/O error - %s", __func__, e.what()); - } - hashBlock = header.GetHash(); - if (txOut.GetHash() != hash) - return error("%s : txid mismatch", __func__); - return true; + if (fTxIndex) { + CDiskTxPos postx; + if (pblocktree->ReadTxIndex(hash, postx)) { + CAutoFile file(OpenBlockFile(postx, true), SER_DISK, CLIENT_VERSION); + if (file.IsNull()) + return error("%s: OpenBlockFile failed", __func__); + CBlockHeader header; + try { + file >> header; + fseek(file.Get(), postx.nTxOffset, SEEK_CUR); + file >> txOut; + } catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s", __func__, e.what()); } + hashBlock = header.GetHash(); + if (txOut.GetHash() != hash) + return error("%s: txid mismatch", __func__); + return true; } + } - if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it - int nHeight = -1; - { - CCoinsViewCache &view = *pcoinsTip; - const CCoins* coins = view.AccessCoins(hash); - if (coins) - nHeight = coins->nHeight; - } - if (nHeight > 0) - pindexSlow = chainActive[nHeight]; + if (fAllowSlow) { // use coin database to locate block that contains transaction, and scan it + int nHeight = -1; + { + const CCoinsViewCache& view = *pcoinsTip; + const CCoins* coins = view.AccessCoins(hash); + if (coins) + nHeight = coins->nHeight; } + if (nHeight > 0) + pindexSlow = chainActive[nHeight]; } if (pindexSlow) { CBlock block; - if (ReadBlockFromDisk(block, pindexSlow)) { + if (ReadBlockFromDisk(block, pindexSlow, consensusParams)) { BOOST_FOREACH(const CTransaction &tx, block.vtx) { if (tx.GetHash() == hash) { txOut = tx; @@ -1133,87 +1636,95 @@ bool GetTransaction(const uint256 &hash, CTransaction &txOut, uint256 &hashBlock // CBlock and CBlockIndex // -bool WriteBlockToDisk(CBlock& block, CDiskBlockPos& pos) +bool WriteBlockToDisk(const CBlock& block, CDiskBlockPos& pos, const CMessageHeader::MessageStartChars& messageStart) { // Open history file to append CAutoFile fileout(OpenBlockFile(pos), SER_DISK, CLIENT_VERSION); if (fileout.IsNull()) - return error("WriteBlockToDisk : OpenBlockFile failed"); + return error("WriteBlockToDisk: OpenBlockFile failed"); // Write index header unsigned int nSize = fileout.GetSerializeSize(block); - fileout << FLATDATA(Params().MessageStart()) << nSize; + fileout << FLATDATA(messageStart) << nSize; // Write block long fileOutPos = ftell(fileout.Get()); if (fileOutPos < 0) - return error("WriteBlockToDisk : ftell failed"); + return error("WriteBlockToDisk: ftell failed"); pos.nPos = (unsigned int)fileOutPos; fileout << block; return true; } -bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos) +bool ReadBlockFromDisk(CBlock& block, const CDiskBlockPos& pos, const Consensus::Params& consensusParams) { block.SetNull(); // Open history file to read CAutoFile filein(OpenBlockFile(pos, true), SER_DISK, CLIENT_VERSION); if (filein.IsNull()) - return error("ReadBlockFromDisk : OpenBlockFile failed"); + return error("ReadBlockFromDisk: OpenBlockFile failed for %s", pos.ToString()); // Read block try { filein >> block; } - catch (std::exception &e) { - return error("%s : Deserialize or I/O error - %s", __func__, e.what()); + catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s at %s", __func__, e.what(), pos.ToString()); } // Check the header - if (!CheckProofOfWork(block.GetHash(), block.nBits)) - return error("ReadBlockFromDisk : Errors in block header"); + if (!CheckProofOfWork(block.GetHash(), block.nBits, consensusParams)) + return error("ReadBlockFromDisk: Errors in block header at %s", pos.ToString()); return true; } -bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex) +bool ReadBlockFromDisk(CBlock& block, const CBlockIndex* pindex, const Consensus::Params& consensusParams) { - if (!ReadBlockFromDisk(block, pindex->GetBlockPos())) + if (!ReadBlockFromDisk(block, pindex->GetBlockPos(), consensusParams)) return false; if (block.GetHash() != pindex->GetBlockHash()) - return error("ReadBlockFromDisk(CBlock&, CBlockIndex*) : GetHash() doesn't match index"); + return error("ReadBlockFromDisk(CBlock&, CBlockIndex*): GetHash() doesn't match index for %s at %s", + pindex->ToString(), pindex->GetBlockPos().ToString()); return true; } -CAmount GetBlockValue(int nHeight, const CAmount& nFees) +CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams) { - CAmount nSubsidy = 50 * COIN; - int halvings = nHeight / Params().SubsidyHalvingInterval(); - + int halvings = nHeight / consensusParams.nSubsidyHalvingInterval; // Force block reward to zero when right shift is undefined. if (halvings >= 64) - return nFees; + return 0; + CAmount nSubsidy = 50 * COIN; // Subsidy is cut in half every 210,000 blocks which will occur approximately every 4 years. nSubsidy >>= halvings; - - return nSubsidy + nFees; + return nSubsidy; } bool IsInitialBlockDownload() { + const CChainParams& chainParams = Params(); + + // Once this function has returned false, it must remain false. + static std::atomic<bool> latchToFalse{false}; + // Optimization: pre-test latch before taking the lock. + if (latchToFalse.load(std::memory_order_relaxed)) + return false; + LOCK(cs_main); - if (fImporting || fReindex || chainActive.Height() < Checkpoints::GetTotalBlocksEstimate()) - return true; - static bool lockIBDState = false; - if (lockIBDState) + if (latchToFalse.load(std::memory_order_relaxed)) return false; + if (fImporting || fReindex) + return true; + if (fCheckpointsEnabled && chainActive.Height() < Checkpoints::GetTotalBlocksEstimate(chainParams.Checkpoints())) + return true; bool state = (chainActive.Height() < pindexBestHeader->nHeight - 24 * 6 || - pindexBestHeader->GetBlockTime() < GetTime() - 24 * 60 * 60); + std::max(chainActive.Tip()->GetBlockTime(), pindexBestHeader->GetBlockTime()) < GetTime() - nMaxTipAge); if (!state) - lockIBDState = true; + latchToFalse.store(true, std::memory_order_relaxed); return state; } @@ -1221,6 +1732,23 @@ bool fLargeWorkForkFound = false; bool fLargeWorkInvalidChainFound = false; CBlockIndex *pindexBestForkTip = NULL, *pindexBestForkBase = NULL; +static void AlertNotify(const std::string& strMessage) +{ + uiInterface.NotifyAlertChanged(); + std::string strCmd = GetArg("-alertnotify", ""); + if (strCmd.empty()) return; + + // Alert text should be plain ascii coming from a trusted source, but to + // be safe we first strip anything not in safeChars, then add single quotes around + // the whole string before passing it to the shell: + std::string singleQuote("'"); + std::string safeStatus = SanitizeString(strMessage); + safeStatus = singleQuote+safeStatus+singleQuote; + boost::replace_all(strCmd, "%s", safeStatus); + + boost::thread t(runCommand, strCmd); // thread runs free +} + void CheckForkWarningConditions() { AssertLockHeld(cs_main); @@ -1240,18 +1768,18 @@ void CheckForkWarningConditions() { std::string warning = std::string("'Warning: Large-work fork detected, forking after block ") + pindexBestForkBase->phashBlock->ToString() + std::string("'"); - CAlert::Notify(warning, true); + AlertNotify(warning); } if (pindexBestForkTip && pindexBestForkBase) { - LogPrintf("CheckForkWarningConditions: Warning: Large valid fork found\n forking the chain at height %d (%s)\n lasting to height %d (%s).\nChain state database corruption likely.\n", + LogPrintf("%s: Warning: Large valid fork found\n forking the chain at height %d (%s)\n lasting to height %d (%s).\nChain state database corruption likely.\n", __func__, pindexBestForkBase->nHeight, pindexBestForkBase->phashBlock->ToString(), pindexBestForkTip->nHeight, pindexBestForkTip->phashBlock->ToString()); fLargeWorkForkFound = true; } else { - LogPrintf("CheckForkWarningConditions: Warning: Found invalid chain at least ~6 blocks longer than our best chain.\nChain state database corruption likely.\n"); + LogPrintf("%s: Warning: Found invalid chain at least ~6 blocks longer than our best chain.\nChain state database corruption likely.\n", __func__); fLargeWorkInvalidChainFound = true; } } @@ -1277,8 +1805,8 @@ void CheckForkWarningConditionsOnNewFork(CBlockIndex* pindexNewForkTip) pfork = pfork->pprev; } - // We define a condition which we should warn the user about as a fork of at least 7 blocks - // who's tip is within 72 blocks (+/- 12 hours if no one mines it) of ours + // We define a condition where we should warn the user about as a fork of at least 7 blocks + // with a tip within 72 blocks (+/- 12 hours if no one mines it) of ours // We use 7 blocks rather arbitrarily as it represents just under 10% of sustained network // hash rate operating on the fork. // or a chain that is entirely longer than ours and invalid (note that this should be detected by both) @@ -1306,13 +1834,13 @@ void Misbehaving(NodeId pnode, int howmuch) return; state->nMisbehavior += howmuch; - int banscore = GetArg("-banscore", 100); + int banscore = GetArg("-banscore", DEFAULT_BANSCORE_THRESHOLD); if (state->nMisbehavior >= banscore && state->nMisbehavior - howmuch < banscore) { - LogPrintf("Misbehaving: %s (%d -> %d) BAN THRESHOLD EXCEEDED\n", state->name, state->nMisbehavior-howmuch, state->nMisbehavior); + LogPrintf("%s: %s (%d -> %d) BAN THRESHOLD EXCEEDED\n", __func__, state->name, state->nMisbehavior-howmuch, state->nMisbehavior); state->fShouldBan = true; } else - LogPrintf("Misbehaving: %s (%d -> %d)\n", state->name, state->nMisbehavior-howmuch, state->nMisbehavior); + LogPrintf("%s: %s (%d -> %d)\n", __func__, state->name, state->nMisbehavior-howmuch, state->nMisbehavior); } void static InvalidChainFound(CBlockIndex* pindexNew) @@ -1320,13 +1848,15 @@ void static InvalidChainFound(CBlockIndex* pindexNew) if (!pindexBestInvalid || pindexNew->nChainWork > pindexBestInvalid->nChainWork) pindexBestInvalid = pindexNew; - LogPrintf("InvalidChainFound: invalid block=%s height=%d log2_work=%.8g date=%s\n", + LogPrintf("%s: invalid block=%s height=%d log2_work=%.8g date=%s\n", __func__, pindexNew->GetBlockHash().ToString(), pindexNew->nHeight, log(pindexNew->nChainWork.getdouble())/log(2.0), DateTimeStrFormat("%Y-%m-%d %H:%M:%S", pindexNew->GetBlockTime())); - LogPrintf("InvalidChainFound: current best=%s height=%d log2_work=%.8g date=%s\n", - chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), - DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime())); + CBlockIndex *tip = chainActive.Tip(); + assert (tip); + LogPrintf("%s: current best=%s height=%d log2_work=%.8g date=%s\n", __func__, + tip->GetBlockHash().ToString(), chainActive.Height(), log(tip->nChainWork.getdouble())/log(2.0), + DateTimeStrFormat("%Y-%m-%d %H:%M:%S", tip->GetBlockTime())); CheckForkWarningConditions(); } @@ -1335,7 +1865,8 @@ void static InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state if (state.IsInvalid(nDoS)) { std::map<uint256, NodeId>::iterator it = mapBlockSource.find(pindex->GetBlockHash()); if (it != mapBlockSource.end() && State(it->second)) { - CBlockReject reject = {state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), pindex->GetBlockHash()}; + assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes + CBlockReject reject = {(unsigned char)state.GetRejectCode(), state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), pindex->GetBlockHash()}; State(it->second)->rejects.push_back(reject); if (nDoS > 0) Misbehaving(it->second, nDoS); @@ -1349,46 +1880,62 @@ void static InvalidBlockFound(CBlockIndex *pindex, const CValidationState &state } } -void UpdateCoins(const CTransaction& tx, CValidationState &state, CCoinsViewCache &inputs, CTxUndo &txundo, int nHeight) +void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, CTxUndo &txundo, int nHeight) { // mark inputs spent if (!tx.IsCoinBase()) { txundo.vprevout.reserve(tx.vin.size()); BOOST_FOREACH(const CTxIn &txin, tx.vin) { - txundo.vprevout.push_back(CTxInUndo()); - bool ret = inputs.ModifyCoins(txin.prevout.hash)->Spend(txin.prevout, txundo.vprevout.back()); - assert(ret); + CCoinsModifier coins = inputs.ModifyCoins(txin.prevout.hash); + unsigned nPos = txin.prevout.n; + + if (nPos >= coins->vout.size() || coins->vout[nPos].IsNull()) + assert(false); + // mark an outpoint spent, and construct undo information + txundo.vprevout.push_back(CTxInUndo(coins->vout[nPos])); + coins->Spend(nPos); + if (coins->vout.size() == 0) { + CTxInUndo& undo = txundo.vprevout.back(); + undo.nHeight = coins->nHeight; + undo.fCoinBase = coins->fCoinBase; + undo.nVersion = coins->nVersion; + } } } - // add outputs - inputs.ModifyCoins(tx.GetHash())->FromTx(tx, nHeight); + inputs.ModifyNewCoins(tx.GetHash(), tx.IsCoinBase())->FromTx(tx, nHeight); +} + +void UpdateCoins(const CTransaction& tx, CCoinsViewCache& inputs, int nHeight) +{ + CTxUndo txundo; + UpdateCoins(tx, inputs, txundo, nHeight); } bool CScriptCheck::operator()() { const CScript &scriptSig = ptxTo->vin[nIn].scriptSig; - if (!VerifyScript(scriptSig, scriptPubKey, nFlags, CachingSignatureChecker(*ptxTo, nIn, cacheStore), &error)) { - return ::error("CScriptCheck() : %s:%d VerifySignature failed: %s", ptxTo->GetHash().ToString(), nIn, ScriptErrorString(error)); + const CScriptWitness *witness = (nIn < ptxTo->wit.vtxinwit.size()) ? &ptxTo->wit.vtxinwit[nIn].scriptWitness : NULL; + if (!VerifyScript(scriptSig, scriptPubKey, witness, nFlags, CachingTransactionSignatureChecker(ptxTo, nIn, amount, cacheStore), &error)) { + return false; } return true; } -bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, bool fScriptChecks, unsigned int flags, bool cacheStore, std::vector<CScriptCheck> *pvChecks) +int GetSpendHeight(const CCoinsViewCache& inputs) { - if (!tx.IsCoinBase()) - { - if (pvChecks) - pvChecks->reserve(tx.vin.size()); + LOCK(cs_main); + CBlockIndex* pindexPrev = mapBlockIndex.find(inputs.GetBestBlock())->second; + return pindexPrev->nHeight + 1; +} +namespace Consensus { +bool CheckTxInputs(const CTransaction& tx, CValidationState& state, const CCoinsViewCache& inputs, int nSpendHeight) +{ // This doesn't trigger the DoS code on purpose; if it did, it would make it easier // for an attacker to attempt to split the network. if (!inputs.HaveInputs(tx)) - return state.Invalid(error("CheckInputs() : %s inputs unavailable", tx.GetHash().ToString())); + return state.Invalid(false, 0, "", "Inputs unavailable"); - // While checking, GetBestBlock() refers to the parent block. - // This is also true for mempool checks. - CBlockIndex *pindexPrev = mapBlockIndex.find(inputs.GetBestBlock())->second; - int nSpendHeight = pindexPrev->nHeight + 1; CAmount nValueIn = 0; CAmount nFees = 0; for (unsigned int i = 0; i < tx.vin.size(); i++) @@ -1400,41 +1947,53 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi // If prev is coinbase, check that it's matured if (coins->IsCoinBase()) { if (nSpendHeight - coins->nHeight < COINBASE_MATURITY) - return state.Invalid( - error("CheckInputs() : tried to spend coinbase at depth %d", nSpendHeight - coins->nHeight), - REJECT_INVALID, "bad-txns-premature-spend-of-coinbase"); + return state.Invalid(false, + REJECT_INVALID, "bad-txns-premature-spend-of-coinbase", + strprintf("tried to spend coinbase at depth %d", nSpendHeight - coins->nHeight)); } // Check for negative or overflow input values nValueIn += coins->vout[prevout.n].nValue; if (!MoneyRange(coins->vout[prevout.n].nValue) || !MoneyRange(nValueIn)) - return state.DoS(100, error("CheckInputs() : txin values out of range"), - REJECT_INVALID, "bad-txns-inputvalues-outofrange"); + return state.DoS(100, false, REJECT_INVALID, "bad-txns-inputvalues-outofrange"); } if (nValueIn < tx.GetValueOut()) - return state.DoS(100, error("CheckInputs() : %s value in (%s) < value out (%s)", - tx.GetHash().ToString(), FormatMoney(nValueIn), FormatMoney(tx.GetValueOut())), - REJECT_INVALID, "bad-txns-in-belowout"); + return state.DoS(100, false, REJECT_INVALID, "bad-txns-in-belowout", false, + strprintf("value in (%s) < value out (%s)", FormatMoney(nValueIn), FormatMoney(tx.GetValueOut()))); // Tally transaction fees CAmount nTxFee = nValueIn - tx.GetValueOut(); if (nTxFee < 0) - return state.DoS(100, error("CheckInputs() : %s nTxFee < 0", tx.GetHash().ToString()), - REJECT_INVALID, "bad-txns-fee-negative"); + return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-negative"); nFees += nTxFee; if (!MoneyRange(nFees)) - return state.DoS(100, error("CheckInputs() : nFees out of range"), - REJECT_INVALID, "bad-txns-fee-outofrange"); + return state.DoS(100, false, REJECT_INVALID, "bad-txns-fee-outofrange"); + return true; +} +}// namespace Consensus + +bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsViewCache &inputs, bool fScriptChecks, unsigned int flags, bool cacheStore, std::vector<CScriptCheck> *pvChecks) +{ + if (!tx.IsCoinBase()) + { + if (!Consensus::CheckTxInputs(tx, state, inputs, GetSpendHeight(inputs))) + return false; + + if (pvChecks) + pvChecks->reserve(tx.vin.size()); // The first loop above does all the inexpensive checks. // Only if ALL inputs pass do we perform expensive ECDSA signature checks. // Helps prevent CPU exhaustion attacks. - // Skip ECDSA signature verification when connecting blocks - // before the last block chain checkpoint. This is safe because block merkle hashes are - // still computed and checked, and any change will be caught at the next checkpoint. + // Skip ECDSA signature verification when connecting blocks before the + // last block chain checkpoint. Assuming the checkpoints are valid this + // is safe because block merkle hashes are still computed and checked, + // and any change will be caught at the next checkpoint. Of course, if + // the checkpoint is for a chain that's invalid due to false scriptSigs + // this optimisation would allow an invalid chain to be accepted. if (fScriptChecks) { for (unsigned int i = 0; i < tx.vin.size(); i++) { const COutPoint &prevout = tx.vin[i].prevout; @@ -1454,9 +2013,9 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi // arguments; if so, don't trigger DoS protection to // avoid splitting the network between upgraded and // non-upgraded nodes. - CScriptCheck check(*coins, tx, i, + CScriptCheck check2(*coins, tx, i, flags & ~STANDARD_NOT_MANDATORY_VERIFY_FLAGS, cacheStore); - if (check()) + if (check2()) return state.Invalid(false, REJECT_NONSTANDARD, strprintf("non-mandatory-script-verify-flag (%s)", ScriptErrorString(check.GetScriptError()))); } // Failures of other flags indicate a transaction that is @@ -1464,8 +2023,8 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi // such nodes as they are not following the protocol. That // said during an upgrade careful thought should be taken // as to the correct behavior - we may want to continue - // peering with non-upgraded nodes even after a soft-fork - // super-majority vote has passed. + // peering with non-upgraded nodes even after soft-fork + // super-majority signaling has occurred. return state.DoS(100,false, REJECT_INVALID, strprintf("mandatory-script-verify-flag-failed (%s)", ScriptErrorString(check.GetScriptError()))); } } @@ -1475,9 +2034,116 @@ bool CheckInputs(const CTransaction& tx, CValidationState &state, const CCoinsVi return true; } +namespace { +bool UndoWriteToDisk(const CBlockUndo& blockundo, CDiskBlockPos& pos, const uint256& hashBlock, const CMessageHeader::MessageStartChars& messageStart) +{ + // Open history file to append + CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); + if (fileout.IsNull()) + return error("%s: OpenUndoFile failed", __func__); + + // Write index header + unsigned int nSize = fileout.GetSerializeSize(blockundo); + fileout << FLATDATA(messageStart) << nSize; + + // Write undo data + long fileOutPos = ftell(fileout.Get()); + if (fileOutPos < 0) + return error("%s: ftell failed", __func__); + pos.nPos = (unsigned int)fileOutPos; + fileout << blockundo; + + // calculate & write checksum + CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); + hasher << hashBlock; + hasher << blockundo; + fileout << hasher.GetHash(); + + return true; +} + +bool UndoReadFromDisk(CBlockUndo& blockundo, const CDiskBlockPos& pos, const uint256& hashBlock) +{ + // Open history file to read + CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION); + if (filein.IsNull()) + return error("%s: OpenBlockFile failed", __func__); + + // Read block + uint256 hashChecksum; + try { + filein >> blockundo; + filein >> hashChecksum; + } + catch (const std::exception& e) { + return error("%s: Deserialize or I/O error - %s", __func__, e.what()); + } + + // Verify checksum + CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); + hasher << hashBlock; + hasher << blockundo; + if (hashChecksum != hasher.GetHash()) + return error("%s: Checksum mismatch", __func__); + + return true; +} -bool DisconnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, bool* pfClean) +/** Abort with a message */ +bool AbortNode(const std::string& strMessage, const std::string& userMessage="") +{ + strMiscWarning = strMessage; + LogPrintf("*** %s\n", strMessage); + uiInterface.ThreadSafeMessageBox( + userMessage.empty() ? _("Error: A fatal internal error occurred, see debug.log for details") : userMessage, + "", CClientUIInterface::MSG_ERROR); + StartShutdown(); + return false; +} + +bool AbortNode(CValidationState& state, const std::string& strMessage, const std::string& userMessage="") +{ + AbortNode(strMessage, userMessage); + return state.Error(strMessage); +} + +} // anon namespace + +/** + * Apply the undo operation of a CTxInUndo to the given chain state. + * @param undo The undo object. + * @param view The coins view to which to apply the changes. + * @param out The out point that corresponds to the tx input. + * @return True on success. + */ +static bool ApplyTxInUndo(const CTxInUndo& undo, CCoinsViewCache& view, const COutPoint& out) +{ + bool fClean = true; + + CCoinsModifier coins = view.ModifyCoins(out.hash); + if (undo.nHeight != 0) { + // undo data contains height: this is the last output of the prevout tx being spent + if (!coins->IsPruned()) + fClean = fClean && error("%s: undo data overwriting existing transaction", __func__); + coins->Clear(); + coins->fCoinBase = undo.fCoinBase; + coins->nHeight = undo.nHeight; + coins->nVersion = undo.nVersion; + } else { + if (coins->IsPruned()) + fClean = fClean && error("%s: undo data adding output to missing transaction", __func__); + } + if (coins->IsAvailable(out.n)) + fClean = fClean && error("%s: undo data overwriting existing output", __func__); + if (coins->vout.size() < out.n+1) + coins->vout.resize(out.n+1); + coins->vout[out.n] = undo.txout; + + return fClean; +} + +bool DisconnectBlock(const CBlock& block, CValidationState& state, const CBlockIndex* pindex, CCoinsViewCache& view, bool* pfClean) { assert(pindex->GetBlockHash() == view.GetBestBlock()); @@ -1489,12 +2155,12 @@ bool DisconnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex CBlockUndo blockUndo; CDiskBlockPos pos = pindex->GetUndoPos(); if (pos.IsNull()) - return error("DisconnectBlock() : no undo data available"); - if (!blockUndo.ReadFromDisk(pos, pindex->pprev->GetBlockHash())) - return error("DisconnectBlock() : failure reading undo data"); + return error("DisconnectBlock(): no undo data available"); + if (!UndoReadFromDisk(blockUndo, pos, pindex->pprev->GetBlockHash())) + return error("DisconnectBlock(): failure reading undo data"); if (blockUndo.vtxundo.size() + 1 != block.vtx.size()) - return error("DisconnectBlock() : block and undo data inconsistent"); + return error("DisconnectBlock(): block and undo data inconsistent"); // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { @@ -1502,11 +2168,8 @@ bool DisconnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex uint256 hash = tx.GetHash(); // Check that all outputs are available and match the outputs in the block itself - // exactly. Note that transactions with only provably unspendable outputs won't - // have outputs available even in the block itself, so we handle that case - // specially with outsEmpty. + // exactly. { - CCoins outsEmpty; CCoinsModifier outs = view.ModifyCoins(hash); outs->ClearUnspendable(); @@ -1517,7 +2180,7 @@ bool DisconnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex if (outsBlock.nVersion < 0) outs->nVersion = outsBlock.nVersion; if (*outs != outsBlock) - fClean = fClean && error("DisconnectBlock() : added transaction mismatch? database corrupted"); + fClean = fClean && error("DisconnectBlock(): added transaction mismatch? database corrupted"); // remove outputs outs->Clear(); @@ -1527,28 +2190,12 @@ bool DisconnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex if (i > 0) { // not coinbases const CTxUndo &txundo = blockUndo.vtxundo[i-1]; if (txundo.vprevout.size() != tx.vin.size()) - return error("DisconnectBlock() : transaction and undo data inconsistent"); + return error("DisconnectBlock(): transaction and undo data inconsistent"); for (unsigned int j = tx.vin.size(); j-- > 0;) { const COutPoint &out = tx.vin[j].prevout; const CTxInUndo &undo = txundo.vprevout[j]; - CCoinsModifier coins = view.ModifyCoins(out.hash); - if (undo.nHeight != 0) { - // undo data contains height: this is the last output of the prevout tx being spent - if (!coins->IsPruned()) - fClean = fClean && error("DisconnectBlock() : undo data overwriting existing transaction"); - coins->Clear(); - coins->fCoinBase = undo.fCoinBase; - coins->nHeight = undo.nHeight; - coins->nVersion = undo.nVersion; - } else { - if (coins->IsPruned()) - fClean = fClean && error("DisconnectBlock() : undo data adding output to missing transaction"); - } - if (coins->IsAvailable(out.n)) - fClean = fClean && error("DisconnectBlock() : undo data overwriting existing output"); - if (coins->vout.size() < out.n+1) - coins->vout.resize(out.n+1); - coins->vout[out.n] = undo.txout; + if (!ApplyTxInUndo(undo, view, out)) + fClean = false; } } } @@ -1559,9 +2206,9 @@ bool DisconnectBlock(CBlock& block, CValidationState& state, CBlockIndex* pindex if (pfClean) { *pfClean = fClean; return true; - } else { - return fClean; } + + return fClean; } void static FlushBlockFile(bool fFinalize = false) @@ -1596,31 +2243,93 @@ void ThreadScriptCheck() { scriptcheckqueue.Thread(); } +// Protected by cs_main +VersionBitsCache versionbitscache; + +int32_t ComputeBlockVersion(const CBlockIndex* pindexPrev, const Consensus::Params& params) +{ + LOCK(cs_main); + int32_t nVersion = VERSIONBITS_TOP_BITS; + + for (int i = 0; i < (int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; i++) { + ThresholdState state = VersionBitsState(pindexPrev, params, (Consensus::DeploymentPos)i, versionbitscache); + if (state == THRESHOLD_LOCKED_IN || state == THRESHOLD_STARTED) { + nVersion |= VersionBitsMask(params, (Consensus::DeploymentPos)i); + } + } + + return nVersion; +} + +/** + * Threshold condition checker that triggers when unknown versionbits are seen on the network. + */ +class WarningBitsConditionChecker : public AbstractThresholdConditionChecker +{ +private: + int bit; + +public: + WarningBitsConditionChecker(int bitIn) : bit(bitIn) {} + + int64_t BeginTime(const Consensus::Params& params) const { return 0; } + int64_t EndTime(const Consensus::Params& params) const { return std::numeric_limits<int64_t>::max(); } + int Period(const Consensus::Params& params) const { return params.nMinerConfirmationWindow; } + int Threshold(const Consensus::Params& params) const { return params.nRuleChangeActivationThreshold; } + + bool Condition(const CBlockIndex* pindex, const Consensus::Params& params) const + { + return ((pindex->nVersion & VERSIONBITS_TOP_MASK) == VERSIONBITS_TOP_BITS) && + ((pindex->nVersion >> bit) & 1) != 0 && + ((ComputeBlockVersion(pindex->pprev, params) >> bit) & 1) == 0; + } +}; + +// Protected by cs_main +static ThresholdConditionCache warningcache[VERSIONBITS_NUM_BITS]; + +static int64_t nTimeCheck = 0; +static int64_t nTimeForks = 0; static int64_t nTimeVerify = 0; static int64_t nTimeConnect = 0; static int64_t nTimeIndex = 0; static int64_t nTimeCallbacks = 0; static int64_t nTimeTotal = 0; -bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, CCoinsViewCache& view, bool fJustCheck) +bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pindex, + CCoinsViewCache& view, const CChainParams& chainparams, bool fJustCheck) { AssertLockHeld(cs_main); + + int64_t nTimeStart = GetTimeMicros(); + // Check it again in case a previous version let a bad block in - if (!CheckBlock(block, state, !fJustCheck, !fJustCheck)) - return false; + if (!CheckBlock(block, state, chainparams.GetConsensus(), !fJustCheck, !fJustCheck)) + return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); // verify that the view's current state corresponds to the previous block - uint256 hashPrevBlock = pindex->pprev == NULL ? uint256(0) : pindex->pprev->GetBlockHash(); + uint256 hashPrevBlock = pindex->pprev == NULL ? uint256() : pindex->pprev->GetBlockHash(); assert(hashPrevBlock == view.GetBestBlock()); // Special case for the genesis block, skipping connection of its transactions // (its coinbase is unspendable) - if (block.GetHash() == Params().HashGenesisBlock()) { - view.SetBestBlock(pindex->GetBlockHash()); + if (block.GetHash() == chainparams.GetConsensus().hashGenesisBlock) { + if (!fJustCheck) + view.SetBestBlock(pindex->GetBlockHash()); return true; } - bool fScriptChecks = pindex->nHeight >= Checkpoints::GetTotalBlocksEstimate(); + bool fScriptChecks = true; + if (fCheckpointsEnabled) { + CBlockIndex *pindexLastCheckpoint = Checkpoints::GetLastCheckpoint(chainparams.Checkpoints()); + if (pindexLastCheckpoint && pindexLastCheckpoint->GetAncestor(pindex->nHeight) == pindex) { + // This block is an ancestor of a checkpoint: disable script checks + fScriptChecks = false; + } + } + + int64_t nTime1 = GetTimeMicros(); nTimeCheck += nTime1 - nTimeStart; + LogPrint("bench", " - Sanity checks: %.2fms [%.2fs]\n", 0.001 * (nTime1 - nTimeStart), nTimeCheck * 0.000001); // Do not allow blocks that contain transactions which 'overwrite' older transactions, // unless those are already completely spent. @@ -1630,18 +2339,29 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin // See BIP30 and http://r6.ca/blog/20120206T005236Z.html for more information. // This logic is not necessary for memory pool transactions, as AcceptToMemoryPool // already refuses previously-known transaction ids entirely. - // This rule was originally applied all blocks whose timestamp was after March 15, 2012, 0:00 UTC. + // This rule was originally applied to all blocks with a timestamp after March 15, 2012, 0:00 UTC. // Now that the whole chain is irreversibly beyond that time it is applied to all blocks except the - // two in the chain that violate it. This prevents exploiting the issue against nodes in their + // two in the chain that violate it. This prevents exploiting the issue against nodes during their // initial block download. bool fEnforceBIP30 = (!pindex->phashBlock) || // Enforce on CreateNewBlock invocations which don't have a hash. - !((pindex->nHeight==91842 && pindex->GetBlockHash() == uint256("0x00000000000a4d0a398161ffc163c503763b1f4360639393e0e4c8e300e0caec")) || - (pindex->nHeight==91880 && pindex->GetBlockHash() == uint256("0x00000000000743f190a18c5577a3c2d2a1f610ae9601ac046a38084ccb7cd721"))); + !((pindex->nHeight==91842 && pindex->GetBlockHash() == uint256S("0x00000000000a4d0a398161ffc163c503763b1f4360639393e0e4c8e300e0caec")) || + (pindex->nHeight==91880 && pindex->GetBlockHash() == uint256S("0x00000000000743f190a18c5577a3c2d2a1f610ae9601ac046a38084ccb7cd721"))); + + // Once BIP34 activated it was not possible to create new duplicate coinbases and thus other than starting + // with the 2 existing duplicate coinbase pairs, not possible to create overwriting txs. But by the + // time BIP34 activated, in each of the existing pairs the duplicate coinbase had overwritten the first + // before the first had been spent. Since those coinbases are sufficiently buried its no longer possible to create further + // duplicate transactions descending from the known pairs either. + // If we're on the known chain at height greater than where BIP34 activated, we can save the db accesses needed for the BIP30 check. + CBlockIndex *pindexBIP34height = pindex->pprev->GetAncestor(chainparams.GetConsensus().BIP34Height); + //Only continue to enforce if we're below BIP34 activation height or the block hash at that height doesn't correspond. + fEnforceBIP30 = fEnforceBIP30 && (!pindexBIP34height || !(pindexBIP34height->GetBlockHash() == chainparams.GetConsensus().BIP34Hash)); + if (fEnforceBIP30) { BOOST_FOREACH(const CTransaction& tx, block.vtx) { const CCoins* coins = view.AccessCoins(tx.GetHash()); if (coins && !coins->IsPruned()) - return state.DoS(100, error("ConnectBlock() : tried to overwrite transaction"), + return state.DoS(100, error("ConnectBlock(): tried to overwrite transaction"), REJECT_INVALID, "bad-txns-BIP30"); } } @@ -1652,14 +2372,42 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin unsigned int flags = fStrictPayToScriptHash ? SCRIPT_VERIFY_P2SH : SCRIPT_VERIFY_NONE; + // Start enforcing the DERSIG (BIP66) rules, for block.nVersion=3 blocks, + // when 75% of the network has upgraded: + if (block.nVersion >= 3 && IsSuperMajority(3, pindex->pprev, chainparams.GetConsensus().nMajorityEnforceBlockUpgrade, chainparams.GetConsensus())) { + flags |= SCRIPT_VERIFY_DERSIG; + } + + // Start enforcing CHECKLOCKTIMEVERIFY, (BIP65) for block.nVersion=4 + // blocks, when 75% of the network has upgraded: + if (block.nVersion >= 4 && IsSuperMajority(4, pindex->pprev, chainparams.GetConsensus().nMajorityEnforceBlockUpgrade, chainparams.GetConsensus())) { + flags |= SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY; + } + + // Start enforcing BIP68 (sequence locks) and BIP112 (CHECKSEQUENCEVERIFY) using versionbits logic. + int nLockTimeFlags = 0; + if (VersionBitsState(pindex->pprev, chainparams.GetConsensus(), Consensus::DEPLOYMENT_CSV, versionbitscache) == THRESHOLD_ACTIVE) { + flags |= SCRIPT_VERIFY_CHECKSEQUENCEVERIFY; + nLockTimeFlags |= LOCKTIME_VERIFY_SEQUENCE; + } + + // Start enforcing WITNESS rules using versionbits logic. + if (IsWitnessEnabled(pindex->pprev, chainparams.GetConsensus())) { + flags |= SCRIPT_VERIFY_WITNESS; + } + + int64_t nTime2 = GetTimeMicros(); nTimeForks += nTime2 - nTime1; + LogPrint("bench", " - Fork checks: %.2fms [%.2fs]\n", 0.001 * (nTime2 - nTime1), nTimeForks * 0.000001); + CBlockUndo blockundo; CCheckQueueControl<CScriptCheck> control(fScriptChecks && nScriptCheckThreads ? &scriptcheckqueue : NULL); - int64_t nTimeStart = GetTimeMicros(); + std::vector<uint256> vOrphanErase; + std::vector<int> prevheights; CAmount nFees = 0; int nInputs = 0; - unsigned int nSigOps = 0; + int64_t nSigOpsCost = 0; CDiskTxPos pos(pindex->GetBlockPos(), GetSizeOfCompactSize(block.vtx.size())); std::vector<std::pair<uint256, CDiskTxPos> > vPos; vPos.reserve(block.vtx.size()); @@ -1669,33 +2417,56 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin const CTransaction &tx = block.vtx[i]; nInputs += tx.vin.size(); - nSigOps += GetLegacySigOpCount(tx); - if (nSigOps > MAX_BLOCK_SIGOPS) - return state.DoS(100, error("ConnectBlock() : too many sigops"), - REJECT_INVALID, "bad-blk-sigops"); if (!tx.IsCoinBase()) { if (!view.HaveInputs(tx)) - return state.DoS(100, error("ConnectBlock() : inputs missing/spent"), + return state.DoS(100, error("ConnectBlock(): inputs missing/spent"), REJECT_INVALID, "bad-txns-inputs-missingorspent"); - if (fStrictPayToScriptHash) - { - // Add in sigops done by pay-to-script-hash inputs; - // this is to prevent a "rogue miner" from creating - // an incredibly-expensive-to-validate block. - nSigOps += GetP2SHSigOpCount(tx, view); - if (nSigOps > MAX_BLOCK_SIGOPS) - return state.DoS(100, error("ConnectBlock() : too many sigops"), - REJECT_INVALID, "bad-blk-sigops"); + // Check that transaction is BIP68 final + // BIP68 lock checks (as opposed to nLockTime checks) must + // be in ConnectBlock because they require the UTXO set + prevheights.resize(tx.vin.size()); + for (size_t j = 0; j < tx.vin.size(); j++) { + prevheights[j] = view.AccessCoins(tx.vin[j].prevout.hash)->nHeight; + } + + // Which orphan pool entries must we evict? + for (size_t j = 0; j < tx.vin.size(); j++) { + auto itByPrev = mapOrphanTransactionsByPrev.find(tx.vin[j].prevout); + if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; + for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { + const CTransaction& orphanTx = (*mi)->second.tx; + const uint256& orphanHash = orphanTx.GetHash(); + vOrphanErase.push_back(orphanHash); + } } + if (!SequenceLocks(tx, nLockTimeFlags, &prevheights, *pindex)) { + return state.DoS(100, error("%s: contains a non-BIP68-final transaction", __func__), + REJECT_INVALID, "bad-txns-nonfinal"); + } + } + + // GetTransactionSigOpCost counts 3 types of sigops: + // * legacy (always) + // * p2sh (when P2SH enabled in flags and excludes coinbase) + // * witness (when witness enabled in flags and excludes coinbase) + nSigOpsCost += GetTransactionSigOpCost(tx, view, flags); + if (nSigOpsCost > MAX_BLOCK_SIGOPS_COST) + return state.DoS(100, error("ConnectBlock(): too many sigops"), + REJECT_INVALID, "bad-blk-sigops"); + + if (!tx.IsCoinBase()) + { nFees += view.GetValueIn(tx)-tx.GetValueOut(); std::vector<CScriptCheck> vChecks; - if (!CheckInputs(tx, state, view, fScriptChecks, flags, false, nScriptCheckThreads ? &vChecks : NULL)) - return false; + bool fCacheResults = fJustCheck; /* Don't cache results if we're actually connecting blocks (still consult the cache, though) */ + if (!CheckInputs(tx, state, view, fScriptChecks, flags, fCacheResults, nScriptCheckThreads ? &vChecks : NULL)) + return error("ConnectBlock(): CheckInputs on %s failed with %s", + tx.GetHash().ToString(), FormatStateMessage(state)); control.Add(vChecks); } @@ -1703,24 +2474,25 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (i > 0) { blockundo.vtxundo.push_back(CTxUndo()); } - UpdateCoins(tx, state, view, i == 0 ? undoDummy : blockundo.vtxundo.back(), pindex->nHeight); + UpdateCoins(tx, view, i == 0 ? undoDummy : blockundo.vtxundo.back(), pindex->nHeight); vPos.push_back(std::make_pair(tx.GetHash(), pos)); pos.nTxOffset += ::GetSerializeSize(tx, SER_DISK, CLIENT_VERSION); } - int64_t nTime1 = GetTimeMicros(); nTimeConnect += nTime1 - nTimeStart; - LogPrint("bench", " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs]\n", (unsigned)block.vtx.size(), 0.001 * (nTime1 - nTimeStart), 0.001 * (nTime1 - nTimeStart) / block.vtx.size(), nInputs <= 1 ? 0 : 0.001 * (nTime1 - nTimeStart) / (nInputs-1), nTimeConnect * 0.000001); + int64_t nTime3 = GetTimeMicros(); nTimeConnect += nTime3 - nTime2; + LogPrint("bench", " - Connect %u transactions: %.2fms (%.3fms/tx, %.3fms/txin) [%.2fs]\n", (unsigned)block.vtx.size(), 0.001 * (nTime3 - nTime2), 0.001 * (nTime3 - nTime2) / block.vtx.size(), nInputs <= 1 ? 0 : 0.001 * (nTime3 - nTime2) / (nInputs-1), nTimeConnect * 0.000001); - if (block.vtx[0].GetValueOut() > GetBlockValue(pindex->nHeight, nFees)) + CAmount blockReward = nFees + GetBlockSubsidy(pindex->nHeight, chainparams.GetConsensus()); + if (block.vtx[0].GetValueOut() > blockReward) return state.DoS(100, - error("ConnectBlock() : coinbase pays too much (actual=%d vs limit=%d)", - block.vtx[0].GetValueOut(), GetBlockValue(pindex->nHeight, nFees)), + error("ConnectBlock(): coinbase pays too much (actual=%d vs limit=%d)", + block.vtx[0].GetValueOut(), blockReward), REJECT_INVALID, "bad-cb-amount"); if (!control.Wait()) return state.DoS(100, false); - int64_t nTime2 = GetTimeMicros(); nTimeVerify += nTime2 - nTimeStart; - LogPrint("bench", " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs]\n", nInputs - 1, 0.001 * (nTime2 - nTimeStart), nInputs <= 1 ? 0 : 0.001 * (nTime2 - nTimeStart) / (nInputs-1), nTimeVerify * 0.000001); + int64_t nTime4 = GetTimeMicros(); nTimeVerify += nTime4 - nTime2; + LogPrint("bench", " - Verify %u txins: %.2fms (%.3fms/txin) [%.2fs]\n", nInputs - 1, 0.001 * (nTime4 - nTime2), nInputs <= 1 ? 0 : 0.001 * (nTime4 - nTime2) / (nInputs-1), nTimeVerify * 0.000001); if (fJustCheck) return true; @@ -1731,9 +2503,9 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (pindex->GetUndoPos().IsNull()) { CDiskBlockPos pos; if (!FindUndoPos(state, pindex->nFile, pos, ::GetSerializeSize(blockundo, SER_DISK, CLIENT_VERSION) + 40)) - return error("ConnectBlock() : FindUndoPos failed"); - if (!blockundo.WriteToDisk(pos, pindex->pprev->GetBlockHash())) - return state.Abort("Failed to write undo data"); + return error("ConnectBlock(): FindUndoPos failed"); + if (!UndoWriteToDisk(blockundo, pos, pindex->pprev->GetBlockHash(), chainparams.MessageStart())) + return AbortNode(state, "Failed to write undo data"); // update nUndoPos in block index pindex->nUndoPos = pos.nPos; @@ -1746,26 +2518,36 @@ bool ConnectBlock(const CBlock& block, CValidationState& state, CBlockIndex* pin if (fTxIndex) if (!pblocktree->WriteTxIndex(vPos)) - return state.Abort("Failed to write transaction index"); + return AbortNode(state, "Failed to write transaction index"); // add this block to the view's block chain view.SetBestBlock(pindex->GetBlockHash()); - int64_t nTime3 = GetTimeMicros(); nTimeIndex += nTime3 - nTime2; - LogPrint("bench", " - Index writing: %.2fms [%.2fs]\n", 0.001 * (nTime3 - nTime2), nTimeIndex * 0.000001); + int64_t nTime5 = GetTimeMicros(); nTimeIndex += nTime5 - nTime4; + LogPrint("bench", " - Index writing: %.2fms [%.2fs]\n", 0.001 * (nTime5 - nTime4), nTimeIndex * 0.000001); // Watch for changes to the previous coinbase transaction. static uint256 hashPrevBestCoinBase; - g_signals.UpdatedTransaction(hashPrevBestCoinBase); + GetMainSignals().UpdatedTransaction(hashPrevBestCoinBase); hashPrevBestCoinBase = block.vtx[0].GetHash(); - int64_t nTime4 = GetTimeMicros(); nTimeCallbacks += nTime4 - nTime3; - LogPrint("bench", " - Callbacks: %.2fms [%.2fs]\n", 0.001 * (nTime4 - nTime3), nTimeCallbacks * 0.000001); + // Erase orphan transactions include or precluded by this block + if (vOrphanErase.size()) { + int nErased = 0; + BOOST_FOREACH(uint256 &orphanHash, vOrphanErase) { + nErased += EraseOrphanTx(orphanHash); + } + LogPrint("mempool", "Erased %d orphan tx included or conflicted by block\n", nErased); + } + + int64_t nTime6 = GetTimeMicros(); nTimeCallbacks += nTime6 - nTime5; + LogPrint("bench", " - Callbacks: %.2fms [%.2fs]\n", 0.001 * (nTime6 - nTime5), nTimeCallbacks * 0.000001); return true; } enum FlushStateMode { + FLUSH_STATE_NONE, FLUSH_STATE_IF_NEEDED, FLUSH_STATE_PERIODIC, FLUSH_STATE_ALWAYS @@ -1773,51 +2555,103 @@ enum FlushStateMode { /** * Update the on-disk chain state. - * The caches and indexes are flushed if either they're too large, forceWrite is set, or - * fast is not set and it's been a while since the last write. + * The caches and indexes are flushed depending on the mode we're called with + * if they're too large, if it's been a while since the last write, + * or always and in all cases if we're in prune mode and are deleting files. */ bool static FlushStateToDisk(CValidationState &state, FlushStateMode mode) { - LOCK(cs_main); + const CChainParams& chainparams = Params(); + LOCK2(cs_main, cs_LastBlockFile); static int64_t nLastWrite = 0; - if ((mode == FLUSH_STATE_ALWAYS) || - ((mode == FLUSH_STATE_PERIODIC || mode == FLUSH_STATE_IF_NEEDED) && pcoinsTip->GetCacheSize() > nCoinCacheSize) || - (mode == FLUSH_STATE_PERIODIC && GetTimeMicros() > nLastWrite + DATABASE_WRITE_INTERVAL * 1000000)) { - // Typical CCoins structures on disk are around 100 bytes in size. - // Pushing a new one to the database can cause it to be written - // twice (once in the log, and once in the tables). This is already - // an overestimation, as most will delete an existing entry or - // overwrite one. Still, use a conservative safety factor of 2. - if (!CheckDiskSpace(100 * 2 * 2 * pcoinsTip->GetCacheSize())) + static int64_t nLastFlush = 0; + static int64_t nLastSetChain = 0; + std::set<int> setFilesToPrune; + bool fFlushForPrune = false; + try { + if (fPruneMode && fCheckForPruning && !fReindex) { + FindFilesToPrune(setFilesToPrune, chainparams.PruneAfterHeight()); + fCheckForPruning = false; + if (!setFilesToPrune.empty()) { + fFlushForPrune = true; + if (!fHavePruned) { + pblocktree->WriteFlag("prunedblockfiles", true); + fHavePruned = true; + } + } + } + int64_t nNow = GetTimeMicros(); + // Avoid writing/flushing immediately after startup. + if (nLastWrite == 0) { + nLastWrite = nNow; + } + if (nLastFlush == 0) { + nLastFlush = nNow; + } + if (nLastSetChain == 0) { + nLastSetChain = nNow; + } + size_t cacheSize = pcoinsTip->DynamicMemoryUsage(); + // The cache is large and close to the limit, but we have time now (not in the middle of a block processing). + bool fCacheLarge = mode == FLUSH_STATE_PERIODIC && cacheSize * (10.0/9) > nCoinCacheUsage; + // The cache is over the limit, we have to write now. + bool fCacheCritical = mode == FLUSH_STATE_IF_NEEDED && cacheSize > nCoinCacheUsage; + // It's been a while since we wrote the block index to disk. Do this frequently, so we don't need to redownload after a crash. + bool fPeriodicWrite = mode == FLUSH_STATE_PERIODIC && nNow > nLastWrite + (int64_t)DATABASE_WRITE_INTERVAL * 1000000; + // It's been very long since we flushed the cache. Do this infrequently, to optimize cache usage. + bool fPeriodicFlush = mode == FLUSH_STATE_PERIODIC && nNow > nLastFlush + (int64_t)DATABASE_FLUSH_INTERVAL * 1000000; + // Combine all conditions that result in a full cache flush. + bool fDoFullFlush = (mode == FLUSH_STATE_ALWAYS) || fCacheLarge || fCacheCritical || fPeriodicFlush || fFlushForPrune; + // Write blocks and block index to disk. + if (fDoFullFlush || fPeriodicWrite) { + // Depend on nMinDiskSpace to ensure we can write block index + if (!CheckDiskSpace(0)) return state.Error("out of disk space"); // First make sure all block and undo data is flushed to disk. FlushBlockFile(); // Then update all block file information (which may refer to block and undo files). - bool fileschanged = false; - for (set<int>::iterator it = setDirtyFileInfo.begin(); it != setDirtyFileInfo.end(); ) { - if (!pblocktree->WriteBlockFileInfo(*it, vinfoBlockFile[*it])) { - return state.Abort("Failed to write to block index"); + { + std::vector<std::pair<int, const CBlockFileInfo*> > vFiles; + vFiles.reserve(setDirtyFileInfo.size()); + for (set<int>::iterator it = setDirtyFileInfo.begin(); it != setDirtyFileInfo.end(); ) { + vFiles.push_back(make_pair(*it, &vinfoBlockFile[*it])); + setDirtyFileInfo.erase(it++); + } + std::vector<const CBlockIndex*> vBlocks; + vBlocks.reserve(setDirtyBlockIndex.size()); + for (set<CBlockIndex*>::iterator it = setDirtyBlockIndex.begin(); it != setDirtyBlockIndex.end(); ) { + vBlocks.push_back(*it); + setDirtyBlockIndex.erase(it++); + } + if (!pblocktree->WriteBatchSync(vFiles, nLastBlockFile, vBlocks)) { + return AbortNode(state, "Files to write to block index database"); } - fileschanged = true; - setDirtyFileInfo.erase(it++); - } - if (fileschanged && !pblocktree->WriteLastBlockFile(nLastBlockFile)) { - return state.Abort("Failed to write to block index"); - } - for (set<CBlockIndex*>::iterator it = setDirtyBlockIndex.begin(); it != setDirtyBlockIndex.end(); ) { - if (!pblocktree->WriteBlockIndex(CDiskBlockIndex(*it))) { - return state.Abort("Failed to write to block index"); - } - setDirtyBlockIndex.erase(it++); } - pblocktree->Sync(); - // Finally flush the chainstate (which may refer to block index entries). + // Finally remove any pruned files + if (fFlushForPrune) + UnlinkPrunedFiles(setFilesToPrune); + nLastWrite = nNow; + } + // Flush best chain related state. This can only be done if the blocks / block index write was also done. + if (fDoFullFlush) { + // Typical CCoins structures on disk are around 128 bytes in size. + // Pushing a new one to the database can cause it to be written + // twice (once in the log, and once in the tables). This is already + // an overestimation, as most will delete an existing entry or + // overwrite one. Still, use a conservative safety factor of 2. + if (!CheckDiskSpace(128 * 2 * 2 * pcoinsTip->GetCacheSize())) + return state.Error("out of disk space"); + // Flush the chainstate (which may refer to block index entries). if (!pcoinsTip->Flush()) - return state.Abort("Failed to write to coin database"); + return AbortNode(state, "Failed to write to coin database"); + nLastFlush = nNow; + } + if (fDoFullFlush || ((mode == FLUSH_STATE_ALWAYS || mode == FLUSH_STATE_PERIODIC) && nNow > nLastSetChain + (int64_t)DATABASE_WRITE_INTERVAL * 1000000)) { // Update best block in wallet (so we can detect restored wallets). - if (mode != FLUSH_STATE_IF_NEEDED) { - g_signals.SetBestChain(chainActive.GetLocator()); - } - nLastWrite = GetTimeMicros(); + GetMainSignals().SetBestChain(chainActive.GetLocator()); + nLastSetChain = nNow; + } + } catch (const std::runtime_error& e) { + return AbortNode(state, std::string("System error while flushing: ") + e.what()); } return true; } @@ -1827,82 +2661,123 @@ void FlushStateToDisk() { FlushStateToDisk(state, FLUSH_STATE_ALWAYS); } +void PruneAndFlush() { + CValidationState state; + fCheckForPruning = true; + FlushStateToDisk(state, FLUSH_STATE_NONE); +} + /** Update chainActive and related internal data structures. */ -void static UpdateTip(CBlockIndex *pindexNew) { +void static UpdateTip(CBlockIndex *pindexNew, const CChainParams& chainParams) { chainActive.SetTip(pindexNew); // New best block nTimeBestReceived = GetTime(); mempool.AddTransactionsUpdated(1); - LogPrintf("UpdateTip: new best=%s height=%d log2_work=%.8g tx=%lu date=%s progress=%f cache=%u\n", - chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, - DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), - Checkpoints::GuessVerificationProgress(chainActive.Tip()), (unsigned int)pcoinsTip->GetCacheSize()); - cvBlockChange.notify_all(); - // Check the version of the last 100 blocks to see if we need to upgrade: static bool fWarned = false; - if (!IsInitialBlockDownload() && !fWarned) + std::vector<std::string> warningMessages; + if (!IsInitialBlockDownload()) { int nUpgraded = 0; const CBlockIndex* pindex = chainActive.Tip(); + for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { + WarningBitsConditionChecker checker(bit); + ThresholdState state = checker.GetStateFor(pindex, chainParams.GetConsensus(), warningcache[bit]); + if (state == THRESHOLD_ACTIVE || state == THRESHOLD_LOCKED_IN) { + if (state == THRESHOLD_ACTIVE) { + strMiscWarning = strprintf(_("Warning: unknown new rules activated (versionbit %i)"), bit); + if (!fWarned) { + AlertNotify(strMiscWarning); + fWarned = true; + } + } else { + warningMessages.push_back(strprintf("unknown new rules are about to activate (versionbit %i)", bit)); + } + } + } + // Check the version of the last 100 blocks to see if we need to upgrade: for (int i = 0; i < 100 && pindex != NULL; i++) { - if (pindex->nVersion > CBlock::CURRENT_VERSION) + int32_t nExpectedVersion = ComputeBlockVersion(pindex->pprev, chainParams.GetConsensus()); + if (pindex->nVersion > VERSIONBITS_LAST_OLD_BLOCK_VERSION && (pindex->nVersion & ~nExpectedVersion) != 0) ++nUpgraded; pindex = pindex->pprev; } if (nUpgraded > 0) - LogPrintf("SetBestChain: %d of last 100 blocks above version %d\n", nUpgraded, (int)CBlock::CURRENT_VERSION); + warningMessages.push_back(strprintf("%d of last 100 blocks have unexpected version", nUpgraded)); if (nUpgraded > 100/2) { // strMiscWarning is read by GetWarnings(), called by Qt and the JSON-RPC code to warn the user: - strMiscWarning = _("Warning: This version is obsolete, upgrade required!"); - CAlert::Notify(strMiscWarning, true); - fWarned = true; + strMiscWarning = _("Warning: Unknown block versions being mined! It's possible unknown rules are in effect"); + if (!fWarned) { + AlertNotify(strMiscWarning); + fWarned = true; + } } } + LogPrintf("%s: new best=%s height=%d version=0x%08x log2_work=%.8g tx=%lu date='%s' progress=%f cache=%.1fMiB(%utx)", __func__, + chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), chainActive.Tip()->nVersion, + log(chainActive.Tip()->nChainWork.getdouble())/log(2.0), (unsigned long)chainActive.Tip()->nChainTx, + DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), + Checkpoints::GuessVerificationProgress(chainParams.Checkpoints(), chainActive.Tip()), pcoinsTip->DynamicMemoryUsage() * (1.0 / (1<<20)), pcoinsTip->GetCacheSize()); + if (!warningMessages.empty()) + LogPrintf(" warning='%s'", boost::algorithm::join(warningMessages, ", ")); + LogPrintf("\n"); + } -/** Disconnect chainActive's tip. */ -bool static DisconnectTip(CValidationState &state) { +/** Disconnect chainActive's tip. You probably want to call mempool.removeForReorg and manually re-limit mempool size after this, with cs_main held. */ +bool static DisconnectTip(CValidationState& state, const CChainParams& chainparams, bool fBare = false) +{ CBlockIndex *pindexDelete = chainActive.Tip(); assert(pindexDelete); - mempool.check(pcoinsTip); // Read block from disk. CBlock block; - if (!ReadBlockFromDisk(block, pindexDelete)) - return state.Abort("Failed to read block"); + if (!ReadBlockFromDisk(block, pindexDelete, chainparams.GetConsensus())) + return AbortNode(state, "Failed to read block"); // Apply the block atomically to the chain state. int64_t nStart = GetTimeMicros(); { CCoinsViewCache view(pcoinsTip); if (!DisconnectBlock(block, state, pindexDelete, view)) - return error("DisconnectTip() : DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); + return error("DisconnectTip(): DisconnectBlock %s failed", pindexDelete->GetBlockHash().ToString()); assert(view.Flush()); } LogPrint("bench", "- Disconnect block: %.2fms\n", (GetTimeMicros() - nStart) * 0.001); // Write the chain state to disk, if necessary. if (!FlushStateToDisk(state, FLUSH_STATE_IF_NEEDED)) return false; - // Resurrect mempool transactions from the disconnected block. - BOOST_FOREACH(const CTransaction &tx, block.vtx) { - // ignore validation errors in resurrected transactions - list<CTransaction> removed; - CValidationState stateDummy; - if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL)) - mempool.remove(tx, removed, true); + + if (!fBare) { + // Resurrect mempool transactions from the disconnected block. + std::vector<uint256> vHashUpdate; + BOOST_FOREACH(const CTransaction &tx, block.vtx) { + // ignore validation errors in resurrected transactions + list<CTransaction> removed; + CValidationState stateDummy; + if (tx.IsCoinBase() || !AcceptToMemoryPool(mempool, stateDummy, tx, false, NULL, true)) { + mempool.removeRecursive(tx, removed); + } else if (mempool.exists(tx.GetHash())) { + vHashUpdate.push_back(tx.GetHash()); + } + } + // AcceptToMemoryPool/addUnchecked all assume that new mempool entries have + // no in-mempool children, which is generally not true when adding + // previously-confirmed transactions back to the mempool. + // UpdateTransactionsFromBlock finds descendants of any transactions in this + // block that were added back and cleans up the mempool state. + mempool.UpdateTransactionsFromBlock(vHashUpdate); } - mempool.removeCoinbaseSpends(pcoinsTip, pindexDelete->nHeight); - mempool.check(pcoinsTip); + // Update chainActive and related variables. - UpdateTip(pindexDelete->pprev); + UpdateTip(pindexDelete->pprev, chainparams); // Let wallets know transactions went from 1-confirmed to // 0-confirmed or conflicted: BOOST_FOREACH(const CTransaction &tx, block.vtx) { - SyncWithWallets(tx, NULL); + SyncWithWallets(tx, pindexDelete->pprev, NULL); } return true; } @@ -1913,19 +2788,19 @@ static int64_t nTimeFlush = 0; static int64_t nTimeChainState = 0; static int64_t nTimePostConnect = 0; -/** +/** * Connect a new block to chainActive. pblock is either NULL or a pointer to a CBlock * corresponding to pindexNew, to bypass loading it again from disk. */ -bool static ConnectTip(CValidationState &state, CBlockIndex *pindexNew, CBlock *pblock) { +bool static ConnectTip(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexNew, const CBlock* pblock) +{ assert(pindexNew->pprev == chainActive.Tip()); - mempool.check(pcoinsTip); // Read block from disk. int64_t nTime1 = GetTimeMicros(); CBlock block; if (!pblock) { - if (!ReadBlockFromDisk(block, pindexNew)) - return state.Abort("Failed to read block"); + if (!ReadBlockFromDisk(block, pindexNew, chainparams.GetConsensus())) + return AbortNode(state, "Failed to read block"); pblock = █ } // Apply the block atomically to the chain state. @@ -1934,15 +2809,14 @@ bool static ConnectTip(CValidationState &state, CBlockIndex *pindexNew, CBlock * LogPrint("bench", " - Load block from disk: %.2fms [%.2fs]\n", (nTime2 - nTime1) * 0.001, nTimeReadFromDisk * 0.000001); { CCoinsViewCache view(pcoinsTip); - CInv inv(MSG_BLOCK, pindexNew->GetBlockHash()); - bool rv = ConnectBlock(*pblock, state, pindexNew, view); - g_signals.BlockChecked(*pblock, state); + bool rv = ConnectBlock(*pblock, state, pindexNew, view, chainparams); + GetMainSignals().BlockChecked(*pblock, state); if (!rv) { if (state.IsInvalid()) InvalidBlockFound(pindexNew, state); - return error("ConnectTip() : ConnectBlock %s failed", pindexNew->GetBlockHash().ToString()); + return error("ConnectTip(): ConnectBlock %s failed", pindexNew->GetBlockHash().ToString()); } - mapBlockSource.erase(inv.hash); + mapBlockSource.erase(pindexNew->GetBlockHash()); nTime3 = GetTimeMicros(); nTimeConnectTotal += nTime3 - nTime2; LogPrint("bench", " - Connect total: %.2fms [%.2fs]\n", (nTime3 - nTime2) * 0.001, nTimeConnectTotal * 0.000001); assert(view.Flush()); @@ -1956,18 +2830,17 @@ bool static ConnectTip(CValidationState &state, CBlockIndex *pindexNew, CBlock * LogPrint("bench", " - Writing chainstate: %.2fms [%.2fs]\n", (nTime5 - nTime4) * 0.001, nTimeChainState * 0.000001); // Remove conflicting transactions from the mempool. list<CTransaction> txConflicted; - mempool.removeForBlock(pblock->vtx, pindexNew->nHeight, txConflicted); - mempool.check(pcoinsTip); + mempool.removeForBlock(pblock->vtx, pindexNew->nHeight, txConflicted, !IsInitialBlockDownload()); // Update chainActive & related variables. - UpdateTip(pindexNew); + UpdateTip(pindexNew, chainparams); // Tell wallet about transactions that went from mempool // to conflicted: BOOST_FOREACH(const CTransaction &tx, txConflicted) { - SyncWithWallets(tx, NULL); + SyncWithWallets(tx, pindexNew, NULL); } // ... and about transactions that got confirmed: BOOST_FOREACH(const CTransaction &tx, pblock->vtx) { - SyncWithWallets(tx, pblock); + SyncWithWallets(tx, pindexNew, pblock); } int64_t nTime6 = GetTimeMicros(); nTimePostConnect += nTime6 - nTime5; nTimeTotal += nTime6 - nTime1; @@ -1997,15 +2870,29 @@ static CBlockIndex* FindMostWorkChain() { CBlockIndex *pindexTest = pindexNew; bool fInvalidAncestor = false; while (pindexTest && !chainActive.Contains(pindexTest)) { - assert(pindexTest->nStatus & BLOCK_HAVE_DATA); assert(pindexTest->nChainTx || pindexTest->nHeight == 0); - if (pindexTest->nStatus & BLOCK_FAILED_MASK) { - // Candidate has an invalid ancestor, remove entire chain from the set. - if (pindexBestInvalid == NULL || pindexNew->nChainWork > pindexBestInvalid->nChainWork) + + // Pruned nodes may have entries in setBlockIndexCandidates for + // which block files have been deleted. Remove those as candidates + // for the most work chain if we come across them; we can't switch + // to a chain unless we have all the non-active-chain parent blocks. + bool fFailedChain = pindexTest->nStatus & BLOCK_FAILED_MASK; + bool fMissingData = !(pindexTest->nStatus & BLOCK_HAVE_DATA); + if (fFailedChain || fMissingData) { + // Candidate chain is not usable (either invalid or missing data) + if (fFailedChain && (pindexBestInvalid == NULL || pindexNew->nChainWork > pindexBestInvalid->nChainWork)) pindexBestInvalid = pindexNew; CBlockIndex *pindexFailed = pindexNew; + // Remove the entire chain from the set. while (pindexTest != pindexFailed) { - pindexFailed->nStatus |= BLOCK_FAILED_CHILD; + if (fFailedChain) { + pindexFailed->nStatus |= BLOCK_FAILED_CHILD; + } else if (fMissingData) { + // If we're missing data, then add back to mapBlocksUnlinked, + // so that if the block arrives in the future we can try adding + // to setBlockIndexCandidates again. + mapBlocksUnlinked.insert(std::make_pair(pindexFailed->pprev, pindexFailed)); + } setBlockIndexCandidates.erase(pindexFailed); pindexFailed = pindexFailed->pprev; } @@ -2036,16 +2923,18 @@ static void PruneBlockIndexCandidates() { * Try to make some progress towards making pindexMostWork the active block. * pblock is either NULL or a pointer to a CBlock corresponding to pindexMostWork. */ -static bool ActivateBestChainStep(CValidationState &state, CBlockIndex *pindexMostWork, CBlock *pblock) { +static bool ActivateBestChainStep(CValidationState& state, const CChainParams& chainparams, CBlockIndex* pindexMostWork, const CBlock* pblock, bool& fInvalidFound) +{ AssertLockHeld(cs_main); - bool fInvalidFound = false; const CBlockIndex *pindexOldTip = chainActive.Tip(); const CBlockIndex *pindexFork = chainActive.FindFork(pindexMostWork); // Disconnect active blocks which are no longer in the best chain. + bool fBlocksDisconnected = false; while (chainActive.Tip() && chainActive.Tip() != pindexFork) { - if (!DisconnectTip(state)) + if (!DisconnectTip(state, chainparams)) return false; + fBlocksDisconnected = true; } // Build list of new blocks to connect. @@ -2053,43 +2942,49 @@ static bool ActivateBestChainStep(CValidationState &state, CBlockIndex *pindexMo bool fContinue = true; int nHeight = pindexFork ? pindexFork->nHeight : -1; while (fContinue && nHeight != pindexMostWork->nHeight) { - // Don't iterate the entire list of potential improvements toward the best tip, as we likely only need - // a few blocks along the way. - int nTargetHeight = std::min(nHeight + 32, pindexMostWork->nHeight); - vpindexToConnect.clear(); - vpindexToConnect.reserve(nTargetHeight - nHeight); - CBlockIndex *pindexIter = pindexMostWork->GetAncestor(nTargetHeight); - while (pindexIter && pindexIter->nHeight != nHeight) { - vpindexToConnect.push_back(pindexIter); - pindexIter = pindexIter->pprev; - } - nHeight = nTargetHeight; - - // Connect new blocks. - BOOST_REVERSE_FOREACH(CBlockIndex *pindexConnect, vpindexToConnect) { - if (!ConnectTip(state, pindexConnect, pindexConnect == pindexMostWork ? pblock : NULL)) { - if (state.IsInvalid()) { - // The block violates a consensus rule. - if (!state.CorruptionPossible()) - InvalidChainFound(vpindexToConnect.back()); - state = CValidationState(); - fInvalidFound = true; - fContinue = false; - break; + // Don't iterate the entire list of potential improvements toward the best tip, as we likely only need + // a few blocks along the way. + int nTargetHeight = std::min(nHeight + 32, pindexMostWork->nHeight); + vpindexToConnect.clear(); + vpindexToConnect.reserve(nTargetHeight - nHeight); + CBlockIndex *pindexIter = pindexMostWork->GetAncestor(nTargetHeight); + while (pindexIter && pindexIter->nHeight != nHeight) { + vpindexToConnect.push_back(pindexIter); + pindexIter = pindexIter->pprev; + } + nHeight = nTargetHeight; + + // Connect new blocks. + BOOST_REVERSE_FOREACH(CBlockIndex *pindexConnect, vpindexToConnect) { + if (!ConnectTip(state, chainparams, pindexConnect, pindexConnect == pindexMostWork ? pblock : NULL)) { + if (state.IsInvalid()) { + // The block violates a consensus rule. + if (!state.CorruptionPossible()) + InvalidChainFound(vpindexToConnect.back()); + state = CValidationState(); + fInvalidFound = true; + fContinue = false; + break; + } else { + // A system error occurred (disk space, database error, ...). + return false; + } } else { - // A system error occurred (disk space, database error, ...). - return false; - } - } else { - PruneBlockIndexCandidates(); - if (!pindexOldTip || chainActive.Tip()->nChainWork > pindexOldTip->nChainWork) { - // We're in a better position than we were. Return temporarily to release the lock. - fContinue = false; - break; + PruneBlockIndexCandidates(); + if (!pindexOldTip || chainActive.Tip()->nChainWork > pindexOldTip->nChainWork) { + // We're in a better position than we were. Return temporarily to release the lock. + fContinue = false; + break; + } } } } + + if (fBlocksDisconnected) { + mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); + LimitMempoolSize(mempool, GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); } + mempool.check(pcoinsTip); // Callbacks/notifications for a new best chain. if (fInvalidFound) @@ -2100,49 +2995,110 @@ static bool ActivateBestChainStep(CValidationState &state, CBlockIndex *pindexMo return true; } +static void NotifyHeaderTip() { + bool fNotify = false; + bool fInitialBlockDownload = false; + static CBlockIndex* pindexHeaderOld = NULL; + CBlockIndex* pindexHeader = NULL; + { + LOCK(cs_main); + if (!setBlockIndexCandidates.empty()) { + pindexHeader = *setBlockIndexCandidates.rbegin(); + } + if (pindexHeader != pindexHeaderOld) { + fNotify = true; + fInitialBlockDownload = IsInitialBlockDownload(); + pindexHeaderOld = pindexHeader; + } + } + // Send block tip changed notifications without cs_main + if (fNotify) { + uiInterface.NotifyHeaderTip(fInitialBlockDownload, pindexHeader); + } +} + /** * Make the best chain active, in multiple steps. The result is either failure * or an activated best chain. pblock is either NULL or a pointer to a block * that is already loaded (to avoid loading it again from disk). */ -bool ActivateBestChain(CValidationState &state, CBlock *pblock) { - CBlockIndex *pindexNewTip = NULL; +bool ActivateBestChain(CValidationState &state, const CChainParams& chainparams, const CBlock *pblock) { CBlockIndex *pindexMostWork = NULL; + CBlockIndex *pindexNewTip = NULL; do { boost::this_thread::interruption_point(); + if (ShutdownRequested()) + break; + const CBlockIndex *pindexFork; bool fInitialDownload; + int nNewHeight; { LOCK(cs_main); - pindexMostWork = FindMostWorkChain(); + CBlockIndex *pindexOldTip = chainActive.Tip(); + if (pindexMostWork == NULL) { + pindexMostWork = FindMostWorkChain(); + } // Whether we have anything to do at all. if (pindexMostWork == NULL || pindexMostWork == chainActive.Tip()) return true; - if (!ActivateBestChainStep(state, pindexMostWork, pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : NULL)) + bool fInvalidFound = false; + if (!ActivateBestChainStep(state, chainparams, pindexMostWork, pblock && pblock->GetHash() == pindexMostWork->GetBlockHash() ? pblock : NULL, fInvalidFound)) return false; + if (fInvalidFound) { + // Wipe cache, we may need another branch now. + pindexMostWork = NULL; + } pindexNewTip = chainActive.Tip(); + pindexFork = chainActive.FindFork(pindexOldTip); fInitialDownload = IsInitialBlockDownload(); + nNewHeight = chainActive.Height(); } // When we reach this point, we switched to a new tip (stored in pindexNewTip). // Notifications/callbacks that can run without cs_main - if (!fInitialDownload) { - uint256 hashNewTip = pindexNewTip->GetBlockHash(); - // Relay inventory, but don't relay old inventory during initial block download. - int nBlockEstimate = Checkpoints::GetTotalBlocksEstimate(); - { - LOCK(cs_vNodes); - BOOST_FOREACH(CNode* pnode, vNodes) - if (chainActive.Height() > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : nBlockEstimate)) - pnode->PushInventory(CInv(MSG_BLOCK, hashNewTip)); + // Always notify the UI if a new block tip was connected + if (pindexFork != pindexNewTip) { + uiInterface.NotifyBlockTip(fInitialDownload, pindexNewTip); + + if (!fInitialDownload) { + // Find the hashes of all blocks that weren't previously in the best chain. + std::vector<uint256> vHashes; + CBlockIndex *pindexToAnnounce = pindexNewTip; + while (pindexToAnnounce != pindexFork) { + vHashes.push_back(pindexToAnnounce->GetBlockHash()); + pindexToAnnounce = pindexToAnnounce->pprev; + if (vHashes.size() == MAX_BLOCKS_TO_ANNOUNCE) { + // Limit announcements in case of a huge reorganization. + // Rely on the peer's synchronization mechanism in that case. + break; + } + } + // Relay inventory, but don't relay old inventory during initial block download. + int nBlockEstimate = 0; + if (fCheckpointsEnabled) + nBlockEstimate = Checkpoints::GetTotalBlocksEstimate(chainparams.Checkpoints()); + { + LOCK(cs_vNodes); + BOOST_FOREACH(CNode* pnode, vNodes) { + if (nNewHeight > (pnode->nStartingHeight != -1 ? pnode->nStartingHeight - 2000 : nBlockEstimate)) { + BOOST_REVERSE_FOREACH(const uint256& hash, vHashes) { + pnode->PushBlockHash(hash); + } + } + } + } + // Notify external listeners about the new tip. + if (!vHashes.empty()) { + GetMainSignals().UpdatedBlockTip(pindexNewTip); + } } - // Notify external listeners about the new tip. - uiInterface.NotifyBlockTip(hashNewTip); } - } while(pindexMostWork != chainActive.Tip()); + } while (pindexNewTip != pindexMostWork); + CheckBlockIndex(chainparams.GetConsensus()); // Write changes periodically to disk, after relay. if (!FlushStateToDisk(state, FLUSH_STATE_PERIODIC)) { @@ -2152,7 +3108,8 @@ bool ActivateBestChain(CValidationState &state, CBlock *pblock) { return true; } -bool InvalidateBlock(CValidationState& state, CBlockIndex *pindex) { +bool InvalidateBlock(CValidationState& state, const CChainParams& chainparams, CBlockIndex *pindex) +{ AssertLockHeld(cs_main); // Mark the block itself as invalid. @@ -2167,26 +3124,30 @@ bool InvalidateBlock(CValidationState& state, CBlockIndex *pindex) { setBlockIndexCandidates.erase(pindexWalk); // ActivateBestChain considers blocks already in chainActive // unconditionally valid already, so force disconnect away from it. - if (!DisconnectTip(state)) { + if (!DisconnectTip(state, chainparams)) { + mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); return false; } } + LimitMempoolSize(mempool, GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000, GetArg("-mempoolexpiry", DEFAULT_MEMPOOL_EXPIRY) * 60 * 60); + // The resulting new best tip may not be in setBlockIndexCandidates anymore, so - // add them again. + // add it again. BlockMap::iterator it = mapBlockIndex.begin(); while (it != mapBlockIndex.end()) { - if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->nChainTx && setBlockIndexCandidates.value_comp()(chainActive.Tip(), it->second)) { - setBlockIndexCandidates.insert(pindex); + if (it->second->IsValid(BLOCK_VALID_TRANSACTIONS) && it->second->nChainTx && !setBlockIndexCandidates.value_comp()(it->second, chainActive.Tip())) { + setBlockIndexCandidates.insert(it->second); } it++; } InvalidChainFound(pindex); + mempool.removeForReorg(pcoinsTip, chainActive.Tip()->nHeight + 1, STANDARD_LOCKTIME_VERIFY_FLAGS); return true; } -bool ReconsiderBlock(CValidationState& state, CBlockIndex *pindex) { +bool ResetBlockFailureFlags(CBlockIndex *pindex) { AssertLockHeld(cs_main); int nHeight = pindex->nHeight; @@ -2262,11 +3223,10 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl pindexNew->nDataPos = pos.nPos; pindexNew->nUndoPos = 0; pindexNew->nStatus |= BLOCK_HAVE_DATA; - pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); - { - LOCK(cs_nBlockSequenceId); - pindexNew->nSequenceId = nBlockSequenceId++; + if (IsWitnessEnabled(pindexNew->pprev, Params().GetConsensus())) { + pindexNew->nStatus |= BLOCK_OPT_WITNESS; } + pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); setDirtyBlockIndex.insert(pindexNew); if (pindexNew->pprev == NULL || pindexNew->pprev->nChainTx) { @@ -2279,7 +3239,13 @@ bool ReceivedBlockTransactions(const CBlock &block, CValidationState& state, CBl CBlockIndex *pindex = queue.front(); queue.pop_front(); pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; - setBlockIndexCandidates.insert(pindex); + { + LOCK(cs_nBlockSequenceId); + pindex->nSequenceId = nBlockSequenceId++; + } + if (chainActive.Tip() == NULL || !setBlockIndexCandidates.value_comp()(pindex, chainActive.Tip())) { + setBlockIndexCandidates.insert(pindex); + } std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> range = mapBlocksUnlinked.equal_range(pindex); while (range.first != range.second) { std::multimap<CBlockIndex*, CBlockIndex*>::iterator it = range.first; @@ -2308,8 +3274,6 @@ bool FindBlockPos(CValidationState &state, CDiskBlockPos &pos, unsigned int nAdd if (!fKnown) { while (vinfoBlockFile[nFile].nSize + nAddSize >= MAX_BLOCKFILE_SIZE) { - LogPrintf("Leaving block file %i: %s\n", nFile, vinfoBlockFile[nFile].ToString()); - FlushBlockFile(true); nFile++; if (vinfoBlockFile.size() <= nFile) { vinfoBlockFile.resize(nFile + 1); @@ -2319,14 +3283,26 @@ bool FindBlockPos(CValidationState &state, CDiskBlockPos &pos, unsigned int nAdd pos.nPos = vinfoBlockFile[nFile].nSize; } - nLastBlockFile = nFile; - vinfoBlockFile[nFile].nSize += nAddSize; + if ((int)nFile != nLastBlockFile) { + if (!fKnown) { + LogPrintf("Leaving block file %i: %s\n", nLastBlockFile, vinfoBlockFile[nLastBlockFile].ToString()); + } + FlushBlockFile(!fKnown); + nLastBlockFile = nFile; + } + vinfoBlockFile[nFile].AddBlock(nHeight, nTime); + if (fKnown) + vinfoBlockFile[nFile].nSize = std::max(pos.nPos + nAddSize, vinfoBlockFile[nFile].nSize); + else + vinfoBlockFile[nFile].nSize += nAddSize; if (!fKnown) { unsigned int nOldChunks = (pos.nPos + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; unsigned int nNewChunks = (vinfoBlockFile[nFile].nSize + BLOCKFILE_CHUNK_SIZE - 1) / BLOCKFILE_CHUNK_SIZE; if (nNewChunks > nOldChunks) { + if (fPruneMode) + fCheckForPruning = true; if (CheckDiskSpace(nNewChunks * BLOCKFILE_CHUNK_SIZE - pos.nPos)) { FILE *file = OpenBlockFile(pos); if (file) { @@ -2358,6 +3334,8 @@ bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigne unsigned int nOldChunks = (pos.nPos + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; unsigned int nNewChunks = (nNewSize + UNDOFILE_CHUNK_SIZE - 1) / UNDOFILE_CHUNK_SIZE; if (nNewChunks > nOldChunks) { + if (fPruneMode) + fCheckForPruning = true; if (CheckDiskSpace(nNewChunks * UNDOFILE_CHUNK_SIZE - pos.nPos)) { FILE *file = OpenUndoFile(pos); if (file) { @@ -2373,119 +3351,176 @@ bool FindUndoPos(CValidationState &state, int nFile, CDiskBlockPos &pos, unsigne return true; } -bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, bool fCheckPOW) +bool CheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW) { // Check proof of work matches claimed amount - if (fCheckPOW && !CheckProofOfWork(block.GetHash(), block.nBits)) - return state.DoS(50, error("CheckBlockHeader() : proof of work failed"), - REJECT_INVALID, "high-hash"); - - // Check timestamp - if (block.GetBlockTime() > GetAdjustedTime() + 2 * 60 * 60) - return state.Invalid(error("CheckBlockHeader() : block timestamp too far in the future"), - REJECT_INVALID, "time-too-new"); + if (fCheckPOW && !CheckProofOfWork(block.GetHash(), block.nBits, consensusParams)) + return state.DoS(50, false, REJECT_INVALID, "high-hash", false, "proof of work failed"); return true; } -bool CheckBlock(const CBlock& block, CValidationState& state, bool fCheckPOW, bool fCheckMerkleRoot) +bool CheckBlock(const CBlock& block, CValidationState& state, const Consensus::Params& consensusParams, bool fCheckPOW, bool fCheckMerkleRoot) { // These are checks that are independent of context. + if (block.fChecked) + return true; + // Check that the header is valid (particularly PoW). This is mostly // redundant with the call in AcceptBlockHeader. - if (!CheckBlockHeader(block, state, fCheckPOW)) + if (!CheckBlockHeader(block, state, consensusParams, fCheckPOW)) return false; // Check the merkle root. if (fCheckMerkleRoot) { bool mutated; - uint256 hashMerkleRoot2 = block.BuildMerkleTree(&mutated); + uint256 hashMerkleRoot2 = BlockMerkleRoot(block, &mutated); if (block.hashMerkleRoot != hashMerkleRoot2) - return state.DoS(100, error("CheckBlock() : hashMerkleRoot mismatch"), - REJECT_INVALID, "bad-txnmrklroot", true); + return state.DoS(100, false, REJECT_INVALID, "bad-txnmrklroot", true, "hashMerkleRoot mismatch"); // Check for merkle tree malleability (CVE-2012-2459): repeating sequences // of transactions in a block without affecting the merkle root of a block, // while still invalidating it. if (mutated) - return state.DoS(100, error("CheckBlock() : duplicate transaction"), - REJECT_INVALID, "bad-txns-duplicate", true); + return state.DoS(100, false, REJECT_INVALID, "bad-txns-duplicate", true, "duplicate transaction"); } // All potential-corruption validation must be done before we do any // transaction validation, as otherwise we may mark the header as invalid // because we receive the wrong transactions for it. + // Note that witness malleability is checked in ContextualCheckBlock, so no + // checks that use witness data may be performed here. // Size limits - if (block.vtx.empty() || block.vtx.size() > MAX_BLOCK_SIZE || ::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION) > MAX_BLOCK_SIZE) - return state.DoS(100, error("CheckBlock() : size limits failed"), - REJECT_INVALID, "bad-blk-length"); + if (block.vtx.empty() || block.vtx.size() > MAX_BLOCK_BASE_SIZE || ::GetSerializeSize(block, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) > MAX_BLOCK_BASE_SIZE) + return state.DoS(100, false, REJECT_INVALID, "bad-blk-length", false, "size limits failed"); // First transaction must be coinbase, the rest must not be if (block.vtx.empty() || !block.vtx[0].IsCoinBase()) - return state.DoS(100, error("CheckBlock() : first tx is not coinbase"), - REJECT_INVALID, "bad-cb-missing"); + return state.DoS(100, false, REJECT_INVALID, "bad-cb-missing", false, "first tx is not coinbase"); for (unsigned int i = 1; i < block.vtx.size(); i++) if (block.vtx[i].IsCoinBase()) - return state.DoS(100, error("CheckBlock() : more than one coinbase"), - REJECT_INVALID, "bad-cb-multiple"); + return state.DoS(100, false, REJECT_INVALID, "bad-cb-multiple", false, "more than one coinbase"); // Check transactions BOOST_FOREACH(const CTransaction& tx, block.vtx) if (!CheckTransaction(tx, state)) - return error("CheckBlock() : CheckTransaction failed"); + return state.Invalid(false, state.GetRejectCode(), state.GetRejectReason(), + strprintf("Transaction check failed (tx hash %s) %s", tx.GetHash().ToString(), state.GetDebugMessage())); unsigned int nSigOps = 0; BOOST_FOREACH(const CTransaction& tx, block.vtx) { nSigOps += GetLegacySigOpCount(tx); } - if (nSigOps > MAX_BLOCK_SIGOPS) - return state.DoS(100, error("CheckBlock() : out-of-bounds SigOpCount"), - REJECT_INVALID, "bad-blk-sigops", true); + if (nSigOps * WITNESS_SCALE_FACTOR > MAX_BLOCK_SIGOPS_COST) + return state.DoS(100, false, REJECT_INVALID, "bad-blk-sigops", false, "out-of-bounds SigOpCount"); + + if (fCheckPOW && fCheckMerkleRoot) + block.fChecked = true; return true; } -bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, CBlockIndex * const pindexPrev) +static bool CheckIndexAgainstCheckpoint(const CBlockIndex* pindexPrev, CValidationState& state, const CChainParams& chainparams, const uint256& hash) { - uint256 hash = block.GetHash(); - if (hash == Params().HashGenesisBlock()) + if (*pindexPrev->phashBlock == chainparams.GetConsensus().hashGenesisBlock) return true; - assert(pindexPrev); - int nHeight = pindexPrev->nHeight+1; + // Don't accept any forks from the main chain prior to last checkpoint + CBlockIndex* pcheckpoint = Checkpoints::GetLastCheckpoint(chainparams.Checkpoints()); + if (pcheckpoint && nHeight < pcheckpoint->nHeight) + return state.DoS(100, error("%s: forked chain older than last checkpoint (height %d)", __func__, nHeight)); + return true; +} + +bool IsWitnessEnabled(const CBlockIndex* pindexPrev, const Consensus::Params& params) +{ + LOCK(cs_main); + return (VersionBitsState(pindexPrev, params, Consensus::DEPLOYMENT_SEGWIT, versionbitscache) == THRESHOLD_ACTIVE); +} + +// Compute at which vout of the block's coinbase transaction the witness +// commitment occurs, or -1 if not found. +static int GetWitnessCommitmentIndex(const CBlock& block) +{ + int commitpos = -1; + for (size_t o = 0; o < block.vtx[0].vout.size(); o++) { + if (block.vtx[0].vout[o].scriptPubKey.size() >= 38 && block.vtx[0].vout[o].scriptPubKey[0] == OP_RETURN && block.vtx[0].vout[o].scriptPubKey[1] == 0x24 && block.vtx[0].vout[o].scriptPubKey[2] == 0xaa && block.vtx[0].vout[o].scriptPubKey[3] == 0x21 && block.vtx[0].vout[o].scriptPubKey[4] == 0xa9 && block.vtx[0].vout[o].scriptPubKey[5] == 0xed) { + commitpos = o; + } + } + return commitpos; +} + +void UpdateUncommittedBlockStructures(CBlock& block, const CBlockIndex* pindexPrev, const Consensus::Params& consensusParams) +{ + int commitpos = GetWitnessCommitmentIndex(block); + static const std::vector<unsigned char> nonce(32, 0x00); + if (commitpos != -1 && IsWitnessEnabled(pindexPrev, consensusParams) && block.vtx[0].wit.IsEmpty()) { + block.vtx[0].wit.vtxinwit.resize(1); + block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.resize(1); + block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0] = nonce; + } +} + +std::vector<unsigned char> GenerateCoinbaseCommitment(CBlock& block, const CBlockIndex* pindexPrev, const Consensus::Params& consensusParams) +{ + std::vector<unsigned char> commitment; + int commitpos = GetWitnessCommitmentIndex(block); + bool fHaveWitness = false; + for (size_t t = 1; t < block.vtx.size(); t++) { + if (!block.vtx[t].wit.IsNull()) { + fHaveWitness = true; + break; + } + } + std::vector<unsigned char> ret(32, 0x00); + if (fHaveWitness && IsWitnessEnabled(pindexPrev, consensusParams)) { + if (commitpos == -1) { + uint256 witnessroot = BlockWitnessMerkleRoot(block, NULL); + CHash256().Write(witnessroot.begin(), 32).Write(&ret[0], 32).Finalize(witnessroot.begin()); + CTxOut out; + out.nValue = 0; + out.scriptPubKey.resize(38); + out.scriptPubKey[0] = OP_RETURN; + out.scriptPubKey[1] = 0x24; + out.scriptPubKey[2] = 0xaa; + out.scriptPubKey[3] = 0x21; + out.scriptPubKey[4] = 0xa9; + out.scriptPubKey[5] = 0xed; + memcpy(&out.scriptPubKey[6], witnessroot.begin(), 32); + commitment = std::vector<unsigned char>(out.scriptPubKey.begin(), out.scriptPubKey.end()); + const_cast<std::vector<CTxOut>*>(&block.vtx[0].vout)->push_back(out); + block.vtx[0].UpdateHash(); + } + } + UpdateUncommittedBlockStructures(block, pindexPrev, consensusParams); + return commitment; +} + +bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& state, const Consensus::Params& consensusParams, CBlockIndex * const pindexPrev, int64_t nAdjustedTime) +{ // Check proof of work - if ((!Params().SkipProofOfWorkCheck()) && - (block.nBits != GetNextWorkRequired(pindexPrev, &block))) - return state.DoS(100, error("%s : incorrect proof of work", __func__), - REJECT_INVALID, "bad-diffbits"); + if (block.nBits != GetNextWorkRequired(pindexPrev, &block, consensusParams)) + return state.DoS(100, false, REJECT_INVALID, "bad-diffbits", false, "incorrect proof of work"); // Check timestamp against prev if (block.GetBlockTime() <= pindexPrev->GetMedianTimePast()) - return state.Invalid(error("%s : block's timestamp is too early", __func__), - REJECT_INVALID, "time-too-old"); - - // Check that the block chain matches the known block chain up to a checkpoint - if (!Checkpoints::CheckBlock(nHeight, hash)) - return state.DoS(100, error("%s : rejected by checkpoint lock-in at %d", __func__, nHeight), - REJECT_CHECKPOINT, "checkpoint mismatch"); + return state.Invalid(false, REJECT_INVALID, "time-too-old", "block's timestamp is too early"); - // Don't accept any forks from the main chain prior to last checkpoint - CBlockIndex* pcheckpoint = Checkpoints::GetLastCheckpoint(); - if (pcheckpoint && nHeight < pcheckpoint->nHeight) - return state.DoS(100, error("%s : forked chain older than last checkpoint (height %d)", __func__, nHeight)); + // Check timestamp + if (block.GetBlockTime() > nAdjustedTime + 2 * 60 * 60) + return state.Invalid(false, REJECT_INVALID, "time-too-new", "block timestamp too far in the future"); - // Reject block.nVersion=1 blocks when 95% (75% on testnet) of the network has upgraded: - if (block.nVersion < 2 && - CBlockIndex::IsSuperMajority(2, pindexPrev, Params().RejectBlockOutdatedMajority())) - { - return state.Invalid(error("%s : rejected nVersion=1 block", __func__), - REJECT_OBSOLETE, "bad-version"); - } + // Reject outdated version blocks when 95% (75% on testnet) of the network has upgraded: + for (int32_t version = 2; version < 5; ++version) // check for version 2, 3 and 4 upgrades + if (block.nVersion < version && IsSuperMajority(version, pindexPrev, consensusParams.nMajorityRejectBlockOutdated, consensusParams)) + return state.Invalid(false, REJECT_OBSOLETE, strprintf("bad-version(0x%08x)", version - 1), + strprintf("rejected nVersion=0x%08x block", version - 1)); return true; } @@ -2493,60 +3528,124 @@ bool ContextualCheckBlockHeader(const CBlockHeader& block, CValidationState& sta bool ContextualCheckBlock(const CBlock& block, CValidationState& state, CBlockIndex * const pindexPrev) { const int nHeight = pindexPrev == NULL ? 0 : pindexPrev->nHeight + 1; + const Consensus::Params& consensusParams = Params().GetConsensus(); + + // Start enforcing BIP113 (Median Time Past) using versionbits logic. + int nLockTimeFlags = 0; + if (VersionBitsState(pindexPrev, consensusParams, Consensus::DEPLOYMENT_CSV, versionbitscache) == THRESHOLD_ACTIVE) { + nLockTimeFlags |= LOCKTIME_MEDIAN_TIME_PAST; + } + + int64_t nLockTimeCutoff = (nLockTimeFlags & LOCKTIME_MEDIAN_TIME_PAST) + ? pindexPrev->GetMedianTimePast() + : block.GetBlockTime(); // Check that all transactions are finalized - BOOST_FOREACH(const CTransaction& tx, block.vtx) - if (!IsFinalTx(tx, nHeight, block.GetBlockTime())) { - return state.DoS(10, error("%s : contains a non-final transaction", __func__), REJECT_INVALID, "bad-txns-nonfinal"); + BOOST_FOREACH(const CTransaction& tx, block.vtx) { + if (!IsFinalTx(tx, nHeight, nLockTimeCutoff)) { + return state.DoS(10, false, REJECT_INVALID, "bad-txns-nonfinal", false, "non-final transaction"); } + } // Enforce block.nVersion=2 rule that the coinbase starts with serialized block height // if 750 of the last 1,000 blocks are version 2 or greater (51/100 if testnet): - if (block.nVersion >= 2 && - CBlockIndex::IsSuperMajority(2, pindexPrev, Params().EnforceBlockUpgradeMajority())) + if (block.nVersion >= 2 && IsSuperMajority(2, pindexPrev, consensusParams.nMajorityEnforceBlockUpgrade, consensusParams)) { CScript expect = CScript() << nHeight; if (block.vtx[0].vin[0].scriptSig.size() < expect.size() || !std::equal(expect.begin(), expect.end(), block.vtx[0].vin[0].scriptSig.begin())) { - return state.DoS(100, error("%s : block height mismatch in coinbase", __func__), REJECT_INVALID, "bad-cb-height"); + return state.DoS(100, false, REJECT_INVALID, "bad-cb-height", false, "block height mismatch in coinbase"); } } + // Validation for witness commitments. + // * We compute the witness hash (which is the hash including witnesses) of all the block's transactions, except the + // coinbase (where 0x0000....0000 is used instead). + // * The coinbase scriptWitness is a stack of a single 32-byte vector, containing a witness nonce (unconstrained). + // * We build a merkle tree with all those witness hashes as leaves (similar to the hashMerkleRoot in the block header). + // * There must be at least one output whose scriptPubKey is a single 36-byte push, the first 4 bytes of which are + // {0xaa, 0x21, 0xa9, 0xed}, and the following 32 bytes are SHA256^2(witness root, witness nonce). In case there are + // multiple, the last one is used. + bool fHaveWitness = false; + if (IsWitnessEnabled(pindexPrev, consensusParams)) { + int commitpos = GetWitnessCommitmentIndex(block); + if (commitpos != -1) { + bool malleated = false; + uint256 hashWitness = BlockWitnessMerkleRoot(block, &malleated); + // The malleation check is ignored; as the transaction tree itself + // already does not permit it, it is impossible to trigger in the + // witness tree. + if (block.vtx[0].wit.vtxinwit.size() != 1 || block.vtx[0].wit.vtxinwit[0].scriptWitness.stack.size() != 1 || block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0].size() != 32) { + return state.DoS(100, error("%s : invalid witness nonce size", __func__), REJECT_INVALID, "bad-witness-nonce-size", true); + } + CHash256().Write(hashWitness.begin(), 32).Write(&block.vtx[0].wit.vtxinwit[0].scriptWitness.stack[0][0], 32).Finalize(hashWitness.begin()); + if (memcmp(hashWitness.begin(), &block.vtx[0].vout[commitpos].scriptPubKey[6], 32)) { + return state.DoS(100, error("%s : witness merkle commitment mismatch", __func__), REJECT_INVALID, "bad-witness-merkle-match", true); + } + fHaveWitness = true; + } + } + + // No witness data is allowed in blocks that don't commit to witness data, as this would otherwise leave room for spam + if (!fHaveWitness) { + for (size_t i = 0; i < block.vtx.size(); i++) { + if (!block.vtx[i].wit.IsNull()) { + return state.DoS(100, error("%s : unexpected witness data found", __func__), REJECT_INVALID, "unexpected-witness", true); + } + } + } + + // After the coinbase witness nonce and commitment are verified, + // we can check if the block cost passes (before we've checked the + // coinbase witness, it would be possible for the cost to be too + // large by filling up the coinbase witness, which doesn't change + // the block hash, so we couldn't mark the block as permanently + // failed). + if (GetBlockCost(block) > MAX_BLOCK_COST) { + return state.DoS(100, error("ContextualCheckBlock(): cost limit failed"), REJECT_INVALID, "bad-blk-cost"); + } + return true; } -bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, CBlockIndex** ppindex) +static bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex=NULL) { AssertLockHeld(cs_main); // Check for duplicate uint256 hash = block.GetHash(); BlockMap::iterator miSelf = mapBlockIndex.find(hash); CBlockIndex *pindex = NULL; - if (miSelf != mapBlockIndex.end()) { - // Block header is already known. - pindex = miSelf->second; - if (ppindex) - *ppindex = pindex; - if (pindex->nStatus & BLOCK_FAILED_MASK) - return state.Invalid(error("%s : block is marked invalid", __func__), 0, "duplicate"); - return true; - } + if (hash != chainparams.GetConsensus().hashGenesisBlock) { + + if (miSelf != mapBlockIndex.end()) { + // Block header is already known. + pindex = miSelf->second; + if (ppindex) + *ppindex = pindex; + if (pindex->nStatus & BLOCK_FAILED_MASK) + return state.Invalid(error("%s: block %s is marked invalid", __func__, hash.ToString()), 0, "duplicate"); + return true; + } - if (!CheckBlockHeader(block, state)) - return false; + if (!CheckBlockHeader(block, state, chainparams.GetConsensus())) + return error("%s: Consensus::CheckBlockHeader: %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); - // Get prev block index - CBlockIndex* pindexPrev = NULL; - if (hash != Params().HashGenesisBlock()) { + // Get prev block index + CBlockIndex* pindexPrev = NULL; BlockMap::iterator mi = mapBlockIndex.find(block.hashPrevBlock); if (mi == mapBlockIndex.end()) - return state.DoS(10, error("%s : prev block not found", __func__), 0, "bad-prevblk"); + return state.DoS(10, error("%s: prev block not found", __func__), 0, "bad-prevblk"); pindexPrev = (*mi).second; - } + if (pindexPrev->nStatus & BLOCK_FAILED_MASK) + return state.DoS(100, error("%s: prev block invalid", __func__), REJECT_INVALID, "bad-prevblk"); - if (!ContextualCheckBlockHeader(block, state, pindexPrev)) - return false; + assert(pindexPrev); + if (fCheckpointsEnabled && !CheckIndexAgainstCheckpoint(pindexPrev, state, chainparams, hash)) + return error("%s: CheckIndexAgainstCheckpoint(): %s", __func__, state.GetRejectReason().c_str()); + if (!ContextualCheckBlockHeader(block, state, chainparams.GetConsensus(), pindexPrev, GetAdjustedTime())) + return error("%s: Consensus::ContextualCheckBlockHeader: %s, %s", __func__, hash.ToString(), FormatStateMessage(state)); + } if (pindex == NULL) pindex = AddToBlockIndex(block); @@ -2556,27 +3655,46 @@ bool AcceptBlockHeader(const CBlockHeader& block, CValidationState& state, CBloc return true; } -bool AcceptBlock(CBlock& block, CValidationState& state, CBlockIndex** ppindex, CDiskBlockPos* dbp) +/** Store block on disk. If dbp is non-NULL, the file is known to already reside on disk */ +static bool AcceptBlock(const CBlock& block, CValidationState& state, const CChainParams& chainparams, CBlockIndex** ppindex, bool fRequested, const CDiskBlockPos* dbp, bool* fNewBlock) { + if (fNewBlock) *fNewBlock = false; AssertLockHeld(cs_main); - CBlockIndex *&pindex = *ppindex; + CBlockIndex *pindexDummy = NULL; + CBlockIndex *&pindex = ppindex ? *ppindex : pindexDummy; - if (!AcceptBlockHeader(block, state, &pindex)) + if (!AcceptBlockHeader(block, state, chainparams, &pindex)) return false; - if (pindex->nStatus & BLOCK_HAVE_DATA) { - // TODO: deal better with duplicate blocks. - // return state.DoS(20, error("AcceptBlock() : already have block %d %s", pindex->nHeight, pindex->GetBlockHash().ToString()), REJECT_DUPLICATE, "duplicate"); - return true; - } - - if ((!CheckBlock(block, state)) || !ContextualCheckBlock(block, state, pindex->pprev)) { + // Try to process all requested blocks that we don't have, but only + // process an unrequested block if it's new and has enough work to + // advance our tip, and isn't too many blocks ahead. + bool fAlreadyHave = pindex->nStatus & BLOCK_HAVE_DATA; + bool fHasMoreWork = (chainActive.Tip() ? pindex->nChainWork > chainActive.Tip()->nChainWork : true); + // Blocks that are too out-of-order needlessly limit the effectiveness of + // pruning, because pruning will not delete block files that contain any + // blocks which are too close in height to the tip. Apply this test + // regardless of whether pruning is enabled; it should generally be safe to + // not process unrequested blocks. + bool fTooFarAhead = (pindex->nHeight > int(chainActive.Height() + MIN_BLOCKS_TO_KEEP)); + + // TODO: deal better with return value and error conditions for duplicate + // and unrequested blocks. + if (fAlreadyHave) return true; + if (!fRequested) { // If we didn't ask for it: + if (pindex->nTx != 0) return true; // This is a previously-processed block that was pruned + if (!fHasMoreWork) return true; // Don't process less-work chains + if (fTooFarAhead) return true; // Block height is too high + } + if (fNewBlock) *fNewBlock = true; + + if ((!CheckBlock(block, state, chainparams.GetConsensus(), GetAdjustedTime())) || !ContextualCheckBlock(block, state, pindex->pprev)) { if (state.IsInvalid() && !state.CorruptionPossible()) { pindex->nStatus |= BLOCK_FAILED_VALID; setDirtyBlockIndex.insert(pindex); } - return false; + return error("%s: %s", __func__, FormatStateMessage(state)); } int nHeight = pindex->nHeight; @@ -2588,24 +3706,26 @@ bool AcceptBlock(CBlock& block, CValidationState& state, CBlockIndex** ppindex, if (dbp != NULL) blockPos = *dbp; if (!FindBlockPos(state, blockPos, nBlockSize+8, nHeight, block.GetBlockTime(), dbp != NULL)) - return error("AcceptBlock() : FindBlockPos failed"); + return error("AcceptBlock(): FindBlockPos failed"); if (dbp == NULL) - if (!WriteBlockToDisk(block, blockPos)) - return state.Abort("Failed to write block"); + if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) + AbortNode(state, "Failed to write block"); if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) - return error("AcceptBlock() : ReceivedBlockTransactions failed"); - } catch(std::runtime_error &e) { - return state.Abort(std::string("System error: ") + e.what()); + return error("AcceptBlock(): ReceivedBlockTransactions failed"); + } catch (const std::runtime_error& e) { + return AbortNode(state, std::string("System error: ") + e.what()); } + if (fCheckForPruning) + FlushStateToDisk(state, FLUSH_STATE_NONE); // we just allocated more disk space for block files + return true; } -bool CBlockIndex::IsSuperMajority(int minVersion, const CBlockIndex* pstart, unsigned int nRequired) +static bool IsSuperMajority(int minVersion, const CBlockIndex* pstart, unsigned nRequired, const Consensus::Params& consensusParams) { - unsigned int nToCheck = Params().ToCheckBlockUpgradeMajority(); unsigned int nFound = 0; - for (unsigned int i = 0; i < nToCheck && nFound < nRequired && pstart != NULL; i++) + for (int i = 0; i < consensusParams.nMajorityWindow && nFound < nRequired && pstart != NULL; i++) { if (pstart->nVersion >= minVersion) ++nFound; @@ -2614,87 +3734,41 @@ bool CBlockIndex::IsSuperMajority(int minVersion, const CBlockIndex* pstart, uns return (nFound >= nRequired); } -/** Turn the lowest '1' bit in the binary representation of a number into a '0'. */ -int static inline InvertLowestOne(int n) { return n & (n - 1); } - -/** Compute what height to jump back to with the CBlockIndex::pskip pointer. */ -int static inline GetSkipHeight(int height) { - if (height < 2) - return 0; - // Determine which height to jump back to. Any number strictly lower than height is acceptable, - // but the following expression seems to perform well in simulations (max 110 steps to go back - // up to 2**18 blocks). - return (height & 1) ? InvertLowestOne(InvertLowestOne(height - 1)) + 1 : InvertLowestOne(height); -} - -CBlockIndex* CBlockIndex::GetAncestor(int height) +bool ProcessNewBlock(CValidationState& state, const CChainParams& chainparams, CNode* pfrom, const CBlock* pblock, bool fForceProcessing, const CDiskBlockPos* dbp) { - if (height > nHeight || height < 0) - return NULL; - - CBlockIndex* pindexWalk = this; - int heightWalk = nHeight; - while (heightWalk > height) { - int heightSkip = GetSkipHeight(heightWalk); - int heightSkipPrev = GetSkipHeight(heightWalk - 1); - if (heightSkip == height || - (heightSkip > height && !(heightSkipPrev < heightSkip - 2 && - heightSkipPrev >= height))) { - // Only follow pskip if pprev->pskip isn't better than pskip->pprev. - pindexWalk = pindexWalk->pskip; - heightWalk = heightSkip; - } else { - pindexWalk = pindexWalk->pprev; - heightWalk--; - } - } - return pindexWalk; -} - -const CBlockIndex* CBlockIndex::GetAncestor(int height) const -{ - return const_cast<CBlockIndex*>(this)->GetAncestor(height); -} - -void CBlockIndex::BuildSkip() -{ - if (pprev) - pskip = pprev->GetAncestor(GetSkipHeight(nHeight)); -} - -bool ProcessNewBlock(CValidationState &state, CNode* pfrom, CBlock* pblock, CDiskBlockPos *dbp) -{ - // Preliminary checks - bool checked = CheckBlock(*pblock, state); - { LOCK(cs_main); - MarkBlockAsReceived(pblock->GetHash()); - if (!checked) { - return error("%s : CheckBlock FAILED", __func__); - } + bool fRequested = MarkBlockAsReceived(pblock->GetHash()); + fRequested |= fForceProcessing; // Store to disk CBlockIndex *pindex = NULL; - bool ret = AcceptBlock(*pblock, state, &pindex, dbp); + bool fNewBlock = false; + bool ret = AcceptBlock(*pblock, state, chainparams, &pindex, fRequested, dbp, &fNewBlock); if (pindex && pfrom) { mapBlockSource[pindex->GetBlockHash()] = pfrom->GetId(); + if (fNewBlock) pfrom->nLastBlockTime = GetTime(); } + CheckBlockIndex(chainparams.GetConsensus()); if (!ret) - return error("%s : AcceptBlock FAILED", __func__); + return error("%s: AcceptBlock FAILED", __func__); } - if (!ActivateBestChain(state, pblock)) - return error("%s : ActivateBestChain failed", __func__); + NotifyHeaderTip(); + + if (!ActivateBestChain(state, chainparams, pblock)) + return error("%s: ActivateBestChain failed", __func__); return true; } -bool TestBlockValidity(CValidationState &state, const CBlock& block, CBlockIndex * const pindexPrev, bool fCheckPOW, bool fCheckMerkleRoot) +bool TestBlockValidity(CValidationState& state, const CChainParams& chainparams, const CBlock& block, CBlockIndex* pindexPrev, bool fCheckPOW, bool fCheckMerkleRoot) { AssertLockHeld(cs_main); - assert(pindexPrev == chainActive.Tip()); + assert(pindexPrev && pindexPrev == chainActive.Tip()); + if (fCheckpointsEnabled && !CheckIndexAgainstCheckpoint(pindexPrev, state, chainparams, block.GetHash())) + return error("%s: CheckIndexAgainstCheckpoint(): %s", __func__, state.GetRejectReason().c_str()); CCoinsViewCache viewNew(pcoinsTip); CBlockIndex indexDummy(block); @@ -2702,39 +3776,127 @@ bool TestBlockValidity(CValidationState &state, const CBlock& block, CBlockIndex indexDummy.nHeight = pindexPrev->nHeight + 1; // NOTE: CheckBlockHeader is called by CheckBlock - if (!ContextualCheckBlockHeader(block, state, pindexPrev)) - return false; - if (!CheckBlock(block, state, fCheckPOW, fCheckMerkleRoot)) - return false; + if (!ContextualCheckBlockHeader(block, state, chainparams.GetConsensus(), pindexPrev, GetAdjustedTime())) + return error("%s: Consensus::ContextualCheckBlockHeader: %s", __func__, FormatStateMessage(state)); + if (!CheckBlock(block, state, chainparams.GetConsensus(), fCheckPOW, fCheckMerkleRoot)) + return error("%s: Consensus::CheckBlock: %s", __func__, FormatStateMessage(state)); if (!ContextualCheckBlock(block, state, pindexPrev)) - return false; - if (!ConnectBlock(block, state, &indexDummy, viewNew, true)) + return error("%s: Consensus::ContextualCheckBlock: %s", __func__, FormatStateMessage(state)); + if (!ConnectBlock(block, state, &indexDummy, viewNew, chainparams, true)) return false; assert(state.IsValid()); return true; } +/** + * BLOCK PRUNING CODE + */ +/* Calculate the amount of disk space the block & undo files currently use */ +uint64_t CalculateCurrentUsage() +{ + uint64_t retval = 0; + BOOST_FOREACH(const CBlockFileInfo &file, vinfoBlockFile) { + retval += file.nSize + file.nUndoSize; + } + return retval; +} +/* Prune a block file (modify associated database entries)*/ +void PruneOneBlockFile(const int fileNumber) +{ + for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); ++it) { + CBlockIndex* pindex = it->second; + if (pindex->nFile == fileNumber) { + pindex->nStatus &= ~BLOCK_HAVE_DATA; + pindex->nStatus &= ~BLOCK_HAVE_UNDO; + pindex->nFile = 0; + pindex->nDataPos = 0; + pindex->nUndoPos = 0; + setDirtyBlockIndex.insert(pindex); + // Prune from mapBlocksUnlinked -- any block we prune would have + // to be downloaded again in order to consider its chain, at which + // point it would be considered as a candidate for + // mapBlocksUnlinked or setBlockIndexCandidates. + std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> range = mapBlocksUnlinked.equal_range(pindex->pprev); + while (range.first != range.second) { + std::multimap<CBlockIndex *, CBlockIndex *>::iterator it = range.first; + range.first++; + if (it->second == pindex) { + mapBlocksUnlinked.erase(it); + } + } + } + } + vinfoBlockFile[fileNumber].SetNull(); + setDirtyFileInfo.insert(fileNumber); +} +void UnlinkPrunedFiles(std::set<int>& setFilesToPrune) +{ + for (set<int>::iterator it = setFilesToPrune.begin(); it != setFilesToPrune.end(); ++it) { + CDiskBlockPos pos(*it, 0); + boost::filesystem::remove(GetBlockPosFilename(pos, "blk")); + boost::filesystem::remove(GetBlockPosFilename(pos, "rev")); + LogPrintf("Prune: %s deleted blk/rev (%05u)\n", __func__, *it); + } +} -bool AbortNode(const std::string &strMessage, const std::string &userMessage) { - strMiscWarning = strMessage; - LogPrintf("*** %s\n", strMessage); - uiInterface.ThreadSafeMessageBox( - userMessage.empty() ? _("Error: A fatal internal error occured, see debug.log for details") : userMessage, - "", CClientUIInterface::MSG_ERROR); - StartShutdown(); - return false; +/* Calculate the block/rev files that should be deleted to remain under target*/ +void FindFilesToPrune(std::set<int>& setFilesToPrune, uint64_t nPruneAfterHeight) +{ + LOCK2(cs_main, cs_LastBlockFile); + if (chainActive.Tip() == NULL || nPruneTarget == 0) { + return; + } + if ((uint64_t)chainActive.Tip()->nHeight <= nPruneAfterHeight) { + return; + } + + unsigned int nLastBlockWeCanPrune = chainActive.Tip()->nHeight - MIN_BLOCKS_TO_KEEP; + uint64_t nCurrentUsage = CalculateCurrentUsage(); + // We don't check to prune until after we've allocated new space for files + // So we should leave a buffer under our target to account for another allocation + // before the next pruning. + uint64_t nBuffer = BLOCKFILE_CHUNK_SIZE + UNDOFILE_CHUNK_SIZE; + uint64_t nBytesToPrune; + int count=0; + + if (nCurrentUsage + nBuffer >= nPruneTarget) { + for (int fileNumber = 0; fileNumber < nLastBlockFile; fileNumber++) { + nBytesToPrune = vinfoBlockFile[fileNumber].nSize + vinfoBlockFile[fileNumber].nUndoSize; + + if (vinfoBlockFile[fileNumber].nSize == 0) + continue; + + if (nCurrentUsage + nBuffer < nPruneTarget) // are we below our target? + break; + + // don't prune files that could have a block within MIN_BLOCKS_TO_KEEP of the main chain's tip but keep scanning + if (vinfoBlockFile[fileNumber].nHeightLast > nLastBlockWeCanPrune) + continue; + + PruneOneBlockFile(fileNumber); + // Queue up the files for removal + setFilesToPrune.insert(fileNumber); + nCurrentUsage -= nBytesToPrune; + count++; + } + } + + LogPrint("prune", "Prune: target=%dMiB actual=%dMiB diff=%dMiB max_prune_height=%d removed %d blk/rev pairs\n", + nPruneTarget/1024/1024, nCurrentUsage/1024/1024, + ((int64_t)nPruneTarget - (int64_t)nCurrentUsage)/1024/1024, + nLastBlockWeCanPrune, count); } bool CheckDiskSpace(uint64_t nAdditionalBytes) { - uint64_t nFreeBytesAvailable = filesystem::space(GetDataDir()).available; + uint64_t nFreeBytesAvailable = boost::filesystem::space(GetDataDir()).available; // Check for nMinDiskSpace bytes (currently 50MB) if (nFreeBytesAvailable < nMinDiskSpace + nAdditionalBytes) @@ -2781,7 +3943,7 @@ boost::filesystem::path GetBlockPosFilename(const CDiskBlockPos &pos, const char CBlockIndex * InsertBlockIndex(uint256 hash) { - if (hash == 0) + if (hash.IsNull()) return NULL; // Return existing @@ -2792,7 +3954,7 @@ CBlockIndex * InsertBlockIndex(uint256 hash) // Create new CBlockIndex* pindexNew = new CBlockIndex(); if (!pindexNew) - throw runtime_error("LoadBlockIndex() : new CBlockIndex failed"); + throw runtime_error("LoadBlockIndex(): new CBlockIndex failed"); mi = mapBlockIndex.insert(make_pair(hash, pindexNew)).first; pindexNew->phashBlock = &((*mi).first); @@ -2801,7 +3963,8 @@ CBlockIndex * InsertBlockIndex(uint256 hash) bool static LoadBlockIndexDB() { - if (!pblocktree->LoadBlockIndexGuts()) + const CChainParams& chainparams = Params(); + if (!pblocktree->LoadBlockIndexGuts(InsertBlockIndex)) return false; boost::this_thread::interruption_point(); @@ -2819,7 +3982,9 @@ bool static LoadBlockIndexDB() { CBlockIndex* pindex = item.second; pindex->nChainWork = (pindex->pprev ? pindex->pprev->nChainWork : 0) + GetBlockProof(*pindex); - if (pindex->nStatus & BLOCK_HAVE_DATA) { + // We can link the chain of blocks for which we've received transactions at some point. + // Pruned nodes may have deleted the block. + if (pindex->nTx > 0) { if (pindex->pprev) { if (pindex->pprev->nChainTx) { pindex->nChainTx = pindex->pprev->nChainTx + pindex->nTx; @@ -2876,6 +4041,11 @@ bool static LoadBlockIndexDB() } } + // Check whether we have ever pruned block & undo files + pblocktree->ReadFlag("prunedblockfiles", fHavePruned); + if (fHavePruned) + LogPrintf("LoadBlockIndexDB(): Block files have previously been pruned\n"); + // Check whether we need to continue reindexing bool fReindexing = false; pblocktree->ReadReindexing(fReindexing); @@ -2883,7 +4053,7 @@ bool static LoadBlockIndexDB() // Check whether we have a transaction index pblocktree->ReadFlag("txindex", fTxIndex); - LogPrintf("LoadBlockIndexDB(): transaction index %s\n", fTxIndex ? "enabled" : "disabled"); + LogPrintf("%s: transaction index %s\n", __func__, fTxIndex ? "enabled" : "disabled"); // Load pointer to end of best chain BlockMap::iterator it = mapBlockIndex.find(pcoinsTip->GetBestBlock()); @@ -2893,10 +4063,10 @@ bool static LoadBlockIndexDB() PruneBlockIndexCandidates(); - LogPrintf("LoadBlockIndexDB(): hashBestChain=%s height=%d date=%s progress=%f\n", + LogPrintf("%s: hashBestChain=%s height=%d date=%s progress=%f\n", __func__, chainActive.Tip()->GetBlockHash().ToString(), chainActive.Height(), DateTimeStrFormat("%Y-%m-%d %H:%M:%S", chainActive.Tip()->GetBlockTime()), - Checkpoints::GuessVerificationProgress(chainActive.Tip())); + Checkpoints::GuessVerificationProgress(chainparams.Checkpoints(), chainActive.Tip())); return true; } @@ -2911,7 +4081,7 @@ CVerifyDB::~CVerifyDB() uiInterface.ShowProgress("", 100); } -bool CVerifyDB::VerifyDB(CCoinsView *coinsview, int nCheckLevel, int nCheckDepth) +bool CVerifyDB::VerifyDB(const CChainParams& chainparams, CCoinsView *coinsview, int nCheckLevel, int nCheckDepth) { LOCK(cs_main); if (chainActive.Tip() == NULL || chainActive.Tip()->pprev == NULL) @@ -2929,33 +4099,47 @@ bool CVerifyDB::VerifyDB(CCoinsView *coinsview, int nCheckLevel, int nCheckDepth CBlockIndex* pindexFailure = NULL; int nGoodTransactions = 0; CValidationState state; + int reportDone = 0; + LogPrintf("[0%]..."); for (CBlockIndex* pindex = chainActive.Tip(); pindex && pindex->pprev; pindex = pindex->pprev) { boost::this_thread::interruption_point(); - uiInterface.ShowProgress(_("Verifying blocks..."), std::max(1, std::min(99, (int)(((double)(chainActive.Height() - pindex->nHeight)) / (double)nCheckDepth * (nCheckLevel >= 4 ? 50 : 100))))); + int percentageDone = std::max(1, std::min(99, (int)(((double)(chainActive.Height() - pindex->nHeight)) / (double)nCheckDepth * (nCheckLevel >= 4 ? 50 : 100)))); + if (reportDone < percentageDone/10) { + // report every 10% step + LogPrintf("[%d%%]...", percentageDone); + reportDone = percentageDone/10; + } + uiInterface.ShowProgress(_("Verifying blocks..."), percentageDone); if (pindex->nHeight < chainActive.Height()-nCheckDepth) break; + if (fPruneMode && !(pindex->nStatus & BLOCK_HAVE_DATA)) { + // If pruning, only go back as far as we have data. + LogPrintf("VerifyDB(): block verification stopping at height %d (pruning, no data)\n", pindex->nHeight); + break; + } CBlock block; // check level 0: read from disk - if (!ReadBlockFromDisk(block, pindex)) - return error("VerifyDB() : *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); + if (!ReadBlockFromDisk(block, pindex, chainparams.GetConsensus())) + return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); // check level 1: verify block validity - if (nCheckLevel >= 1 && !CheckBlock(block, state)) - return error("VerifyDB() : *** found bad block at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); + if (nCheckLevel >= 1 && !CheckBlock(block, state, chainparams.GetConsensus())) + return error("%s: *** found bad block at %d, hash=%s (%s)\n", __func__, + pindex->nHeight, pindex->GetBlockHash().ToString(), FormatStateMessage(state)); // check level 2: verify undo validity if (nCheckLevel >= 2 && pindex) { CBlockUndo undo; CDiskBlockPos pos = pindex->GetUndoPos(); if (!pos.IsNull()) { - if (!undo.ReadFromDisk(pos, pindex->pprev->GetBlockHash())) - return error("VerifyDB() : *** found bad undo data at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); + if (!UndoReadFromDisk(undo, pos, pindex->pprev->GetBlockHash())) + return error("VerifyDB(): *** found bad undo data at %d, hash=%s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); } } // check level 3: check for inconsistencies during memory-only disconnect of tip blocks - if (nCheckLevel >= 3 && pindex == pindexState && (coins.GetCacheSize() + pcoinsTip->GetCacheSize()) <= nCoinCacheSize) { + if (nCheckLevel >= 3 && pindex == pindexState && (coins.DynamicMemoryUsage() + pcoinsTip->DynamicMemoryUsage()) <= nCoinCacheUsage) { bool fClean = true; if (!DisconnectBlock(block, state, pindex, coins, &fClean)) - return error("VerifyDB() : *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); + return error("VerifyDB(): *** irrecoverable inconsistency in block data at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); pindexState = pindex->pprev; if (!fClean) { nGoodTransactions = 0; @@ -2963,9 +4147,11 @@ bool CVerifyDB::VerifyDB(CCoinsView *coinsview, int nCheckLevel, int nCheckDepth } else nGoodTransactions += block.vtx.size(); } + if (ShutdownRequested()) + return true; } if (pindexFailure) - return error("VerifyDB() : *** coin database inconsistencies found (last %i blocks, %i good transactions before that)\n", chainActive.Height() - pindexFailure->nHeight + 1, nGoodTransactions); + return error("VerifyDB(): *** coin database inconsistencies found (last %i blocks, %i good transactions before that)\n", chainActive.Height() - pindexFailure->nHeight + 1, nGoodTransactions); // check level 4: try reconnecting blocks if (nCheckLevel >= 4) { @@ -2975,24 +4161,135 @@ bool CVerifyDB::VerifyDB(CCoinsView *coinsview, int nCheckLevel, int nCheckDepth uiInterface.ShowProgress(_("Verifying blocks..."), std::max(1, std::min(99, 100 - (int)(((double)(chainActive.Height() - pindex->nHeight)) / (double)nCheckDepth * 50)))); pindex = chainActive.Next(pindex); CBlock block; - if (!ReadBlockFromDisk(block, pindex)) - return error("VerifyDB() : *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); - if (!ConnectBlock(block, state, pindex, coins)) - return error("VerifyDB() : *** found unconnectable block at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); + if (!ReadBlockFromDisk(block, pindex, chainparams.GetConsensus())) + return error("VerifyDB(): *** ReadBlockFromDisk failed at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); + if (!ConnectBlock(block, state, pindex, coins, chainparams)) + return error("VerifyDB(): *** found unconnectable block at %d, hash=%s", pindex->nHeight, pindex->GetBlockHash().ToString()); } } + LogPrintf("[DONE].\n"); LogPrintf("No coin database inconsistencies in last %i blocks (%i transactions)\n", chainActive.Height() - pindexState->nHeight, nGoodTransactions); return true; } +bool RewindBlockIndex(const CChainParams& params) +{ + LOCK(cs_main); + + int nHeight = 1; + while (nHeight <= chainActive.Height()) { + if (IsWitnessEnabled(chainActive[nHeight - 1], params.GetConsensus()) && !(chainActive[nHeight]->nStatus & BLOCK_OPT_WITNESS)) { + break; + } + nHeight++; + } + + // nHeight is now the height of the first insufficiently-validated block, or tipheight + 1 + CValidationState state; + CBlockIndex* pindex = chainActive.Tip(); + while (chainActive.Height() >= nHeight) { + if (fPruneMode && !(chainActive.Tip()->nStatus & BLOCK_HAVE_DATA)) { + // If pruning, don't try rewinding past the HAVE_DATA point; + // since older blocks can't be served anyway, there's + // no need to walk further, and trying to DisconnectTip() + // will fail (and require a needless reindex/redownload + // of the blockchain). + break; + } + if (!DisconnectTip(state, params, true)) { + return error("RewindBlockIndex: unable to disconnect block at height %i", pindex->nHeight); + } + // Occasionally flush state to disk. + if (!FlushStateToDisk(state, FLUSH_STATE_PERIODIC)) + return false; + } + + // Reduce validity flag and have-data flags. + // We do this after actual disconnecting, otherwise we'll end up writing the lack of data + // to disk before writing the chainstate, resulting in a failure to continue if interrupted. + for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); it++) { + CBlockIndex* pindexIter = it->second; + + // Note: If we encounter an insufficiently validated block that + // is on chainActive, it must be because we are a pruning node, and + // this block or some successor doesn't HAVE_DATA, so we were unable to + // rewind all the way. Blocks remaining on chainActive at this point + // must not have their validity reduced. + if (IsWitnessEnabled(pindexIter->pprev, params.GetConsensus()) && !(pindexIter->nStatus & BLOCK_OPT_WITNESS) && !chainActive.Contains(pindexIter)) { + // Reduce validity + pindexIter->nStatus = std::min<unsigned int>(pindexIter->nStatus & BLOCK_VALID_MASK, BLOCK_VALID_TREE) | (pindexIter->nStatus & ~BLOCK_VALID_MASK); + // Remove have-data flags. + pindexIter->nStatus &= ~(BLOCK_HAVE_DATA | BLOCK_HAVE_UNDO); + // Remove storage location. + pindexIter->nFile = 0; + pindexIter->nDataPos = 0; + pindexIter->nUndoPos = 0; + // Remove various other things + pindexIter->nTx = 0; + pindexIter->nChainTx = 0; + pindexIter->nSequenceId = 0; + // Make sure it gets written. + setDirtyBlockIndex.insert(pindexIter); + // Update indexes + setBlockIndexCandidates.erase(pindexIter); + std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> ret = mapBlocksUnlinked.equal_range(pindexIter->pprev); + while (ret.first != ret.second) { + if (ret.first->second == pindexIter) { + mapBlocksUnlinked.erase(ret.first++); + } else { + ++ret.first; + } + } + } else if (pindexIter->IsValid(BLOCK_VALID_TRANSACTIONS) && pindexIter->nChainTx) { + setBlockIndexCandidates.insert(pindexIter); + } + } + + PruneBlockIndexCandidates(); + + CheckBlockIndex(params.GetConsensus()); + + if (!FlushStateToDisk(state, FLUSH_STATE_ALWAYS)) { + return false; + } + + return true; +} + void UnloadBlockIndex() { - mapBlockIndex.clear(); + LOCK(cs_main); setBlockIndexCandidates.clear(); chainActive.SetTip(NULL); pindexBestInvalid = NULL; + pindexBestHeader = NULL; + mempool.clear(); + mapOrphanTransactions.clear(); + mapOrphanTransactionsByPrev.clear(); + nSyncStarted = 0; + mapBlocksUnlinked.clear(); + vinfoBlockFile.clear(); + nLastBlockFile = 0; + nBlockSequenceId = 1; + mapBlockSource.clear(); + mapBlocksInFlight.clear(); + nPreferredDownload = 0; + setDirtyBlockIndex.clear(); + setDirtyFileInfo.clear(); + mapNodeState.clear(); + recentRejects.reset(NULL); + versionbitscache.Clear(); + for (int b = 0; b < VERSIONBITS_NUM_BITS; b++) { + warningcache[b].clear(); + } + + BOOST_FOREACH(BlockMap::value_type& entry, mapBlockIndex) { + delete entry.second; + } + mapBlockIndex.clear(); + fHavePruned = false; } bool LoadBlockIndex() @@ -3003,48 +4300,50 @@ bool LoadBlockIndex() return true; } - -bool InitBlockIndex() { +bool InitBlockIndex(const CChainParams& chainparams) +{ LOCK(cs_main); + + // Initialize global variables that cannot be constructed at startup. + recentRejects.reset(new CRollingBloomFilter(120000, 0.000001)); + // Check whether we're already initialized if (chainActive.Genesis() != NULL) return true; // Use the provided setting for -txindex in the new database - fTxIndex = GetBoolArg("-txindex", false); + fTxIndex = GetBoolArg("-txindex", DEFAULT_TXINDEX); pblocktree->WriteFlag("txindex", fTxIndex); LogPrintf("Initializing databases...\n"); // Only add the genesis block if not reindexing (in which case we reuse the one already on disk) if (!fReindex) { try { - CBlock &block = const_cast<CBlock&>(Params().GenesisBlock()); + CBlock &block = const_cast<CBlock&>(chainparams.GenesisBlock()); // Start new block file unsigned int nBlockSize = ::GetSerializeSize(block, SER_DISK, CLIENT_VERSION); CDiskBlockPos blockPos; CValidationState state; if (!FindBlockPos(state, blockPos, nBlockSize+8, 0, block.GetBlockTime())) - return error("LoadBlockIndex() : FindBlockPos failed"); - if (!WriteBlockToDisk(block, blockPos)) - return error("LoadBlockIndex() : writing genesis block to disk failed"); + return error("LoadBlockIndex(): FindBlockPos failed"); + if (!WriteBlockToDisk(block, blockPos, chainparams.MessageStart())) + return error("LoadBlockIndex(): writing genesis block to disk failed"); CBlockIndex *pindex = AddToBlockIndex(block); if (!ReceivedBlockTransactions(block, state, pindex, blockPos)) - return error("LoadBlockIndex() : genesis block not accepted"); - if (!ActivateBestChain(state, &block)) - return error("LoadBlockIndex() : genesis block cannot be activated"); - // Force a chainstate write so that when we VerifyDB in a moment, it doesnt check stale data + return error("LoadBlockIndex(): genesis block not accepted"); + if (!ActivateBestChain(state, chainparams, &block)) + return error("LoadBlockIndex(): genesis block cannot be activated"); + // Force a chainstate write so that when we VerifyDB in a moment, it doesn't check stale data return FlushStateToDisk(state, FLUSH_STATE_ALWAYS); - } catch(std::runtime_error &e) { - return error("LoadBlockIndex() : failed to initialize block database: %s", e.what()); + } catch (const std::runtime_error& e) { + return error("LoadBlockIndex(): failed to initialize block database: %s", e.what()); } } return true; } - - -bool LoadExternalBlockFile(FILE* fileIn, CDiskBlockPos *dbp) +bool LoadExternalBlockFile(const CChainParams& chainparams, FILE* fileIn, CDiskBlockPos *dbp) { // Map of disk positions for blocks with unknown parent (only used for reindex) static std::multimap<uint256, CDiskBlockPos> mapBlocksUnknownParent; @@ -3053,7 +4352,7 @@ bool LoadExternalBlockFile(FILE* fileIn, CDiskBlockPos *dbp) int nLoaded = 0; try { // This takes over fileIn and calls fclose() on it in the CBufferedFile destructor - CBufferedFile blkdat(fileIn, 2*MAX_BLOCK_SIZE, MAX_BLOCK_SIZE+8, SER_DISK, CLIENT_VERSION); + CBufferedFile blkdat(fileIn, 2*MAX_BLOCK_SERIALIZED_SIZE, MAX_BLOCK_SERIALIZED_SIZE+8, SER_DISK, CLIENT_VERSION); uint64_t nRewind = blkdat.GetPos(); while (!blkdat.eof()) { boost::this_thread::interruption_point(); @@ -3065,16 +4364,16 @@ bool LoadExternalBlockFile(FILE* fileIn, CDiskBlockPos *dbp) try { // locate a header unsigned char buf[MESSAGE_START_SIZE]; - blkdat.FindByte(Params().MessageStart()[0]); + blkdat.FindByte(chainparams.MessageStart()[0]); nRewind = blkdat.GetPos()+1; blkdat >> FLATDATA(buf); - if (memcmp(buf, Params().MessageStart(), MESSAGE_START_SIZE)) + if (memcmp(buf, chainparams.MessageStart(), MESSAGE_START_SIZE)) continue; // read size blkdat >> nSize; - if (nSize < 80 || nSize > MAX_BLOCK_SIZE) + if (nSize < 80 || nSize > MAX_BLOCK_SERIALIZED_SIZE) continue; - } catch (const std::exception &) { + } catch (const std::exception&) { // no valid block header found; don't complain break; } @@ -3091,7 +4390,7 @@ bool LoadExternalBlockFile(FILE* fileIn, CDiskBlockPos *dbp) // detect out of order blocks, and store them for later uint256 hash = block.GetHash(); - if (hash != Params().HashGenesisBlock() && mapBlockIndex.find(block.hashPrevBlock) == mapBlockIndex.end()) { + if (hash != chainparams.GetConsensus().hashGenesisBlock && mapBlockIndex.find(block.hashPrevBlock) == mapBlockIndex.end()) { LogPrint("reindex", "%s: Out of order block %s, parent %s not known\n", __func__, hash.ToString(), block.hashPrevBlock.ToString()); if (dbp) @@ -3101,15 +4400,26 @@ bool LoadExternalBlockFile(FILE* fileIn, CDiskBlockPos *dbp) // process in case the block isn't known yet if (mapBlockIndex.count(hash) == 0 || (mapBlockIndex[hash]->nStatus & BLOCK_HAVE_DATA) == 0) { + LOCK(cs_main); CValidationState state; - if (ProcessNewBlock(state, NULL, &block, dbp)) + if (AcceptBlock(block, state, chainparams, NULL, true, dbp, NULL)) nLoaded++; if (state.IsError()) break; - } else if (hash != Params().HashGenesisBlock() && mapBlockIndex[hash]->nHeight % 1000 == 0) { - LogPrintf("Block Import: already had block %s at height %d\n", hash.ToString(), mapBlockIndex[hash]->nHeight); + } else if (hash != chainparams.GetConsensus().hashGenesisBlock && mapBlockIndex[hash]->nHeight % 1000 == 0) { + LogPrint("reindex", "Block Import: already had block %s at height %d\n", hash.ToString(), mapBlockIndex[hash]->nHeight); } + // Activate the genesis block so normal node progress can continue + if (hash == chainparams.GetConsensus().hashGenesisBlock) { + CValidationState state; + if (!ActivateBestChain(state, chainparams)) { + break; + } + } + + NotifyHeaderTip(); + // Recursively process earlier encountered successors of this block deque<uint256> queue; queue.push_back(hash); @@ -3119,12 +4429,13 @@ bool LoadExternalBlockFile(FILE* fileIn, CDiskBlockPos *dbp) std::pair<std::multimap<uint256, CDiskBlockPos>::iterator, std::multimap<uint256, CDiskBlockPos>::iterator> range = mapBlocksUnknownParent.equal_range(head); while (range.first != range.second) { std::multimap<uint256, CDiskBlockPos>::iterator it = range.first; - if (ReadBlockFromDisk(block, it->second)) + if (ReadBlockFromDisk(block, it->second, chainparams.GetConsensus())) { - LogPrintf("%s: Processing out of order child %s of %s\n", __func__, block.GetHash().ToString(), + LogPrint("reindex", "%s: Processing out of order child %s of %s\n", __func__, block.GetHash().ToString(), head.ToString()); + LOCK(cs_main); CValidationState dummy; - if (ProcessNewBlock(dummy, NULL, &block, &it->second)) + if (AcceptBlock(block, dummy, chainparams, NULL, true, &it->second, NULL)) { nLoaded++; queue.push_back(block.GetHash()); @@ -3132,13 +4443,14 @@ bool LoadExternalBlockFile(FILE* fileIn, CDiskBlockPos *dbp) } range.first++; mapBlocksUnknownParent.erase(it); + NotifyHeaderTip(); } } - } catch (std::exception &e) { - LogPrintf("%s : Deserialize or I/O error - %s", __func__, e.what()); + } catch (const std::exception& e) { + LogPrintf("%s: Deserialize or I/O error - %s\n", __func__, e.what()); } } - } catch(std::runtime_error &e) { + } catch (const std::runtime_error& e) { AbortNode(std::string("System error: ") + e.what()); } if (nLoaded > 0) @@ -3146,60 +4458,228 @@ bool LoadExternalBlockFile(FILE* fileIn, CDiskBlockPos *dbp) return nLoaded > 0; } -////////////////////////////////////////////////////////////////////////////// -// -// CAlert -// +void static CheckBlockIndex(const Consensus::Params& consensusParams) +{ + if (!fCheckBlockIndex) { + return; + } + + LOCK(cs_main); + + // During a reindex, we read the genesis block and call CheckBlockIndex before ActivateBestChain, + // so we have the genesis block in mapBlockIndex but no active chain. (A few of the tests when + // iterating the block tree require that chainActive has been initialized.) + if (chainActive.Height() < 0) { + assert(mapBlockIndex.size() <= 1); + return; + } + + // Build forward-pointing map of the entire block tree. + std::multimap<CBlockIndex*,CBlockIndex*> forward; + for (BlockMap::iterator it = mapBlockIndex.begin(); it != mapBlockIndex.end(); it++) { + forward.insert(std::make_pair(it->second->pprev, it->second)); + } + + assert(forward.size() == mapBlockIndex.size()); + + std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> rangeGenesis = forward.equal_range(NULL); + CBlockIndex *pindex = rangeGenesis.first->second; + rangeGenesis.first++; + assert(rangeGenesis.first == rangeGenesis.second); // There is only one index entry with parent NULL. + + // Iterate over the entire block tree, using depth-first search. + // Along the way, remember whether there are blocks on the path from genesis + // block being explored which are the first to have certain properties. + size_t nNodes = 0; + int nHeight = 0; + CBlockIndex* pindexFirstInvalid = NULL; // Oldest ancestor of pindex which is invalid. + CBlockIndex* pindexFirstMissing = NULL; // Oldest ancestor of pindex which does not have BLOCK_HAVE_DATA. + CBlockIndex* pindexFirstNeverProcessed = NULL; // Oldest ancestor of pindex for which nTx == 0. + CBlockIndex* pindexFirstNotTreeValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_TREE (regardless of being valid or not). + CBlockIndex* pindexFirstNotTransactionsValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_TRANSACTIONS (regardless of being valid or not). + CBlockIndex* pindexFirstNotChainValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_CHAIN (regardless of being valid or not). + CBlockIndex* pindexFirstNotScriptsValid = NULL; // Oldest ancestor of pindex which does not have BLOCK_VALID_SCRIPTS (regardless of being valid or not). + while (pindex != NULL) { + nNodes++; + if (pindexFirstInvalid == NULL && pindex->nStatus & BLOCK_FAILED_VALID) pindexFirstInvalid = pindex; + if (pindexFirstMissing == NULL && !(pindex->nStatus & BLOCK_HAVE_DATA)) pindexFirstMissing = pindex; + if (pindexFirstNeverProcessed == NULL && pindex->nTx == 0) pindexFirstNeverProcessed = pindex; + if (pindex->pprev != NULL && pindexFirstNotTreeValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TREE) pindexFirstNotTreeValid = pindex; + if (pindex->pprev != NULL && pindexFirstNotTransactionsValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_TRANSACTIONS) pindexFirstNotTransactionsValid = pindex; + if (pindex->pprev != NULL && pindexFirstNotChainValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_CHAIN) pindexFirstNotChainValid = pindex; + if (pindex->pprev != NULL && pindexFirstNotScriptsValid == NULL && (pindex->nStatus & BLOCK_VALID_MASK) < BLOCK_VALID_SCRIPTS) pindexFirstNotScriptsValid = pindex; + + // Begin: actual consistency checks. + if (pindex->pprev == NULL) { + // Genesis block checks. + assert(pindex->GetBlockHash() == consensusParams.hashGenesisBlock); // Genesis block's hash must match. + assert(pindex == chainActive.Genesis()); // The current active chain's genesis block must be this block. + } + if (pindex->nChainTx == 0) assert(pindex->nSequenceId == 0); // nSequenceId can't be set for blocks that aren't linked + // VALID_TRANSACTIONS is equivalent to nTx > 0 for all nodes (whether or not pruning has occurred). + // HAVE_DATA is only equivalent to nTx > 0 (or VALID_TRANSACTIONS) if no pruning has occurred. + if (!fHavePruned) { + // If we've never pruned, then HAVE_DATA should be equivalent to nTx > 0 + assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0)); + assert(pindexFirstMissing == pindexFirstNeverProcessed); + } else { + // If we have pruned, then we can only say that HAVE_DATA implies nTx > 0 + if (pindex->nStatus & BLOCK_HAVE_DATA) assert(pindex->nTx > 0); + } + if (pindex->nStatus & BLOCK_HAVE_UNDO) assert(pindex->nStatus & BLOCK_HAVE_DATA); + assert(((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TRANSACTIONS) == (pindex->nTx > 0)); // This is pruning-independent. + // All parents having had data (at some point) is equivalent to all parents being VALID_TRANSACTIONS, which is equivalent to nChainTx being set. + assert((pindexFirstNeverProcessed != NULL) == (pindex->nChainTx == 0)); // nChainTx != 0 is used to signal that all parent blocks have been processed (but may have been pruned). + assert((pindexFirstNotTransactionsValid != NULL) == (pindex->nChainTx == 0)); + assert(pindex->nHeight == nHeight); // nHeight must be consistent. + assert(pindex->pprev == NULL || pindex->nChainWork >= pindex->pprev->nChainWork); // For every block except the genesis block, the chainwork must be larger than the parent's. + assert(nHeight < 2 || (pindex->pskip && (pindex->pskip->nHeight < nHeight))); // The pskip pointer must point back for all but the first 2 blocks. + assert(pindexFirstNotTreeValid == NULL); // All mapBlockIndex entries must at least be TREE valid + if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_TREE) assert(pindexFirstNotTreeValid == NULL); // TREE valid implies all parents are TREE valid + if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_CHAIN) assert(pindexFirstNotChainValid == NULL); // CHAIN valid implies all parents are CHAIN valid + if ((pindex->nStatus & BLOCK_VALID_MASK) >= BLOCK_VALID_SCRIPTS) assert(pindexFirstNotScriptsValid == NULL); // SCRIPTS valid implies all parents are SCRIPTS valid + if (pindexFirstInvalid == NULL) { + // Checks for not-invalid blocks. + assert((pindex->nStatus & BLOCK_FAILED_MASK) == 0); // The failed mask cannot be set for blocks without invalid parents. + } + if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && pindexFirstNeverProcessed == NULL) { + if (pindexFirstInvalid == NULL) { + // If this block sorts at least as good as the current tip and + // is valid and we have all data for its parents, it must be in + // setBlockIndexCandidates. chainActive.Tip() must also be there + // even if some data has been pruned. + if (pindexFirstMissing == NULL || pindex == chainActive.Tip()) { + assert(setBlockIndexCandidates.count(pindex)); + } + // If some parent is missing, then it could be that this block was in + // setBlockIndexCandidates but had to be removed because of the missing data. + // In this case it must be in mapBlocksUnlinked -- see test below. + } + } else { // If this block sorts worse than the current tip or some ancestor's block has never been seen, it cannot be in setBlockIndexCandidates. + assert(setBlockIndexCandidates.count(pindex) == 0); + } + // Check whether this block is in mapBlocksUnlinked. + std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> rangeUnlinked = mapBlocksUnlinked.equal_range(pindex->pprev); + bool foundInUnlinked = false; + while (rangeUnlinked.first != rangeUnlinked.second) { + assert(rangeUnlinked.first->first == pindex->pprev); + if (rangeUnlinked.first->second == pindex) { + foundInUnlinked = true; + break; + } + rangeUnlinked.first++; + } + if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed != NULL && pindexFirstInvalid == NULL) { + // If this block has block data available, some parent was never received, and has no invalid parents, it must be in mapBlocksUnlinked. + assert(foundInUnlinked); + } + if (!(pindex->nStatus & BLOCK_HAVE_DATA)) assert(!foundInUnlinked); // Can't be in mapBlocksUnlinked if we don't HAVE_DATA + if (pindexFirstMissing == NULL) assert(!foundInUnlinked); // We aren't missing data for any parent -- cannot be in mapBlocksUnlinked. + if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed == NULL && pindexFirstMissing != NULL) { + // We HAVE_DATA for this block, have received data for all parents at some point, but we're currently missing data for some parent. + assert(fHavePruned); // We must have pruned. + // This block may have entered mapBlocksUnlinked if: + // - it has a descendant that at some point had more work than the + // tip, and + // - we tried switching to that descendant but were missing + // data for some intermediate block between chainActive and the + // tip. + // So if this block is itself better than chainActive.Tip() and it wasn't in + // setBlockIndexCandidates, then it must be in mapBlocksUnlinked. + if (!CBlockIndexWorkComparator()(pindex, chainActive.Tip()) && setBlockIndexCandidates.count(pindex) == 0) { + if (pindexFirstInvalid == NULL) { + assert(foundInUnlinked); + } + } + } + // assert(pindex->GetBlockHash() == pindex->GetBlockHeader().GetHash()); // Perhaps too slow + // End: actual consistency checks. + + // Try descending into the first subnode. + std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> range = forward.equal_range(pindex); + if (range.first != range.second) { + // A subnode was found. + pindex = range.first->second; + nHeight++; + continue; + } + // This is a leaf node. + // Move upwards until we reach a node of which we have not yet visited the last child. + while (pindex) { + // We are going to either move to a parent or a sibling of pindex. + // If pindex was the first with a certain property, unset the corresponding variable. + if (pindex == pindexFirstInvalid) pindexFirstInvalid = NULL; + if (pindex == pindexFirstMissing) pindexFirstMissing = NULL; + if (pindex == pindexFirstNeverProcessed) pindexFirstNeverProcessed = NULL; + if (pindex == pindexFirstNotTreeValid) pindexFirstNotTreeValid = NULL; + if (pindex == pindexFirstNotTransactionsValid) pindexFirstNotTransactionsValid = NULL; + if (pindex == pindexFirstNotChainValid) pindexFirstNotChainValid = NULL; + if (pindex == pindexFirstNotScriptsValid) pindexFirstNotScriptsValid = NULL; + // Find our parent. + CBlockIndex* pindexPar = pindex->pprev; + // Find which child we just visited. + std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> rangePar = forward.equal_range(pindexPar); + while (rangePar.first->second != pindex) { + assert(rangePar.first != rangePar.second); // Our parent must have at least the node we're coming from as child. + rangePar.first++; + } + // Proceed to the next one. + rangePar.first++; + if (rangePar.first != rangePar.second) { + // Move to the sibling. + pindex = rangePar.first->second; + break; + } else { + // Move up further. + pindex = pindexPar; + nHeight--; + continue; + } + } + } + + // Check that we actually traversed the entire map. + assert(nNodes == forward.size()); +} -string GetWarnings(string strFor) +std::string GetWarnings(const std::string& strFor) { - int nPriority = 0; string strStatusBar; string strRPC; + string strGUI; - if (!CLIENT_VERSION_IS_RELEASE) - strStatusBar = _("This is a pre-release test build - use at your own risk - do not use for mining or merchant applications"); + if (!CLIENT_VERSION_IS_RELEASE) { + strStatusBar = "This is a pre-release test build - use at your own risk - do not use for mining or merchant applications"; + strGUI = _("This is a pre-release test build - use at your own risk - do not use for mining or merchant applications"); + } - if (GetBoolArg("-testsafemode", false)) - strStatusBar = strRPC = "testsafemode enabled"; + if (GetBoolArg("-testsafemode", DEFAULT_TESTSAFEMODE)) + strStatusBar = strRPC = strGUI = "testsafemode enabled"; // Misc warnings like out of disk space and clock is wrong if (strMiscWarning != "") { - nPriority = 1000; - strStatusBar = strMiscWarning; + strStatusBar = strGUI = strMiscWarning; } if (fLargeWorkForkFound) { - nPriority = 2000; - strStatusBar = strRPC = _("Warning: The network does not appear to fully agree! Some miners appear to be experiencing issues."); + strStatusBar = strRPC = "Warning: The network does not appear to fully agree! Some miners appear to be experiencing issues."; + strGUI = _("Warning: The network does not appear to fully agree! Some miners appear to be experiencing issues."); } else if (fLargeWorkInvalidChainFound) { - nPriority = 2000; - strStatusBar = strRPC = _("Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade."); - } - - // Alerts - { - LOCK(cs_mapAlerts); - BOOST_FOREACH(PAIRTYPE(const uint256, CAlert)& item, mapAlerts) - { - const CAlert& alert = item.second; - if (alert.AppliesToMe() && alert.nPriority > nPriority) - { - nPriority = alert.nPriority; - strStatusBar = alert.strStatusBar; - } - } + strStatusBar = strRPC = "Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade."; + strGUI = _("Warning: We do not appear to fully agree with our peers! You may need to upgrade, or other nodes may need to upgrade."); } - if (strFor == "statusbar") + if (strFor == "gui") + return strGUI; + else if (strFor == "statusbar") return strStatusBar; else if (strFor == "rpc") return strRPC; - assert(!"GetWarnings() : invalid parameter"); + assert(!"GetWarnings(): invalid parameter"); return "error"; } @@ -3216,26 +4696,40 @@ string GetWarnings(string strFor) // -bool static AlreadyHave(const CInv& inv) +bool static AlreadyHave(const CInv& inv) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { switch (inv.type) { case MSG_TX: + case MSG_WITNESS_TX: { - bool txInMap = false; - txInMap = mempool.exists(inv.hash); - return txInMap || mapOrphanTransactions.count(inv.hash) || - pcoinsTip->HaveCoins(inv.hash); + assert(recentRejects); + if (chainActive.Tip()->GetBlockHash() != hashRecentRejectsChainTip) + { + // If the chain tip has changed previously rejected transactions + // might be now valid, e.g. due to a nLockTime'd tx becoming valid, + // or a double-spend. Reset the rejects filter and give those + // txs a second chance. + hashRecentRejectsChainTip = chainActive.Tip()->GetBlockHash(); + recentRejects->reset(); + } + + // Use pcoinsTip->HaveCoinsInCache as a quick approximation to exclude + // requesting or processing some txs which have already been included in a block + return recentRejects->contains(inv.hash) || + mempool.exists(inv.hash) || + mapOrphanTransactions.count(inv.hash) || + pcoinsTip->HaveCoinsInCache(inv.hash); } case MSG_BLOCK: + case MSG_WITNESS_BLOCK: return mapBlockIndex.count(inv.hash); } // Don't know what it is, just say we already got one return true; } - -void static ProcessGetData(CNode* pfrom) +void static ProcessGetData(CNode* pfrom, const Consensus::Params& consensusParams) { std::deque<CInv>::iterator it = pfrom->vRecvGetData.begin(); @@ -3253,58 +4747,84 @@ void static ProcessGetData(CNode* pfrom) boost::this_thread::interruption_point(); it++; - if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK) + if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK || inv.type == MSG_CMPCT_BLOCK || inv.type == MSG_WITNESS_BLOCK) { bool send = false; BlockMap::iterator mi = mapBlockIndex.find(inv.hash); if (mi != mapBlockIndex.end()) { - // If the requested block is at a height below our last - // checkpoint, only serve it if it's in the checkpointed chain - int nHeight = mi->second->nHeight; - CBlockIndex* pcheckpoint = Checkpoints::GetLastCheckpoint(); - if (pcheckpoint && nHeight < pcheckpoint->nHeight) { - if (!chainActive.Contains(mi->second)) - { - LogPrintf("ProcessGetData(): ignoring request for old block that isn't in the main chain\n"); - } else { - send = true; - } - } else { + if (chainActive.Contains(mi->second)) { send = true; + } else { + static const int nOneMonth = 30 * 24 * 60 * 60; + // To prevent fingerprinting attacks, only send blocks outside of the active + // chain if they are valid, and no more than a month older (both in time, and in + // best equivalent proof of work) than the best header chain we know about. + send = mi->second->IsValid(BLOCK_VALID_SCRIPTS) && (pindexBestHeader != NULL) && + (pindexBestHeader->GetBlockTime() - mi->second->GetBlockTime() < nOneMonth) && + (GetBlockProofEquivalentTime(*pindexBestHeader, *mi->second, *pindexBestHeader, consensusParams) < nOneMonth); + if (!send) { + LogPrintf("%s: ignoring request from peer=%i for old block that isn't in the main chain\n", __func__, pfrom->GetId()); + } } } - if (send) + // disconnect node in case we have reached the outbound limit for serving historical blocks + // never disconnect whitelisted nodes + static const int nOneWeek = 7 * 24 * 60 * 60; // assume > 1 week = historical + if (send && CNode::OutboundTargetReached(true) && ( ((pindexBestHeader != NULL) && (pindexBestHeader->GetBlockTime() - mi->second->GetBlockTime() > nOneWeek)) || inv.type == MSG_FILTERED_BLOCK) && !pfrom->fWhitelisted) + { + LogPrint("net", "historical block serving limit reached, disconnect peer=%d\n", pfrom->GetId()); + + //disconnect node + pfrom->fDisconnect = true; + send = false; + } + // Pruned nodes may have deleted the block, so check whether + // it's available before trying to send. + if (send && (mi->second->nStatus & BLOCK_HAVE_DATA)) { // Send block from disk CBlock block; - if (!ReadBlockFromDisk(block, (*mi).second)) + if (!ReadBlockFromDisk(block, (*mi).second, consensusParams)) assert(!"cannot load block from disk"); if (inv.type == MSG_BLOCK) - pfrom->PushMessage("block", block); - else // MSG_FILTERED_BLOCK) + pfrom->PushMessageWithFlag(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, block); + else if (inv.type == MSG_WITNESS_BLOCK) + pfrom->PushMessage(NetMsgType::BLOCK, block); + else if (inv.type == MSG_FILTERED_BLOCK) { LOCK(pfrom->cs_filter); if (pfrom->pfilter) { CMerkleBlock merkleBlock(block, *pfrom->pfilter); - pfrom->PushMessage("merkleblock", merkleBlock); + pfrom->PushMessage(NetMsgType::MERKLEBLOCK, merkleBlock); // CMerkleBlock just contains hashes, so also push any transactions in the block the client did not see // This avoids hurting performance by pointlessly requiring a round-trip - // Note that there is currently no way for a node to request any single transactions we didnt send here - + // Note that there is currently no way for a node to request any single transactions we didn't send here - // they must either disconnect and retry or request the full block. // Thus, the protocol spec specified allows for us to provide duplicate txn here, // however we MUST always provide at least what the remote peer needs typedef std::pair<unsigned int, uint256> PairType; BOOST_FOREACH(PairType& pair, merkleBlock.vMatchedTxn) - if (!pfrom->setInventoryKnown.count(CInv(MSG_TX, pair.second))) - pfrom->PushMessage("tx", block.vtx[pair.first]); + pfrom->PushMessageWithFlag(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, block.vtx[pair.first]); } // else // no response } + else if (inv.type == MSG_CMPCT_BLOCK) + { + // If a peer is asking for old blocks, we're almost guaranteed + // they wont have a useful mempool to match against a compact block, + // and we dont feel like constructing the object for them, so + // instead we respond with the full, non-compact block. + if (mi->second->nHeight >= chainActive.Height() - 10) { + CBlockHeaderAndShortTxIDs cmpctblock(block); + pfrom->PushMessageWithFlag(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::CMPCTBLOCK, cmpctblock); + } else + pfrom->PushMessageWithFlag(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, block); + } - // Trigger them to send a getblocks request for the next batch of inventory + // Trigger the peer node to send a getblocks request for the next batch of inventory if (inv.hash == pfrom->hashContinue) { // Bypass PushInventory, this must send even if redundant, @@ -3312,42 +4832,37 @@ void static ProcessGetData(CNode* pfrom) // wait for other stuff first. vector<CInv> vInv; vInv.push_back(CInv(MSG_BLOCK, chainActive.Tip()->GetBlockHash())); - pfrom->PushMessage("inv", vInv); - pfrom->hashContinue = 0; + pfrom->PushMessage(NetMsgType::INV, vInv); + pfrom->hashContinue.SetNull(); } } } - else if (inv.IsKnownType()) + else if (inv.type == MSG_TX || inv.type == MSG_WITNESS_TX) { // Send stream from relay memory - bool pushed = false; - { - LOCK(cs_mapRelay); - map<CInv, CDataStream>::iterator mi = mapRelay.find(inv); - if (mi != mapRelay.end()) { - pfrom->PushMessage(inv.GetCommand(), (*mi).second); - pushed = true; + bool push = false; + auto mi = mapRelay.find(inv.hash); + if (mi != mapRelay.end()) { + pfrom->PushMessageWithFlag(inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0, NetMsgType::TX, *mi->second); + push = true; + } else if (pfrom->timeLastMempoolReq) { + auto txinfo = mempool.info(inv.hash); + // To protect privacy, do not answer getdata using the mempool when + // that TX couldn't have been INVed in reply to a MEMPOOL request. + if (txinfo.tx && txinfo.nTime <= pfrom->timeLastMempoolReq) { + pfrom->PushMessageWithFlag(inv.type == MSG_TX ? SERIALIZE_TRANSACTION_NO_WITNESS : 0, NetMsgType::TX, *txinfo.tx); + push = true; } } - if (!pushed && inv.type == MSG_TX) { - CTransaction tx; - if (mempool.lookup(inv.hash, tx)) { - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); - ss.reserve(1000); - ss << tx; - pfrom->PushMessage("tx", ss); - pushed = true; - } - } - if (!pushed) { + if (!push) { vNotFound.push_back(inv); } } // Track requests for our stuff. - g_signals.Inventory(inv.hash); + GetMainSignals().Inventory(inv.hash); - if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK) + if (inv.type == MSG_BLOCK || inv.type == MSG_FILTERED_BLOCK || inv.type == MSG_CMPCT_BLOCK || inv.type == MSG_WITNESS_BLOCK) break; } } @@ -3362,14 +4877,21 @@ void static ProcessGetData(CNode* pfrom) // do that because they want to know about (and store and rebroadcast and // risk analyze) the dependencies of transactions relevant to them, without // having to download the entire memory pool. - pfrom->PushMessage("notfound", vNotFound); + pfrom->PushMessage(NetMsgType::NOTFOUND, vNotFound); + } +} + +uint32_t GetFetchFlags(CNode* pfrom, CBlockIndex* pprev, const Consensus::Params& chainparams) { + uint32_t nFetchFlags = 0; + if (IsWitnessEnabled(pprev, chainparams) && State(pfrom->GetId())->fHaveWitness) { + nFetchFlags |= MSG_WITNESS_FLAG; } + return nFetchFlags; } -bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, int64_t nTimeReceived) +bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, int64_t nTimeReceived, const CChainParams& chainparams) { - RandAddSeedPerfmon(); - LogPrint("net", "received: %s (%u bytes) peer=%d\n", strCommand, vRecv.size(), pfrom->id); + LogPrint("net", "received: %s (%u bytes) peer=%d\n", SanitizeString(strCommand), vRecv.size(), pfrom->id); if (mapArgs.count("-dropmessagestest") && GetRand(atoi(mapArgs["-dropmessagestest"])) == 0) { LogPrintf("dropmessagestest DROPPING RECV MESSAGE\n"); @@ -3377,14 +4899,29 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } + if (!(nLocalServices & NODE_BLOOM) && + (strCommand == NetMsgType::FILTERLOAD || + strCommand == NetMsgType::FILTERADD || + strCommand == NetMsgType::FILTERCLEAR)) + { + if (pfrom->nVersion >= NO_BLOOM_VERSION) { + LOCK(cs_main); + Misbehaving(pfrom->GetId(), 100); + return false; + } else { + pfrom->fDisconnect = true; + return false; + } + } - if (strCommand == "version") + if (strCommand == NetMsgType::VERSION) { // Each connection can only send one version message if (pfrom->nVersion != 0) { - pfrom->PushMessage("reject", strCommand, REJECT_DUPLICATE, string("Duplicate version message")); + pfrom->PushMessage(NetMsgType::REJECT, strCommand, REJECT_DUPLICATE, string("Duplicate version message")); + LOCK(cs_main); Misbehaving(pfrom->GetId(), 1); return false; } @@ -3393,12 +4930,27 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, CAddress addrMe; CAddress addrFrom; uint64_t nNonce = 1; - vRecv >> pfrom->nVersion >> pfrom->nServices >> nTime >> addrMe; + uint64_t nServiceInt; + vRecv >> pfrom->nVersion >> nServiceInt >> nTime >> addrMe; + pfrom->nServices = ServiceFlags(nServiceInt); + if (!pfrom->fInbound) + { + addrman.SetServices(pfrom->addr, pfrom->nServices); + } + if (pfrom->nServicesExpected & ~pfrom->nServices) + { + LogPrint("net", "peer=%d does not offer the expected services (%08x offered, %08x expected); disconnecting\n", pfrom->id, pfrom->nServices, pfrom->nServicesExpected); + pfrom->PushMessage(NetMsgType::REJECT, strCommand, REJECT_NONSTANDARD, + strprintf("Expected to offer services %08x", pfrom->nServicesExpected)); + pfrom->fDisconnect = true; + return false; + } + if (pfrom->nVersion < MIN_PEER_PROTO_VERSION) { // disconnect from peers older than this proto version LogPrintf("peer=%d using obsolete version %i; disconnecting\n", pfrom->id, pfrom->nVersion); - pfrom->PushMessage("reject", strCommand, REJECT_OBSOLETE, + pfrom->PushMessage(NetMsgType::REJECT, strCommand, REJECT_OBSOLETE, strprintf("Version must be %d or greater", MIN_PEER_PROTO_VERSION)); pfrom->fDisconnect = true; return false; @@ -3409,15 +4961,19 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (!vRecv.empty()) vRecv >> addrFrom >> nNonce; if (!vRecv.empty()) { - vRecv >> LIMITED_STRING(pfrom->strSubVer, 256); + vRecv >> LIMITED_STRING(pfrom->strSubVer, MAX_SUBVERSION_LENGTH); pfrom->cleanSubVer = SanitizeString(pfrom->strSubVer); } - if (!vRecv.empty()) + if (!vRecv.empty()) { vRecv >> pfrom->nStartingHeight; - if (!vRecv.empty()) - vRecv >> pfrom->fRelayTxes; // set to true after we get the first filter* message - else - pfrom->fRelayTxes = true; + } + { + LOCK(pfrom->cs_filter); + if (!vRecv.empty()) + vRecv >> pfrom->fRelayTxes; // set to true after we get the first filter* message + else + pfrom->fRelayTxes = true; + } // Disconnect if we connected to ourself if (nNonce == nLocalHostNonce && nNonce > 1) @@ -3439,11 +4995,20 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, pfrom->fClient = !(pfrom->nServices & NODE_NETWORK); + if((pfrom->nServices & NODE_WITNESS)) + { + LOCK(cs_main); + State(pfrom->GetId())->fHaveWitness = true; + } + // Potentially mark this peer as a preferred download peer. + { + LOCK(cs_main); UpdatePreferredDownload(pfrom, State(pfrom->GetId())); + } // Change version - pfrom->PushMessage("verack"); + pfrom->PushMessage(NetMsgType::VERACK); pfrom->ssSend.SetVersion(min(pfrom->nVersion, PROTOCOL_VERSION)); if (!pfrom->fInbound) @@ -3454,9 +5019,11 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, CAddress addr = GetLocalAddress(&pfrom->addr); if (addr.IsRoutable()) { + LogPrintf("ProcessMessages: advertising address %s\n", addr.ToString()); pfrom->PushAddress(addr); } else if (IsPeerAddrLocalGood(pfrom)) { addr.SetIP(pfrom->addrLocal); + LogPrintf("ProcessMessages: advertising address %s\n", addr.ToString()); pfrom->PushAddress(addr); } } @@ -3464,7 +5031,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // Get recent addresses if (pfrom->fOneShot || pfrom->nVersion >= CADDR_TIME_VERSION || addrman.size() < 1000) { - pfrom->PushMessage("getaddr"); + pfrom->PushMessage(NetMsgType::GETADDR); pfrom->fGetAddr = true; } addrman.Good(pfrom->addr); @@ -3476,13 +5043,6 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } } - // Relay alerts - { - LOCK(cs_mapAlerts); - BOOST_FOREACH(PAIRTYPE(const uint256, CAlert)& item, mapAlerts) - item.second.RelayTo(pfrom); - } - pfrom->fSuccessfullyConnected = true; string remoteAddr; @@ -3494,25 +5054,52 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, pfrom->nStartingHeight, addrMe.ToString(), pfrom->id, remoteAddr); - AddTimeData(pfrom->addr, nTime); + int64_t nTimeOffset = nTime - GetTime(); + pfrom->nTimeOffset = nTimeOffset; + AddTimeData(pfrom->addr, nTimeOffset); } else if (pfrom->nVersion == 0) { // Must have a version message before anything else + LOCK(cs_main); Misbehaving(pfrom->GetId(), 1); return false; } - else if (strCommand == "verack") + else if (strCommand == NetMsgType::VERACK) { pfrom->SetRecvVersion(min(pfrom->nVersion, PROTOCOL_VERSION)); + + // Mark this node as currently connected, so we update its timestamp later. + if (pfrom->fNetworkNode) { + LOCK(cs_main); + State(pfrom->GetId())->fCurrentlyConnected = true; + } + + if (pfrom->nVersion >= SENDHEADERS_VERSION) { + // Tell our peer we prefer to receive headers rather than inv's + // We send this to non-NODE NETWORK peers as well, because even + // non-NODE NETWORK peers can announce blocks (such as pruning + // nodes) + pfrom->PushMessage(NetMsgType::SENDHEADERS); + } + if (pfrom->nVersion >= SHORT_IDS_BLOCKS_VERSION) { + // Tell our peer we are willing to provide version-1 cmpctblocks + // However, we do not request new block announcements using + // cmpctblock messages. + // We send this to non-NODE NETWORK peers as well, because + // they may wish to request compact blocks from us + bool fAnnounceUsingCMPCTBLOCK = false; + uint64_t nCMPCTBLOCKVersion = 1; + pfrom->PushMessage(NetMsgType::SENDCMPCT, fAnnounceUsingCMPCTBLOCK, nCMPCTBLOCKVersion); + } } - else if (strCommand == "addr") + else if (strCommand == NetMsgType::ADDR) { vector<CAddress> vAddr; vRecv >> vAddr; @@ -3522,6 +5109,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, return true; if (vAddr.size() > 1000) { + LOCK(cs_main); Misbehaving(pfrom->GetId(), 20); return error("message addr size() = %u", vAddr.size()); } @@ -3534,6 +5122,9 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, { boost::this_thread::interruption_point(); + if ((addr.nServices & REQUIRED_SERVICES) != REQUIRED_SERVICES) + continue; + if (addr.nTime <= 100000000 || addr.nTime > nNow + 10 * 60) addr.nTime = nNow - 5 * 24 * 60 * 60; pfrom->AddAddressKnown(addr); @@ -3544,26 +5135,21 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, { LOCK(cs_vNodes); // Use deterministic randomness to send to the same nodes for 24 hours - // at a time so the setAddrKnowns of the chosen nodes prevent repeats - static uint256 hashSalt; - if (hashSalt == 0) - hashSalt = GetRandHash(); + // at a time so the addrKnowns of the chosen nodes prevent repeats + static const uint64_t salt0 = GetRand(std::numeric_limits<uint64_t>::max()); + static const uint64_t salt1 = GetRand(std::numeric_limits<uint64_t>::max()); uint64_t hashAddr = addr.GetHash(); - uint256 hashRand = hashSalt ^ (hashAddr<<32) ^ ((GetTime()+hashAddr)/(24*60*60)); - hashRand = Hash(BEGIN(hashRand), END(hashRand)); - multimap<uint256, CNode*> mapMix; + multimap<uint64_t, CNode*> mapMix; + const CSipHasher hasher = CSipHasher(salt0, salt1).Write(hashAddr << 32).Write((GetTime() + hashAddr) / (24*60*60)); BOOST_FOREACH(CNode* pnode, vNodes) { if (pnode->nVersion < CADDR_TIME_VERSION) continue; - unsigned int nPointer; - memcpy(&nPointer, &pnode, sizeof(nPointer)); - uint256 hashKey = hashRand ^ nPointer; - hashKey = Hash(BEGIN(hashKey), END(hashKey)); + uint64_t hashKey = CSipHasher(hasher).Write(pnode->id).Finalize(); mapMix.insert(make_pair(hashKey, pnode)); } int nRelayNodes = fReachable ? 2 : 1; // limited relaying of addresses outside our network(s) - for (multimap<uint256, CNode*>::iterator mi = mapMix.begin(); mi != mapMix.end() && nRelayNodes-- > 0; ++mi) + for (multimap<uint64_t, CNode*>::iterator mi = mapMix.begin(); mi != mapMix.end() && nRelayNodes-- > 0; ++mi) ((*mi).second)->PushAddress(addr); } } @@ -3578,38 +5164,65 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, pfrom->fDisconnect = true; } + else if (strCommand == NetMsgType::SENDHEADERS) + { + LOCK(cs_main); + State(pfrom->GetId())->fPreferHeaders = true; + } + + else if (strCommand == NetMsgType::SENDCMPCT) + { + bool fAnnounceUsingCMPCTBLOCK = false; + uint64_t nCMPCTBLOCKVersion = 1; + vRecv >> fAnnounceUsingCMPCTBLOCK >> nCMPCTBLOCKVersion; + if (nCMPCTBLOCKVersion == 1) { + LOCK(cs_main); + State(pfrom->GetId())->fProvidesHeaderAndIDs = true; + State(pfrom->GetId())->fPreferHeaderAndIDs = fAnnounceUsingCMPCTBLOCK; + } + } + - else if (strCommand == "inv") + else if (strCommand == NetMsgType::INV) { vector<CInv> vInv; vRecv >> vInv; if (vInv.size() > MAX_INV_SZ) { + LOCK(cs_main); Misbehaving(pfrom->GetId(), 20); return error("message inv size() = %u", vInv.size()); } + bool fBlocksOnly = !fRelayTxes; + + // Allow whitelisted peers to send data other than blocks in blocks only mode if whitelistrelay is true + if (pfrom->fWhitelisted && GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY)) + fBlocksOnly = false; + LOCK(cs_main); + uint32_t nFetchFlags = GetFetchFlags(pfrom, chainActive.Tip(), chainparams.GetConsensus()); + std::vector<CInv> vToFetch; for (unsigned int nInv = 0; nInv < vInv.size(); nInv++) { - const CInv &inv = vInv[nInv]; + CInv &inv = vInv[nInv]; boost::this_thread::interruption_point(); - pfrom->AddInventoryKnown(inv); bool fAlreadyHave = AlreadyHave(inv); LogPrint("net", "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom->id); - if (!fAlreadyHave && !fImporting && !fReindex && inv.type != MSG_BLOCK) - pfrom->AskFor(inv); + if (inv.type == MSG_TX) { + inv.type |= nFetchFlags; + } if (inv.type == MSG_BLOCK) { UpdateBlockAvailability(pfrom->GetId(), inv.hash); if (!fAlreadyHave && !fImporting && !fReindex && !mapBlocksInFlight.count(inv.hash)) { - // First request the headers preceeding the announced block. In the normal fully-synced + // First request the headers preceding the announced block. In the normal fully-synced // case where a new block is announced that succeeds the current tip (no reorganization), // there are no such headers. // Secondly, and only when we are close to being synced, we request the announced block directly, @@ -3617,19 +5230,34 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // time the block arrives, the header chain leading up to it is already validated. Not // doing this will result in the received block being rejected as an orphan in case it is // not a direct successor. - pfrom->PushMessage("getheaders", chainActive.GetLocator(pindexBestHeader), inv.hash); - if (chainActive.Tip()->GetBlockTime() > GetAdjustedTime() - Params().TargetSpacing() * 20) { - vToFetch.push_back(inv); + pfrom->PushMessage(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), inv.hash); + CNodeState *nodestate = State(pfrom->GetId()); + if (CanDirectFetch(chainparams.GetConsensus()) && + nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER && + (!IsWitnessEnabled(chainActive.Tip(), chainparams.GetConsensus()) || State(pfrom->GetId())->fHaveWitness)) { + inv.type |= nFetchFlags; + if (nodestate->fProvidesHeaderAndIDs && !(nLocalServices & NODE_WITNESS)) + vToFetch.push_back(CInv(MSG_CMPCT_BLOCK, inv.hash)); + else + vToFetch.push_back(inv); // Mark block as in flight already, even though the actual "getdata" message only goes out // later (within the same cs_main lock, though). - MarkBlockAsInFlight(pfrom->GetId(), inv.hash); + MarkBlockAsInFlight(pfrom->GetId(), inv.hash, chainparams.GetConsensus()); } LogPrint("net", "getheaders (%d) %s to peer=%d\n", pindexBestHeader->nHeight, inv.hash.ToString(), pfrom->id); } } + else + { + pfrom->AddInventoryKnown(inv); + if (fBlocksOnly) + LogPrint("net", "transaction (%s) inv sent in violation of protocol peer=%d\n", inv.hash.ToString(), pfrom->id); + else if (!fAlreadyHave && !fImporting && !fReindex && !IsInitialBlockDownload()) + pfrom->AskFor(inv); + } // Track requests for our stuff - g_signals.Inventory(inv.hash); + GetMainSignals().Inventory(inv.hash); if (pfrom->nSendSize > (SendBufferSize() * 2)) { Misbehaving(pfrom->GetId(), 50); @@ -3638,16 +5266,17 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } if (!vToFetch.empty()) - pfrom->PushMessage("getdata", vToFetch); + pfrom->PushMessage(NetMsgType::GETDATA, vToFetch); } - else if (strCommand == "getdata") + else if (strCommand == NetMsgType::GETDATA) { vector<CInv> vInv; vRecv >> vInv; if (vInv.size() > MAX_INV_SZ) { + LOCK(cs_main); Misbehaving(pfrom->GetId(), 20); return error("message getdata size() = %u", vInv.size()); } @@ -3659,11 +5288,11 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, LogPrint("net", "received getdata for: %s peer=%d\n", vInv[0].ToString(), pfrom->id); pfrom->vRecvGetData.insert(pfrom->vRecvGetData.end(), vInv.begin(), vInv.end()); - ProcessGetData(pfrom); + ProcessGetData(pfrom, chainparams.GetConsensus()); } - else if (strCommand == "getblocks") + else if (strCommand == NetMsgType::GETBLOCKS) { CBlockLocator locator; uint256 hashStop; @@ -3678,7 +5307,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (pindex) pindex = chainActive.Next(pindex); int nLimit = 500; - LogPrint("net", "getblocks %d to %s limit %d from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop==uint256(0) ? "end" : hashStop.ToString(), nLimit, pfrom->id); + LogPrint("net", "getblocks %d to %s limit %d from peer=%d\n", (pindex ? pindex->nHeight : -1), hashStop.IsNull() ? "end" : hashStop.ToString(), nLimit, pfrom->id); for (; pindex; pindex = chainActive.Next(pindex)) { if (pindex->GetBlockHash() == hashStop) @@ -3686,11 +5315,19 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, LogPrint("net", " getblocks stopping at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); break; } + // If pruning, don't inv blocks unless we have on disk and are likely to still have + // for some reasonable time window (1 hour) that block relay might require. + const int nPrunedBlocksLikelyToHave = MIN_BLOCKS_TO_KEEP - 3600 / chainparams.GetConsensus().nPowTargetSpacing; + if (fPruneMode && (!(pindex->nStatus & BLOCK_HAVE_DATA) || pindex->nHeight <= chainActive.Tip()->nHeight - nPrunedBlocksLikelyToHave)) + { + LogPrint("net", " getblocks stopping, pruned or too old block at %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); + break; + } pfrom->PushInventory(CInv(MSG_BLOCK, pindex->GetBlockHash())); if (--nLimit <= 0) { - // When this block is requested, we'll send an inv that'll make them - // getblocks the next batch of inventory. + // When this block is requested, we'll send an inv that'll + // trigger the peer to getblocks the next batch of inventory. LogPrint("net", " getblocks stopping at limit %d %s\n", pindex->nHeight, pindex->GetBlockHash().ToString()); pfrom->hashContinue = pindex->GetBlockHash(); break; @@ -3699,14 +5336,52 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } - else if (strCommand == "getheaders") + else if (strCommand == NetMsgType::GETBLOCKTXN) + { + BlockTransactionsRequest req; + vRecv >> req; + + BlockMap::iterator it = mapBlockIndex.find(req.blockhash); + if (it == mapBlockIndex.end() || !(it->second->nStatus & BLOCK_HAVE_DATA)) { + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us a getblocktxn for a block we don't have", pfrom->id); + return true; + } + + if (it->second->nHeight < chainActive.Height() - 15) { + LogPrint("net", "Peer %d sent us a getblocktxn for a block > 15 deep", pfrom->id); + return true; + } + + CBlock block; + assert(ReadBlockFromDisk(block, it->second, chainparams.GetConsensus())); + + BlockTransactions resp(req); + for (size_t i = 0; i < req.indexes.size(); i++) { + if (req.indexes[i] >= block.vtx.size()) { + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us a getblocktxn with out-of-bounds tx indices", pfrom->id); + return true; + } + resp.txn[i] = block.vtx[req.indexes[i]]; + } + pfrom->PushMessageWithFlag(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCKTXN, resp); + } + + + else if (strCommand == NetMsgType::GETHEADERS) { CBlockLocator locator; uint256 hashStop; vRecv >> locator >> hashStop; LOCK(cs_main); + if (IsInitialBlockDownload() && !pfrom->fWhitelisted) { + LogPrint("net", "Ignoring getheaders from peer=%d because node is in initial block download\n", pfrom->id); + return true; + } + CNodeState *nodestate = State(pfrom->GetId()); CBlockIndex* pindex = NULL; if (locator.IsNull()) { @@ -3734,13 +5409,26 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (--nLimit <= 0 || pindex->GetBlockHash() == hashStop) break; } - pfrom->PushMessage("headers", vHeaders); + // pindex can be NULL either if we sent chainActive.Tip() OR + // if our peer has chainActive.Tip() (and thus we are sending an empty + // headers message). In both cases it's safe to update + // pindexBestHeaderSent to be our tip. + nodestate->pindexBestHeaderSent = pindex ? pindex : chainActive.Tip(); + pfrom->PushMessage(NetMsgType::HEADERS, vHeaders); } - else if (strCommand == "tx") + else if (strCommand == NetMsgType::TX) { - vector<uint256> vWorkQueue; + // Stop processing the transaction early if + // We are in blocks only mode and peer is either not whitelisted or whitelistrelay is off + if (!fRelayTxes && (!pfrom->fWhitelisted || !GetBoolArg("-whitelistrelay", DEFAULT_WHITELISTRELAY))) + { + LogPrint("net", "transaction sent in violation of protocol peer=%d\n", pfrom->id); + return true; + } + + deque<COutPoint> vWorkQueue; vector<uint256> vEraseQueue; CTransaction tx; vRecv >> tx; @@ -3753,62 +5441,72 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, bool fMissingInputs = false; CValidationState state; - mapAlreadyAskedFor.erase(inv); + pfrom->setAskFor.erase(inv.hash); + mapAlreadyAskedFor.erase(inv.hash); - if (AcceptToMemoryPool(mempool, state, tx, true, &fMissingInputs)) - { + if (!AlreadyHave(inv) && AcceptToMemoryPool(mempool, state, tx, true, &fMissingInputs)) { mempool.check(pcoinsTip); RelayTransaction(tx); - vWorkQueue.push_back(inv.hash); - vEraseQueue.push_back(inv.hash); + for (unsigned int i = 0; i < tx.vout.size(); i++) { + vWorkQueue.emplace_back(inv.hash, i); + } + + pfrom->nLastTXTime = GetTime(); - LogPrint("mempool", "AcceptToMemoryPool: peer=%d %s : accepted %s (poolsz %u)\n", - pfrom->id, pfrom->cleanSubVer, + LogPrint("mempool", "AcceptToMemoryPool: peer=%d: accepted %s (poolsz %u txn, %u kB)\n", + pfrom->id, tx.GetHash().ToString(), - mempool.mapTx.size()); + mempool.size(), mempool.DynamicMemoryUsage() / 1000); // Recursively process any orphan transactions that depended on this one set<NodeId> setMisbehaving; - for (unsigned int i = 0; i < vWorkQueue.size(); i++) - { - map<uint256, set<uint256> >::iterator itByPrev = mapOrphanTransactionsByPrev.find(vWorkQueue[i]); + while (!vWorkQueue.empty()) { + auto itByPrev = mapOrphanTransactionsByPrev.find(vWorkQueue.front()); + vWorkQueue.pop_front(); if (itByPrev == mapOrphanTransactionsByPrev.end()) continue; - for (set<uint256>::iterator mi = itByPrev->second.begin(); + for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { - const uint256& orphanHash = *mi; - const CTransaction& orphanTx = mapOrphanTransactions[orphanHash].tx; - NodeId fromPeer = mapOrphanTransactions[orphanHash].fromPeer; + const CTransaction& orphanTx = (*mi)->second.tx; + const uint256& orphanHash = orphanTx.GetHash(); + NodeId fromPeer = (*mi)->second.fromPeer; bool fMissingInputs2 = false; // Use a dummy CValidationState so someone can't setup nodes to counter-DoS based on orphan // resolution (that is, feeding people an invalid transaction based on LegitTxX in order to get // anyone relaying LegitTxX banned) CValidationState stateDummy; - vEraseQueue.push_back(orphanHash); if (setMisbehaving.count(fromPeer)) continue; - if (AcceptToMemoryPool(mempool, stateDummy, orphanTx, true, &fMissingInputs2)) - { + if (AcceptToMemoryPool(mempool, stateDummy, orphanTx, true, &fMissingInputs2)) { LogPrint("mempool", " accepted orphan tx %s\n", orphanHash.ToString()); RelayTransaction(orphanTx); - vWorkQueue.push_back(orphanHash); + for (unsigned int i = 0; i < orphanTx.vout.size(); i++) { + vWorkQueue.emplace_back(orphanHash, i); + } + vEraseQueue.push_back(orphanHash); } else if (!fMissingInputs2) { int nDos = 0; - if (stateDummy.IsInvalid(nDos) && nDos > 0) + if (stateDummy.IsInvalid(nDos) && nDos > 0 && (!state.CorruptionPossible() || State(fromPeer)->fHaveWitness)) { // Punish peer that gave us an invalid orphan tx Misbehaving(fromPeer, nDos); setMisbehaving.insert(fromPeer); LogPrint("mempool", " invalid orphan tx %s\n", orphanHash.ToString()); } - // too-little-fee orphan + // Has inputs but not accepted to mempool + // Probably non-standard or insufficient fee/priority LogPrint("mempool", " removed orphan tx %s\n", orphanHash.ToString()); + vEraseQueue.push_back(orphanHash); + if (!stateDummy.CorruptionPossible()) { + assert(recentRejects); + recentRejects->insert(orphanHash); + } } mempool.check(pcoinsTip); } @@ -3819,40 +5517,248 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } else if (fMissingInputs) { - AddOrphanTx(tx, pfrom->GetId()); - - // DoS prevention: do not allow mapOrphanTransactions to grow unbounded - unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, GetArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS)); - unsigned int nEvicted = LimitOrphanTxSize(nMaxOrphanTx); - if (nEvicted > 0) - LogPrint("mempool", "mapOrphan overflow, removed %u tx\n", nEvicted); - } else if (pfrom->fWhitelisted) { - // Always relay transactions received from whitelisted peers, even - // if they are already in the mempool (allowing the node to function - // as a gateway for nodes hidden behind it). - RelayTransaction(tx); + bool fRejectedParents = false; // It may be the case that the orphans parents have all been rejected + BOOST_FOREACH(const CTxIn& txin, tx.vin) { + if (recentRejects->contains(txin.prevout.hash)) { + fRejectedParents = true; + break; + } + } + if (!fRejectedParents) { + BOOST_FOREACH(const CTxIn& txin, tx.vin) { + CInv inv(MSG_TX, txin.prevout.hash); + pfrom->AddInventoryKnown(inv); + if (!AlreadyHave(inv)) pfrom->AskFor(inv); + } + AddOrphanTx(tx, pfrom->GetId()); + + // DoS prevention: do not allow mapOrphanTransactions to grow unbounded + unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, GetArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS)); + unsigned int nEvicted = LimitOrphanTxSize(nMaxOrphanTx); + if (nEvicted > 0) + LogPrint("mempool", "mapOrphan overflow, removed %u tx\n", nEvicted); + } else { + LogPrint("mempool", "not keeping orphan with rejected parents %s\n",tx.GetHash().ToString()); + } + } else { + if (!state.CorruptionPossible()) { + assert(recentRejects); + recentRejects->insert(tx.GetHash()); + } + + if (pfrom->fWhitelisted && GetBoolArg("-whitelistforcerelay", DEFAULT_WHITELISTFORCERELAY)) { + // Always relay transactions received from whitelisted peers, even + // if they were already in the mempool or rejected from it due + // to policy, allowing the node to function as a gateway for + // nodes hidden behind it. + // + // Never relay transactions that we would assign a non-zero DoS + // score for, as we expect peers to do the same with us in that + // case. + int nDoS = 0; + if (!state.IsInvalid(nDoS) || nDoS == 0) { + LogPrintf("Force relaying tx %s from whitelisted peer=%d\n", tx.GetHash().ToString(), pfrom->id); + RelayTransaction(tx); + } else { + LogPrintf("Not relaying invalid transaction %s from whitelisted peer=%d (%s)\n", tx.GetHash().ToString(), pfrom->id, FormatStateMessage(state)); + } + } } int nDoS = 0; if (state.IsInvalid(nDoS)) { - LogPrint("mempool", "%s from peer=%d %s was not accepted into the memory pool: %s\n", tx.GetHash().ToString(), - pfrom->id, pfrom->cleanSubVer, - state.GetRejectReason()); - pfrom->PushMessage("reject", strCommand, state.GetRejectCode(), - state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash); - if (nDoS > 0) + LogPrint("mempoolrej", "%s from peer=%d was not accepted: %s\n", tx.GetHash().ToString(), + pfrom->id, + FormatStateMessage(state)); + if (state.GetRejectCode() < REJECT_INTERNAL) // Never send AcceptToMemoryPool's internal codes over P2P + pfrom->PushMessage(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), + state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash); + if (nDoS > 0 && (!state.CorruptionPossible() || State(pfrom->id)->fHaveWitness)) { + // When a non-witness-supporting peer gives us a transaction that would + // be accepted if witness validation was off, we can't blame them for it. Misbehaving(pfrom->GetId(), nDoS); + } } + FlushStateToDisk(state, FLUSH_STATE_PERIODIC); } - else if (strCommand == "headers" && !fImporting && !fReindex) // Ignore headers received while importing + else if (strCommand == NetMsgType::CMPCTBLOCK && !fImporting && !fReindex) // Ignore blocks received while importing + { + CBlockHeaderAndShortTxIDs cmpctblock; + vRecv >> cmpctblock; + + LOCK(cs_main); + + if (mapBlockIndex.find(cmpctblock.header.hashPrevBlock) == mapBlockIndex.end()) { + // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers + if (!IsInitialBlockDownload()) + pfrom->PushMessage(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256()); + return true; + } + + CBlockIndex *pindex = NULL; + CValidationState state; + if (!AcceptBlockHeader(cmpctblock.header, state, chainparams, &pindex)) { + int nDoS; + if (state.IsInvalid(nDoS)) { + if (nDoS > 0) + Misbehaving(pfrom->GetId(), nDoS); + LogPrintf("Peer %d sent us invalid header via cmpctblock\n", pfrom->id); + return true; + } + } + + // If AcceptBlockHeader returned true, it set pindex + assert(pindex); + UpdateBlockAvailability(pfrom->GetId(), pindex->GetBlockHash()); + + std::map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator blockInFlightIt = mapBlocksInFlight.find(pindex->GetBlockHash()); + bool fAlreadyInFlight = blockInFlightIt != mapBlocksInFlight.end(); + + if (pindex->nStatus & BLOCK_HAVE_DATA) // Nothing to do here + return true; + + if (pindex->nChainWork <= chainActive.Tip()->nChainWork || // We know something better + pindex->nTx != 0) { // We had this block at some point, but pruned it + if (fAlreadyInFlight) { + // We requested this block for some reason, but our mempool will probably be useless + // so we just grab the block via normal getdata + std::vector<CInv> vInv(1); + vInv[0] = CInv(MSG_BLOCK, cmpctblock.header.GetHash()); + pfrom->PushMessage(NetMsgType::GETDATA, vInv); + return true; + } + } + + // If we're not close to tip yet, give up and let parallel block fetch work its magic + if (!fAlreadyInFlight && !CanDirectFetch(chainparams.GetConsensus())) + return true; + + CNodeState *nodestate = State(pfrom->GetId()); + + // We want to be a bit conservative just to be extra careful about DoS + // possibilities in compact block processing... + if (pindex->nHeight <= chainActive.Height() + 2) { + if ((!fAlreadyInFlight && nodestate->nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) || + (fAlreadyInFlight && blockInFlightIt->second.first == pfrom->GetId())) { + list<QueuedBlock>::iterator *queuedBlockIt = NULL; + if (!MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), chainparams.GetConsensus(), pindex, &queuedBlockIt)) { + if (!(*queuedBlockIt)->partialBlock) + (*queuedBlockIt)->partialBlock.reset(new PartiallyDownloadedBlock(&mempool)); + else { + // The block was already in flight using compact blocks from the same peer + LogPrint("net", "Peer sent us compact block we were already syncing!\n"); + return true; + } + } + + PartiallyDownloadedBlock& partialBlock = *(*queuedBlockIt)->partialBlock; + ReadStatus status = partialBlock.InitData(cmpctblock); + if (status == READ_STATUS_INVALID) { + MarkBlockAsReceived(pindex->GetBlockHash()); // Reset in-flight state in case of whitelist + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us invalid compact block\n", pfrom->id); + return true; + } else if (status == READ_STATUS_FAILED) { + // Duplicate txindexes, the block is now in-flight, so just request it + std::vector<CInv> vInv(1); + vInv[0] = CInv(MSG_BLOCK, cmpctblock.header.GetHash()); + pfrom->PushMessage(NetMsgType::GETDATA, vInv); + return true; + } + + BlockTransactionsRequest req; + for (size_t i = 0; i < cmpctblock.BlockTxCount(); i++) { + if (!partialBlock.IsTxAvailable(i)) + req.indexes.push_back(i); + } + if (req.indexes.empty()) { + // Dirty hack to jump to BLOCKTXN code (TODO: move message handling into their own functions) + BlockTransactions txn; + txn.blockhash = cmpctblock.header.GetHash(); + CDataStream blockTxnMsg(SER_NETWORK, PROTOCOL_VERSION); + blockTxnMsg << txn; + return ProcessMessage(pfrom, NetMsgType::BLOCKTXN, blockTxnMsg, nTimeReceived, chainparams); + } else { + req.blockhash = pindex->GetBlockHash(); + pfrom->PushMessage(NetMsgType::GETBLOCKTXN, req); + } + } + } else { + if (fAlreadyInFlight) { + // We requested this block, but its far into the future, so our + // mempool will probably be useless - request the block normally + std::vector<CInv> vInv(1); + vInv[0] = CInv(MSG_BLOCK, cmpctblock.header.GetHash()); + pfrom->PushMessage(NetMsgType::GETDATA, vInv); + return true; + } else { + // If this was an announce-cmpctblock, we want the same treatment as a header message + // Dirty hack to process as if it were just a headers message (TODO: move message handling into their own functions) + std::vector<CBlock> headers; + headers.push_back(cmpctblock.header); + CDataStream vHeadersMsg(SER_NETWORK, PROTOCOL_VERSION); + vHeadersMsg << headers; + return ProcessMessage(pfrom, NetMsgType::HEADERS, vHeadersMsg, nTimeReceived, chainparams); + } + } + + CheckBlockIndex(chainparams.GetConsensus()); + } + + else if (strCommand == NetMsgType::BLOCKTXN && !fImporting && !fReindex) // Ignore blocks received while importing + { + BlockTransactions resp; + vRecv >> resp; + + LOCK(cs_main); + + map<uint256, pair<NodeId, list<QueuedBlock>::iterator> >::iterator it = mapBlocksInFlight.find(resp.blockhash); + if (it == mapBlocksInFlight.end() || !it->second.second->partialBlock || + it->second.first != pfrom->GetId()) { + LogPrint("net", "Peer %d sent us block transactions for block we weren't expecting\n", pfrom->id); + return true; + } + + PartiallyDownloadedBlock& partialBlock = *it->second.second->partialBlock; + CBlock block; + ReadStatus status = partialBlock.FillBlock(block, resp.txn); + if (status == READ_STATUS_INVALID) { + MarkBlockAsReceived(resp.blockhash); // Reset in-flight state in case of whitelist + Misbehaving(pfrom->GetId(), 100); + LogPrintf("Peer %d sent us invalid compact block/non-matching block transactions\n", pfrom->id); + return true; + } else if (status == READ_STATUS_FAILED) { + // Might have collided, fall back to getdata now :( + std::vector<CInv> invs; + invs.push_back(CInv(MSG_BLOCK, resp.blockhash)); + pfrom->PushMessage(NetMsgType::GETDATA, invs); + } else { + CValidationState state; + ProcessNewBlock(state, chainparams, pfrom, &block, false, NULL); + int nDoS; + if (state.IsInvalid(nDoS)) { + assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes + pfrom->PushMessage(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), + state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), block.GetHash()); + if (nDoS > 0) { + LOCK(cs_main); + Misbehaving(pfrom->GetId(), nDoS); + } + } + } + } + + + else if (strCommand == NetMsgType::HEADERS && !fImporting && !fReindex) // Ignore headers received while importing { std::vector<CBlockHeader> headers; // Bypass the normal CBlock deserialization, as we don't want to risk deserializing 2000 full blocks. unsigned int nCount = ReadCompactSize(vRecv); if (nCount > MAX_HEADERS_RESULTS) { + LOCK(cs_main); Misbehaving(pfrom->GetId(), 20); return error("headers message size = %u", nCount); } @@ -3862,6 +5768,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, ReadCompactSize(vRecv); // ignore tx count; assume it is 0. } + { LOCK(cs_main); if (nCount == 0) { @@ -3869,6 +5776,35 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, return true; } + CNodeState *nodestate = State(pfrom->GetId()); + + // If this looks like it could be a block announcement (nCount < + // MAX_BLOCKS_TO_ANNOUNCE), use special logic for handling headers that + // don't connect: + // - Send a getheaders message in response to try to connect the chain. + // - The peer can send up to MAX_UNCONNECTING_HEADERS in a row that + // don't connect before giving DoS points + // - Once a headers message is received that is valid and does connect, + // nUnconnectingHeaders gets reset back to 0. + if (mapBlockIndex.find(headers[0].hashPrevBlock) == mapBlockIndex.end() && nCount < MAX_BLOCKS_TO_ANNOUNCE) { + nodestate->nUnconnectingHeaders++; + pfrom->PushMessage(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexBestHeader), uint256()); + LogPrint("net", "received header %s: missing prev block %s, sending getheaders (%d) to end (peer=%d, nUnconnectingHeaders=%d)\n", + headers[0].GetHash().ToString(), + headers[0].hashPrevBlock.ToString(), + pindexBestHeader->nHeight, + pfrom->id, nodestate->nUnconnectingHeaders); + // Set hashLastUnknownBlock for this peer, so that if we + // eventually get the headers - even from a different peer - + // we can use this peer to download. + UpdateBlockAvailability(pfrom->GetId(), headers.back().GetHash()); + + if (nodestate->nUnconnectingHeaders % MAX_UNCONNECTING_HEADERS == 0) { + Misbehaving(pfrom->GetId(), 20); + } + return true; + } + CBlockIndex *pindexLast = NULL; BOOST_FOREACH(const CBlockHeader& header, headers) { CValidationState state; @@ -3876,7 +5812,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, Misbehaving(pfrom->GetId(), 20); return error("non-continuous headers sequence"); } - if (!AcceptBlockHeader(header, state, &pindexLast)) { + if (!AcceptBlockHeader(header, state, chainparams, &pindexLast)) { int nDoS; if (state.IsInvalid(nDoS)) { if (nDoS > 0) @@ -3886,34 +5822,102 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } } - if (pindexLast) - UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); + if (nodestate->nUnconnectingHeaders > 0) { + LogPrint("net", "peer=%d: resetting nUnconnectingHeaders (%d -> 0)\n", pfrom->id, nodestate->nUnconnectingHeaders); + } + nodestate->nUnconnectingHeaders = 0; + + assert(pindexLast); + UpdateBlockAvailability(pfrom->GetId(), pindexLast->GetBlockHash()); - if (nCount == MAX_HEADERS_RESULTS && pindexLast) { + if (nCount == MAX_HEADERS_RESULTS) { // Headers message had its maximum size; the peer may have more headers. // TODO: optimize: if pindexLast is an ancestor of chainActive.Tip or pindexBestHeader, continue // from there instead. LogPrint("net", "more getheaders (%d) to end to peer=%d (startheight:%d)\n", pindexLast->nHeight, pfrom->id, pfrom->nStartingHeight); - pfrom->PushMessage("getheaders", chainActive.GetLocator(pindexLast), uint256(0)); + pfrom->PushMessage(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexLast), uint256()); + } + + bool fCanDirectFetch = CanDirectFetch(chainparams.GetConsensus()); + // If this set of headers is valid and ends in a block with at least as + // much work as our tip, download as much as possible. + if (fCanDirectFetch && pindexLast->IsValid(BLOCK_VALID_TREE) && chainActive.Tip()->nChainWork <= pindexLast->nChainWork) { + vector<CBlockIndex *> vToFetch; + CBlockIndex *pindexWalk = pindexLast; + // Calculate all the blocks we'd need to switch to pindexLast, up to a limit. + while (pindexWalk && !chainActive.Contains(pindexWalk) && vToFetch.size() <= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (!(pindexWalk->nStatus & BLOCK_HAVE_DATA) && + !mapBlocksInFlight.count(pindexWalk->GetBlockHash()) && + (!IsWitnessEnabled(pindexWalk->pprev, chainparams.GetConsensus()) || State(pfrom->GetId())->fHaveWitness)) { + // We don't have this block, and it's not yet in flight. + vToFetch.push_back(pindexWalk); + } + pindexWalk = pindexWalk->pprev; + } + // If pindexWalk still isn't on our main chain, we're looking at a + // very large reorg at a time we think we're close to caught up to + // the main chain -- this shouldn't really happen. Bail out on the + // direct fetch and rely on parallel download instead. + if (!chainActive.Contains(pindexWalk)) { + LogPrint("net", "Large reorg, won't direct fetch to %s (%d)\n", + pindexLast->GetBlockHash().ToString(), + pindexLast->nHeight); + } else { + vector<CInv> vGetData; + // Download as much as possible, from earliest to latest. + BOOST_REVERSE_FOREACH(CBlockIndex *pindex, vToFetch) { + if (nodestate->nBlocksInFlight >= MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + // Can't download any more from this peer + break; + } + uint32_t nFetchFlags = GetFetchFlags(pfrom, pindex->pprev, chainparams.GetConsensus()); + vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); + MarkBlockAsInFlight(pfrom->GetId(), pindex->GetBlockHash(), chainparams.GetConsensus(), pindex); + LogPrint("net", "Requesting block %s from peer=%d\n", + pindex->GetBlockHash().ToString(), pfrom->id); + } + if (vGetData.size() > 1) { + LogPrint("net", "Downloading blocks toward %s (%d) via headers direct fetch\n", + pindexLast->GetBlockHash().ToString(), pindexLast->nHeight); + } + if (vGetData.size() > 0) { + if (nodestate->fProvidesHeaderAndIDs && vGetData.size() == 1 && mapBlocksInFlight.size() == 1 && pindexLast->pprev->IsValid(BLOCK_VALID_CHAIN) && !(nLocalServices & NODE_WITNESS)) { + // We seem to be rather well-synced, so it appears pfrom was the first to provide us + // with this block! Let's get them to announce using compact blocks in the future. + MaybeSetPeerAsAnnouncingHeaderAndIDs(nodestate, pfrom); + // In any case, we want to download using a compact block, not a regular one + vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); + } + pfrom->PushMessage(NetMsgType::GETDATA, vGetData); + } + } } + + CheckBlockIndex(chainparams.GetConsensus()); + } + + NotifyHeaderTip(); } - else if (strCommand == "block" && !fImporting && !fReindex) // Ignore blocks received while importing + else if (strCommand == NetMsgType::BLOCK && !fImporting && !fReindex) // Ignore blocks received while importing { CBlock block; vRecv >> block; - CInv inv(MSG_BLOCK, block.GetHash()); - LogPrint("net", "received block %s peer=%d\n", inv.hash.ToString(), pfrom->id); - - pfrom->AddInventoryKnown(inv); + LogPrint("net", "received block %s peer=%d\n", block.GetHash().ToString(), pfrom->id); CValidationState state; - ProcessNewBlock(state, pfrom, &block); + // Process all blocks from whitelisted peers, even if not requested, + // unless we're still syncing with the network. + // Such an unrequested block may still be processed, subject to the + // conditions in AcceptBlock(). + bool forceProcessing = pfrom->fWhitelisted && !IsInitialBlockDownload(); + ProcessNewBlock(state, chainparams, pfrom, &block, forceProcessing, NULL); int nDoS; if (state.IsInvalid(nDoS)) { - pfrom->PushMessage("reject", strCommand, state.GetRejectCode(), - state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), inv.hash); + assert (state.GetRejectCode() < REJECT_INTERNAL); // Blocks are never rejected with internal reject codes + pfrom->PushMessage(NetMsgType::REJECT, strCommand, (unsigned char)state.GetRejectCode(), + state.GetRejectReason().substr(0, MAX_REJECT_MESSAGE_LENGTH), block.GetHash()); if (nDoS > 0) { LOCK(cs_main); Misbehaving(pfrom->GetId(), nDoS); @@ -3923,8 +5927,26 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } - else if (strCommand == "getaddr") + else if (strCommand == NetMsgType::GETADDR) { + // This asymmetric behavior for inbound and outbound connections was introduced + // to prevent a fingerprinting attack: an attacker can send specific fake addresses + // to users' AddrMan and later request them by sending getaddr messages. + // Making nodes which are behind NAT and can only make outgoing connections ignore + // the getaddr message mitigates the attack. + if (!pfrom->fInbound) { + LogPrint("net", "Ignoring \"getaddr\" from outbound connection. peer=%d\n", pfrom->id); + return true; + } + + // Only send one GetAddr response per connection to reduce resource waste + // and discourage addr stamping of INV announcements. + if (pfrom->fSentAddr) { + LogPrint("net", "Ignoring repeated \"getaddr\". peer=%d\n", pfrom->id); + return true; + } + pfrom->fSentAddr = true; + pfrom->vAddrToSend.clear(); vector<CAddress> vAddr = addrman.GetAddr(); BOOST_FOREACH(const CAddress &addr, vAddr) @@ -3932,32 +5954,28 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } - else if (strCommand == "mempool") + else if (strCommand == NetMsgType::MEMPOOL) { - LOCK2(cs_main, pfrom->cs_filter); + if (!(nLocalServices & NODE_BLOOM) && !pfrom->fWhitelisted) + { + LogPrint("net", "mempool request with bloom filters disabled, disconnect peer=%d\n", pfrom->GetId()); + pfrom->fDisconnect = true; + return true; + } - std::vector<uint256> vtxid; - mempool.queryHashes(vtxid); - vector<CInv> vInv; - BOOST_FOREACH(uint256& hash, vtxid) { - CInv inv(MSG_TX, hash); - CTransaction tx; - bool fInMemPool = mempool.lookup(hash, tx); - if (!fInMemPool) continue; // another thread removed since queryHashes, maybe... - if ((pfrom->pfilter && pfrom->pfilter->IsRelevantAndUpdate(tx)) || - (!pfrom->pfilter)) - vInv.push_back(inv); - if (vInv.size() == MAX_INV_SZ) { - pfrom->PushMessage("inv", vInv); - vInv.clear(); - } + if (CNode::OutboundTargetReached(false) && !pfrom->fWhitelisted) + { + LogPrint("net", "mempool request with bandwidth limit reached, disconnect peer=%d\n", pfrom->GetId()); + pfrom->fDisconnect = true; + return true; } - if (vInv.size() > 0) - pfrom->PushMessage("inv", vInv); + + LOCK(pfrom->cs_inventory); + pfrom->fSendMempool = true; } - else if (strCommand == "ping") + else if (strCommand == NetMsgType::PING) { if (pfrom->nVersion > BIP0031_VERSION) { @@ -3974,12 +5992,12 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // it, if the remote node sends a ping once per second and this node takes 5 // seconds to respond to each, the 5th ping the remote sends would appear to // return very quickly. - pfrom->PushMessage("pong", nonce); + pfrom->PushMessage(NetMsgType::PONG, nonce); } } - else if (strCommand == "pong") + else if (strCommand == NetMsgType::PONG) { int64_t pingUsecEnd = nTimeReceived; uint64_t nonce = 0; @@ -3999,6 +6017,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, if (pingUsecTime > 0) { // Successful ping time measurement, replace previous pfrom->nPingUsecTime = pingUsecTime; + pfrom->nMinPingUsecTime = std::min(pfrom->nMinPingUsecTime, pingUsecTime); } else { // This should never happen sProblem = "Timing mishap"; @@ -4007,7 +6026,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // Nonce mismatches are normal when pings are overlapping sProblem = "Nonce mismatch"; if (nonce == 0) { - // This is most likely a bug in another implementation somewhere, cancel this ping + // This is most likely a bug in another implementation somewhere; cancel this ping bPingFinished = true; sProblem = "Nonce zero"; } @@ -4016,15 +6035,14 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, sProblem = "Unsolicited pong without ping"; } } else { - // This is most likely a bug in another implementation somewhere, cancel this ping + // This is most likely a bug in another implementation somewhere; cancel this ping bPingFinished = true; sProblem = "Short payload"; } if (!(sProblem.empty())) { - LogPrint("net", "pong peer=%d %s: %s, %x expected, %x received, %u bytes\n", + LogPrint("net", "pong peer=%d: %s, %x expected, %x received, %u bytes\n", pfrom->id, - pfrom->cleanSubVer, sProblem, pfrom->nPingNonceSent, nonce, @@ -4036,48 +6054,21 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } - else if (strCommand == "alert") - { - CAlert alert; - vRecv >> alert; - - uint256 alertHash = alert.GetHash(); - if (pfrom->setKnown.count(alertHash) == 0) - { - if (alert.ProcessAlert()) - { - // Relay - pfrom->setKnown.insert(alertHash); - { - LOCK(cs_vNodes); - BOOST_FOREACH(CNode* pnode, vNodes) - alert.RelayTo(pnode); - } - } - else { - // Small DoS penalty so peers that send us lots of - // duplicate/expired/invalid-signature/whatever alerts - // eventually get banned. - // This isn't a Misbehaving(100) (immediate ban) because the - // peer might be an older or different implementation with - // a different signature key, etc. - Misbehaving(pfrom->GetId(), 10); - } - } - } - - - else if (strCommand == "filterload") + else if (strCommand == NetMsgType::FILTERLOAD) { CBloomFilter filter; vRecv >> filter; + LOCK(pfrom->cs_filter); + if (!filter.IsWithinSizeConstraints()) + { // There is no excuse for sending a too-large filter + LOCK(cs_main); Misbehaving(pfrom->GetId(), 100); + } else { - LOCK(pfrom->cs_filter); delete pfrom->pfilter; pfrom->pfilter = new CBloomFilter(filter); pfrom->pfilter->UpdateEmptyFull(); @@ -4086,7 +6077,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } - else if (strCommand == "filteradd") + else if (strCommand == NetMsgType::FILTERADD) { vector<unsigned char> vData; vRecv >> vData; @@ -4095,18 +6086,22 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // and thus, the maximum size any matched object can have) in a filteradd message if (vData.size() > MAX_SCRIPT_ELEMENT_SIZE) { + LOCK(cs_main); Misbehaving(pfrom->GetId(), 100); } else { LOCK(pfrom->cs_filter); if (pfrom->pfilter) pfrom->pfilter->insert(vData); else + { + LOCK(cs_main); Misbehaving(pfrom->GetId(), 100); + } } } - else if (strCommand == "filterclear") + else if (strCommand == NetMsgType::FILTERCLEAR) { LOCK(pfrom->cs_filter); delete pfrom->pfilter; @@ -4115,7 +6110,7 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, } - else if (strCommand == "reject") + else if (strCommand == NetMsgType::REJECT) { if (fDebug) { try { @@ -4125,32 +6120,38 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, ostringstream ss; ss << strMsg << " code " << itostr(ccode) << ": " << strReason; - if (strMsg == "block" || strMsg == "tx") + if (strMsg == NetMsgType::BLOCK || strMsg == NetMsgType::TX) { uint256 hash; vRecv >> hash; ss << ": hash " << hash.ToString(); } LogPrint("net", "Reject %s\n", SanitizeString(ss.str())); - } catch (std::ios_base::failure& e) { + } catch (const std::ios_base::failure&) { // Avoid feedback loops by preventing reject messages from triggering a new reject message. LogPrint("net", "Unparseable reject message received\n"); } } } - else - { + else if (strCommand == NetMsgType::FEEFILTER) { + CAmount newFeeFilter = 0; + vRecv >> newFeeFilter; + if (MoneyRange(newFeeFilter)) { + { + LOCK(pfrom->cs_feeFilter); + pfrom->minFeeFilter = newFeeFilter; + } + LogPrint("net", "received: feefilter of %s from peer=%d\n", CFeeRate(newFeeFilter).ToString(), pfrom->id); + } + } + + else { // Ignore unknown commands for extensibility LogPrint("net", "Unknown command \"%s\" from peer=%d\n", SanitizeString(strCommand), pfrom->id); } - // Update the last seen time for this node's address - if (pfrom->fNetworkNode) - if (strCommand == "version" || strCommand == "addr" || strCommand == "inv" || strCommand == "getdata" || strCommand == "ping") - AddressCurrentlyConnected(pfrom->addr); - return true; } @@ -4158,8 +6159,9 @@ bool static ProcessMessage(CNode* pfrom, string strCommand, CDataStream& vRecv, // requires LOCK(cs_vRecvMsg) bool ProcessMessages(CNode* pfrom) { + const CChainParams& chainparams = Params(); //if (fDebug) - // LogPrintf("ProcessMessages(%u messages)\n", pfrom->vRecvMsg.size()); + // LogPrintf("%s(%u messages)\n", __func__, pfrom->vRecvMsg.size()); // // Message format @@ -4172,7 +6174,7 @@ bool ProcessMessages(CNode* pfrom) bool fOk = true; if (!pfrom->vRecvGetData.empty()) - ProcessGetData(pfrom); + ProcessGetData(pfrom, chainparams.GetConsensus()); // this maintains the order of responses if (!pfrom->vRecvGetData.empty()) return fOk; @@ -4187,7 +6189,7 @@ bool ProcessMessages(CNode* pfrom) CNetMessage& msg = *it; //if (fDebug) - // LogPrintf("ProcessMessages(message %u msgsz, %u bytes, complete:%s)\n", + // LogPrintf("%s(message %u msgsz, %u bytes, complete:%s)\n", __func__, // msg.hdr.nMessageSize, msg.vRecv.size(), // msg.complete() ? "Y" : "N"); @@ -4199,17 +6201,17 @@ bool ProcessMessages(CNode* pfrom) it++; // Scan for message start - if (memcmp(msg.hdr.pchMessageStart, Params().MessageStart(), MESSAGE_START_SIZE) != 0) { - LogPrintf("PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", msg.hdr.GetCommand(), pfrom->id); + if (memcmp(msg.hdr.pchMessageStart, chainparams.MessageStart(), MESSAGE_START_SIZE) != 0) { + LogPrintf("PROCESSMESSAGE: INVALID MESSAGESTART %s peer=%d\n", SanitizeString(msg.hdr.GetCommand()), pfrom->id); fOk = false; break; } // Read header CMessageHeader& hdr = msg.hdr; - if (!hdr.IsValid()) + if (!hdr.IsValid(chainparams.MessageStart())) { - LogPrintf("PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", hdr.GetCommand(), pfrom->id); + LogPrintf("PROCESSMESSAGE: ERRORS IN HEADER %s peer=%d\n", SanitizeString(hdr.GetCommand()), pfrom->id); continue; } string strCommand = hdr.GetCommand(); @@ -4220,12 +6222,11 @@ bool ProcessMessages(CNode* pfrom) // Checksum CDataStream& vRecv = msg.vRecv; uint256 hash = Hash(vRecv.begin(), vRecv.begin() + nMessageSize); - unsigned int nChecksum = 0; - memcpy(&nChecksum, &hash, sizeof(nChecksum)); + unsigned int nChecksum = ReadLE32((unsigned char*)&hash); if (nChecksum != hdr.nChecksum) { - LogPrintf("ProcessMessages(%s, %u bytes) : CHECKSUM ERROR nChecksum=%08x hdr.nChecksum=%08x\n", - strCommand, nMessageSize, nChecksum, hdr.nChecksum); + LogPrintf("%s(%s, %u bytes): CHECKSUM ERROR nChecksum=%08x hdr.nChecksum=%08x\n", __func__, + SanitizeString(strCommand), nMessageSize, nChecksum, hdr.nChecksum); continue; } @@ -4233,38 +6234,43 @@ bool ProcessMessages(CNode* pfrom) bool fRet = false; try { - fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.nTime); + fRet = ProcessMessage(pfrom, strCommand, vRecv, msg.nTime, chainparams); boost::this_thread::interruption_point(); } - catch (std::ios_base::failure& e) + catch (const std::ios_base::failure& e) { - pfrom->PushMessage("reject", strCommand, REJECT_MALFORMED, string("error parsing message")); + pfrom->PushMessage(NetMsgType::REJECT, strCommand, REJECT_MALFORMED, string("error parsing message")); if (strstr(e.what(), "end of data")) { // Allow exceptions from under-length message on vRecv - LogPrintf("ProcessMessages(%s, %u bytes) : Exception '%s' caught, normally caused by a message being shorter than its stated length\n", strCommand, nMessageSize, e.what()); + LogPrintf("%s(%s, %u bytes): Exception '%s' caught, normally caused by a message being shorter than its stated length\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } else if (strstr(e.what(), "size too large")) { // Allow exceptions from over-long size - LogPrintf("ProcessMessages(%s, %u bytes) : Exception '%s' caught\n", strCommand, nMessageSize, e.what()); + LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); + } + else if (strstr(e.what(), "non-canonical ReadCompactSize()")) + { + // Allow exceptions from non-canonical encoding + LogPrintf("%s(%s, %u bytes): Exception '%s' caught\n", __func__, SanitizeString(strCommand), nMessageSize, e.what()); } else { PrintExceptionContinue(&e, "ProcessMessages()"); } } - catch (boost::thread_interrupted) { + catch (const boost::thread_interrupted&) { throw; } - catch (std::exception& e) { + catch (const std::exception& e) { PrintExceptionContinue(&e, "ProcessMessages()"); } catch (...) { PrintExceptionContinue(NULL, "ProcessMessages()"); } if (!fRet) - LogPrintf("ProcessMessage(%s, %u bytes) FAILED peer=%d\n", strCommand, nMessageSize, pfrom->id); + LogPrintf("%s(%s, %u bytes) FAILED peer=%d\n", __func__, SanitizeString(strCommand), nMessageSize, pfrom->id); break; } @@ -4276,11 +6282,28 @@ bool ProcessMessages(CNode* pfrom) return fOk; } +class CompareInvMempoolOrder +{ + CTxMemPool *mp; +public: + CompareInvMempoolOrder(CTxMemPool *mempool) + { + mp = mempool; + } + + bool operator()(std::set<uint256>::iterator a, std::set<uint256>::iterator b) + { + /* As std::make_heap produces a max-heap, we want the entries with the + * fewest ancestors/highest fee to sort later. */ + return mp->CompareDepthAndScore(*b, *a); + } +}; -bool SendMessages(CNode* pto, bool fSendTrickle) +bool SendMessages(CNode* pto) { + const Consensus::Params& consensusParams = Params().GetConsensus(); { - // Don't send anything until we get their version message + // Don't send anything until we get its version message if (pto->nVersion == 0) return true; @@ -4305,11 +6328,11 @@ bool SendMessages(CNode* pto, bool fSendTrickle) pto->nPingUsecStart = GetTimeMicros(); if (pto->nVersion > BIP0031_VERSION) { pto->nPingNonceSent = nonce; - pto->PushMessage("ping", nonce); + pto->PushMessage(NetMsgType::PING, nonce); } else { // Peer is too old to support ping command with nonce, pong will never arrive. pto->nPingNonceSent = 0; - pto->PushMessage("ping"); + pto->PushMessage(NetMsgType::PING); } } @@ -4318,47 +6341,39 @@ bool SendMessages(CNode* pto, bool fSendTrickle) return true; // Address refresh broadcast - static int64_t nLastRebroadcast; - if (!IsInitialBlockDownload() && (GetTime() - nLastRebroadcast > 24 * 60 * 60)) - { - LOCK(cs_vNodes); - BOOST_FOREACH(CNode* pnode, vNodes) - { - // Periodically clear setAddrKnown to allow refresh broadcasts - if (nLastRebroadcast) - pnode->setAddrKnown.clear(); - - // Rebroadcast our address - AdvertizeLocal(pnode); - } - if (!vNodes.empty()) - nLastRebroadcast = GetTime(); + int64_t nNow = GetTimeMicros(); + if (!IsInitialBlockDownload() && pto->nNextLocalAddrSend < nNow) { + AdvertiseLocal(pto); + pto->nNextLocalAddrSend = PoissonNextSend(nNow, AVG_LOCAL_ADDRESS_BROADCAST_INTERVAL); } // // Message: addr // - if (fSendTrickle) - { + if (pto->nNextAddrSend < nNow) { + pto->nNextAddrSend = PoissonNextSend(nNow, AVG_ADDRESS_BROADCAST_INTERVAL); vector<CAddress> vAddr; vAddr.reserve(pto->vAddrToSend.size()); BOOST_FOREACH(const CAddress& addr, pto->vAddrToSend) { - // returns true if wasn't already contained in the set - if (pto->setAddrKnown.insert(addr).second) + if (!pto->addrKnown.contains(addr.GetKey())) { + pto->addrKnown.insert(addr.GetKey()); vAddr.push_back(addr); // receiver rejects addr messages larger than 1000 if (vAddr.size() >= 1000) { - pto->PushMessage("addr", vAddr); + pto->PushMessage(NetMsgType::ADDR, vAddr); vAddr.clear(); } } } pto->vAddrToSend.clear(); if (!vAddr.empty()) - pto->PushMessage("addr", vAddr); + pto->PushMessage(NetMsgType::ADDR, vAddr); + // we only send the big addr message once + if (pto->vAddrToSend.capacity() > 40) + pto->vAddrToSend.shrink_to_fit(); } CNodeState &state = *State(pto->GetId()); @@ -4371,28 +6386,37 @@ bool SendMessages(CNode* pto, bool fSendTrickle) LogPrintf("Warning: not banning local peer %s!\n", pto->addr.ToString()); else { - CNode::Ban(pto->addr); + CNode::Ban(pto->addr, BanReasonNodeMisbehaving); } } state.fShouldBan = false; } BOOST_FOREACH(const CBlockReject& reject, state.rejects) - pto->PushMessage("reject", (string)"block", reject.chRejectCode, reject.strRejectReason, reject.hashBlock); + pto->PushMessage(NetMsgType::REJECT, (string)NetMsgType::BLOCK, reject.chRejectCode, reject.strRejectReason, reject.hashBlock); state.rejects.clear(); // Start block sync if (pindexBestHeader == NULL) pindexBestHeader = chainActive.Tip(); bool fFetch = state.fPreferredDownload || (nPreferredDownload == 0 && !pto->fClient && !pto->fOneShot); // Download if this is a nice peer, or we have no nice peers and this one might do. - if (!state.fSyncStarted && !pto->fClient && fFetch && !fImporting && !fReindex) { + if (!state.fSyncStarted && !pto->fClient && !fImporting && !fReindex) { // Only actively request headers from a single peer, unless we're close to today. - if (nSyncStarted == 0 || pindexBestHeader->GetBlockTime() > GetAdjustedTime() - 24 * 60 * 60) { + if ((nSyncStarted == 0 && fFetch) || pindexBestHeader->GetBlockTime() > GetAdjustedTime() - 24 * 60 * 60) { state.fSyncStarted = true; nSyncStarted++; - CBlockIndex *pindexStart = pindexBestHeader->pprev ? pindexBestHeader->pprev : pindexBestHeader; + const CBlockIndex *pindexStart = pindexBestHeader; + /* If possible, start at the block preceding the currently + best known header. This ensures that we always get a + non-empty list of headers back as long as the peer + is up-to-date. With a non-empty response, we can initialise + the peer's known best block. This wouldn't be possible + if we requested starting at pindexBestHeader and + got back an empty response. */ + if (pindexStart->pprev) + pindexStart = pindexStart->pprev; LogPrint("net", "initial getheaders (%d) to peer=%d (startheight:%d)\n", pindexStart->nHeight, pto->id, pto->nStartingHeight); - pto->PushMessage("getheaders", chainActive.GetLocator(pindexStart), uint256(0)); + pto->PushMessage(NetMsgType::GETHEADERS, chainActive.GetLocator(pindexStart), uint256()); } } @@ -4401,59 +6425,268 @@ bool SendMessages(CNode* pto, bool fSendTrickle) // transactions become unconfirmed and spams other nodes. if (!fReindex && !fImporting && !IsInitialBlockDownload()) { - g_signals.Broadcast(); + GetMainSignals().Broadcast(nTimeBestReceived); + } + + // + // Try sending block announcements via headers + // + { + // If we have less than MAX_BLOCKS_TO_ANNOUNCE in our + // list of block hashes we're relaying, and our peer wants + // headers announcements, then find the first header + // not yet known to our peer but would connect, and send. + // If no header would connect, or if we have too many + // blocks, or if the peer doesn't want headers, just + // add all to the inv queue. + LOCK(pto->cs_inventory); + vector<CBlock> vHeaders; + bool fRevertToInv = ((!state.fPreferHeaders && + (!state.fPreferHeaderAndIDs || pto->vBlockHashesToAnnounce.size() > 1)) || + pto->vBlockHashesToAnnounce.size() > MAX_BLOCKS_TO_ANNOUNCE); + CBlockIndex *pBestIndex = NULL; // last header queued for delivery + ProcessBlockAvailability(pto->id); // ensure pindexBestKnownBlock is up-to-date + + if (!fRevertToInv) { + bool fFoundStartingHeader = false; + // Try to find first header that our peer doesn't have, and + // then send all headers past that one. If we come across any + // headers that aren't on chainActive, give up. + BOOST_FOREACH(const uint256 &hash, pto->vBlockHashesToAnnounce) { + BlockMap::iterator mi = mapBlockIndex.find(hash); + assert(mi != mapBlockIndex.end()); + CBlockIndex *pindex = mi->second; + if (chainActive[pindex->nHeight] != pindex) { + // Bail out if we reorged away from this block + fRevertToInv = true; + break; + } + if (pBestIndex != NULL && pindex->pprev != pBestIndex) { + // This means that the list of blocks to announce don't + // connect to each other. + // This shouldn't really be possible to hit during + // regular operation (because reorgs should take us to + // a chain that has some block not on the prior chain, + // which should be caught by the prior check), but one + // way this could happen is by using invalidateblock / + // reconsiderblock repeatedly on the tip, causing it to + // be added multiple times to vBlockHashesToAnnounce. + // Robustly deal with this rare situation by reverting + // to an inv. + fRevertToInv = true; + break; + } + pBestIndex = pindex; + if (fFoundStartingHeader) { + // add this to the headers message + vHeaders.push_back(pindex->GetBlockHeader()); + } else if (PeerHasHeader(&state, pindex)) { + continue; // keep looking for the first new block + } else if (pindex->pprev == NULL || PeerHasHeader(&state, pindex->pprev)) { + // Peer doesn't have this header but they do have the prior one. + // Start sending headers. + fFoundStartingHeader = true; + vHeaders.push_back(pindex->GetBlockHeader()); + } else { + // Peer doesn't have this header or the prior one -- nothing will + // connect, so bail out. + fRevertToInv = true; + break; + } + } + } + if (!fRevertToInv && !vHeaders.empty()) { + if (vHeaders.size() == 1 && state.fPreferHeaderAndIDs) { + // We only send up to 1 block as header-and-ids, as otherwise + // probably means we're doing an initial-ish-sync or they're slow + LogPrint("net", "%s sending header-and-ids %s to peer %d\n", __func__, + vHeaders.front().GetHash().ToString(), pto->id); + //TODO: Shouldn't need to reload block from disk, but requires refactor + CBlock block; + assert(ReadBlockFromDisk(block, pBestIndex, consensusParams)); + CBlockHeaderAndShortTxIDs cmpctblock(block); + pto->PushMessageWithFlag(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::CMPCTBLOCK, cmpctblock); + state.pindexBestHeaderSent = pBestIndex; + } else if (state.fPreferHeaders) { + if (vHeaders.size() > 1) { + LogPrint("net", "%s: %u headers, range (%s, %s), to peer=%d\n", __func__, + vHeaders.size(), + vHeaders.front().GetHash().ToString(), + vHeaders.back().GetHash().ToString(), pto->id); + } else { + LogPrint("net", "%s: sending header %s to peer=%d\n", __func__, + vHeaders.front().GetHash().ToString(), pto->id); + } + pto->PushMessage(NetMsgType::HEADERS, vHeaders); + state.pindexBestHeaderSent = pBestIndex; + } else + fRevertToInv = true; + } + if (fRevertToInv) { + // If falling back to using an inv, just try to inv the tip. + // The last entry in vBlockHashesToAnnounce was our tip at some point + // in the past. + if (!pto->vBlockHashesToAnnounce.empty()) { + const uint256 &hashToAnnounce = pto->vBlockHashesToAnnounce.back(); + BlockMap::iterator mi = mapBlockIndex.find(hashToAnnounce); + assert(mi != mapBlockIndex.end()); + CBlockIndex *pindex = mi->second; + + // Warn if we're announcing a block that is not on the main chain. + // This should be very rare and could be optimized out. + // Just log for now. + if (chainActive[pindex->nHeight] != pindex) { + LogPrint("net", "Announcing block %s not on main chain (tip=%s)\n", + hashToAnnounce.ToString(), chainActive.Tip()->GetBlockHash().ToString()); + } + + // If the peer's chain has this block, don't inv it back. + if (!PeerHasHeader(&state, pindex)) { + pto->PushInventory(CInv(MSG_BLOCK, hashToAnnounce)); + LogPrint("net", "%s: sending inv peer=%d hash=%s\n", __func__, + pto->id, hashToAnnounce.ToString()); + } + } + } + pto->vBlockHashesToAnnounce.clear(); } // // Message: inventory // vector<CInv> vInv; - vector<CInv> vInvWait; { LOCK(pto->cs_inventory); - vInv.reserve(pto->vInventoryToSend.size()); - vInvWait.reserve(pto->vInventoryToSend.size()); - BOOST_FOREACH(const CInv& inv, pto->vInventoryToSend) - { - if (pto->setInventoryKnown.count(inv)) - continue; + vInv.reserve(std::max<size_t>(pto->vInventoryBlockToSend.size(), INVENTORY_BROADCAST_MAX)); + + // Add blocks + BOOST_FOREACH(const uint256& hash, pto->vInventoryBlockToSend) { + vInv.push_back(CInv(MSG_BLOCK, hash)); + if (vInv.size() == MAX_INV_SZ) { + pto->PushMessage(NetMsgType::INV, vInv); + vInv.clear(); + } + } + pto->vInventoryBlockToSend.clear(); + + // Check whether periodic sends should happen + bool fSendTrickle = pto->fWhitelisted; + if (pto->nNextInvSend < nNow) { + fSendTrickle = true; + // Use half the delay for outbound peers, as there is less privacy concern for them. + pto->nNextInvSend = PoissonNextSend(nNow, INVENTORY_BROADCAST_INTERVAL >> !pto->fInbound); + } + + // Time to send but the peer has requested we not relay transactions. + if (fSendTrickle) { + LOCK(pto->cs_filter); + if (!pto->fRelayTxes) pto->setInventoryTxToSend.clear(); + } - // trickle out tx inv to protect privacy - if (inv.type == MSG_TX && !fSendTrickle) + // Respond to BIP35 mempool requests + if (fSendTrickle && pto->fSendMempool) { + auto vtxinfo = mempool.infoAll(); + pto->fSendMempool = false; + CAmount filterrate = 0; { - // 1/4 of tx invs blast to all immediately - static uint256 hashSalt; - if (hashSalt == 0) - hashSalt = GetRandHash(); - uint256 hashRand = inv.hash ^ hashSalt; - hashRand = Hash(BEGIN(hashRand), END(hashRand)); - bool fTrickleWait = ((hashRand & 3) != 0); - - if (fTrickleWait) - { - vInvWait.push_back(inv); - continue; + LOCK(pto->cs_feeFilter); + filterrate = pto->minFeeFilter; + } + + LOCK(pto->cs_filter); + + for (const auto& txinfo : vtxinfo) { + const uint256& hash = txinfo.tx->GetHash(); + CInv inv(MSG_TX, hash); + pto->setInventoryTxToSend.erase(hash); + if (filterrate) { + if (txinfo.feeRate.GetFeePerK() < filterrate) + continue; + } + if (pto->pfilter) { + if (!pto->pfilter->IsRelevantAndUpdate(*txinfo.tx)) continue; + } + pto->filterInventoryKnown.insert(hash); + vInv.push_back(inv); + if (vInv.size() == MAX_INV_SZ) { + pto->PushMessage(NetMsgType::INV, vInv); + vInv.clear(); } } + pto->timeLastMempoolReq = GetTime(); + } - // returns true if wasn't already contained in the set - if (pto->setInventoryKnown.insert(inv).second) + // Determine transactions to relay + if (fSendTrickle) { + // Produce a vector with all candidates for sending + vector<std::set<uint256>::iterator> vInvTx; + vInvTx.reserve(pto->setInventoryTxToSend.size()); + for (std::set<uint256>::iterator it = pto->setInventoryTxToSend.begin(); it != pto->setInventoryTxToSend.end(); it++) { + vInvTx.push_back(it); + } + CAmount filterrate = 0; { - vInv.push_back(inv); - if (vInv.size() >= 1000) + LOCK(pto->cs_feeFilter); + filterrate = pto->minFeeFilter; + } + // Topologically and fee-rate sort the inventory we send for privacy and priority reasons. + // A heap is used so that not all items need sorting if only a few are being sent. + CompareInvMempoolOrder compareInvMempoolOrder(&mempool); + std::make_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); + // No reason to drain out at many times the network's capacity, + // especially since we have many peers and some will draw much shorter delays. + unsigned int nRelayedTransactions = 0; + LOCK(pto->cs_filter); + while (!vInvTx.empty() && nRelayedTransactions < INVENTORY_BROADCAST_MAX) { + // Fetch the top element from the heap + std::pop_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); + std::set<uint256>::iterator it = vInvTx.back(); + vInvTx.pop_back(); + uint256 hash = *it; + // Remove it from the to-be-sent set + pto->setInventoryTxToSend.erase(it); + // Check if not in the filter already + if (pto->filterInventoryKnown.contains(hash)) { + continue; + } + // Not in the mempool anymore? don't bother sending it. + auto txinfo = mempool.info(hash); + if (!txinfo.tx) { + continue; + } + if (filterrate && txinfo.feeRate.GetFeePerK() < filterrate) { + continue; + } + if (pto->pfilter && !pto->pfilter->IsRelevantAndUpdate(*txinfo.tx)) continue; + // Send + vInv.push_back(CInv(MSG_TX, hash)); + nRelayedTransactions++; { - pto->PushMessage("inv", vInv); + // Expire old relay messages + while (!vRelayExpiration.empty() && vRelayExpiration.front().first < nNow) + { + mapRelay.erase(vRelayExpiration.front().second); + vRelayExpiration.pop_front(); + } + + auto ret = mapRelay.insert(std::make_pair(hash, std::move(txinfo.tx))); + if (ret.second) { + vRelayExpiration.push_back(std::make_pair(nNow + 15 * 60 * 1000000, ret.first)); + } + } + if (vInv.size() == MAX_INV_SZ) { + pto->PushMessage(NetMsgType::INV, vInv); vInv.clear(); } + pto->filterInventoryKnown.insert(hash); } } - pto->vInventoryToSend = vInvWait; } if (!vInv.empty()) - pto->PushMessage("inv", vInv); + pto->PushMessage(NetMsgType::INV, vInv); // Detect whether we're stalling - int64_t nNow = GetTimeMicros(); + nNow = GetTimeMicros(); if (!pto->fDisconnect && state.nStallingSince && state.nStallingSince < nNow - 1000000 * BLOCK_STALLING_TIMEOUT) { // Stalling only triggers when the block download window cannot move. During normal steady state, // the download window should be much larger than the to-be-downloaded set of blocks, so disconnection @@ -4461,20 +6694,36 @@ bool SendMessages(CNode* pto, bool fSendTrickle) LogPrintf("Peer=%d is stalling block download, disconnecting\n", pto->id); pto->fDisconnect = true; } + // In case there is a block that has been in flight from this peer for 2 + 0.5 * N times the block interval + // (with N the number of peers from which we're downloading validated blocks), disconnect due to timeout. + // We compensate for other peers to prevent killing off peers due to our own downstream link + // being saturated. We only count validated in-flight blocks so peers can't advertise non-existing block hashes + // to unreasonably increase our timeout. + if (!pto->fDisconnect && state.vBlocksInFlight.size() > 0) { + QueuedBlock &queuedBlock = state.vBlocksInFlight.front(); + int nOtherPeersWithValidatedDownloads = nPeersWithValidatedDownloads - (state.nBlocksInFlightValidHeaders > 0); + if (nNow > state.nDownloadingSince + consensusParams.nPowTargetSpacing * (BLOCK_DOWNLOAD_TIMEOUT_BASE + BLOCK_DOWNLOAD_TIMEOUT_PER_PEER * nOtherPeersWithValidatedDownloads)) { + LogPrintf("Timeout downloading block %s from peer=%d, disconnecting\n", queuedBlock.hash.ToString(), pto->id); + pto->fDisconnect = true; + } + } // // Message: getdata (blocks) // vector<CInv> vGetData; - if (!pto->fDisconnect && !pto->fClient && fFetch && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (!pto->fDisconnect && !pto->fClient && (fFetch || !IsInitialBlockDownload()) && state.nBlocksInFlight < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { vector<CBlockIndex*> vToDownload; NodeId staller = -1; FindNextBlocksToDownload(pto->GetId(), MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.nBlocksInFlight, vToDownload, staller); BOOST_FOREACH(CBlockIndex *pindex, vToDownload) { - vGetData.push_back(CInv(MSG_BLOCK, pindex->GetBlockHash())); - MarkBlockAsInFlight(pto->GetId(), pindex->GetBlockHash(), pindex); - LogPrint("net", "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), - pindex->nHeight, pto->id); + if (State(pto->GetId())->fHaveWitness || !IsWitnessEnabled(pindex->pprev, consensusParams)) { + uint32_t nFetchFlags = GetFetchFlags(pto, pindex->pprev, consensusParams); + vGetData.push_back(CInv(MSG_BLOCK | nFetchFlags, pindex->GetBlockHash())); + MarkBlockAsInFlight(pto->GetId(), pindex->GetBlockHash(), consensusParams, pindex); + LogPrint("net", "Requesting block %s (%d) peer=%d\n", pindex->GetBlockHash().ToString(), + pindex->nHeight, pto->id); + } } if (state.nBlocksInFlight == 0 && staller != -1) { if (State(staller)->nStallingSince == 0) { @@ -4497,71 +6746,42 @@ bool SendMessages(CNode* pto, bool fSendTrickle) vGetData.push_back(inv); if (vGetData.size() >= 1000) { - pto->PushMessage("getdata", vGetData); + pto->PushMessage(NetMsgType::GETDATA, vGetData); vGetData.clear(); } + } else { + //If we're not going to ask, don't expect a response. + pto->setAskFor.erase(inv.hash); } pto->mapAskFor.erase(pto->mapAskFor.begin()); } if (!vGetData.empty()) - pto->PushMessage("getdata", vGetData); - - } - return true; -} - - -bool CBlockUndo::WriteToDisk(CDiskBlockPos &pos, const uint256 &hashBlock) -{ - // Open history file to append - CAutoFile fileout(OpenUndoFile(pos), SER_DISK, CLIENT_VERSION); - if (fileout.IsNull()) - return error("CBlockUndo::WriteToDisk : OpenUndoFile failed"); - - // Write index header - unsigned int nSize = fileout.GetSerializeSize(*this); - fileout << FLATDATA(Params().MessageStart()) << nSize; - - // Write undo data - long fileOutPos = ftell(fileout.Get()); - if (fileOutPos < 0) - return error("CBlockUndo::WriteToDisk : ftell failed"); - pos.nPos = (unsigned int)fileOutPos; - fileout << *this; - - // calculate & write checksum - CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); - hasher << hashBlock; - hasher << *this; - fileout << hasher.GetHash(); + pto->PushMessage(NetMsgType::GETDATA, vGetData); - return true; -} - -bool CBlockUndo::ReadFromDisk(const CDiskBlockPos &pos, const uint256 &hashBlock) -{ - // Open history file to read - CAutoFile filein(OpenUndoFile(pos, true), SER_DISK, CLIENT_VERSION); - if (filein.IsNull()) - return error("CBlockUndo::ReadFromDisk : OpenBlockFile failed"); - - // Read block - uint256 hashChecksum; - try { - filein >> *this; - filein >> hashChecksum; - } - catch (std::exception &e) { - return error("%s : Deserialize or I/O error - %s", __func__, e.what()); + // + // Message: feefilter + // + // We don't want white listed peers to filter txs to us if we have -whitelistforcerelay + if (pto->nVersion >= FEEFILTER_VERSION && GetBoolArg("-feefilter", DEFAULT_FEEFILTER) && + !(pto->fWhitelisted && GetBoolArg("-whitelistforcerelay", DEFAULT_WHITELISTFORCERELAY))) { + CAmount currentFilter = mempool.GetMinFee(GetArg("-maxmempool", DEFAULT_MAX_MEMPOOL_SIZE) * 1000000).GetFeePerK(); + int64_t timeNow = GetTimeMicros(); + if (timeNow > pto->nextSendTimeFeeFilter) { + CAmount filterToSend = filterRounder.round(currentFilter); + if (filterToSend != pto->lastSentFeeFilter) { + pto->PushMessage(NetMsgType::FEEFILTER, filterToSend); + pto->lastSentFeeFilter = filterToSend; + } + pto->nextSendTimeFeeFilter = PoissonNextSend(timeNow, AVG_FEEFILTER_BROADCAST_INTERVAL); + } + // If the fee filter has changed substantially and it's still more than MAX_FEEFILTER_CHANGE_DELAY + // until scheduled broadcast, then move the broadcast to within MAX_FEEFILTER_CHANGE_DELAY. + else if (timeNow + MAX_FEEFILTER_CHANGE_DELAY * 1000000 < pto->nextSendTimeFeeFilter && + (currentFilter < 3 * pto->lastSentFeeFilter / 4 || currentFilter > 4 * pto->lastSentFeeFilter / 3)) { + pto->nextSendTimeFeeFilter = timeNow + (insecure_rand() % MAX_FEEFILTER_CHANGE_DELAY) * 1000000; + } + } } - - // Verify checksum - CHashWriter hasher(SER_GETHASH, PROTOCOL_VERSION); - hasher << hashBlock; - hasher << *this; - if (hashChecksum != hasher.GetHash()) - return error("CBlockUndo::ReadFromDisk : Checksum mismatch"); - return true; } @@ -4569,7 +6789,11 @@ bool CBlockUndo::ReadFromDisk(const CDiskBlockPos &pos, const uint256 &hashBlock return strprintf("CBlockFileInfo(blocks=%u, size=%u, heights=%u...%u, time=%s...%s)", nBlocks, nSize, nHeightFirst, nHeightLast, DateTimeStrFormat("%Y-%m-%d", nTimeFirst), DateTimeStrFormat("%Y-%m-%d", nTimeLast)); } - +ThresholdState VersionBitsTipState(const Consensus::Params& params, Consensus::DeploymentPos pos) +{ + LOCK(cs_main); + return VersionBitsState(chainActive.Tip(), params, pos, versionbitscache); +} class CMainCleanup { |