diff options
Diffstat (limited to 'src')
58 files changed, 989 insertions, 541 deletions
diff --git a/src/Makefile.am b/src/Makefile.am index e1ae049b15..b48d723bc9 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -224,6 +224,7 @@ BITCOIN_CORE_H = \ node/miner.h \ node/mini_miner.h \ node/minisketchwrapper.h \ + node/peerman_args.h \ node/psbt.h \ node/transaction.h \ node/txreconciliation.h \ @@ -421,6 +422,7 @@ libbitcoin_node_a_SOURCES = \ node/miner.cpp \ node/mini_miner.cpp \ node/minisketchwrapper.cpp \ + node/peerman_args.cpp \ node/psbt.cpp \ node/transaction.cpp \ node/txreconciliation.cpp \ @@ -715,6 +717,7 @@ libbitcoin_util_a_SOURCES = \ logging.cpp \ random.cpp \ randomenv.cpp \ + streams.cpp \ support/cleanse.cpp \ sync.cpp \ util/asmap.cpp \ @@ -896,8 +899,8 @@ if BUILD_BITCOIN_KERNEL_LIB lib_LTLIBRARIES += $(LIBBITCOINKERNEL) libbitcoinkernel_la_LDFLAGS = $(AM_LDFLAGS) -no-undefined $(RELDFLAGS) $(PTHREAD_FLAGS) -libbitcoinkernel_la_LIBADD = $(LIBBITCOIN_CRYPTO) $(LIBUNIVALUE) $(LIBLEVELDB) $(LIBMEMENV) $(LIBSECP256K1) -libbitcoinkernel_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) -I$(srcdir)/$(UNIVALUE_INCLUDE_DIR_INT) +libbitcoinkernel_la_LIBADD = $(LIBBITCOIN_CRYPTO) $(LIBLEVELDB) $(LIBMEMENV) $(LIBSECP256K1) +libbitcoinkernel_la_CPPFLAGS = $(AM_CPPFLAGS) -I$(builddir)/obj -I$(srcdir)/secp256k1/include -DBUILD_BITCOIN_INTERNAL $(BOOST_CPPFLAGS) $(LEVELDB_CPPFLAGS) # libbitcoinkernel requires default symbol visibility, explicitly specify that # here so that things still work even when user configures with @@ -957,6 +960,7 @@ libbitcoinkernel_la_SOURCES = \ script/sigcache.cpp \ script/standard.cpp \ signet.cpp \ + streams.cpp \ support/cleanse.cpp \ support/lockedpool.cpp \ sync.cpp \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 10c8389c80..51bfb1e459 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -52,7 +52,8 @@ bench_bench_bitcoin_SOURCES = \ bench/streams_findbyte.cpp \ bench/strencodings.cpp \ bench/util_time.cpp \ - bench/verify_script.cpp + bench/verify_script.cpp \ + bench/xor.cpp nodist_bench_bench_bitcoin_SOURCES = $(GENERATED_BENCH_FILES) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 224f1fe301..f2a82ce73b 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -427,8 +427,9 @@ endif %.json.h: %.json @$(MKDIR_P) $(@D) $(AM_V_GEN) { \ + echo "#include <string>" && \ echo "namespace json_tests{" && \ - echo "static unsigned const char $(*F)[] = {" && \ + echo "static const std::string $(*F){" && \ $(HEXDUMP) -v -e '8/1 "0x%02x, "' -e '"\n"' $< | $(SED) -e 's/0x ,//g' && \ echo "};};"; \ } > "$@.new" && mv -f "$@.new" "$@" diff --git a/src/bench/bench.h b/src/bench/bench.h index 78196134e7..6065ddf3fc 100644 --- a/src/bench/bench.h +++ b/src/bench/bench.h @@ -14,7 +14,7 @@ #include <string> #include <vector> -#include <bench/nanobench.h> +#include <bench/nanobench.h> // IWYU pragma: export /* * Usage: diff --git a/src/bench/load_external.cpp b/src/bench/load_external.cpp index 1378a7b20a..252cbb163b 100644 --- a/src/bench/load_external.cpp +++ b/src/bench/load_external.cpp @@ -49,14 +49,13 @@ static void LoadExternalBlockFile(benchmark::Bench& bench) fclose(file); } - Chainstate& chainstate{testing_setup->m_node.chainman->ActiveChainstate()}; std::multimap<uint256, FlatFilePos> blocks_with_unknown_parent; FlatFilePos pos; bench.run([&] { // "rb" is "binary, O_RDONLY", positioned to the start of the file. // The file will be closed by LoadExternalBlockFile(). FILE* file{fsbridge::fopen(blkfile, "rb")}; - chainstate.LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent); + testing_setup->m_node.chainman->LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent); }); fs::remove(blkfile); } diff --git a/src/bench/xor.cpp b/src/bench/xor.cpp new file mode 100644 index 0000000000..edda74214a --- /dev/null +++ b/src/bench/xor.cpp @@ -0,0 +1,24 @@ +// Copyright (c) The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit/. + +#include <bench/bench.h> + +#include <random.h> +#include <streams.h> + +#include <cstddef> +#include <vector> + +static void Xor(benchmark::Bench& bench) +{ + FastRandomContext frc{/*fDeterministic=*/true}; + auto data{frc.randbytes<std::byte>(1024)}; + auto key{frc.randbytes<std::byte>(31)}; + + bench.batch(data.size()).unit("byte").run([&] { + util::Xor(data, key); + }); +} + +BENCHMARK(Xor, benchmark::PriorityLevel::HIGH); diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 0c25ddf373..103d8885db 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -562,6 +562,16 @@ static CAmount AmountFromValue(const UniValue& value) return amount; } +static std::vector<unsigned char> ParseHexUV(const UniValue& v, const std::string& strName) +{ + std::string strHex; + if (v.isStr()) + strHex = v.getValStr(); + if (!IsHex(strHex)) + throw std::runtime_error(strName + " must be hexadecimal string (not '" + strHex + "')"); + return ParseHex(strHex); +} + static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) { int nHashType = SIGHASH_ALL; diff --git a/src/chain.h b/src/chain.h index f5dd0fd315..2e1fb37bec 100644 --- a/src/chain.h +++ b/src/chain.h @@ -113,10 +113,10 @@ enum BlockStatus : uint32_t { BLOCK_VALID_TRANSACTIONS = 3, //! Outputs do not overspend inputs, no double spends, coinbase output ok, no immature coinbase spends, BIP30. - //! Implies all parents are also at least CHAIN. + //! Implies all parents are either at least VALID_CHAIN, or are ASSUMED_VALID BLOCK_VALID_CHAIN = 4, - //! Scripts & signatures ok. Implies all parents are also at least SCRIPTS. + //! Scripts & signatures ok. Implies all parents are either at least VALID_SCRIPTS, or are ASSUMED_VALID. BLOCK_VALID_SCRIPTS = 5, //! All validity bits. @@ -134,10 +134,18 @@ enum BlockStatus : uint32_t { BLOCK_OPT_WITNESS = 128, //!< block data in blk*.dat was received with a witness-enforcing client /** - * If set, this indicates that the block index entry is assumed-valid. - * Certain diagnostics will be skipped in e.g. CheckBlockIndex(). - * It almost certainly means that the block's full validation is pending - * on a background chainstate. See `doc/design/assumeutxo.md`. + * If ASSUMED_VALID is set, it means that this block has not been validated + * and has validity status less than VALID_SCRIPTS. Also that it may have + * descendant blocks with VALID_SCRIPTS set, because they can be validated + * based on an assumeutxo snapshot. + * + * When an assumeutxo snapshot is loaded, the ASSUMED_VALID flag is added to + * unvalidated blocks at the snapshot height and below. Then, as the background + * validation progresses, and these blocks are validated, the ASSUMED_VALID + * flags are removed. See `doc/design/assumeutxo.md` for details. + * + * This flag is only used to implement checks in CheckBlockIndex() and + * should not be used elsewhere. */ BLOCK_ASSUMED_VALID = 256, }; diff --git a/src/core_io.h b/src/core_io.h index 997f3bfd5b..1f5ecbaea6 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -6,6 +6,7 @@ #define BITCOIN_CORE_IO_H #include <consensus/amount.h> +#include <util/result.h> #include <string> #include <vector> @@ -45,8 +46,7 @@ bool DecodeHexBlockHeader(CBlockHeader&, const std::string& hex_header); * @see ParseHashV for an RPC-oriented version of this */ bool ParseHashStr(const std::string& strHex, uint256& result); -std::vector<unsigned char> ParseHexUV(const UniValue& v, const std::string& strName); -int ParseSighashString(const UniValue& sighash); +[[nodiscard]] util::Result<int> SighashFromStr(const std::string& sighash); // core_write.cpp UniValue ValueFromAmount(const CAmount amount); diff --git a/src/core_read.cpp b/src/core_read.cpp index 84cd559b7f..dfabf3a0c2 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -10,7 +10,7 @@ #include <script/sign.h> #include <serialize.h> #include <streams.h> -#include <univalue.h> +#include <util/result.h> #include <util/strencodings.h> #include <version.h> @@ -242,36 +242,21 @@ bool ParseHashStr(const std::string& strHex, uint256& result) return true; } -std::vector<unsigned char> ParseHexUV(const UniValue& v, const std::string& strName) +util::Result<int> SighashFromStr(const std::string& sighash) { - std::string strHex; - if (v.isStr()) - strHex = v.getValStr(); - if (!IsHex(strHex)) - throw std::runtime_error(strName + " must be hexadecimal string (not '" + strHex + "')"); - return ParseHex(strHex); -} - -int ParseSighashString(const UniValue& sighash) -{ - int hash_type = SIGHASH_DEFAULT; - if (!sighash.isNull()) { - static std::map<std::string, int> map_sighash_values = { - {std::string("DEFAULT"), int(SIGHASH_DEFAULT)}, - {std::string("ALL"), int(SIGHASH_ALL)}, - {std::string("ALL|ANYONECANPAY"), int(SIGHASH_ALL|SIGHASH_ANYONECANPAY)}, - {std::string("NONE"), int(SIGHASH_NONE)}, - {std::string("NONE|ANYONECANPAY"), int(SIGHASH_NONE|SIGHASH_ANYONECANPAY)}, - {std::string("SINGLE"), int(SIGHASH_SINGLE)}, - {std::string("SINGLE|ANYONECANPAY"), int(SIGHASH_SINGLE|SIGHASH_ANYONECANPAY)}, - }; - const std::string& strHashType = sighash.get_str(); - const auto& it = map_sighash_values.find(strHashType); - if (it != map_sighash_values.end()) { - hash_type = it->second; - } else { - throw std::runtime_error(strHashType + " is not a valid sighash parameter."); - } + static std::map<std::string, int> map_sighash_values = { + {std::string("DEFAULT"), int(SIGHASH_DEFAULT)}, + {std::string("ALL"), int(SIGHASH_ALL)}, + {std::string("ALL|ANYONECANPAY"), int(SIGHASH_ALL|SIGHASH_ANYONECANPAY)}, + {std::string("NONE"), int(SIGHASH_NONE)}, + {std::string("NONE|ANYONECANPAY"), int(SIGHASH_NONE|SIGHASH_ANYONECANPAY)}, + {std::string("SINGLE"), int(SIGHASH_SINGLE)}, + {std::string("SINGLE|ANYONECANPAY"), int(SIGHASH_SINGLE|SIGHASH_ANYONECANPAY)}, + }; + const auto& it = map_sighash_values.find(sighash); + if (it != map_sighash_values.end()) { + return it->second; + } else { + return util::Error{Untranslated(sighash + " is not a valid sighash parameter.")}; } - return hash_type; } diff --git a/src/crypto/common.h b/src/crypto/common.h index dc12ed9942..6ae5d4cd24 100644 --- a/src/crypto/common.h +++ b/src/crypto/common.h @@ -17,73 +17,73 @@ uint16_t static inline ReadLE16(const unsigned char* ptr) { uint16_t x; - memcpy((char*)&x, ptr, 2); + memcpy(&x, ptr, 2); return le16toh(x); } uint32_t static inline ReadLE32(const unsigned char* ptr) { uint32_t x; - memcpy((char*)&x, ptr, 4); + memcpy(&x, ptr, 4); return le32toh(x); } uint64_t static inline ReadLE64(const unsigned char* ptr) { uint64_t x; - memcpy((char*)&x, ptr, 8); + memcpy(&x, ptr, 8); return le64toh(x); } void static inline WriteLE16(unsigned char* ptr, uint16_t x) { uint16_t v = htole16(x); - memcpy(ptr, (char*)&v, 2); + memcpy(ptr, &v, 2); } void static inline WriteLE32(unsigned char* ptr, uint32_t x) { uint32_t v = htole32(x); - memcpy(ptr, (char*)&v, 4); + memcpy(ptr, &v, 4); } void static inline WriteLE64(unsigned char* ptr, uint64_t x) { uint64_t v = htole64(x); - memcpy(ptr, (char*)&v, 8); + memcpy(ptr, &v, 8); } uint16_t static inline ReadBE16(const unsigned char* ptr) { uint16_t x; - memcpy((char*)&x, ptr, 2); + memcpy(&x, ptr, 2); return be16toh(x); } uint32_t static inline ReadBE32(const unsigned char* ptr) { uint32_t x; - memcpy((char*)&x, ptr, 4); + memcpy(&x, ptr, 4); return be32toh(x); } uint64_t static inline ReadBE64(const unsigned char* ptr) { uint64_t x; - memcpy((char*)&x, ptr, 8); + memcpy(&x, ptr, 8); return be64toh(x); } void static inline WriteBE32(unsigned char* ptr, uint32_t x) { uint32_t v = htobe32(x); - memcpy(ptr, (char*)&v, 4); + memcpy(ptr, &v, 4); } void static inline WriteBE64(unsigned char* ptr, uint64_t x) { uint64_t v = htobe64(x); - memcpy(ptr, (char*)&v, 8); + memcpy(ptr, &v, 8); } /** Return the smallest number n such that (x >> n) == 0 (or 64 if the highest bit in x is set. */ diff --git a/src/dbwrapper.h b/src/dbwrapper.h index 35782edca6..4ae2106211 100644 --- a/src/dbwrapper.h +++ b/src/dbwrapper.h @@ -54,6 +54,8 @@ struct DBParams { DBOptions options{}; }; +inline auto CharCast(const std::byte* data) { return reinterpret_cast<const char*>(data); } + class dbwrapper_error : public std::runtime_error { public: @@ -113,12 +115,12 @@ public: { ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); + leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); ssValue.reserve(DBWRAPPER_PREALLOC_VALUE_SIZE); ssValue << value; ssValue.Xor(dbwrapper_private::GetObfuscateKey(parent)); - leveldb::Slice slValue((const char*)ssValue.data(), ssValue.size()); + leveldb::Slice slValue(CharCast(ssValue.data()), ssValue.size()); batch.Put(slKey, slValue); // LevelDB serializes writes as: @@ -138,7 +140,7 @@ public: { ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); + leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); batch.Delete(slKey); // LevelDB serializes erases as: @@ -177,7 +179,7 @@ public: DataStream ssKey{}; ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); + leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); piter->Seek(slKey); } @@ -265,7 +267,7 @@ public: DataStream ssKey{}; ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); + leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); std::string strValue; leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); @@ -307,7 +309,7 @@ public: DataStream ssKey{}; ssKey.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey << key; - leveldb::Slice slKey((const char*)ssKey.data(), ssKey.size()); + leveldb::Slice slKey(CharCast(ssKey.data()), ssKey.size()); std::string strValue; leveldb::Status status = pdb->Get(readoptions, slKey, &strValue); @@ -351,8 +353,8 @@ public: ssKey2.reserve(DBWRAPPER_PREALLOC_KEY_SIZE); ssKey1 << key_begin; ssKey2 << key_end; - leveldb::Slice slKey1((const char*)ssKey1.data(), ssKey1.size()); - leveldb::Slice slKey2((const char*)ssKey2.data(), ssKey2.size()); + leveldb::Slice slKey1(CharCast(ssKey1.data()), ssKey1.size()); + leveldb::Slice slKey2(CharCast(ssKey2.data()), ssKey2.size()); uint64_t size = 0; leveldb::Range range(slKey1, slKey2); pdb->GetApproximateSizes(&range, 1, &size); diff --git a/src/hash.h b/src/hash.h index 2e3ed11b43..89c6f0dab9 100644 --- a/src/hash.h +++ b/src/hash.h @@ -160,7 +160,6 @@ public: template<typename T> CHashWriter& operator<<(const T& obj) { - // Serialize to this stream ::Serialize(*this, obj); return (*this); } @@ -228,7 +227,6 @@ public: template<typename T> CHashVerifier<Source>& operator>>(T&& obj) { - // Unserialize from this stream ::Unserialize(*this, obj); return (*this); } diff --git a/src/init.cpp b/src/init.cpp index f726fe54ca..c11f100139 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -50,7 +50,7 @@ #include <node/mempool_args.h> #include <node/mempool_persist_args.h> #include <node/miner.h> -#include <node/txreconciliation.h> +#include <node/peerman_args.h> #include <node/validation_cache_args.h> #include <policy/feerate.h> #include <policy/fees.h> @@ -1168,7 +1168,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) fListen = args.GetBoolArg("-listen", DEFAULT_LISTEN); fDiscover = args.GetBoolArg("-discover", true); - const bool ignores_incoming_txs{args.GetBoolArg("-blocksonly", DEFAULT_BLOCKSONLY)}; + + PeerManager::Options peerman_opts{}; + ApplyArgsManOptions(args, peerman_opts); { @@ -1216,7 +1218,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) assert(!node.fee_estimator); // Don't initialize fee estimation with old data if we don't relay transactions, // as they would never get updated. - if (!ignores_incoming_txs) { + if (!peerman_opts.ignore_incoming_txs) { bool read_stale_estimates = args.GetBoolArg("-acceptstalefeeestimates", DEFAULT_ACCEPT_STALE_FEE_ESTIMATES); if (read_stale_estimates && (chainparams.GetChainType() != ChainType::REGTEST)) { return InitError(strprintf(_("acceptstalefeeestimates is not supported on %s chain."), chainparams.GetChainTypeString())); @@ -1540,8 +1542,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) ChainstateManager& chainman = *Assert(node.chainman); assert(!node.peerman); - node.peerman = PeerManager::make(*node.connman, *node.addrman, node.banman.get(), - chainman, *node.mempool, ignores_incoming_txs); + node.peerman = PeerManager::make(*node.connman, *node.addrman, + node.banman.get(), chainman, + *node.mempool, peerman_opts); RegisterValidationInterface(node.peerman.get()); // ********************************************************* Step 8: start indexers diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 8da2c701d3..be6777d14b 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -10,7 +10,6 @@ #include <blockencodings.h> #include <blockfilter.h> #include <chainparams.h> -#include <common/args.h> #include <consensus/amount.h> #include <consensus/validation.h> #include <deploymentstatus.h> @@ -487,7 +486,7 @@ class PeerManagerImpl final : public PeerManager public: PeerManagerImpl(CConnman& connman, AddrMan& addrman, BanMan* banman, ChainstateManager& chainman, - CTxMemPool& pool, bool ignore_incoming_txs); + CTxMemPool& pool, Options opts); /** Overridden from CValidationInterface. */ void BlockConnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected) override @@ -515,7 +514,7 @@ public: std::optional<std::string> FetchBlock(NodeId peer_id, const CBlockIndex& block_index) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); bool GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) const override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); - bool IgnoresIncomingTxs() override { return m_ignore_incoming_txs; } + bool IgnoresIncomingTxs() override { return m_opts.ignore_incoming_txs; } void SendPings() override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); void RelayTransaction(const uint256& txid, const uint256& wtxid) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); void SetBestHeight(int height) override { m_best_height = height; }; @@ -718,8 +717,7 @@ private: /** Next time to check for stale tip */ std::chrono::seconds m_stale_tip_check_time GUARDED_BY(cs_main){0s}; - /** Whether this node is running in -blocksonly mode */ - const bool m_ignore_incoming_txs; + const Options m_opts; bool RejectIncomingTxs(const CNode& peer) const; @@ -1212,7 +1210,7 @@ void PeerManagerImpl::MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid) // When in -blocksonly mode, never request high-bandwidth mode from peers. Our // mempool will not contain the transactions necessary to reconstruct the // compact block. - if (m_ignore_incoming_txs) return; + if (m_opts.ignore_incoming_txs) return; CNodeState* nodestate = State(nodeid); if (!nodestate || !nodestate->m_provides_cmpctblocks) { @@ -1650,13 +1648,12 @@ bool PeerManagerImpl::GetNodeStateStats(NodeId nodeid, CNodeStateStats& stats) c void PeerManagerImpl::AddToCompactExtraTransactions(const CTransactionRef& tx) { - size_t max_extra_txn = gArgs.GetIntArg("-blockreconstructionextratxn", DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN); - if (max_extra_txn <= 0) + if (m_opts.max_extra_txs <= 0) return; if (!vExtraTxnForCompact.size()) - vExtraTxnForCompact.resize(max_extra_txn); + vExtraTxnForCompact.resize(m_opts.max_extra_txs); vExtraTxnForCompact[vExtraTxnForCompactIt] = std::make_pair(tx->GetWitnessHash(), tx); - vExtraTxnForCompactIt = (vExtraTxnForCompactIt + 1) % max_extra_txn; + vExtraTxnForCompactIt = (vExtraTxnForCompactIt + 1) % m_opts.max_extra_txs; } void PeerManagerImpl::Misbehaving(Peer& peer, int howmuch, const std::string& message) @@ -1809,25 +1806,25 @@ std::optional<std::string> PeerManagerImpl::FetchBlock(NodeId peer_id, const CBl std::unique_ptr<PeerManager> PeerManager::make(CConnman& connman, AddrMan& addrman, BanMan* banman, ChainstateManager& chainman, - CTxMemPool& pool, bool ignore_incoming_txs) + CTxMemPool& pool, Options opts) { - return std::make_unique<PeerManagerImpl>(connman, addrman, banman, chainman, pool, ignore_incoming_txs); + return std::make_unique<PeerManagerImpl>(connman, addrman, banman, chainman, pool, opts); } PeerManagerImpl::PeerManagerImpl(CConnman& connman, AddrMan& addrman, BanMan* banman, ChainstateManager& chainman, - CTxMemPool& pool, bool ignore_incoming_txs) + CTxMemPool& pool, Options opts) : m_chainparams(chainman.GetParams()), m_connman(connman), m_addrman(addrman), m_banman(banman), m_chainman(chainman), m_mempool(pool), - m_ignore_incoming_txs(ignore_incoming_txs) + m_opts{opts} { // While Erlay support is incomplete, it must be enabled explicitly via -txreconciliation. // This argument can go away after Erlay support is complete. - if (gArgs.GetBoolArg("-txreconciliation", DEFAULT_TXRECONCILIATION_ENABLE)) { + if (opts.reconcile_txs) { m_txreconciliation = std::make_unique<TxReconciliationTracker>(TXRECONCILIATION_VERSION); } } @@ -2729,7 +2726,7 @@ void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const Peer& peer, c last_header.nHeight); } if (vGetData.size() > 0) { - if (!m_ignore_incoming_txs && + if (!m_opts.ignore_incoming_txs && nodestate->m_provides_cmpctblocks && vGetData.size() == 1 && mapBlocksInFlight.size() == 1 && @@ -3434,7 +3431,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // - we are not in -blocksonly mode. const auto* tx_relay = peer->GetTxRelay(); if (tx_relay && WITH_LOCK(tx_relay->m_bloom_filter_mutex, return tx_relay->m_relay_txs) && - !pfrom.IsAddrFetchConn() && !m_ignore_incoming_txs) { + !pfrom.IsAddrFetchConn() && !m_opts.ignore_incoming_txs) { const uint64_t recon_salt = m_txreconciliation->PreRegisterPeer(pfrom.GetId()); m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::SENDTXRCNCL, TXRECONCILIATION_VERSION, recon_salt)); @@ -4239,8 +4236,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, m_txrequest.ForgetTxHash(tx.GetWitnessHash()); // DoS prevention: do not allow m_orphanage to grow unbounded (see CVE-2012-3789) - unsigned int nMaxOrphanTx = (unsigned int)std::max((int64_t)0, gArgs.GetIntArg("-maxorphantx", DEFAULT_MAX_ORPHAN_TRANSACTIONS)); - m_orphanage.LimitOrphans(nMaxOrphanTx); + m_orphanage.LimitOrphans(m_opts.max_orphan_txs); } else { LogPrint(BCLog::MEMPOOL, "not keeping orphan with rejected parents %s\n",tx.GetHash().ToString()); // We will continue to reject this tx since it has rejected @@ -5008,7 +5004,7 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt msg.m_recv.data() ); - if (gArgs.GetBoolArg("-capturemessages", false)) { + if (m_opts.capture_messages) { CaptureMessage(pfrom->addr, msg.m_type, MakeUCharSpan(msg.m_recv), /*is_incoming=*/true); } @@ -5358,7 +5354,7 @@ void PeerManagerImpl::MaybeSendSendHeaders(CNode& node, Peer& peer) void PeerManagerImpl::MaybeSendFeefilter(CNode& pto, Peer& peer, std::chrono::microseconds current_time) { - if (m_ignore_incoming_txs) return; + if (m_opts.ignore_incoming_txs) return; if (pto.GetCommonVersion() < FEEFILTER_VERSION) return; // peers with the forcerelay permission should not filter txs to us if (pto.HasPermission(NetPermissionFlags::ForceRelay)) return; @@ -5426,7 +5422,7 @@ bool PeerManagerImpl::RejectIncomingTxs(const CNode& peer) const if (peer.IsBlockOnlyConn()) return true; if (peer.IsFeelerConn()) return true; // In -blocksonly mode, peers need the 'relay' permission to send txs to us - if (m_ignore_incoming_txs && !peer.HasPermission(NetPermissionFlags::Relay)) return true; + if (m_opts.ignore_incoming_txs && !peer.HasPermission(NetPermissionFlags::Relay)) return true; return false; } diff --git a/src/net_processing.h b/src/net_processing.h index deebb24c94..a0cbe92289 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -14,6 +14,8 @@ class CChainParams; class CTxMemPool; class ChainstateManager; +/** Whether transaction reconciliation protocol should be enabled by default. */ +static constexpr bool DEFAULT_TXRECONCILIATION_ENABLE{false}; /** Default for -maxorphantx, maximum number of orphan transactions kept in memory */ static const unsigned int DEFAULT_MAX_ORPHAN_TRANSACTIONS = 100; /** Default number of orphan+recently-replaced txn to keep around for block reconstruction */ @@ -43,9 +45,18 @@ struct CNodeStateStats { class PeerManager : public CValidationInterface, public NetEventsInterface { public: + struct Options { + /** Whether this node is running in -blocksonly mode */ + bool ignore_incoming_txs{DEFAULT_BLOCKSONLY}; + bool reconcile_txs{DEFAULT_TXRECONCILIATION_ENABLE}; + uint32_t max_orphan_txs{DEFAULT_MAX_ORPHAN_TRANSACTIONS}; + size_t max_extra_txs{DEFAULT_BLOCK_RECONSTRUCTION_EXTRA_TXN}; + bool capture_messages{false}; + }; + static std::unique_ptr<PeerManager> make(CConnman& connman, AddrMan& addrman, BanMan* banman, ChainstateManager& chainman, - CTxMemPool& pool, bool ignore_incoming_txs); + CTxMemPool& pool, Options opts); virtual ~PeerManager() { } /** diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 78416ec576..0d25c798ce 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -618,7 +618,7 @@ fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const return BlockFileSeq().FileName(pos); } -bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown) +bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown) { LOCK(cs_LastBlockFile); @@ -644,7 +644,7 @@ bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigne // when the undo file is keeping up with the block file, we want to flush it explicitly // when it is lagging behind (more blocks arrive than are being connected), we let the // undo block write case handle it - finalize_undo = (m_blockfile_info[nFile].nHeightLast == (unsigned int)active_chain.Tip()->nHeight); + finalize_undo = (m_blockfile_info[nFile].nHeightLast == m_undo_height_in_last_blockfile); nFile++; if (m_blockfile_info.size() <= nFile) { m_blockfile_info.resize(nFile + 1); @@ -660,6 +660,7 @@ bool BlockManager::FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigne } FlushBlockFile(!fKnown, finalize_undo); m_last_blockfile = nFile; + m_undo_height_in_last_blockfile = 0; // No undo data yet in the new file, so reset our undo-height tracking. } m_blockfile_info[nFile].AddBlock(nHeight, nTime); @@ -749,8 +750,9 @@ bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValid // the FindBlockPos function if (_pos.nFile < m_last_blockfile && static_cast<uint32_t>(block.nHeight) == m_blockfile_info[_pos.nFile].nHeightLast) { FlushUndoFile(_pos.nFile, true); + } else if (_pos.nFile == m_last_blockfile && static_cast<uint32_t>(block.nHeight) > m_undo_height_in_last_blockfile) { + m_undo_height_in_last_blockfile = block.nHeight; } - // update nUndoPos in block index block.nUndoPos = _pos.nPos; block.nStatus |= BLOCK_HAVE_UNDO; @@ -839,7 +841,7 @@ bool BlockManager::ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatF return true; } -FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const FlatFilePos* dbp) +FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, const FlatFilePos* dbp) { unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); FlatFilePos blockPos; @@ -852,7 +854,7 @@ FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, CCha // we add BLOCK_SERIALIZATION_HEADER_SIZE only for new blocks since they will have the serialization header added when written to disk. nBlockSize += static_cast<unsigned int>(BLOCK_SERIALIZATION_HEADER_SIZE); } - if (!FindBlockPos(blockPos, nBlockSize, nHeight, active_chain, block.GetBlockTime(), position_known)) { + if (!FindBlockPos(blockPos, nBlockSize, nHeight, block.GetBlockTime(), position_known)) { error("%s: FindBlockPos failed", __func__); return FlatFilePos(); } @@ -905,7 +907,7 @@ void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFile break; // This error is logged in OpenBlockFile } LogPrintf("Reindexing block file blk%05u.dat...\n", (unsigned int)nFile); - chainman.ActiveChainstate().LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent); + chainman.LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent); if (chainman.m_interrupt) { LogPrintf("Interrupt requested. Exit %s\n", __func__); return; @@ -924,7 +926,7 @@ void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFile FILE* file = fsbridge::fopen(path, "rb"); if (file) { LogPrintf("Importing blocks file %s...\n", fs::PathToString(path)); - chainman.ActiveChainstate().LoadExternalBlockFile(file); + chainman.LoadExternalBlockFile(file); if (chainman.m_interrupt) { LogPrintf("Interrupt requested. Exit %s\n", __func__); return; diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index c2e903e470..eb40d45aba 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -24,7 +24,6 @@ class BlockValidationState; class CBlock; class CBlockFileInfo; class CBlockUndo; -class CChain; class CChainParams; class Chainstate; class ChainstateManager; @@ -94,7 +93,7 @@ private: EXCLUSIVE_LOCKS_REQUIRED(cs_main); void FlushBlockFile(bool fFinalize = false, bool finalize_undo = false); void FlushUndoFile(int block_file, bool finalize = false); - bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, CChain& active_chain, uint64_t nTime, bool fKnown); + bool FindBlockPos(FlatFilePos& pos, unsigned int nAddSize, unsigned int nHeight, uint64_t nTime, bool fKnown); bool FindUndoPos(BlockValidationState& state, int nFile, FlatFilePos& pos, unsigned int nAddSize); FlatFileSeq BlockFileSeq() const; @@ -128,6 +127,19 @@ private: RecursiveMutex cs_LastBlockFile; std::vector<CBlockFileInfo> m_blockfile_info; int m_last_blockfile = 0; + + // Track the height of the highest block in m_last_blockfile whose undo + // data has been written. Block data is written to block files in download + // order, but is written to undo files in validation order, which is + // usually in order by height. To avoid wasting disk space, undo files will + // be trimmed whenever the corresponding block file is finalized and + // the height of the highest block written to the block file equals the + // height of the highest block written to the undo file. This is a + // heuristic and can sometimes preemptively trim undo files that will write + // more data later, and sometimes fail to trim undo files that can't have + // more data written later. + unsigned int m_undo_height_in_last_blockfile = 0; + /** Global flag to indicate we should check to see if there are * block/undo files that should be deleted. Set on startup * or if we allocate more file space when we're in prune mode @@ -202,7 +214,7 @@ public: EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** Store block on disk. If dbp is not nullptr, then it provides the known position of the block within a block file on disk. */ - FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, CChain& active_chain, const FlatFilePos* dbp); + FlatFilePos SaveBlockToDisk(const CBlock& block, int nHeight, const FlatFilePos* dbp); /** Whether running in -prune mode. */ [[nodiscard]] bool IsPruneMode() const { return m_prune_mode; } diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index 255d8be0ec..0828f64856 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -221,7 +221,7 @@ ChainstateLoadResult LoadChainstate(ChainstateManager& chainman, const CacheSize // A reload of the block index is required to recompute setBlockIndexCandidates // for the fully validated chainstate. - chainman.ActiveChainstate().UnloadBlockIndex(); + chainman.ActiveChainstate().ClearBlockIndexCandidates(); auto [init_status, init_error] = CompleteChainstateInitialization(chainman, cache_sizes, options); if (init_status != ChainstateLoadStatus::SUCCESS) { diff --git a/src/node/peerman_args.cpp b/src/node/peerman_args.cpp new file mode 100644 index 0000000000..e0dcf21c33 --- /dev/null +++ b/src/node/peerman_args.cpp @@ -0,0 +1,26 @@ +#include <node/peerman_args.h> + +#include <common/args.h> +#include <net_processing.h> + +namespace node { + +void ApplyArgsManOptions(const ArgsManager& argsman, PeerManager::Options& options) +{ + if (auto value{argsman.GetBoolArg("-txreconciliation")}) options.reconcile_txs = *value; + + if (auto value{argsman.GetIntArg("-maxorphantx")}) { + options.max_orphan_txs = uint32_t(std::max(int64_t{0}, *value)); + } + + if (auto value{argsman.GetIntArg("-blockreconstructionextratxn")}) { + options.max_extra_txs = size_t(std::max(int64_t{0}, *value)); + } + + if (auto value{argsman.GetBoolArg("-capturemessages")}) options.capture_messages = *value; + + if (auto value{argsman.GetBoolArg("-blocksonly")}) options.ignore_incoming_txs = *value; +} + +} // namespace node + diff --git a/src/node/peerman_args.h b/src/node/peerman_args.h new file mode 100644 index 0000000000..73dbdb446c --- /dev/null +++ b/src/node/peerman_args.h @@ -0,0 +1,12 @@ +#ifndef BITCOIN_NODE_PEERMAN_ARGS_H +#define BITCOIN_NODE_PEERMAN_ARGS_H + +#include <net_processing.h> + +class ArgsManager; + +namespace node { +void ApplyArgsManOptions(const ArgsManager& argsman, PeerManager::Options& options); +} // namespace node + +#endif // BITCOIN_NODE_PEERMAN_ARGS_H diff --git a/src/node/txreconciliation.h b/src/node/txreconciliation.h index 4591dd5df7..3bbb077366 100644 --- a/src/node/txreconciliation.h +++ b/src/node/txreconciliation.h @@ -11,8 +11,6 @@ #include <memory> #include <tuple> -/** Whether transaction reconciliation protocol should be enabled by default. */ -static constexpr bool DEFAULT_TXRECONCILIATION_ENABLE{false}; /** Supported transaction reconciliation protocol version */ static constexpr uint32_t TXRECONCILIATION_VERSION{1}; diff --git a/src/rpc/node.cpp b/src/rpc/node.cpp index 3828401642..dc5ecb9cbc 100644 --- a/src/rpc/node.cpp +++ b/src/rpc/node.cpp @@ -57,11 +57,9 @@ static RPCHelpMan setmocktime() throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("Mocktime cannot be negative: %s.", time)); } SetMockTime(time); - auto node_context = util::AnyPtr<NodeContext>(request.context); - if (node_context) { - for (const auto& chain_client : node_context->chain_clients) { - chain_client->setMockTime(time); - } + const NodeContext& node_context{EnsureAnyNodeContext(request.context)}; + for (const auto& chain_client : node_context.chain_clients) { + chain_client->setMockTime(time); } return UniValue::VNULL; @@ -89,10 +87,9 @@ static RPCHelpMan mockscheduler() throw std::runtime_error("delta_time must be between 1 and 3600 seconds (1 hr)"); } - auto node_context = CHECK_NONFATAL(util::AnyPtr<NodeContext>(request.context)); - // protect against null pointer dereference - CHECK_NONFATAL(node_context->scheduler); - node_context->scheduler->MockForward(std::chrono::seconds(delta_seconds)); + const NodeContext& node_context{EnsureAnyNodeContext(request.context)}; + CHECK_NONFATAL(node_context.scheduler)->MockForward(std::chrono::seconds{delta_seconds}); + SyncWithValidationInterfaceQueue(); return UniValue::VNULL; }, diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 19e14f88df..faae840d40 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -3,8 +3,10 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <clientversion.h> +#include <core_io.h> #include <common/args.h> #include <consensus/amount.h> +#include <script/interpreter.h> #include <key_io.h> #include <outputtype.h> #include <rpc/util.h> @@ -12,6 +14,7 @@ #include <script/signingprovider.h> #include <tinyformat.h> #include <util/check.h> +#include <util/result.h> #include <util/strencodings.h> #include <util/string.h> #include <util/translation.h> @@ -310,6 +313,23 @@ UniValue DescribeAddress(const CTxDestination& dest) return std::visit(DescribeAddressVisitor(), dest); } +/** + * Returns a sighash value corresponding to the passed in argument. + * + * @pre The sighash argument should be string or null. +*/ +int ParseSighashString(const UniValue& sighash) +{ + if (sighash.isNull()) { + return SIGHASH_DEFAULT; + } + const auto result{SighashFromStr(sighash.get_str())}; + if (!result) { + throw JSONRPCError(RPC_INVALID_PARAMETER, util::ErrorString(result).original); + } + return result.value(); +} + unsigned int ParseConfirmTarget(const UniValue& value, unsigned int max_target) { const int target{value.getInt<int>()}; diff --git a/src/rpc/util.h b/src/rpc/util.h index 4cba5a9818..02d26f1ab7 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -100,6 +100,9 @@ CTxDestination AddAndGetMultisigDestination(const int required, const std::vecto UniValue DescribeAddress(const CTxDestination& dest); +/** Parse a sighash string representation and raise an RPC error if it is invalid. */ +int ParseSighashString(const UniValue& sighash); + //! Parse a confirm target option and raise an RPC error if it is invalid. unsigned int ParseConfirmTarget(const UniValue& value, unsigned int max_target); diff --git a/src/span.h b/src/span.h index 0563556dc2..7209d21a58 100644 --- a/src/span.h +++ b/src/span.h @@ -5,10 +5,10 @@ #ifndef BITCOIN_SPAN_H #define BITCOIN_SPAN_H -#include <type_traits> -#include <cstddef> #include <algorithm> -#include <assert.h> +#include <cassert> +#include <cstddef> +#include <type_traits> #ifdef DEBUG #define CONSTEXPR_IF_NOT_DEBUG @@ -267,10 +267,10 @@ Span<std::byte> MakeWritableByteSpan(V&& v) noexcept } // Helper functions to safely cast to unsigned char pointers. -inline unsigned char* UCharCast(char* c) { return (unsigned char*)c; } +inline unsigned char* UCharCast(char* c) { return reinterpret_cast<unsigned char*>(c); } inline unsigned char* UCharCast(unsigned char* c) { return c; } -inline unsigned char* UCharCast(std::byte* c) { return (unsigned char*)c; } -inline const unsigned char* UCharCast(const char* c) { return (unsigned char*)c; } +inline unsigned char* UCharCast(std::byte* c) { return reinterpret_cast<unsigned char*>(c); } +inline const unsigned char* UCharCast(const char* c) { return reinterpret_cast<const unsigned char*>(c); } inline const unsigned char* UCharCast(const unsigned char* c) { return c; } inline const unsigned char* UCharCast(const std::byte* c) { return reinterpret_cast<const unsigned char*>(c); } diff --git a/src/streams.cpp b/src/streams.cpp new file mode 100644 index 0000000000..6921dad677 --- /dev/null +++ b/src/streams.cpp @@ -0,0 +1,66 @@ +// Copyright (c) 2009-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit/. + +#include <span.h> +#include <streams.h> + +#include <array> + +std::size_t AutoFile::detail_fread(Span<std::byte> dst) +{ + if (!m_file) throw std::ios_base::failure("AutoFile::read: file handle is nullptr"); + if (m_xor.empty()) { + return std::fread(dst.data(), 1, dst.size(), m_file); + } else { + const auto init_pos{std::ftell(m_file)}; + if (init_pos < 0) throw std::ios_base::failure("AutoFile::read: ftell failed"); + std::size_t ret{std::fread(dst.data(), 1, dst.size(), m_file)}; + util::Xor(dst.subspan(0, ret), m_xor, init_pos); + return ret; + } +} + +void AutoFile::read(Span<std::byte> dst) +{ + if (detail_fread(dst) != dst.size()) { + throw std::ios_base::failure(feof() ? "AutoFile::read: end of file" : "AutoFile::read: fread failed"); + } +} + +void AutoFile::ignore(size_t nSize) +{ + if (!m_file) throw std::ios_base::failure("AutoFile::ignore: file handle is nullptr"); + unsigned char data[4096]; + while (nSize > 0) { + size_t nNow = std::min<size_t>(nSize, sizeof(data)); + if (std::fread(data, 1, nNow, m_file) != nNow) { + throw std::ios_base::failure(feof() ? "AutoFile::ignore: end of file" : "AutoFile::ignore: fread failed"); + } + nSize -= nNow; + } +} + +void AutoFile::write(Span<const std::byte> src) +{ + if (!m_file) throw std::ios_base::failure("AutoFile::write: file handle is nullptr"); + if (m_xor.empty()) { + if (std::fwrite(src.data(), 1, src.size(), m_file) != src.size()) { + throw std::ios_base::failure("AutoFile::write: write failed"); + } + } else { + auto current_pos{std::ftell(m_file)}; + if (current_pos < 0) throw std::ios_base::failure("AutoFile::write: ftell failed"); + std::array<std::byte, 4096> buf; + while (src.size() > 0) { + auto buf_now{Span{buf}.first(std::min<size_t>(src.size(), buf.size()))}; + std::copy(src.begin(), src.begin() + buf_now.size(), buf_now.begin()); + util::Xor(buf_now, m_xor, current_pos); + if (std::fwrite(buf_now.data(), 1, buf_now.size(), m_file) != buf_now.size()) { + throw std::ios_base::failure{"XorFile::write: failed"}; + } + src = src.subspan(buf_now.size()); + current_pos += buf_now.size(); + } + } +} diff --git a/src/streams.h b/src/streams.h index 03df20b5db..5ff952be76 100644 --- a/src/streams.h +++ b/src/streams.h @@ -13,6 +13,7 @@ #include <algorithm> #include <assert.h> +#include <cstddef> #include <cstdio> #include <ios> #include <limits> @@ -23,6 +24,27 @@ #include <utility> #include <vector> +namespace util { +inline void Xor(Span<std::byte> write, Span<const std::byte> key, size_t key_offset = 0) +{ + if (key.size() == 0) { + return; + } + key_offset %= key.size(); + + for (size_t i = 0, j = key_offset; i != write.size(); i++) { + write[i] ^= key[j++]; + + // This potentially acts on very many bytes of data, so it's + // important that we calculate `j`, i.e. the `key` index in this + // way instead of doing a %, which would effectively be a division + // for each byte Xor'd -- much slower than need be. + if (j == key.size()) + j = 0; + } +} +} // namespace util + template<typename Stream> class OverrideStream { @@ -37,7 +59,6 @@ public: template<typename T> OverrideStream<Stream>& operator<<(const T& obj) { - // Serialize to this stream ::Serialize(*this, obj); return (*this); } @@ -45,7 +66,6 @@ public: template<typename T> OverrideStream<Stream>& operator>>(T&& obj) { - // Unserialize from this stream ::Unserialize(*this, obj); return (*this); } @@ -110,7 +130,6 @@ class CVectorWriter template<typename T> CVectorWriter& operator<<(const T& obj) { - // Serialize to this stream ::Serialize(*this, obj); return (*this); } @@ -151,7 +170,6 @@ public: template<typename T> SpanReader& operator>>(T&& obj) { - // Unserialize from this stream ::Unserialize(*this, obj); return (*this); } @@ -296,7 +314,6 @@ public: template<typename T> DataStream& operator<<(const T& obj) { - // Serialize to this stream ::Serialize(*this, obj); return (*this); } @@ -304,7 +321,6 @@ public: template<typename T> DataStream& operator>>(T&& obj) { - // Unserialize from this stream ::Unserialize(*this, obj); return (*this); } @@ -316,20 +332,7 @@ public: */ void Xor(const std::vector<unsigned char>& key) { - if (key.size() == 0) { - return; - } - - for (size_type i = 0, j = 0; i != size(); i++) { - vch[i] ^= std::byte{key[j++]}; - - // This potentially acts on very many bytes of data, so it's - // important that we calculate `j`, i.e. the `key` index in this - // way instead of doing a %, which would effectively be a division - // for each byte Xor'd -- much slower than need be. - if (j == key.size()) - j = 0; - } + util::Xor(MakeWritableByteSpan(*this), MakeByteSpan(key)); } }; @@ -469,7 +472,6 @@ public: } }; - /** Non-refcounted RAII wrapper for FILE* * * Will automatically close the file when it goes out of scope if not null. @@ -479,81 +481,60 @@ public: class AutoFile { protected: - FILE* file; + std::FILE* m_file; + const std::vector<std::byte> m_xor; public: - explicit AutoFile(FILE* filenew) : file{filenew} {} + explicit AutoFile(std::FILE* file, std::vector<std::byte> data_xor={}) : m_file{file}, m_xor{std::move(data_xor)} {} - ~AutoFile() - { - fclose(); - } + ~AutoFile() { fclose(); } // Disallow copies AutoFile(const AutoFile&) = delete; AutoFile& operator=(const AutoFile&) = delete; + bool feof() const { return std::feof(m_file); } + int fclose() { - int retval{0}; - if (file) { - retval = ::fclose(file); - file = nullptr; - } - return retval; + if (auto rel{release()}) return std::fclose(rel); + return 0; } /** Get wrapped FILE* with transfer of ownership. * @note This will invalidate the AutoFile object, and makes it the responsibility of the caller * of this function to clean up the returned FILE*. */ - FILE* release() { FILE* ret = file; file = nullptr; return ret; } + std::FILE* release() + { + std::FILE* ret{m_file}; + m_file = nullptr; + return ret; + } /** Get wrapped FILE* without transfer of ownership. * @note Ownership of the FILE* will remain with this class. Use this only if the scope of the * AutoFile outlives use of the passed pointer. */ - FILE* Get() const { return file; } + std::FILE* Get() const { return m_file; } /** Return true if the wrapped FILE* is nullptr, false otherwise. */ - bool IsNull() const { return (file == nullptr); } + bool IsNull() const { return m_file == nullptr; } + + /** Implementation detail, only used internally. */ + std::size_t detail_fread(Span<std::byte> dst); // // Stream subset // - void read(Span<std::byte> dst) - { - if (!file) throw std::ios_base::failure("AutoFile::read: file handle is nullptr"); - if (fread(dst.data(), 1, dst.size(), file) != dst.size()) { - throw std::ios_base::failure(feof(file) ? "AutoFile::read: end of file" : "AutoFile::read: fread failed"); - } - } - - void ignore(size_t nSize) - { - if (!file) throw std::ios_base::failure("AutoFile::ignore: file handle is nullptr"); - unsigned char data[4096]; - while (nSize > 0) { - size_t nNow = std::min<size_t>(nSize, sizeof(data)); - if (fread(data, 1, nNow, file) != nNow) - throw std::ios_base::failure(feof(file) ? "AutoFile::ignore: end of file" : "AutoFile::read: fread failed"); - nSize -= nNow; - } - } - - void write(Span<const std::byte> src) - { - if (!file) throw std::ios_base::failure("AutoFile::write: file handle is nullptr"); - if (fwrite(src.data(), 1, src.size(), file) != src.size()) { - throw std::ios_base::failure("AutoFile::write: write failed"); - } - } + void read(Span<std::byte> dst); + void ignore(size_t nSize); + void write(Span<const std::byte> src); template <typename T> AutoFile& operator<<(const T& obj) { - if (!file) throw std::ios_base::failure("AutoFile::operator<<: file handle is nullptr"); ::Serialize(*this, obj); return *this; } @@ -561,7 +542,6 @@ public: template <typename T> AutoFile& operator>>(T&& obj) { - if (!file) throw std::ios_base::failure("AutoFile::operator>>: file handle is nullptr"); ::Unserialize(*this, obj); return *this; } @@ -574,16 +554,13 @@ private: const int nVersion; public: - CAutoFile(FILE* filenew, int nTypeIn, int nVersionIn) : AutoFile{filenew}, nType(nTypeIn), nVersion(nVersionIn) {} + explicit CAutoFile(std::FILE* file, int type, int version, std::vector<std::byte> data_xor = {}) : AutoFile{file, std::move(data_xor)}, nType{type}, nVersion{version} {} int GetType() const { return nType; } int GetVersion() const { return nVersion; } template<typename T> CAutoFile& operator<<(const T& obj) { - // Serialize to this stream - if (!file) - throw std::ios_base::failure("CAutoFile::operator<<: file handle is nullptr"); ::Serialize(*this, obj); return (*this); } @@ -591,9 +568,6 @@ public: template<typename T> CAutoFile& operator>>(T&& obj) { - // Unserialize from this stream - if (!file) - throw std::ios_base::failure("CAutoFile::operator>>: file handle is nullptr"); ::Unserialize(*this, obj); return (*this); } @@ -742,7 +716,6 @@ public: template<typename T> CBufferedFile& operator>>(T&& obj) { - // Unserialize from this stream ::Unserialize(*this, obj); return (*this); } diff --git a/src/support/allocators/secure.h b/src/support/allocators/secure.h index b2076bea07..558f835f11 100644 --- a/src/support/allocators/secure.h +++ b/src/support/allocators/secure.h @@ -17,27 +17,14 @@ // out of memory and clears its contents before deletion. // template <typename T> -struct secure_allocator : public std::allocator<T> { - using base = std::allocator<T>; - using traits = std::allocator_traits<base>; - using size_type = typename traits::size_type; - using difference_type = typename traits::difference_type; - using pointer = typename traits::pointer; - using const_pointer = typename traits::const_pointer; - using value_type = typename traits::value_type; - secure_allocator() noexcept {} - secure_allocator(const secure_allocator& a) noexcept : base(a) {} +struct secure_allocator { + using value_type = T; + + secure_allocator() = default; template <typename U> - secure_allocator(const secure_allocator<U>& a) noexcept : base(a) - { - } - ~secure_allocator() noexcept {} - template <typename Other> - struct rebind { - typedef secure_allocator<Other> other; - }; + secure_allocator(const secure_allocator<U>&) noexcept {} - T* allocate(std::size_t n, const void* hint = nullptr) + T* allocate(std::size_t n) { T* allocation = static_cast<T*>(LockedPoolManager::Instance().alloc(sizeof(T) * n)); if (!allocation) { @@ -53,6 +40,17 @@ struct secure_allocator : public std::allocator<T> { } LockedPoolManager::Instance().free(p); } + + template <typename U> + friend bool operator==(const secure_allocator&, const secure_allocator<U>&) noexcept + { + return true; + } + template <typename U> + friend bool operator!=(const secure_allocator&, const secure_allocator<U>&) noexcept + { + return false; + } }; // This is exactly like std::string, but with a custom allocator. diff --git a/src/support/allocators/zeroafterfree.h b/src/support/allocators/zeroafterfree.h index 2dc644c242..6d50eb70a6 100644 --- a/src/support/allocators/zeroafterfree.h +++ b/src/support/allocators/zeroafterfree.h @@ -12,31 +12,36 @@ #include <vector> template <typename T> -struct zero_after_free_allocator : public std::allocator<T> { - using base = std::allocator<T>; - using traits = std::allocator_traits<base>; - using size_type = typename traits::size_type; - using difference_type = typename traits::difference_type; - using pointer = typename traits::pointer; - using const_pointer = typename traits::const_pointer; - using value_type = typename traits::value_type; - zero_after_free_allocator() noexcept {} - zero_after_free_allocator(const zero_after_free_allocator& a) noexcept : base(a) {} +struct zero_after_free_allocator { + using value_type = T; + + zero_after_free_allocator() noexcept = default; template <typename U> - zero_after_free_allocator(const zero_after_free_allocator<U>& a) noexcept : base(a) + zero_after_free_allocator(const zero_after_free_allocator<U>&) noexcept + { + } + + T* allocate(std::size_t n) { + return std::allocator<T>{}.allocate(n); } - ~zero_after_free_allocator() noexcept {} - template <typename Other> - struct rebind { - typedef zero_after_free_allocator<Other> other; - }; void deallocate(T* p, std::size_t n) { if (p != nullptr) memory_cleanse(p, sizeof(T) * n); - std::allocator<T>::deallocate(p, n); + std::allocator<T>{}.deallocate(p, n); + } + + template <typename U> + friend bool operator==(const zero_after_free_allocator&, const zero_after_free_allocator<U>&) noexcept + { + return true; + } + template <typename U> + friend bool operator!=(const zero_after_free_allocator&, const zero_after_free_allocator<U>&) noexcept + { + return false; } }; diff --git a/src/test/argsman_tests.cpp b/src/test/argsman_tests.cpp index 0b789e7f5c..1f46efe464 100644 --- a/src/test/argsman_tests.cpp +++ b/src/test/argsman_tests.cpp @@ -112,7 +112,7 @@ public: test.SetupArgs({{"-value", flags}}); const char* argv[] = {"ignored", arg}; std::string error; - bool success = test.ParseParameters(arg ? 2 : 1, (char**)argv, error); + bool success = test.ParseParameters(arg ? 2 : 1, argv, error); BOOST_CHECK_EQUAL(test.GetSetting("-value").write(), expect.setting.write()); auto settings_list = test.GetSettingsList("-value"); @@ -217,13 +217,13 @@ BOOST_AUTO_TEST_CASE(util_ParseParameters) std::string error; LOCK(testArgs.cs_args); testArgs.SetupArgs({a, b, ccc, d}); - BOOST_CHECK(testArgs.ParseParameters(0, (char**)argv_test, error)); + BOOST_CHECK(testArgs.ParseParameters(0, argv_test, error)); BOOST_CHECK(testArgs.m_settings.command_line_options.empty() && testArgs.m_settings.ro_config.empty()); - BOOST_CHECK(testArgs.ParseParameters(1, (char**)argv_test, error)); + BOOST_CHECK(testArgs.ParseParameters(1, argv_test, error)); BOOST_CHECK(testArgs.m_settings.command_line_options.empty() && testArgs.m_settings.ro_config.empty()); - BOOST_CHECK(testArgs.ParseParameters(7, (char**)argv_test, error)); + BOOST_CHECK(testArgs.ParseParameters(7, argv_test, error)); // expectation: -ignored is ignored (program name argument), // -a, -b and -ccc end up in map, -d ignored because it is after // a non-option argument (non-GNU option parsing) @@ -248,17 +248,17 @@ BOOST_AUTO_TEST_CASE(util_ParseInvalidParameters) const char* argv[] = {"ignored", "-registered"}; std::string error; - BOOST_CHECK(test.ParseParameters(2, (char**)argv, error)); + BOOST_CHECK(test.ParseParameters(2, argv, error)); BOOST_CHECK_EQUAL(error, ""); argv[1] = "-unregistered"; - BOOST_CHECK(!test.ParseParameters(2, (char**)argv, error)); + BOOST_CHECK(!test.ParseParameters(2, argv, error)); BOOST_CHECK_EQUAL(error, "Invalid parameter -unregistered"); // Make sure registered parameters prefixed with a chain type trigger errors. // (Previously, they were accepted and ignored.) argv[1] = "-test.registered"; - BOOST_CHECK(!test.ParseParameters(2, (char**)argv, error)); + BOOST_CHECK(!test.ParseParameters(2, argv, error)); BOOST_CHECK_EQUAL(error, "Invalid parameter -test.registered"); } @@ -269,7 +269,7 @@ static void TestParse(const std::string& str, bool expected_bool, int64_t expect std::string arg = "-value=" + str; const char* argv[] = {"ignored", arg.c_str()}; std::string error; - BOOST_CHECK(test.ParseParameters(2, (char**)argv, error)); + BOOST_CHECK(test.ParseParameters(2, argv, error)); BOOST_CHECK_EQUAL(test.GetBoolArg("-value", false), expected_bool); BOOST_CHECK_EQUAL(test.GetBoolArg("-value", true), expected_bool); BOOST_CHECK_EQUAL(test.GetIntArg("-value", 99998), expected_int); @@ -331,7 +331,7 @@ BOOST_AUTO_TEST_CASE(util_GetBoolArg) std::string error; LOCK(testArgs.cs_args); testArgs.SetupArgs({a, b, c, d, e, f}); - BOOST_CHECK(testArgs.ParseParameters(7, (char**)argv_test, error)); + BOOST_CHECK(testArgs.ParseParameters(7, argv_test, error)); // Each letter should be set. for (const char opt : "abcdef") @@ -368,7 +368,7 @@ BOOST_AUTO_TEST_CASE(util_GetBoolArgEdgeCases) const char *argv_test[] = {"ignored", "-nofoo", "-foo", "-nobar=0"}; testArgs.SetupArgs({foo, bar}); std::string error; - BOOST_CHECK(testArgs.ParseParameters(4, (char**)argv_test, error)); + BOOST_CHECK(testArgs.ParseParameters(4, argv_test, error)); // This was passed twice, second one overrides the negative setting. BOOST_CHECK(!testArgs.IsArgNegated("-foo")); @@ -380,7 +380,7 @@ BOOST_AUTO_TEST_CASE(util_GetBoolArgEdgeCases) // Config test const char *conf_test = "nofoo=1\nfoo=1\nnobar=0\n"; - BOOST_CHECK(testArgs.ParseParameters(1, (char**)argv_test, error)); + BOOST_CHECK(testArgs.ParseParameters(1, argv_test, error)); testArgs.ReadConfigString(conf_test); // This was passed twice, second one overrides the negative setting, @@ -395,7 +395,7 @@ BOOST_AUTO_TEST_CASE(util_GetBoolArgEdgeCases) // Combined test const char *combo_test_args[] = {"ignored", "-nofoo", "-bar"}; const char *combo_test_conf = "foo=1\nnobar=1\n"; - BOOST_CHECK(testArgs.ParseParameters(3, (char**)combo_test_args, error)); + BOOST_CHECK(testArgs.ParseParameters(3, combo_test_args, error)); testArgs.ReadConfigString(combo_test_conf); // Command line overrides, but doesn't erase old setting @@ -655,38 +655,38 @@ BOOST_AUTO_TEST_CASE(util_GetChainTypeString) const char* testnetconf = "testnet=1\nregtest=0\n[test]\nregtest=1"; std::string error; - BOOST_CHECK(test_args.ParseParameters(0, (char**)argv_testnet, error)); + BOOST_CHECK(test_args.ParseParameters(0, argv_testnet, error)); BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "main"); - BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_testnet, error)); + BOOST_CHECK(test_args.ParseParameters(2, argv_testnet, error)); BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); - BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_regtest, error)); + BOOST_CHECK(test_args.ParseParameters(2, argv_regtest, error)); BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "regtest"); - BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_test_no_reg, error)); + BOOST_CHECK(test_args.ParseParameters(3, argv_test_no_reg, error)); BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); - BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_both, error)); + BOOST_CHECK(test_args.ParseParameters(3, argv_both, error)); BOOST_CHECK_THROW(test_args.GetChainTypeString(), std::runtime_error); - BOOST_CHECK(test_args.ParseParameters(0, (char**)argv_testnet, error)); + BOOST_CHECK(test_args.ParseParameters(0, argv_testnet, error)); test_args.ReadConfigString(testnetconf); BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); - BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_testnet, error)); + BOOST_CHECK(test_args.ParseParameters(2, argv_testnet, error)); test_args.ReadConfigString(testnetconf); BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); - BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_regtest, error)); + BOOST_CHECK(test_args.ParseParameters(2, argv_regtest, error)); test_args.ReadConfigString(testnetconf); BOOST_CHECK_THROW(test_args.GetChainTypeString(), std::runtime_error); - BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_test_no_reg, error)); + BOOST_CHECK(test_args.ParseParameters(3, argv_test_no_reg, error)); test_args.ReadConfigString(testnetconf); BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); - BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_both, error)); + BOOST_CHECK(test_args.ParseParameters(3, argv_both, error)); test_args.ReadConfigString(testnetconf); BOOST_CHECK_THROW(test_args.GetChainTypeString(), std::runtime_error); @@ -694,23 +694,23 @@ BOOST_AUTO_TEST_CASE(util_GetChainTypeString) // [test] regtest=1 potentially relevant) doesn't break things test_args.SelectConfigNetwork("test"); - BOOST_CHECK(test_args.ParseParameters(0, (char**)argv_testnet, error)); + BOOST_CHECK(test_args.ParseParameters(0, argv_testnet, error)); test_args.ReadConfigString(testnetconf); BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); - BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_testnet, error)); + BOOST_CHECK(test_args.ParseParameters(2, argv_testnet, error)); test_args.ReadConfigString(testnetconf); BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); - BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_regtest, error)); + BOOST_CHECK(test_args.ParseParameters(2, argv_regtest, error)); test_args.ReadConfigString(testnetconf); BOOST_CHECK_THROW(test_args.GetChainTypeString(), std::runtime_error); - BOOST_CHECK(test_args.ParseParameters(2, (char**)argv_test_no_reg, error)); + BOOST_CHECK(test_args.ParseParameters(2, argv_test_no_reg, error)); test_args.ReadConfigString(testnetconf); BOOST_CHECK_EQUAL(test_args.GetChainTypeString(), "test"); - BOOST_CHECK(test_args.ParseParameters(3, (char**)argv_both, error)); + BOOST_CHECK(test_args.ParseParameters(3, argv_both, error)); test_args.ReadConfigString(testnetconf); BOOST_CHECK_THROW(test_args.GetChainTypeString(), std::runtime_error); } diff --git a/src/test/base58_tests.cpp b/src/test/base58_tests.cpp index 7f3ca6bf93..bb3defb93e 100644 --- a/src/test/base58_tests.cpp +++ b/src/test/base58_tests.cpp @@ -23,7 +23,7 @@ BOOST_FIXTURE_TEST_SUITE(base58_tests, BasicTestingSetup) // Goal: test low-level base58 encoding functionality BOOST_AUTO_TEST_CASE(base58_EncodeBase58) { - UniValue tests = read_json(std::string(json_tests::base58_encode_decode, json_tests::base58_encode_decode + sizeof(json_tests::base58_encode_decode))); + UniValue tests = read_json(json_tests::base58_encode_decode); for (unsigned int idx = 0; idx < tests.size(); idx++) { const UniValue& test = tests[idx]; std::string strTest = test.write(); @@ -43,7 +43,7 @@ BOOST_AUTO_TEST_CASE(base58_EncodeBase58) // Goal: test low-level base58 decoding functionality BOOST_AUTO_TEST_CASE(base58_DecodeBase58) { - UniValue tests = read_json(std::string(json_tests::base58_encode_decode, json_tests::base58_encode_decode + sizeof(json_tests::base58_encode_decode))); + UniValue tests = read_json(json_tests::base58_encode_decode); std::vector<unsigned char> result; for (unsigned int idx = 0; idx < tests.size(); idx++) { diff --git a/src/test/blockfilter_tests.cpp b/src/test/blockfilter_tests.cpp index 9388b4c96a..dfeac6ca42 100644 --- a/src/test/blockfilter_tests.cpp +++ b/src/test/blockfilter_tests.cpp @@ -128,9 +128,7 @@ BOOST_AUTO_TEST_CASE(blockfilter_basic_test) BOOST_AUTO_TEST_CASE(blockfilters_json_test) { UniValue json; - std::string json_data(json_tests::blockfilters, - json_tests::blockfilters + sizeof(json_tests::blockfilters)); - if (!json.read(json_data) || !json.isArray()) { + if (!json.read(json_tests::blockfilters) || !json.isArray()) { BOOST_ERROR("Parse error."); return; } diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp index e4ed861b12..f52c692649 100644 --- a/src/test/blockmanager_tests.cpp +++ b/src/test/blockmanager_tests.cpp @@ -30,22 +30,21 @@ BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos) .notifications = notifications, }; BlockManager blockman{m_node.kernel->interrupt, blockman_opts}; - CChain chain {}; // simulate adding a genesis block normally - BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, nullptr).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); + BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, nullptr).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); // simulate what happens during reindex // simulate a well-formed genesis block being found at offset 8 in the blk00000.dat file // the block is found at offset 8 because there is an 8 byte serialization header // consisting of 4 magic bytes + 4 length bytes before each block in a well-formed blk file. FlatFilePos pos{0, BLOCK_SERIALIZATION_HEADER_SIZE}; - BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, chain, &pos).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); + BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, &pos).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); // now simulate what happens after reindex for the first new block processed // the actual block contents don't matter, just that it's a block. // verify that the write position is at offset 0x12d. // this is a check to make sure that https://github.com/bitcoin/bitcoin/issues/21379 does not recur // 8 bytes (for serialization header) + 285 (for serialized genesis block) = 293 // add another 8 bytes for the second block's serialization header and we get 293 + 8 = 301 - FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1, chain, nullptr)}; + FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1, nullptr)}; BOOST_CHECK_EQUAL(actual.nPos, BLOCK_SERIALIZATION_HEADER_SIZE + ::GetSerializeSize(params->GenesisBlock(), CLIENT_VERSION) + BLOCK_SERIALIZATION_HEADER_SIZE); } diff --git a/src/test/coinstatsindex_tests.cpp b/src/test/coinstatsindex_tests.cpp index 74d6d7231a..787a196a0c 100644 --- a/src/test/coinstatsindex_tests.cpp +++ b/src/test/coinstatsindex_tests.cpp @@ -98,7 +98,7 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_unclean_shutdown, TestChain100Setup) LOCK(cs_main); BlockValidationState state; BOOST_CHECK(CheckBlock(block, state, params.GetConsensus())); - BOOST_CHECK(chainstate.AcceptBlock(new_block, state, &new_block_index, true, nullptr, nullptr, true)); + BOOST_CHECK(m_node.chainman->AcceptBlock(new_block, state, &new_block_index, true, nullptr, nullptr, true)); CCoinsViewCache view(&chainstate.CoinsTip()); BOOST_CHECK(chainstate.ConnectBlock(block, state, new_block_index, view)); } diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 13349329ff..9193d9a8b3 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -132,8 +132,7 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) { NodeId id{0}; auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman); - auto peerLogic = PeerManager::make(*connman, *m_node.addrman, nullptr, - *m_node.chainman, *m_node.mempool, false); + auto peerLogic = PeerManager::make(*connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, {}); constexpr int max_outbound_full_relay = MAX_OUTBOUND_FULL_RELAY_CONNECTIONS; CConnman::Options options; @@ -209,8 +208,7 @@ BOOST_AUTO_TEST_CASE(block_relay_only_eviction) { NodeId id{0}; auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman); - auto peerLogic = PeerManager::make(*connman, *m_node.addrman, nullptr, - *m_node.chainman, *m_node.mempool, false); + auto peerLogic = PeerManager::make(*connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, {}); constexpr int max_outbound_block_relay{MAX_BLOCK_RELAY_ONLY_CONNECTIONS}; constexpr int64_t MINIMUM_CONNECT_TIME{30}; @@ -273,8 +271,7 @@ BOOST_AUTO_TEST_CASE(peer_discouragement) auto banman = std::make_unique<BanMan>(m_args.GetDataDirBase() / "banlist", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman); - auto peerLogic = PeerManager::make(*connman, *m_node.addrman, banman.get(), - *m_node.chainman, *m_node.mempool, false); + auto peerLogic = PeerManager::make(*connman, *m_node.addrman, banman.get(), *m_node.chainman, *m_node.mempool, {}); CNetAddr tor_netaddr; BOOST_REQUIRE( @@ -376,8 +373,7 @@ BOOST_AUTO_TEST_CASE(DoS_bantime) auto banman = std::make_unique<BanMan>(m_args.GetDataDirBase() / "banlist", nullptr, DEFAULT_MISBEHAVING_BANTIME); auto connman = std::make_unique<CConnman>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman); - auto peerLogic = PeerManager::make(*connman, *m_node.addrman, banman.get(), - *m_node.chainman, *m_node.mempool, false); + auto peerLogic = PeerManager::make(*connman, *m_node.addrman, banman.get(), *m_node.chainman, *m_node.mempool, {}); banman->ClearBanned(); int64_t nStartTime = GetTime(); diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index 23fadd8984..cdf240dc59 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -32,7 +32,7 @@ FUZZ_TARGET(connman, .init = initialize_connman) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; SetMockTime(ConsumeTime(fuzzed_data_provider)); - CConnman connman{fuzzed_data_provider.ConsumeIntegral<uint64_t>(), + ConnmanTestMsg connman{fuzzed_data_provider.ConsumeIntegral<uint64_t>(), fuzzed_data_provider.ConsumeIntegral<uint64_t>(), *g_setup->m_node.addrman, *g_setup->m_node.netgroupman, @@ -41,6 +41,12 @@ FUZZ_TARGET(connman, .init = initialize_connman) CNode random_node = ConsumeNode(fuzzed_data_provider); CSubNet random_subnet; std::string random_string; + + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) { + CNode& p2p_node{*ConsumeNodeAsUniquePtr(fuzzed_data_provider).release()}; + connman.AddTestNode(p2p_node); + } + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, @@ -128,4 +134,6 @@ FUZZ_TARGET(connman, .init = initialize_connman) (void)connman.GetTotalBytesSent(); (void)connman.GetTryNewOutboundPeer(); (void)connman.GetUseAddrmanOutgoing(); + + connman.ClearTestNodes(); } diff --git a/src/test/fuzz/descriptor_parse.cpp b/src/test/fuzz/descriptor_parse.cpp index 579942c3b5..8ed659323c 100644 --- a/src/test/fuzz/descriptor_parse.cpp +++ b/src/test/fuzz/descriptor_parse.cpp @@ -3,17 +3,160 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> +#include <key_io.h> #include <pubkey.h> #include <script/descriptor.h> #include <test/fuzz/fuzz.h> #include <util/chaintype.h> +//! Types are raw (un)compressed pubkeys, raw xonly pubkeys, raw privkeys (WIF), xpubs, xprvs. +static constexpr uint8_t KEY_TYPES_COUNT{6}; +//! How many keys we'll generate in total. +static constexpr size_t TOTAL_KEYS_GENERATED{std::numeric_limits<uint8_t>::max() + 1}; + +/** + * Converts a mocked descriptor string to a valid one. Every key in a mocked descriptor key is + * represented by 2 hex characters preceded by the '%' character. We parse the two hex characters + * as an index in a list of pre-generated keys. This list contains keys of the various types + * accepted in descriptor keys expressions. + */ +class MockedDescriptorConverter { + //! 256 keys of various types. + std::array<std::string, TOTAL_KEYS_GENERATED> keys_str; + +public: + // We derive the type of key to generate from the 1-byte id parsed from hex. + bool IdIsCompPubKey(uint8_t idx) const { return idx % KEY_TYPES_COUNT == 0; } + bool IdIsUnCompPubKey(uint8_t idx) const { return idx % KEY_TYPES_COUNT == 1; } + bool IdIsXOnlyPubKey(uint8_t idx) const { return idx % KEY_TYPES_COUNT == 2; } + bool IdIsConstPrivKey(uint8_t idx) const { return idx % KEY_TYPES_COUNT == 3; } + bool IdIsXpub(uint8_t idx) const { return idx % KEY_TYPES_COUNT == 4; } + bool IdIsXprv(uint8_t idx) const { return idx % KEY_TYPES_COUNT == 5; } + + //! When initializing the target, populate the list of keys. + void Init() { + // The data to use as a private key or a seed for an xprv. + std::array<std::byte, 32> key_data{std::byte{1}}; + // Generate keys of all kinds and store them in the keys array. + for (size_t i{0}; i < TOTAL_KEYS_GENERATED; i++) { + key_data[31] = std::byte(i); + + // If this is a "raw" key, generate a normal privkey. Otherwise generate + // an extended one. + if (IdIsCompPubKey(i) || IdIsUnCompPubKey(i) || IdIsXOnlyPubKey(i) || IdIsConstPrivKey(i)) { + CKey privkey; + privkey.Set(UCharCast(key_data.begin()), UCharCast(key_data.end()), !IdIsUnCompPubKey(i)); + if (IdIsCompPubKey(i) || IdIsUnCompPubKey(i)) { + CPubKey pubkey{privkey.GetPubKey()}; + keys_str[i] = HexStr(pubkey); + } else if (IdIsXOnlyPubKey(i)) { + const XOnlyPubKey pubkey{privkey.GetPubKey()}; + keys_str[i] = HexStr(pubkey); + } else { + keys_str[i] = EncodeSecret(privkey); + } + } else { + CExtKey ext_privkey; + ext_privkey.SetSeed(key_data); + if (IdIsXprv(i)) { + keys_str[i] = EncodeExtKey(ext_privkey); + } else { + const CExtPubKey ext_pubkey{ext_privkey.Neuter()}; + keys_str[i] = EncodeExtPubKey(ext_pubkey); + } + } + } + } + + //! Parse an id in the keys vectors from a 2-characters hex string. + std::optional<uint8_t> IdxFromHex(std::string_view hex_characters) const { + if (hex_characters.size() != 2) return {}; + auto idx = ParseHex(hex_characters); + if (idx.size() != 1) return {}; + return idx[0]; + } + + //! Get an actual descriptor string from a descriptor string whose keys were mocked. + std::optional<std::string> GetDescriptor(std::string_view mocked_desc) const { + // The smallest fragment would be "pk(%00)" + if (mocked_desc.size() < 7) return {}; + + // The actual descriptor string to be returned. + std::string desc; + desc.reserve(mocked_desc.size()); + + // Replace all occurrences of '%' followed by two hex characters with the corresponding key. + for (size_t i = 0; i < mocked_desc.size();) { + if (mocked_desc[i] == '%') { + if (i + 3 >= mocked_desc.size()) return {}; + if (const auto idx = IdxFromHex(mocked_desc.substr(i + 1, 2))) { + desc += keys_str[*idx]; + i += 3; + } else { + return {}; + } + } else { + desc += mocked_desc[i++]; + } + } + + return desc; + } +}; + +//! The converter of mocked descriptors, needs to be initialized when the target is. +MockedDescriptorConverter MOCKED_DESC_CONVERTER; + +/** Test a successfully parsed descriptor. */ +static void TestDescriptor(const Descriptor& desc, FlatSigningProvider& sig_provider, std::string& dummy) +{ + // Trivial helpers. + (void)desc.IsRange(); + (void)desc.IsSolvable(); + (void)desc.IsSingleType(); + (void)desc.GetOutputType(); + + // Serialization to string representation. + (void)desc.ToString(); + (void)desc.ToPrivateString(sig_provider, dummy); + (void)desc.ToNormalizedString(sig_provider, dummy); + + // Serialization to Script. + DescriptorCache cache; + std::vector<CScript> out_scripts; + (void)desc.Expand(0, sig_provider, out_scripts, sig_provider, &cache); + (void)desc.ExpandPrivate(0, sig_provider, sig_provider); + (void)desc.ExpandFromCache(0, cache, out_scripts, sig_provider); + + // If we could serialize to script we must be able to infer using the same provider. + if (!out_scripts.empty()) { + assert(InferDescriptor(out_scripts.back(), sig_provider)); + } +} + void initialize_descriptor_parse() { ECC_Start(); SelectParams(ChainType::MAIN); } +void initialize_mocked_descriptor_parse() +{ + initialize_descriptor_parse(); + MOCKED_DESC_CONVERTER.Init(); +} + +FUZZ_TARGET(mocked_descriptor_parse, .init = initialize_mocked_descriptor_parse) +{ + const std::string mocked_descriptor{buffer.begin(), buffer.end()}; + if (const auto descriptor = MOCKED_DESC_CONVERTER.GetDescriptor(mocked_descriptor)) { + FlatSigningProvider signing_provider; + std::string error; + const auto desc = Parse(*descriptor, signing_provider, error); + if (desc) TestDescriptor(*desc, signing_provider, error); + } +} + FUZZ_TARGET(descriptor_parse, .init = initialize_descriptor_parse) { const std::string descriptor(buffer.begin(), buffer.end()); @@ -21,10 +164,6 @@ FUZZ_TARGET(descriptor_parse, .init = initialize_descriptor_parse) std::string error; for (const bool require_checksum : {true, false}) { const auto desc = Parse(descriptor, signing_provider, error, require_checksum); - if (desc) { - (void)desc->ToString(); - (void)desc->IsRange(); - (void)desc->IsSolvable(); - } + if (desc) TestDescriptor(*desc, signing_provider, error); } } diff --git a/src/test/fuzz/load_external_block_file.cpp b/src/test/fuzz/load_external_block_file.cpp index 502f7b897c..bdaa4ad1b8 100644 --- a/src/test/fuzz/load_external_block_file.cpp +++ b/src/test/fuzz/load_external_block_file.cpp @@ -35,9 +35,9 @@ FUZZ_TARGET(load_external_block_file, .init = initialize_load_external_block_fil // Corresponds to the -reindex case (track orphan blocks across files). FlatFilePos flat_file_pos; std::multimap<uint256, FlatFilePos> blocks_with_unknown_parent; - g_setup->m_node.chainman->ActiveChainstate().LoadExternalBlockFile(fuzzed_block_file, &flat_file_pos, &blocks_with_unknown_parent); + g_setup->m_node.chainman->LoadExternalBlockFile(fuzzed_block_file, &flat_file_pos, &blocks_with_unknown_parent); } else { // Corresponds to the -loadblock= case (orphan blocks aren't tracked across files). - g_setup->m_node.chainman->ActiveChainstate().LoadExternalBlockFile(fuzzed_block_file); + g_setup->m_node.chainman->LoadExternalBlockFile(fuzzed_block_file); } } diff --git a/src/test/fuzz/parse_univalue.cpp b/src/test/fuzz/parse_univalue.cpp index bfa856211d..a3d6ab6375 100644 --- a/src/test/fuzz/parse_univalue.cpp +++ b/src/test/fuzz/parse_univalue.cpp @@ -3,7 +3,6 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <chainparams.h> -#include <core_io.h> #include <rpc/client.h> #include <rpc/util.h> #include <test/fuzz/fuzz.h> @@ -58,12 +57,6 @@ FUZZ_TARGET(parse_univalue, .init = initialize_parse_univalue) } catch (const UniValue&) { } try { - (void)ParseHexUV(univalue, "A"); - (void)ParseHexUV(univalue, random_string); - } catch (const UniValue&) { - } catch (const std::runtime_error&) { - } - try { (void)ParseHexV(univalue, "A"); } catch (const UniValue&) { } catch (const std::runtime_error&) { @@ -74,8 +67,8 @@ FUZZ_TARGET(parse_univalue, .init = initialize_parse_univalue) } catch (const std::runtime_error&) { } try { - (void)ParseSighashString(univalue); - } catch (const std::runtime_error&) { + if (univalue.isNull() || univalue.isStr()) (void)ParseSighashString(univalue); + } catch (const UniValue&) { } try { (void)AmountFromValue(univalue); diff --git a/src/test/key_io_tests.cpp b/src/test/key_io_tests.cpp index 92bdbae3c4..66b4e09ebf 100644 --- a/src/test/key_io_tests.cpp +++ b/src/test/key_io_tests.cpp @@ -22,7 +22,7 @@ BOOST_FIXTURE_TEST_SUITE(key_io_tests, BasicTestingSetup) // Goal: check that parsed keys match test payload BOOST_AUTO_TEST_CASE(key_io_valid_parse) { - UniValue tests = read_json(std::string(json_tests::key_io_valid, json_tests::key_io_valid + sizeof(json_tests::key_io_valid))); + UniValue tests = read_json(json_tests::key_io_valid); CKey privkey; CTxDestination destination; SelectParams(ChainType::MAIN); @@ -83,7 +83,7 @@ BOOST_AUTO_TEST_CASE(key_io_valid_parse) // Goal: check that generated keys match test vectors BOOST_AUTO_TEST_CASE(key_io_valid_gen) { - UniValue tests = read_json(std::string(json_tests::key_io_valid, json_tests::key_io_valid + sizeof(json_tests::key_io_valid))); + UniValue tests = read_json(json_tests::key_io_valid); for (unsigned int idx = 0; idx < tests.size(); idx++) { const UniValue& test = tests[idx]; @@ -121,7 +121,7 @@ BOOST_AUTO_TEST_CASE(key_io_valid_gen) // Goal: check that base58 parsing code is robust against a variety of corrupted data BOOST_AUTO_TEST_CASE(key_io_invalid) { - UniValue tests = read_json(std::string(json_tests::key_io_invalid, json_tests::key_io_invalid + sizeof(json_tests::key_io_invalid))); // Negative testcases + UniValue tests = read_json(json_tests::key_io_invalid); // Negative testcases CKey privkey; CTxDestination destination; diff --git a/src/test/rpc_tests.cpp b/src/test/rpc_tests.cpp index 2f783a4b95..4cad3ec68e 100644 --- a/src/test/rpc_tests.cpp +++ b/src/test/rpc_tests.cpp @@ -20,7 +20,7 @@ static UniValue JSON(std::string_view json) { UniValue value; - BOOST_CHECK(value.read(json.data(), json.size())); + BOOST_CHECK(value.read(json)); return value; } diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index 7bebadf224..884e3d0634 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -394,7 +394,7 @@ BOOST_AUTO_TEST_CASE(bip341_spk_test_vectors) using control_set = decltype(TaprootSpendData::scripts)::mapped_type; UniValue tests; - tests.read((const char*)json_tests::bip341_wallet_vectors, sizeof(json_tests::bip341_wallet_vectors)); + tests.read(json_tests::bip341_wallet_vectors); const auto& vectors = tests["scriptPubKey"]; diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index c89f2c1f5b..411924496c 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -890,7 +890,7 @@ BOOST_AUTO_TEST_CASE(script_build) std::set<std::string> tests_set; { - UniValue json_tests = read_json(std::string(json_tests::script_tests, json_tests::script_tests + sizeof(json_tests::script_tests))); + UniValue json_tests = read_json(json_tests::script_tests); for (unsigned int idx = 0; idx < json_tests.size(); idx++) { const UniValue& tv = json_tests[idx]; @@ -929,7 +929,7 @@ BOOST_AUTO_TEST_CASE(script_json_test) // scripts. // If a witness is given, then the last value in the array should be the // amount (nValue) to use in the crediting tx - UniValue tests = read_json(std::string(json_tests::script_tests, json_tests::script_tests + sizeof(json_tests::script_tests))); + UniValue tests = read_json(json_tests::script_tests); for (unsigned int idx = 0; idx < tests.size(); idx++) { const UniValue& test = tests[idx]; @@ -1743,7 +1743,7 @@ BOOST_AUTO_TEST_CASE(script_assets_test) BOOST_AUTO_TEST_CASE(bip341_keypath_test_vectors) { UniValue tests; - tests.read((const char*)json_tests::bip341_wallet_vectors, sizeof(json_tests::bip341_wallet_vectors)); + tests.read(json_tests::bip341_wallet_vectors); const auto& vectors = tests["keyPathSpending"]; diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 68ef719c71..d1c0e1349e 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -162,7 +162,7 @@ BOOST_AUTO_TEST_CASE(sighash_test) // Goal: check that SignatureHash generates correct hash BOOST_AUTO_TEST_CASE(sighash_from_data) { - UniValue tests = read_json(std::string(json_tests::sighash, json_tests::sighash + sizeof(json_tests::sighash))); + UniValue tests = read_json(json_tests::sighash); for (unsigned int idx = 0; idx < tests.size(); idx++) { const UniValue& test = tests[idx]; diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index 55e4f200b1..5232175824 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -6,6 +6,7 @@ #include <test/util/random.h> #include <test/util/setup_common.h> #include <util/fs.h> +#include <util/strencodings.h> #include <boost/test/unit_test.hpp> @@ -13,6 +14,55 @@ using namespace std::string_literals; BOOST_FIXTURE_TEST_SUITE(streams_tests, BasicTestingSetup) +BOOST_AUTO_TEST_CASE(xor_file) +{ + fs::path xor_path{m_args.GetDataDirBase() / "test_xor.bin"}; + auto raw_file{[&](const auto& mode) { return fsbridge::fopen(xor_path, mode); }}; + const std::vector<uint8_t> test1{1, 2, 3}; + const std::vector<uint8_t> test2{4, 5}; + const std::vector<std::byte> xor_pat{std::byte{0xff}, std::byte{0x00}}; + { + // Check errors for missing file + AutoFile xor_file{raw_file("rb"), xor_pat}; + BOOST_CHECK_EXCEPTION(xor_file << std::byte{}, std::ios_base::failure, HasReason{"AutoFile::write: file handle is nullpt"}); + BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: file handle is nullpt"}); + BOOST_CHECK_EXCEPTION(xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: file handle is nullpt"}); + } + { + AutoFile xor_file{raw_file("wbx"), xor_pat}; + xor_file << test1 << test2; + } + { + // Read raw from disk + AutoFile non_xor_file{raw_file("rb")}; + std::vector<std::byte> raw(7); + non_xor_file >> Span{raw}; + BOOST_CHECK_EQUAL(HexStr(raw), "fc01fd03fd04fa"); + // Check that no padding exists + BOOST_CHECK_EXCEPTION(non_xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: end of file"}); + } + { + AutoFile xor_file{raw_file("rb"), xor_pat}; + std::vector<std::byte> read1, read2; + xor_file >> read1 >> read2; + BOOST_CHECK_EQUAL(HexStr(read1), HexStr(test1)); + BOOST_CHECK_EQUAL(HexStr(read2), HexStr(test2)); + // Check that eof was reached + BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: end of file"}); + } + { + AutoFile xor_file{raw_file("rb"), xor_pat}; + std::vector<std::byte> read2; + // Check that ignore works + xor_file.ignore(4); + xor_file >> read2; + BOOST_CHECK_EQUAL(HexStr(read2), HexStr(test2)); + // Check that ignore and read fail now + BOOST_CHECK_EXCEPTION(xor_file.ignore(1), std::ios_base::failure, HasReason{"AutoFile::ignore: end of file"}); + BOOST_CHECK_EXCEPTION(xor_file >> std::byte{}, std::ios_base::failure, HasReason{"AutoFile::read: end of file"}); + } +} + BOOST_AUTO_TEST_CASE(streams_vector_writer) { unsigned char a(1); diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 7511dff9e8..34dd2c3e9f 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -190,7 +190,7 @@ BOOST_AUTO_TEST_CASE(tx_valid) { BOOST_CHECK_MESSAGE(CheckMapFlagNames(), "mapFlagNames is missing a script verification flag"); // Read tests from test/data/tx_valid.json - UniValue tests = read_json(std::string(json_tests::tx_valid, json_tests::tx_valid + sizeof(json_tests::tx_valid))); + UniValue tests = read_json(json_tests::tx_valid); for (unsigned int idx = 0; idx < tests.size(); idx++) { const UniValue& test = tests[idx]; @@ -278,7 +278,7 @@ BOOST_AUTO_TEST_CASE(tx_valid) BOOST_AUTO_TEST_CASE(tx_invalid) { // Read tests from test/data/tx_invalid.json - UniValue tests = read_json(std::string(json_tests::tx_invalid, json_tests::tx_invalid + sizeof(json_tests::tx_invalid))); + UniValue tests = read_json(json_tests::tx_invalid); for (unsigned int idx = 0; idx < tests.size(); idx++) { const UniValue& test = tests[idx]; diff --git a/src/test/util/chainstate.h b/src/test/util/chainstate.h index bf8f8b5819..9ff2c08807 100644 --- a/src/test/util/chainstate.h +++ b/src/test/util/chainstate.h @@ -71,6 +71,7 @@ CreateAndActivateUTXOSnapshot( // This is a stripped-down version of node::LoadChainstate which // preserves the block index. LOCK(::cs_main); + CBlockIndex *orig_tip = node.chainman->ActiveChainstate().m_chain.Tip(); uint256 gen_hash = node.chainman->ActiveChainstate().m_chain[0]->GetBlockHash(); node.chainman->ResetChainstates(); node.chainman->InitializeChainstate(node.mempool.get()); @@ -83,6 +84,22 @@ CreateAndActivateUTXOSnapshot( chain.setBlockIndexCandidates.insert(node.chainman->m_blockman.LookupBlockIndex(gen_hash)); chain.LoadChainTip(); node.chainman->MaybeRebalanceCaches(); + + // Reset the HAVE_DATA flags below the snapshot height, simulating + // never-having-downloaded them in the first place. + // TODO: perhaps we could improve this by using pruning to delete + // these blocks instead + CBlockIndex *pindex = orig_tip; + while (pindex && pindex != chain.m_chain.Tip()) { + pindex->nStatus &= ~BLOCK_HAVE_DATA; + pindex->nStatus &= ~BLOCK_HAVE_UNDO; + // We have to set the ASSUMED_VALID flag, because otherwise it + // would not be possible to have a block index entry without HAVE_DATA + // and with nTx > 0 (since we aren't setting the pruned flag); + // see CheckBlockIndex(). + pindex->nStatus |= BLOCK_ASSUMED_VALID; + pindex = pindex->pprev; + } } BlockValidationState state; if (!node.chainman->ActiveChainstate().ActivateBestChain(state)) { diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 6ae2187974..65c657da96 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -27,6 +27,7 @@ #include <node/kernel_notifications.h> #include <node/mempool_args.h> #include <node/miner.h> +#include <node/peerman_args.h> #include <node/validation_cache_args.h> #include <noui.h> #include <policy/fees.h> @@ -251,9 +252,12 @@ TestingSetup::TestingSetup( m_node.args->GetIntArg("-checkaddrman", 0)); m_node.banman = std::make_unique<BanMan>(m_args.GetDataDirBase() / "banlist", nullptr, DEFAULT_MISBEHAVING_BANTIME); m_node.connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman); // Deterministic randomness for tests. + PeerManager::Options peerman_opts; + ApplyArgsManOptions(*m_node.args, peerman_opts); m_node.peerman = PeerManager::make(*m_node.connman, *m_node.addrman, m_node.banman.get(), *m_node.chainman, - *m_node.mempool, false); + *m_node.mempool, peerman_opts); + { CConnman::Options options; options.m_msgproc = m_node.peerman.get(); diff --git a/src/test/validation_chainstate_tests.cpp b/src/test/validation_chainstate_tests.cpp index 2078fcd8f8..fe2d2ba592 100644 --- a/src/test/validation_chainstate_tests.cpp +++ b/src/test/validation_chainstate_tests.cpp @@ -77,6 +77,13 @@ BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) // After adding some blocks to the tip, best block should have changed. BOOST_CHECK(::g_best_block != curr_tip); + // Grab block 1 from disk; we'll add it to the background chain later. + std::shared_ptr<CBlock> pblockone = std::make_shared<CBlock>(); + { + LOCK(::cs_main); + chainman.m_blockman.ReadBlockFromDisk(*pblockone, *chainman.ActiveChain()[1]); + } + BOOST_REQUIRE(CreateAndActivateUTXOSnapshot( this, NoMalleation, /*reset_chainstate=*/ true)); @@ -104,11 +111,7 @@ BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) assert(false); }()}; - // Create a block to append to the validation chain. - std::vector<CMutableTransaction> noTxns; - CScript scriptPubKey = CScript() << ToByteVector(coinbaseKey.GetPubKey()) << OP_CHECKSIG; - CBlock validation_block = this->CreateBlock(noTxns, scriptPubKey, background_cs); - auto pblock = std::make_shared<const CBlock>(validation_block); + // Append the first block to the background chain. BlockValidationState state; CBlockIndex* pindex = nullptr; const CChainParams& chainparams = Params(); @@ -118,17 +121,18 @@ BOOST_FIXTURE_TEST_CASE(chainstate_update_tip, TestChain100Setup) // once it is changed to support multiple chainstates. { LOCK(::cs_main); - bool checked = CheckBlock(*pblock, state, chainparams.GetConsensus()); + bool checked = CheckBlock(*pblockone, state, chainparams.GetConsensus()); BOOST_CHECK(checked); - bool accepted = background_cs.AcceptBlock( - pblock, state, &pindex, true, nullptr, &newblock, true); + bool accepted = chainman.AcceptBlock( + pblockone, state, &pindex, true, nullptr, &newblock, true); BOOST_CHECK(accepted); } + // UpdateTip is called here - bool block_added = background_cs.ActivateBestChain(state, pblock); + bool block_added = background_cs.ActivateBestChain(state, pblockone); // Ensure tip is as expected - BOOST_CHECK_EQUAL(background_cs.m_chain.Tip()->GetBlockHash(), validation_block.GetHash()); + BOOST_CHECK_EQUAL(background_cs.m_chain.Tip()->GetBlockHash(), pblockone->GetHash()); // g_best_block should be unchanged after adding a block to the background // validation chain. diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 99860961a2..160a807f69 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -49,6 +49,9 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) c1.InitCoinsDB( /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); WITH_LOCK(::cs_main, c1.InitCoinsCache(1 << 23)); + c1.LoadGenesisBlock(); + BlockValidationState val_state; + BOOST_CHECK(c1.ActivateBestChain(val_state, nullptr)); BOOST_CHECK(!manager.IsSnapshotActive()); BOOST_CHECK(WITH_LOCK(::cs_main, return !manager.IsSnapshotValidated())); @@ -58,7 +61,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) auto& active_chain = WITH_LOCK(manager.GetMutex(), return manager.ActiveChain()); BOOST_CHECK_EQUAL(&active_chain, &c1.m_chain); - BOOST_CHECK_EQUAL(WITH_LOCK(manager.GetMutex(), return manager.ActiveHeight()), -1); + BOOST_CHECK_EQUAL(WITH_LOCK(manager.GetMutex(), return manager.ActiveHeight()), 0); auto active_tip = WITH_LOCK(manager.GetMutex(), return manager.ActiveTip()); auto exp_tip = c1.m_chain.Tip(); @@ -68,7 +71,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) // Create a snapshot-based chainstate. // - const uint256 snapshot_blockhash = GetRandHash(); + const uint256 snapshot_blockhash = active_tip->GetBlockHash(); Chainstate& c2 = WITH_LOCK(::cs_main, return manager.ActivateExistingSnapshot( &mempool, snapshot_blockhash)); chainstates.push_back(&c2); @@ -78,8 +81,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) c2.InitCoinsDB( /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); WITH_LOCK(::cs_main, c2.InitCoinsCache(1 << 23)); - // Unlike c1, which doesn't have any blocks. Gets us different tip, height. - c2.LoadGenesisBlock(); + c2.m_chain.SetTip(*active_tip); BlockValidationState _; BOOST_CHECK(c2.ActivateBestChain(_, nullptr)); @@ -99,16 +101,14 @@ BOOST_AUTO_TEST_CASE(chainstatemanager) auto exp_tip2 = c2.m_chain.Tip(); BOOST_CHECK_EQUAL(active_tip2, exp_tip2); - // Ensure that these pointers actually correspond to different - // CCoinsViewCache instances. - BOOST_CHECK(exp_tip != exp_tip2); + BOOST_CHECK_EQUAL(exp_tip, exp_tip2); // Let scheduler events finish running to avoid accessing memory that is going to be unloaded SyncWithValidationInterfaceQueue(); } //! Test rebalancing the caches associated with each chainstate. -BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) +BOOST_FIXTURE_TEST_CASE(chainstatemanager_rebalance_caches, TestChain100Setup) { ChainstateManager& manager = *m_node.chainman; CTxMemPool& mempool = *m_node.mempool; @@ -121,7 +121,7 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) // Create a legacy (IBD) chainstate. // - Chainstate& c1 = WITH_LOCK(::cs_main, return manager.InitializeChainstate(&mempool)); + Chainstate& c1 = manager.ActiveChainstate(); chainstates.push_back(&c1); c1.InitCoinsDB( /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); @@ -129,8 +129,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) { LOCK(::cs_main); c1.InitCoinsCache(1 << 23); - BOOST_REQUIRE(c1.LoadGenesisBlock()); - c1.CoinsTip().SetBestBlock(InsecureRand256()); manager.MaybeRebalanceCaches(); } @@ -139,7 +137,8 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) // Create a snapshot-based chainstate. // - Chainstate& c2 = WITH_LOCK(cs_main, return manager.ActivateExistingSnapshot(&mempool, GetRandHash())); + CBlockIndex* snapshot_base{WITH_LOCK(manager.GetMutex(), return manager.ActiveChain()[manager.ActiveChain().Height() / 2])}; + Chainstate& c2 = WITH_LOCK(cs_main, return manager.ActivateExistingSnapshot(&mempool, *snapshot_base->phashBlock)); chainstates.push_back(&c2); c2.InitCoinsDB( /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); @@ -147,8 +146,6 @@ BOOST_AUTO_TEST_CASE(chainstatemanager_rebalance_caches) { LOCK(::cs_main); c2.InitCoinsCache(1 << 23); - BOOST_REQUIRE(c2.LoadGenesisBlock()); - c2.CoinsTip().SetBestBlock(InsecureRand256()); manager.MaybeRebalanceCaches(); } @@ -415,7 +412,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_activate_snapshot, SnapshotTestSetup) //! - Then mark a region of the chain BLOCK_ASSUMED_VALID and introduce a second chainstate //! that will tolerate assumed-valid blocks. Run LoadBlockIndex() and ensure that the first //! chainstate only contains fully validated blocks and the other chainstate contains all blocks, -//! even those assumed-valid. +//! except those marked assume-valid, because those entries don't HAVE_DATA. //! BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) { @@ -430,28 +427,34 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) const int assumed_valid_start_idx = last_assumed_valid_idx - expected_assumed_valid; CBlockIndex* validated_tip{nullptr}; + CBlockIndex* assumed_base{nullptr}; CBlockIndex* assumed_tip{WITH_LOCK(chainman.GetMutex(), return chainman.ActiveChain().Tip())}; auto reload_all_block_indexes = [&]() { + // For completeness, we also reset the block sequence counters to + // ensure that no state which affects the ranking of tip-candidates is + // retained (even though this isn't strictly necessary). + WITH_LOCK(::cs_main, return chainman.ResetBlockSequenceCounters()); for (Chainstate* cs : chainman.GetAll()) { LOCK(::cs_main); - cs->UnloadBlockIndex(); + cs->ClearBlockIndexCandidates(); BOOST_CHECK(cs->setBlockIndexCandidates.empty()); } WITH_LOCK(::cs_main, chainman.LoadBlockIndex()); }; - // Ensure that without any assumed-valid BlockIndex entries, all entries are considered - // tip candidates. + // Ensure that without any assumed-valid BlockIndex entries, only the current tip is + // considered as a candidate. reload_all_block_indexes(); - BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.size(), cs1.m_chain.Height() + 1); + BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.size(), 1); - // Mark some region of the chain assumed-valid. + // Mark some region of the chain assumed-valid, and remove the HAVE_DATA flag. for (int i = 0; i <= cs1.m_chain.Height(); ++i) { LOCK(::cs_main); auto index = cs1.m_chain[i]; + // Blocks with heights in range [20, 40) are marked ASSUMED_VALID if (i < last_assumed_valid_idx && i >= assumed_valid_start_idx) { index->nStatus = BlockStatus::BLOCK_VALID_TREE | BlockStatus::BLOCK_ASSUMED_VALID; } @@ -464,25 +467,41 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_loadblockindex, TestChain100Setup) validated_tip = index; BOOST_CHECK(!index->IsAssumedValid()); } + // Note the last assumed valid block as the snapshot base + if (i == last_assumed_valid_idx - 1) { + assumed_base = index; + BOOST_CHECK(index->IsAssumedValid()); + } else if (i == last_assumed_valid_idx) { + BOOST_CHECK(!index->IsAssumedValid()); + } } BOOST_CHECK_EQUAL(expected_assumed_valid, num_assumed_valid); + // Note: cs2's tip is not set when ActivateExistingSnapshot is called. Chainstate& cs2 = WITH_LOCK(::cs_main, - return chainman.ActivateExistingSnapshot(&mempool, GetRandHash())); + return chainman.ActivateExistingSnapshot(&mempool, *assumed_base->phashBlock)); + + // Set tip of the fully validated chain to be the validated tip + cs1.m_chain.SetTip(*validated_tip); + + // Set tip of the assume-valid-based chain to the assume-valid block + cs2.m_chain.SetTip(*assumed_base); reload_all_block_indexes(); - // The fully validated chain only has candidates up to the start of the assumed-valid - // blocks. + // The fully validated chain should have the current validated tip + // and the assumed valid base as candidates. + BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.size(), 2); BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.count(validated_tip), 1); - BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.count(assumed_tip), 0); - BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.size(), assumed_valid_start_idx); + BOOST_CHECK_EQUAL(cs1.setBlockIndexCandidates.count(assumed_base), 1); - // The assumed-valid tolerant chain has all blocks as candidates. - BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.count(validated_tip), 1); + // The assumed-valid tolerant chain has the assumed valid base as a + // candidate, but otherwise has none of the assumed-valid (which do not + // HAVE_DATA) blocks as candidates. + BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.count(validated_tip), 0); BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.count(assumed_tip), 1); - BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.size(), num_indexes); + BOOST_CHECK_EQUAL(cs2.setBlockIndexCandidates.size(), num_indexes - last_assumed_valid_idx + 1); } //! Ensure that snapshot chainstates initialize properly when found on disk. diff --git a/src/test/validationinterface_tests.cpp b/src/test/validationinterface_tests.cpp index 576768a625..fcd0b25b38 100644 --- a/src/test/validationinterface_tests.cpp +++ b/src/test/validationinterface_tests.cpp @@ -12,7 +12,7 @@ #include <atomic> -BOOST_FIXTURE_TEST_SUITE(validationinterface_tests, TestingSetup) +BOOST_FIXTURE_TEST_SUITE(validationinterface_tests, ChainTestingSetup) struct TestSubscriberNoop final : public CValidationInterface { void BlockChecked(const CBlock&, const BlockValidationState&) override {} diff --git a/src/univalue/include/univalue.h b/src/univalue/include/univalue.h index 94f80f9c27..acbce25741 100644 --- a/src/univalue/include/univalue.h +++ b/src/univalue/include/univalue.h @@ -7,13 +7,15 @@ #define BITCOIN_UNIVALUE_INCLUDE_UNIVALUE_H #include <charconv> +#include <cstddef> #include <cstdint> -#include <cstring> #include <map> #include <stdexcept> #include <string> #include <string_view> +#include <system_error> #include <type_traits> +#include <utility> #include <vector> class UniValue { @@ -94,9 +96,7 @@ public: std::string write(unsigned int prettyIndent = 0, unsigned int indentLevel = 0) const; - bool read(const char *raw, size_t len); - bool read(const char *raw) { return read(raw, strlen(raw)); } - bool read(std::string_view raw) { return read(raw.data(), raw.size()); } + bool read(std::string_view raw); private: UniValue::VType typ; diff --git a/src/univalue/lib/univalue_read.cpp b/src/univalue/lib/univalue_read.cpp index 2f2385383c..40d465f438 100644 --- a/src/univalue/lib/univalue_read.cpp +++ b/src/univalue/lib/univalue_read.cpp @@ -5,10 +5,11 @@ #include <univalue.h> #include <univalue_utffilter.h> -#include <cstdio> #include <cstdint> +#include <cstdio> #include <cstring> #include <string> +#include <string_view> #include <vector> /* @@ -259,7 +260,7 @@ enum expect_bits : unsigned { #define setExpect(bit) (expectMask |= EXP_##bit) #define clearExpect(bit) (expectMask &= ~EXP_##bit) -bool UniValue::read(const char *raw, size_t size) +bool UniValue::read(std::string_view str_in) { clear(); @@ -270,7 +271,8 @@ bool UniValue::read(const char *raw, size_t size) unsigned int consumed; enum jtokentype tok = JTOK_NONE; enum jtokentype last_tok = JTOK_NONE; - const char* end = raw + size; + const char* raw{str_in.data()}; + const char* end{raw + str_in.size()}; do { last_tok = tok; diff --git a/src/univalue/test/unitester.cpp b/src/univalue/test/unitester.cpp index 6344a5a0ab..d27e647e5c 100644 --- a/src/univalue/test/unitester.cpp +++ b/src/univalue/test/unitester.cpp @@ -153,7 +153,7 @@ void no_nul_test() { char buf[] = "___[1,2,3]___"; UniValue val; - assert(val.read(buf + 3, 7)); + assert(val.read({buf + 3, 7})); } int main (int argc, char *argv[]) diff --git a/src/validation.cpp b/src/validation.cpp index e6def01db5..cd6654abe4 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -1579,6 +1579,13 @@ Chainstate::Chainstate( m_chainman(chainman), m_from_snapshot_blockhash(from_snapshot_blockhash) {} +const CBlockIndex* Chainstate::SnapshotBase() +{ + if (!m_from_snapshot_blockhash) return nullptr; + if (!m_cached_snapshot_base) m_cached_snapshot_base = Assert(m_chainman.m_blockman.LookupBlockIndex(*m_from_snapshot_blockhash)); + return m_cached_snapshot_base; +} + void Chainstate::InitCoinsDB( size_t cache_size_bytes, bool in_memory, @@ -3193,7 +3200,8 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< // that the best block hash is non-null. if (m_chainman.m_interrupt) break; } while (pindexNewTip != pindexMostWork); - CheckBlockIndex(); + + m_chainman.CheckBlockIndex(); // Write changes periodically to disk, after relay. if (!FlushStateToDisk(state, FlushStateMode::PERIODIC)) { @@ -3213,17 +3221,17 @@ bool Chainstate::PreciousBlock(BlockValidationState& state, CBlockIndex* pindex) // Nothing to do, this block is not at the tip. return true; } - if (m_chain.Tip()->nChainWork > nLastPreciousChainwork) { + if (m_chain.Tip()->nChainWork > m_chainman.nLastPreciousChainwork) { // The chain has been extended since the last call, reset the counter. - nBlockReverseSequenceId = -1; + m_chainman.nBlockReverseSequenceId = -1; } - nLastPreciousChainwork = m_chain.Tip()->nChainWork; + m_chainman.nLastPreciousChainwork = m_chain.Tip()->nChainWork; setBlockIndexCandidates.erase(pindex); - pindex->nSequenceId = nBlockReverseSequenceId; - if (nBlockReverseSequenceId > std::numeric_limits<int32_t>::min()) { + pindex->nSequenceId = m_chainman.nBlockReverseSequenceId; + if (m_chainman.nBlockReverseSequenceId > std::numeric_limits<int32_t>::min()) { // We can't keep reducing the counter if somebody really wants to // call preciousblock 2**31-1 times on the same set of tips... - nBlockReverseSequenceId--; + m_chainman.nBlockReverseSequenceId--; } if (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && pindex->HaveTxsDownloaded()) { setBlockIndexCandidates.insert(pindex); @@ -3339,7 +3347,7 @@ bool Chainstate::InvalidateBlock(BlockValidationState& state, CBlockIndex* pinde to_mark_failed = invalid_walk_tip; } - CheckBlockIndex(); + m_chainman.CheckBlockIndex(); { LOCK(cs_main); @@ -3416,8 +3424,32 @@ void Chainstate::ResetBlockFailureFlags(CBlockIndex *pindex) { } } +void Chainstate::TryAddBlockIndexCandidate(CBlockIndex* pindex) +{ + AssertLockHeld(cs_main); + // The block only is a candidate for the most-work-chain if it has more work than our current tip. + if (m_chain.Tip() != nullptr && setBlockIndexCandidates.value_comp()(pindex, m_chain.Tip())) { + return; + } + + bool is_active_chainstate = this == &m_chainman.ActiveChainstate(); + if (is_active_chainstate) { + // The active chainstate should always add entries that have more + // work than the tip. + setBlockIndexCandidates.insert(pindex); + } else if (!m_disabled) { + // For the background chainstate, we only consider connecting blocks + // towards the snapshot base (which can't be nullptr or else we'll + // never make progress). + const CBlockIndex* snapshot_base{Assert(m_chainman.GetSnapshotBaseBlock())}; + if (snapshot_base->GetAncestor(pindex->nHeight) == pindex) { + setBlockIndexCandidates.insert(pindex); + } + } +} + /** Mark a block as having its data received and checked (up to BLOCK_VALID_TRANSACTIONS). */ -void Chainstate::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos) +void ChainstateManager::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos) { AssertLockHeld(cs_main); pindexNew->nTx = block.vtx.size(); @@ -3426,7 +3458,7 @@ void Chainstate::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pin pindexNew->nDataPos = pos.nPos; pindexNew->nUndoPos = 0; pindexNew->nStatus |= BLOCK_HAVE_DATA; - if (DeploymentActiveAt(*pindexNew, m_chainman, Consensus::DEPLOYMENT_SEGWIT)) { + if (DeploymentActiveAt(*pindexNew, *this, Consensus::DEPLOYMENT_SEGWIT)) { pindexNew->nStatus |= BLOCK_OPT_WITNESS; } pindexNew->RaiseValidity(BLOCK_VALID_TRANSACTIONS); @@ -3443,8 +3475,8 @@ void Chainstate::ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pin queue.pop_front(); pindex->nChainTx = (pindex->pprev ? pindex->pprev->nChainTx : 0) + pindex->nTx; pindex->nSequenceId = nBlockSequenceId++; - if (m_chain.Tip() == nullptr || !setBlockIndexCandidates.value_comp()(pindex, m_chain.Tip())) { - setBlockIndexCandidates.insert(pindex); + for (Chainstate *c : GetAll()) { + c->TryAddBlockIndexCandidate(pindex); } std::pair<std::multimap<CBlockIndex*, CBlockIndex*>::iterator, std::multimap<CBlockIndex*, CBlockIndex*>::iterator> range = m_blockman.m_blocks_unlinked.equal_range(pindex); while (range.first != range.second) { @@ -3858,7 +3890,7 @@ bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& for (const CBlockHeader& header : headers) { CBlockIndex *pindex = nullptr; // Use a temp pindex instead of ppindex to avoid a const_cast bool accepted{AcceptBlockHeader(header, state, &pindex, min_pow_checked)}; - ActiveChainstate().CheckBlockIndex(); + CheckBlockIndex(); if (!accepted) { return false; @@ -3905,7 +3937,7 @@ void ChainstateManager::ReportHeadersPresync(const arith_uint256& work, int64_t } /** Store block on disk. If dbp is non-nullptr, the file is known to already reside on disk */ -bool Chainstate::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock, bool min_pow_checked) +bool ChainstateManager::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock, bool min_pow_checked) { const CBlock& block = *pblock; @@ -3915,23 +3947,24 @@ bool Chainstate::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockV CBlockIndex *pindexDummy = nullptr; CBlockIndex *&pindex = ppindex ? *ppindex : pindexDummy; - bool accepted_header{m_chainman.AcceptBlockHeader(block, state, &pindex, min_pow_checked)}; + bool accepted_header{AcceptBlockHeader(block, state, &pindex, min_pow_checked)}; CheckBlockIndex(); if (!accepted_header) return false; - // Try to process all requested blocks that we don't have, but only - // process an unrequested block if it's new and has enough work to - // advance our tip, and isn't too many blocks ahead. + // Check all requested blocks that we do not already have for validity and + // save them to disk. Skip processing of unrequested blocks as an anti-DoS + // measure, unless the blocks have more work than the active chain tip, and + // aren't too far ahead of it, so are likely to be attached soon. bool fAlreadyHave = pindex->nStatus & BLOCK_HAVE_DATA; - bool fHasMoreOrSameWork = (m_chain.Tip() ? pindex->nChainWork >= m_chain.Tip()->nChainWork : true); + bool fHasMoreOrSameWork = (ActiveTip() ? pindex->nChainWork >= ActiveTip()->nChainWork : true); // Blocks that are too out-of-order needlessly limit the effectiveness of // pruning, because pruning will not delete block files that contain any // blocks which are too close in height to the tip. Apply this test // regardless of whether pruning is enabled; it should generally be safe to // not process unrequested blocks. - bool fTooFarAhead{pindex->nHeight > m_chain.Height() + int(MIN_BLOCKS_TO_KEEP)}; + bool fTooFarAhead{pindex->nHeight > ActiveHeight() + int(MIN_BLOCKS_TO_KEEP)}; // TODO: Decouple this function from the block download logic by removing fRequested // This requires some new chain data structure to efficiently look up if a @@ -3951,13 +3984,13 @@ bool Chainstate::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockV // If our tip is behind, a peer could try to send us // low-work blocks on a fake chain that we would never // request; don't process these. - if (pindex->nChainWork < m_chainman.MinimumChainWork()) return true; + if (pindex->nChainWork < MinimumChainWork()) return true; } - const CChainParams& params{m_chainman.GetParams()}; + const CChainParams& params{GetParams()}; if (!CheckBlock(block, state, params.GetConsensus()) || - !ContextualCheckBlock(block, state, m_chainman, pindex->pprev)) { + !ContextualCheckBlock(block, state, *this, pindex->pprev)) { if (state.IsInvalid() && state.GetResult() != BlockValidationResult::BLOCK_MUTATED) { pindex->nStatus |= BLOCK_FAILED_VALID; m_blockman.m_dirty_blockindex.insert(pindex); @@ -3967,23 +4000,30 @@ bool Chainstate::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockV // Header is valid/has work, merkle tree and segwit merkle tree are good...RELAY NOW // (but if it does not build on our best tip, let the SendMessages loop relay it) - if (!IsInitialBlockDownload() && m_chain.Tip() == pindex->pprev) + if (!ActiveChainstate().IsInitialBlockDownload() && ActiveTip() == pindex->pprev) GetMainSignals().NewPoWValidBlock(pindex, pblock); // Write block to history file if (fNewBlock) *fNewBlock = true; try { - FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, pindex->nHeight, m_chain, dbp)}; + FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, pindex->nHeight, dbp)}; if (blockPos.IsNull()) { state.Error(strprintf("%s: Failed to find position to write new block to disk", __func__)); return false; } ReceivedBlockTransactions(block, pindex, blockPos); } catch (const std::runtime_error& e) { - return FatalError(m_chainman.GetNotifications(), state, std::string("System error: ") + e.what()); + return FatalError(GetNotifications(), state, std::string("System error: ") + e.what()); } - FlushStateToDisk(state, FlushStateMode::NONE); + // TODO: FlushStateToDisk() handles flushing of both block and chainstate + // data, so we should move this to ChainstateManager so that we can be more + // intelligent about how we flush. + // For now, since FlushStateMode::NONE is used, all that can happen is that + // the block files may be pruned, so we can just call this on one + // chainstate (particularly if we haven't implemented pruning with + // background validation yet). + ActiveChainstate().FlushStateToDisk(state, FlushStateMode::NONE); CheckBlockIndex(); @@ -4011,7 +4051,7 @@ bool ChainstateManager::ProcessNewBlock(const std::shared_ptr<const CBlock>& blo bool ret = CheckBlock(*block, state, GetConsensus()); if (ret) { // Store to disk - ret = ActiveChainstate().AcceptBlock(block, state, &pindex, force_processing, nullptr, new_block, min_pow_checked); + ret = AcceptBlock(block, state, &pindex, force_processing, nullptr, new_block, min_pow_checked); } if (!ret) { GetMainSignals().BlockChecked(*block, state); @@ -4379,10 +4419,9 @@ bool Chainstate::NeedsRedownload() const return false; } -void Chainstate::UnloadBlockIndex() +void Chainstate::ClearBlockIndexCandidates() { AssertLockHeld(::cs_main); - nBlockSequenceId = 1; setBlockIndexCandidates.clear(); } @@ -4401,62 +4440,19 @@ bool ChainstateManager::LoadBlockIndex() std::sort(vSortedByHeight.begin(), vSortedByHeight.end(), CBlockIndexHeightOnlyComparator()); - // Find start of assumed-valid region. - int first_assumed_valid_height = std::numeric_limits<int>::max(); - - for (const CBlockIndex* block : vSortedByHeight) { - if (block->IsAssumedValid()) { - auto chainstates = GetAll(); - - // If we encounter an assumed-valid block index entry, ensure that we have - // one chainstate that tolerates assumed-valid entries and another that does - // not (i.e. the background validation chainstate), since assumed-valid - // entries should always be pending validation by a fully-validated chainstate. - auto any_chain = [&](auto fnc) { return std::any_of(chainstates.cbegin(), chainstates.cend(), fnc); }; - assert(any_chain([](auto chainstate) { return chainstate->reliesOnAssumedValid(); })); - assert(any_chain([](auto chainstate) { return !chainstate->reliesOnAssumedValid(); })); - - first_assumed_valid_height = block->nHeight; - LogPrintf("Saw first assumedvalid block at height %d (%s)\n", - first_assumed_valid_height, block->ToString()); - break; - } - } - for (CBlockIndex* pindex : vSortedByHeight) { if (m_interrupt) return false; - if (pindex->IsAssumedValid() || + // If we have an assumeutxo-based chainstate, then the snapshot + // block will be a candidate for the tip, but it may not be + // VALID_TRANSACTIONS (eg if we haven't yet downloaded the block), + // so we special-case the snapshot block as a potential candidate + // here. + if (pindex == GetSnapshotBaseBlock() || (pindex->IsValid(BLOCK_VALID_TRANSACTIONS) && (pindex->HaveTxsDownloaded() || pindex->pprev == nullptr))) { - // Fill each chainstate's block candidate set. Only add assumed-valid - // blocks to the tip candidate set if the chainstate is allowed to rely on - // assumed-valid blocks. - // - // If all setBlockIndexCandidates contained the assumed-valid blocks, the - // background chainstate's ActivateBestChain() call would add assumed-valid - // blocks to the chain (based on how FindMostWorkChain() works). Obviously - // we don't want this since the purpose of the background validation chain - // is to validate assued-valid blocks. - // - // Note: This is considering all blocks whose height is greater or equal to - // the first assumed-valid block to be assumed-valid blocks, and excluding - // them from the background chainstate's setBlockIndexCandidates set. This - // does mean that some blocks which are not technically assumed-valid - // (later blocks on a fork beginning before the first assumed-valid block) - // might not get added to the background chainstate, but this is ok, - // because they will still be attached to the active chainstate if they - // actually contain more work. - // - // Instead of this height-based approach, an earlier attempt was made at - // detecting "holistically" whether the block index under consideration - // relied on an assumed-valid ancestor, but this proved to be too slow to - // be practical. for (Chainstate* chainstate : GetAll()) { - if (chainstate->reliesOnAssumedValid() || - pindex->nHeight < first_assumed_valid_height) { - chainstate->setBlockIndexCandidates.insert(pindex); - } + chainstate->TryAddBlockIndexCandidate(pindex); } } if (pindex->nStatus & BLOCK_FAILED_MASK && (!m_best_invalid || pindex->nChainWork > m_best_invalid->nChainWork)) { @@ -4496,12 +4492,12 @@ bool Chainstate::LoadGenesisBlock() try { const CBlock& block = params.GenesisBlock(); - FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, 0, m_chain, nullptr)}; + FlatFilePos blockPos{m_blockman.SaveBlockToDisk(block, 0, nullptr)}; if (blockPos.IsNull()) { return error("%s: writing genesis block to disk failed", __func__); } CBlockIndex* pindex = m_blockman.AddToBlockIndex(block, m_chainman.m_best_header); - ReceivedBlockTransactions(block, pindex, blockPos); + m_chainman.ReceivedBlockTransactions(block, pindex, blockPos); } catch (const std::runtime_error& e) { return error("%s: failed to write genesis block: %s", __func__, e.what()); } @@ -4509,18 +4505,16 @@ bool Chainstate::LoadGenesisBlock() return true; } -void Chainstate::LoadExternalBlockFile( +void ChainstateManager::LoadExternalBlockFile( FILE* fileIn, FlatFilePos* dbp, std::multimap<uint256, FlatFilePos>* blocks_with_unknown_parent) { - AssertLockNotHeld(m_chainstate_mutex); - // Either both should be specified (-reindex), or neither (-loadblock). assert(!dbp == !blocks_with_unknown_parent); const auto start{SteadyClock::now()}; - const CChainParams& params{m_chainman.GetParams()}; + const CChainParams& params{GetParams()}; int nLoaded = 0; try { @@ -4530,7 +4524,7 @@ void Chainstate::LoadExternalBlockFile( // such as a block fails to deserialize. uint64_t nRewind = blkdat.GetPos(); while (!blkdat.eof()) { - if (m_chainman.m_interrupt) return; + if (m_interrupt) return; blkdat.SetPos(nRewind); nRewind++; // start one byte further next time, in case of failure @@ -4605,8 +4599,15 @@ void Chainstate::LoadExternalBlockFile( // Activate the genesis block so normal node progress can continue if (hash == params.GetConsensus().hashGenesisBlock) { - BlockValidationState state; - if (!ActivateBestChain(state, nullptr)) { + bool genesis_activation_failure = false; + for (auto c : GetAll()) { + BlockValidationState state; + if (!c->ActivateBestChain(state, nullptr)) { + genesis_activation_failure = true; + break; + } + } + if (genesis_activation_failure) { break; } } @@ -4619,14 +4620,21 @@ void Chainstate::LoadExternalBlockFile( // until after all of the block files are loaded. ActivateBestChain can be // called by concurrent network message processing. but, that is not // reliable for the purpose of pruning while importing. - BlockValidationState state; - if (!ActivateBestChain(state, pblock)) { - LogPrint(BCLog::REINDEX, "failed to activate chain (%s)\n", state.ToString()); + bool activation_failure = false; + for (auto c : GetAll()) { + BlockValidationState state; + if (!c->ActivateBestChain(state, pblock)) { + LogPrint(BCLog::REINDEX, "failed to activate chain (%s)\n", state.ToString()); + activation_failure = true; + break; + } + } + if (activation_failure) { break; } } - NotifyHeaderTip(*this); + NotifyHeaderTip(ActiveChainstate()); if (!blocks_with_unknown_parent) continue; @@ -4652,7 +4660,7 @@ void Chainstate::LoadExternalBlockFile( } range.first++; blocks_with_unknown_parent->erase(it); - NotifyHeaderTip(*this); + NotifyHeaderTip(ActiveChainstate()); } } } catch (const std::exception& e) { @@ -4671,14 +4679,14 @@ void Chainstate::LoadExternalBlockFile( } } } catch (const std::runtime_error& e) { - m_chainman.GetNotifications().fatalError(std::string("System error: ") + e.what()); + GetNotifications().fatalError(std::string("System error: ") + e.what()); } LogPrintf("Loaded %i blocks from external file in %dms\n", nLoaded, Ticks<std::chrono::milliseconds>(SteadyClock::now() - start)); } -void Chainstate::CheckBlockIndex() +void ChainstateManager::CheckBlockIndex() { - if (!m_chainman.ShouldCheckBlockIndex()) { + if (!ShouldCheckBlockIndex()) { return; } @@ -4687,7 +4695,7 @@ void Chainstate::CheckBlockIndex() // During a reindex, we read the genesis block and call CheckBlockIndex before ActivateBestChain, // so we have the genesis block in m_blockman.m_block_index but no active chain. (A few of the // tests when iterating the block tree require that m_chain has been initialized.) - if (m_chain.Height() < 0) { + if (ActiveChain().Height() < 0) { assert(m_blockman.m_block_index.size() <= 1); return; } @@ -4717,12 +4725,12 @@ void Chainstate::CheckBlockIndex() CBlockIndex* pindexFirstNotTransactionsValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_TRANSACTIONS (regardless of being valid or not). CBlockIndex* pindexFirstNotChainValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_CHAIN (regardless of being valid or not). CBlockIndex* pindexFirstNotScriptsValid = nullptr; // Oldest ancestor of pindex which does not have BLOCK_VALID_SCRIPTS (regardless of being valid or not). + CBlockIndex* pindexFirstAssumeValid = nullptr; // Oldest ancestor of pindex which has BLOCK_ASSUMED_VALID while (pindex != nullptr) { nNodes++; + if (pindexFirstAssumeValid == nullptr && pindex->nStatus & BLOCK_ASSUMED_VALID) pindexFirstAssumeValid = pindex; if (pindexFirstInvalid == nullptr && pindex->nStatus & BLOCK_FAILED_VALID) pindexFirstInvalid = pindex; - // Assumed-valid index entries will not have data since we haven't downloaded the - // full block yet. - if (pindexFirstMissing == nullptr && !(pindex->nStatus & BLOCK_HAVE_DATA) && !pindex->IsAssumedValid()) { + if (pindexFirstMissing == nullptr && !(pindex->nStatus & BLOCK_HAVE_DATA)) { pindexFirstMissing = pindex; } if (pindexFirstNeverProcessed == nullptr && pindex->nTx == 0) pindexFirstNeverProcessed = pindex; @@ -4751,8 +4759,12 @@ void Chainstate::CheckBlockIndex() // Begin: actual consistency checks. if (pindex->pprev == nullptr) { // Genesis block checks. - assert(pindex->GetBlockHash() == m_chainman.GetConsensus().hashGenesisBlock); // Genesis block's hash must match. - assert(pindex == m_chain.Genesis()); // The current active chain's genesis block must be this block. + assert(pindex->GetBlockHash() == GetConsensus().hashGenesisBlock); // Genesis block's hash must match. + for (auto c : GetAll()) { + if (c->m_chain.Genesis() != nullptr) { + assert(pindex == c->m_chain.Genesis()); // The chain's genesis block must be this block. + } + } } if (!pindex->HaveTxsDownloaded()) assert(pindex->nSequenceId <= 0); // nSequenceId can't be set positive for blocks that aren't linked (negative is used for preciousblock) // VALID_TRANSACTIONS is equivalent to nTx > 0 for all nodes (whether or not pruning has occurred). @@ -4762,7 +4774,13 @@ void Chainstate::CheckBlockIndex() if (!m_blockman.m_have_pruned && !pindex->IsAssumedValid()) { // If we've never pruned, then HAVE_DATA should be equivalent to nTx > 0 assert(!(pindex->nStatus & BLOCK_HAVE_DATA) == (pindex->nTx == 0)); - assert(pindexFirstMissing == pindexFirstNeverProcessed); + if (pindexFirstAssumeValid == nullptr) { + // If we've got some assume valid blocks, then we might have + // missing blocks (not HAVE_DATA) but still treat them as + // having been processed (with a fake nTx value). Otherwise, we + // can assert that these are the same. + assert(pindexFirstMissing == pindexFirstNeverProcessed); + } } else { // If we have pruned, then we can only say that HAVE_DATA implies nTx > 0 if (pindex->nStatus & BLOCK_HAVE_DATA) assert(pindex->nTx > 0); @@ -4792,27 +4810,32 @@ void Chainstate::CheckBlockIndex() // Checks for not-invalid blocks. assert((pindex->nStatus & BLOCK_FAILED_MASK) == 0); // The failed mask cannot be set for blocks without invalid parents. } - if (!CBlockIndexWorkComparator()(pindex, m_chain.Tip()) && pindexFirstNeverProcessed == nullptr) { - if (pindexFirstInvalid == nullptr) { - const bool is_active = this == &m_chainman.ActiveChainstate(); - - // If this block sorts at least as good as the current tip and - // is valid and we have all data for its parents, it must be in - // setBlockIndexCandidates. m_chain.Tip() must also be there - // even if some data has been pruned. - // - // Don't perform this check for the background chainstate since - // its setBlockIndexCandidates shouldn't have some entries (i.e. those past the - // snapshot block) which do exist in the block index for the active chainstate. - if (is_active && (pindexFirstMissing == nullptr || pindex == m_chain.Tip())) { - assert(setBlockIndexCandidates.count(pindex)); + // Chainstate-specific checks on setBlockIndexCandidates + for (auto c : GetAll()) { + if (c->m_chain.Tip() == nullptr) continue; + if (!CBlockIndexWorkComparator()(pindex, c->m_chain.Tip()) && pindexFirstNeverProcessed == nullptr) { + if (pindexFirstInvalid == nullptr) { + const bool is_active = c == &ActiveChainstate(); + // If this block sorts at least as good as the current tip and + // is valid and we have all data for its parents, it must be in + // setBlockIndexCandidates. m_chain.Tip() must also be there + // even if some data has been pruned. + // + if ((pindexFirstMissing == nullptr || pindex == c->m_chain.Tip())) { + // The active chainstate should always have this block + // as a candidate, but a background chainstate should + // only have it if it is an ancestor of the snapshot base. + if (is_active || GetSnapshotBaseBlock()->GetAncestor(pindex->nHeight) == pindex) { + assert(c->setBlockIndexCandidates.count(pindex)); + } + } + // If some parent is missing, then it could be that this block was in + // setBlockIndexCandidates but had to be removed because of the missing data. + // In this case it must be in m_blocks_unlinked -- see test below. } - // If some parent is missing, then it could be that this block was in - // setBlockIndexCandidates but had to be removed because of the missing data. - // In this case it must be in m_blocks_unlinked -- see test below. + } else { // If this block sorts worse than the current tip or some ancestor's block has never been seen, it cannot be in setBlockIndexCandidates. + assert(c->setBlockIndexCandidates.count(pindex) == 0); } - } else { // If this block sorts worse than the current tip or some ancestor's block has never been seen, it cannot be in setBlockIndexCandidates. - assert(setBlockIndexCandidates.count(pindex) == 0); } // Check whether this block is in m_blocks_unlinked. std::pair<std::multimap<CBlockIndex*,CBlockIndex*>::iterator,std::multimap<CBlockIndex*,CBlockIndex*>::iterator> rangeUnlinked = m_blockman.m_blocks_unlinked.equal_range(pindex->pprev); @@ -4833,18 +4856,23 @@ void Chainstate::CheckBlockIndex() if (pindexFirstMissing == nullptr) assert(!foundInUnlinked); // We aren't missing data for any parent -- cannot be in m_blocks_unlinked. if (pindex->pprev && (pindex->nStatus & BLOCK_HAVE_DATA) && pindexFirstNeverProcessed == nullptr && pindexFirstMissing != nullptr) { // We HAVE_DATA for this block, have received data for all parents at some point, but we're currently missing data for some parent. - assert(m_blockman.m_have_pruned); // We must have pruned. + assert(m_blockman.m_have_pruned || pindexFirstAssumeValid != nullptr); // We must have pruned, or else we're using a snapshot (causing us to have faked the received data for some parent(s)). // This block may have entered m_blocks_unlinked if: // - it has a descendant that at some point had more work than the // tip, and // - we tried switching to that descendant but were missing // data for some intermediate block between m_chain and the // tip. - // So if this block is itself better than m_chain.Tip() and it wasn't in + // So if this block is itself better than any m_chain.Tip() and it wasn't in // setBlockIndexCandidates, then it must be in m_blocks_unlinked. - if (!CBlockIndexWorkComparator()(pindex, m_chain.Tip()) && setBlockIndexCandidates.count(pindex) == 0) { - if (pindexFirstInvalid == nullptr) { - assert(foundInUnlinked); + for (auto c : GetAll()) { + const bool is_active = c == &ActiveChainstate(); + if (!CBlockIndexWorkComparator()(pindex, c->m_chain.Tip()) && c->setBlockIndexCandidates.count(pindex) == 0) { + if (pindexFirstInvalid == nullptr) { + if (is_active || GetSnapshotBaseBlock()->GetAncestor(pindex->nHeight) == pindex) { + assert(foundInUnlinked); + } + } } } } @@ -4871,6 +4899,7 @@ void Chainstate::CheckBlockIndex() if (pindex == pindexFirstNotTransactionsValid) pindexFirstNotTransactionsValid = nullptr; if (pindex == pindexFirstNotChainValid) pindexFirstNotChainValid = nullptr; if (pindex == pindexFirstNotScriptsValid) pindexFirstNotScriptsValid = nullptr; + if (pindex == pindexFirstAssumeValid) pindexFirstAssumeValid = nullptr; // Find our parent. CBlockIndex* pindexPar = pindex->pprev; // Find which child we just visited. @@ -5682,9 +5711,7 @@ util::Result<void> Chainstate::InvalidateCoinsDBOnDisk() const CBlockIndex* ChainstateManager::GetSnapshotBaseBlock() const { - const auto blockhash_op = this->SnapshotBlockhash(); - if (!blockhash_op) return nullptr; - return Assert(m_blockman.LookupBlockIndex(*blockhash_op)); + return m_active_chainstate ? m_active_chainstate->SnapshotBase() : nullptr; } std::optional<int> ChainstateManager::GetSnapshotBaseHeight() const diff --git a/src/validation.h b/src/validation.h index af8ceb5dfa..d7ad86a5e8 100644 --- a/src/validation.h +++ b/src/validation.h @@ -466,17 +466,6 @@ class Chainstate { protected: /** - * Every received block is assigned a unique and increasing identifier, so we - * know which one to give priority in case of a fork. - */ - /** Blocks loaded from disk are assigned id 0, so start the counter at 1. */ - int32_t nBlockSequenceId GUARDED_BY(::cs_main) = 1; - /** Decreasing counter (used by subsequent preciousblock calls). */ - int32_t nBlockReverseSequenceId = -1; - /** chainwork for the last block that preciousblock has been applied to. */ - arith_uint256 nLastPreciousChainwork = 0; - - /** * The ChainState Mutex * A lock that must be held when modifying this ChainState - held in ActivateBestChain() and * InvalidateBlock() @@ -511,6 +500,9 @@ protected: //! is set to true on the snapshot chainstate. bool m_disabled GUARDED_BY(::cs_main) {false}; + //! Cached result of LookupBlockIndex(*m_from_snapshot_blockhash) + const CBlockIndex* m_cached_snapshot_base GUARDED_BY(::cs_main) {nullptr}; + public: //! Reference to a BlockManager instance which itself is shared across all //! Chainstate instances. @@ -562,9 +554,12 @@ public: */ const std::optional<uint256> m_from_snapshot_blockhash; - //! Return true if this chainstate relies on blocks that are assumed-valid. In - //! practice this means it was created based on a UTXO snapshot. - bool reliesOnAssumedValid() { return m_from_snapshot_blockhash.has_value(); } + /** + * The base of the snapshot this chainstate was created from. + * + * nullptr if this chainstate was not created from a snapshot. + */ + const CBlockIndex* SnapshotBase() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** * The set of all CBlockIndex entries with either BLOCK_VALID_TRANSACTIONS (for @@ -620,37 +615,6 @@ public: bool ResizeCoinsCaches(size_t coinstip_size, size_t coinsdb_size) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - /** - * Import blocks from an external file - * - * During reindexing, this function is called for each block file (datadir/blocks/blk?????.dat). - * It reads all blocks contained in the given file and attempts to process them (add them to the - * block index). The blocks may be out of order within each file and across files. Often this - * function reads a block but finds that its parent hasn't been read yet, so the block can't be - * processed yet. The function will add an entry to the blocks_with_unknown_parent map (which is - * passed as an argument), so that when the block's parent is later read and processed, this - * function can re-read the child block from disk and process it. - * - * Because a block's parent may be in a later file, not just later in the same file, the - * blocks_with_unknown_parent map must be passed in and out with each call. It's a multimap, - * rather than just a map, because multiple blocks may have the same parent (when chain splits - * or stale blocks exist). It maps from parent-hash to child-disk-position. - * - * This function can also be used to read blocks from user-specified block files using the - * -loadblock= option. There's no unknown-parent tracking, so the last two arguments are omitted. - * - * - * @param[in] fileIn FILE handle to file containing blocks to read - * @param[in] dbp (optional) Disk block position (only for reindex) - * @param[in,out] blocks_with_unknown_parent (optional) Map of disk positions for blocks with - * unknown parent, key is parent block hash - * (only used for reindex) - * */ - void LoadExternalBlockFile( - FILE* fileIn, - FlatFilePos* dbp = nullptr, - std::multimap<uint256, FlatFilePos>* blocks_with_unknown_parent = nullptr) - EXCLUSIVE_LOCKS_REQUIRED(!m_chainstate_mutex); /** * Update the on-disk chain state. @@ -702,8 +666,6 @@ public: EXCLUSIVE_LOCKS_REQUIRED(!m_chainstate_mutex) LOCKS_EXCLUDED(::cs_main); - bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock, bool min_pow_checked) EXCLUSIVE_LOCKS_REQUIRED(cs_main); - // Block (dis)connection on a given view: DisconnectResult DisconnectBlock(const CBlock& block, const CBlockIndex* pindex, CCoinsViewCache& view) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); @@ -738,9 +700,11 @@ public: /** Ensures we have a genesis block in the block tree, possibly writing one to disk. */ bool LoadGenesisBlock(); + void TryAddBlockIndexCandidate(CBlockIndex* pindex) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + void PruneBlockIndexCandidates(); - void UnloadBlockIndex() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + void ClearBlockIndexCandidates() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** Check whether we are doing an initial block download (synchronizing from disk or network) */ bool IsInitialBlockDownload() const; @@ -748,13 +712,6 @@ public: /** Find the last common block of this chain and a locator. */ const CBlockIndex* FindForkInGlobalIndex(const CBlockLocator& locator) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); - /** - * Make various assertions about the state of the block index. - * - * By default this only executes fully when using the Regtest chain; see: m_options.check_block_index. - */ - void CheckBlockIndex(); - /** Load the persisted mempool from disk */ void LoadMempool(const fs::path& load_path, fsbridge::FopenFn mockable_fopen_function = fsbridge::fopen); @@ -784,7 +741,6 @@ private: void InvalidBlockFound(CBlockIndex* pindex, const BlockValidationState& state) EXCLUSIVE_LOCKS_REQUIRED(cs_main); CBlockIndex* FindMostWorkChain() EXCLUSIVE_LOCKS_REQUIRED(cs_main); - void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos) EXCLUSIVE_LOCKS_REQUIRED(cs_main); bool RollforwardBlock(const CBlockIndex* pindex, CCoinsViewCache& inputs) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -971,6 +927,13 @@ public: kernel::Notifications& GetNotifications() const { return m_options.notifications; }; /** + * Make various assertions about the state of the block index. + * + * By default this only executes fully when using the Regtest chain; see: m_options.check_block_index. + */ + void CheckBlockIndex(); + + /** * Alias for ::cs_main. * Should be used in new code to make it easier to make ::cs_main a member * of this class. @@ -991,6 +954,27 @@ public: node::BlockManager m_blockman; /** + * Every received block is assigned a unique and increasing identifier, so we + * know which one to give priority in case of a fork. + */ + /** Blocks loaded from disk are assigned id 0, so start the counter at 1. */ + int32_t nBlockSequenceId GUARDED_BY(::cs_main) = 1; + /** Decreasing counter (used by subsequent preciousblock calls). */ + int32_t nBlockReverseSequenceId = -1; + /** chainwork for the last block that preciousblock has been applied to. */ + arith_uint256 nLastPreciousChainwork = 0; + + // Reset the memory-only sequence counters we use to track block arrival + // (used by tests to reset state) + void ResetBlockSequenceCounters() EXCLUSIVE_LOCKS_REQUIRED(::cs_main) + { + AssertLockHeld(::cs_main); + nBlockSequenceId = 1; + nBlockReverseSequenceId = -1; + } + + + /** * In order to efficiently track invalidity of headers, we keep the set of * blocks which we tried to connect and found to be invalid here (ie which * were set to BLOCK_FAILED_VALID since the last restart). We can then @@ -1086,6 +1070,37 @@ public: } /** + * Import blocks from an external file + * + * During reindexing, this function is called for each block file (datadir/blocks/blk?????.dat). + * It reads all blocks contained in the given file and attempts to process them (add them to the + * block index). The blocks may be out of order within each file and across files. Often this + * function reads a block but finds that its parent hasn't been read yet, so the block can't be + * processed yet. The function will add an entry to the blocks_with_unknown_parent map (which is + * passed as an argument), so that when the block's parent is later read and processed, this + * function can re-read the child block from disk and process it. + * + * Because a block's parent may be in a later file, not just later in the same file, the + * blocks_with_unknown_parent map must be passed in and out with each call. It's a multimap, + * rather than just a map, because multiple blocks may have the same parent (when chain splits + * or stale blocks exist). It maps from parent-hash to child-disk-position. + * + * This function can also be used to read blocks from user-specified block files using the + * -loadblock= option. There's no unknown-parent tracking, so the last two arguments are omitted. + * + * + * @param[in] fileIn FILE handle to file containing blocks to read + * @param[in] dbp (optional) Disk block position (only for reindex) + * @param[in,out] blocks_with_unknown_parent (optional) Map of disk positions for blocks with + * unknown parent, key is parent block hash + * (only used for reindex) + * */ + void LoadExternalBlockFile( + FILE* fileIn, + FlatFilePos* dbp = nullptr, + std::multimap<uint256, FlatFilePos>* blocks_with_unknown_parent = nullptr); + + /** * Process an incoming block. This only returns after the best known valid * block is made active. Note that it does not, however, guarantee that the * specific block passed to it has been checked for validity! @@ -1125,6 +1140,29 @@ public: bool ProcessNewBlockHeaders(const std::vector<CBlockHeader>& block, bool min_pow_checked, BlockValidationState& state, const CBlockIndex** ppindex = nullptr) LOCKS_EXCLUDED(cs_main); /** + * Sufficiently validate a block for disk storage (and store on disk). + * + * @param[in] pblock The block we want to process. + * @param[in] fRequested Whether we requested this block from a + * peer. + * @param[in] dbp The location on disk, if we are importing + * this block from prior storage. + * @param[in] min_pow_checked True if proof-of-work anti-DoS checks have + * been done by caller for headers chain + * + * @param[out] state The state of the block validation. + * @param[out] ppindex Optional return parameter to get the + * CBlockIndex pointer for this block. + * @param[out] fNewBlock Optional return parameter to indicate if the + * block is new to our storage. + * + * @returns False if the block or header is invalid, or if saving to disk fails (likely a fatal error); true otherwise. + */ + bool AcceptBlock(const std::shared_ptr<const CBlock>& pblock, BlockValidationState& state, CBlockIndex** ppindex, bool fRequested, const FlatFilePos* dbp, bool* fNewBlock, bool min_pow_checked) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + void ReceivedBlockTransactions(const CBlock& block, CBlockIndex* pindexNew, const FlatFilePos& pos) EXCLUSIVE_LOCKS_REQUIRED(cs_main); + + /** * Try to add a transaction to the memory pool. * * @param[in] tx The transaction to submit for mempool acceptance. diff --git a/src/wallet/crypter.cpp b/src/wallet/crypter.cpp index e2799c2d05..57a19fb5f2 100644 --- a/src/wallet/crypter.cpp +++ b/src/wallet/crypter.cpp @@ -112,7 +112,7 @@ bool EncryptSecret(const CKeyingMaterial& vMasterKey, const CKeyingMaterial &vch memcpy(chIV.data(), &nIV, WALLET_CRYPTO_IV_SIZE); if(!cKeyCrypter.SetKey(vMasterKey, chIV)) return false; - return cKeyCrypter.Encrypt(*((const CKeyingMaterial*)&vchPlaintext), vchCiphertext); + return cKeyCrypter.Encrypt(vchPlaintext, vchCiphertext); } bool DecryptSecret(const CKeyingMaterial& vMasterKey, const std::vector<unsigned char>& vchCiphertext, const uint256& nIV, CKeyingMaterial& vchPlaintext) |