diff options
Diffstat (limited to 'src')
137 files changed, 2405 insertions, 1726 deletions
diff --git a/src/.clang-tidy b/src/.clang-tidy index b4d50135dd..4deb5a85a5 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -1,6 +1,6 @@ Checks: ' -*, -bitcoin-unterminated-logprintf, +bitcoin-*, bugprone-argument-comment, bugprone-use-after-move, misc-unused-using-decls, diff --git a/src/Makefile.am b/src/Makefile.am index 06c156a8c0..feed4a0061 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -117,6 +117,7 @@ endif .PHONY: FORCE check-symbols check-security # bitcoin core # BITCOIN_CORE_H = \ + addresstype.h \ addrdb.h \ addrman.h \ addrman_impl.h \ @@ -189,6 +190,7 @@ BITCOIN_CORE_H = \ kernel/mempool_limits.h \ kernel/mempool_options.h \ kernel/mempool_persist.h \ + kernel/mempool_removal_reason.h \ kernel/notifications_interface.h \ kernel/validation_cache_sizes.h \ key.h \ @@ -265,7 +267,7 @@ BITCOIN_CORE_H = \ script/sigcache.h \ script/sign.h \ script/signingprovider.h \ - script/standard.h \ + script/solver.h \ shutdown.h \ signet.h \ streams.h \ @@ -400,6 +402,7 @@ libbitcoin_node_a_SOURCES = \ kernel/context.cpp \ kernel/cs_main.cpp \ kernel/mempool_persist.cpp \ + kernel/mempool_removal_reason.cpp \ mapport.cpp \ net.cpp \ net_processing.cpp \ @@ -659,6 +662,7 @@ libbitcoin_consensus_a_SOURCES = \ libbitcoin_common_a_CPPFLAGS = $(AM_CPPFLAGS) $(BITCOIN_INCLUDES) $(BOOST_CPPFLAGS) libbitcoin_common_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libbitcoin_common_a_SOURCES = \ + addresstype.cpp \ base58.cpp \ bech32.cpp \ chainparams.cpp \ @@ -699,7 +703,7 @@ libbitcoin_common_a_SOURCES = \ script/miniscript.cpp \ script/sign.cpp \ script/signingprovider.cpp \ - script/standard.cpp \ + script/solver.cpp \ warnings.cpp \ $(BITCOIN_CORE_H) @@ -938,6 +942,7 @@ libbitcoinkernel_la_SOURCES = \ kernel/context.cpp \ kernel/cs_main.cpp \ kernel/mempool_persist.cpp \ + kernel/mempool_removal_reason.cpp \ key.cpp \ logging.cpp \ node/blockstorage.cpp \ @@ -960,7 +965,7 @@ libbitcoinkernel_la_SOURCES = \ script/script.cpp \ script/script_error.cpp \ script/sigcache.cpp \ - script/standard.cpp \ + script/solver.cpp \ signet.cpp \ streams.cpp \ support/cleanse.cpp \ diff --git a/src/addresstype.cpp b/src/addresstype.cpp new file mode 100644 index 0000000000..2454cfb5d9 --- /dev/null +++ b/src/addresstype.cpp @@ -0,0 +1,153 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. + +#include <addresstype.h> + +#include <crypto/sha256.h> +#include <hash.h> +#include <pubkey.h> +#include <script/script.h> +#include <script/solver.h> +#include <uint256.h> +#include <util/hash_type.h> + +#include <cassert> +#include <vector> + +typedef std::vector<unsigned char> valtype; + +ScriptHash::ScriptHash(const CScript& in) : BaseHash(Hash160(in)) {} +ScriptHash::ScriptHash(const CScriptID& in) : BaseHash{in} {} + +PKHash::PKHash(const CPubKey& pubkey) : BaseHash(pubkey.GetID()) {} +PKHash::PKHash(const CKeyID& pubkey_id) : BaseHash(pubkey_id) {} + +WitnessV0KeyHash::WitnessV0KeyHash(const CPubKey& pubkey) : BaseHash(pubkey.GetID()) {} +WitnessV0KeyHash::WitnessV0KeyHash(const PKHash& pubkey_hash) : BaseHash{pubkey_hash} {} + +CKeyID ToKeyID(const PKHash& key_hash) +{ + return CKeyID{uint160{key_hash}}; +} + +CKeyID ToKeyID(const WitnessV0KeyHash& key_hash) +{ + return CKeyID{uint160{key_hash}}; +} + +CScriptID ToScriptID(const ScriptHash& script_hash) +{ + return CScriptID{uint160{script_hash}}; +} + +WitnessV0ScriptHash::WitnessV0ScriptHash(const CScript& in) +{ + CSHA256().Write(in.data(), in.size()).Finalize(begin()); +} + +bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) +{ + std::vector<valtype> vSolutions; + TxoutType whichType = Solver(scriptPubKey, vSolutions); + + switch (whichType) { + case TxoutType::PUBKEY: { + CPubKey pubKey(vSolutions[0]); + if (!pubKey.IsValid()) + return false; + + addressRet = PKHash(pubKey); + return true; + } + case TxoutType::PUBKEYHASH: { + addressRet = PKHash(uint160(vSolutions[0])); + return true; + } + case TxoutType::SCRIPTHASH: { + addressRet = ScriptHash(uint160(vSolutions[0])); + return true; + } + case TxoutType::WITNESS_V0_KEYHASH: { + WitnessV0KeyHash hash; + std::copy(vSolutions[0].begin(), vSolutions[0].end(), hash.begin()); + addressRet = hash; + return true; + } + case TxoutType::WITNESS_V0_SCRIPTHASH: { + WitnessV0ScriptHash hash; + std::copy(vSolutions[0].begin(), vSolutions[0].end(), hash.begin()); + addressRet = hash; + return true; + } + case TxoutType::WITNESS_V1_TAPROOT: { + WitnessV1Taproot tap; + std::copy(vSolutions[0].begin(), vSolutions[0].end(), tap.begin()); + addressRet = tap; + return true; + } + case TxoutType::WITNESS_UNKNOWN: { + WitnessUnknown unk; + unk.version = vSolutions[0][0]; + std::copy(vSolutions[1].begin(), vSolutions[1].end(), unk.program); + unk.length = vSolutions[1].size(); + addressRet = unk; + return true; + } + case TxoutType::MULTISIG: + case TxoutType::NULL_DATA: + case TxoutType::NONSTANDARD: + return false; + } // no default case, so the compiler can warn about missing cases + assert(false); +} + +namespace { +class CScriptVisitor +{ +public: + CScript operator()(const CNoDestination& dest) const + { + return CScript(); + } + + CScript operator()(const PKHash& keyID) const + { + return CScript() << OP_DUP << OP_HASH160 << ToByteVector(keyID) << OP_EQUALVERIFY << OP_CHECKSIG; + } + + CScript operator()(const ScriptHash& scriptID) const + { + return CScript() << OP_HASH160 << ToByteVector(scriptID) << OP_EQUAL; + } + + CScript operator()(const WitnessV0KeyHash& id) const + { + return CScript() << OP_0 << ToByteVector(id); + } + + CScript operator()(const WitnessV0ScriptHash& id) const + { + return CScript() << OP_0 << ToByteVector(id); + } + + CScript operator()(const WitnessV1Taproot& tap) const + { + return CScript() << OP_1 << ToByteVector(tap); + } + + CScript operator()(const WitnessUnknown& id) const + { + return CScript() << CScript::EncodeOP_N(id.version) << std::vector<unsigned char>(id.program, id.program + id.length); + } +}; +} // namespace + +CScript GetScriptForDestination(const CTxDestination& dest) +{ + return std::visit(CScriptVisitor(), dest); +} + +bool IsValidDestination(const CTxDestination& dest) { + return dest.index() != 0; +} diff --git a/src/addresstype.h b/src/addresstype.h new file mode 100644 index 0000000000..6b651e9014 --- /dev/null +++ b/src/addresstype.h @@ -0,0 +1,121 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_ADDRESSTYPE_H +#define BITCOIN_ADDRESSTYPE_H + +#include <pubkey.h> +#include <script/script.h> +#include <uint256.h> +#include <util/hash_type.h> + +#include <variant> +#include <algorithm> + +class CNoDestination { +public: + friend bool operator==(const CNoDestination &a, const CNoDestination &b) { return true; } + friend bool operator<(const CNoDestination &a, const CNoDestination &b) { return true; } +}; + +struct PKHash : public BaseHash<uint160> +{ + PKHash() : BaseHash() {} + explicit PKHash(const uint160& hash) : BaseHash(hash) {} + explicit PKHash(const CPubKey& pubkey); + explicit PKHash(const CKeyID& pubkey_id); +}; +CKeyID ToKeyID(const PKHash& key_hash); + +struct WitnessV0KeyHash; + +struct ScriptHash : public BaseHash<uint160> +{ + ScriptHash() : BaseHash() {} + // These don't do what you'd expect. + // Use ScriptHash(GetScriptForDestination(...)) instead. + explicit ScriptHash(const WitnessV0KeyHash& hash) = delete; + explicit ScriptHash(const PKHash& hash) = delete; + + explicit ScriptHash(const uint160& hash) : BaseHash(hash) {} + explicit ScriptHash(const CScript& script); + explicit ScriptHash(const CScriptID& script); +}; +CScriptID ToScriptID(const ScriptHash& script_hash); + +struct WitnessV0ScriptHash : public BaseHash<uint256> +{ + WitnessV0ScriptHash() : BaseHash() {} + explicit WitnessV0ScriptHash(const uint256& hash) : BaseHash(hash) {} + explicit WitnessV0ScriptHash(const CScript& script); +}; + +struct WitnessV0KeyHash : public BaseHash<uint160> +{ + WitnessV0KeyHash() : BaseHash() {} + explicit WitnessV0KeyHash(const uint160& hash) : BaseHash(hash) {} + explicit WitnessV0KeyHash(const CPubKey& pubkey); + explicit WitnessV0KeyHash(const PKHash& pubkey_hash); +}; +CKeyID ToKeyID(const WitnessV0KeyHash& key_hash); + +struct WitnessV1Taproot : public XOnlyPubKey +{ + WitnessV1Taproot() : XOnlyPubKey() {} + explicit WitnessV1Taproot(const XOnlyPubKey& xpk) : XOnlyPubKey(xpk) {} +}; + +//! CTxDestination subtype to encode any future Witness version +struct WitnessUnknown +{ + unsigned int version; + unsigned int length; + unsigned char program[40]; + + friend bool operator==(const WitnessUnknown& w1, const WitnessUnknown& w2) { + if (w1.version != w2.version) return false; + if (w1.length != w2.length) return false; + return std::equal(w1.program, w1.program + w1.length, w2.program); + } + + friend bool operator<(const WitnessUnknown& w1, const WitnessUnknown& w2) { + if (w1.version < w2.version) return true; + if (w1.version > w2.version) return false; + if (w1.length < w2.length) return true; + if (w1.length > w2.length) return false; + return std::lexicographical_compare(w1.program, w1.program + w1.length, w2.program, w2.program + w2.length); + } +}; + +/** + * A txout script template with a specific destination. It is either: + * * CNoDestination: no destination set + * * PKHash: TxoutType::PUBKEYHASH destination (P2PKH) + * * ScriptHash: TxoutType::SCRIPTHASH destination (P2SH) + * * WitnessV0ScriptHash: TxoutType::WITNESS_V0_SCRIPTHASH destination (P2WSH) + * * WitnessV0KeyHash: TxoutType::WITNESS_V0_KEYHASH destination (P2WPKH) + * * WitnessV1Taproot: TxoutType::WITNESS_V1_TAPROOT destination (P2TR) + * * WitnessUnknown: TxoutType::WITNESS_UNKNOWN destination (P2W???) + * A CTxDestination is the internal data type encoded in a bitcoin address + */ +using CTxDestination = std::variant<CNoDestination, PKHash, ScriptHash, WitnessV0ScriptHash, WitnessV0KeyHash, WitnessV1Taproot, WitnessUnknown>; + +/** Check whether a CTxDestination is a CNoDestination. */ +bool IsValidDestination(const CTxDestination& dest); + +/** + * Parse a standard scriptPubKey for the destination address. Assigns result to + * the addressRet parameter and returns true if successful. Currently only works for P2PK, + * P2PKH, P2SH, P2WPKH, and P2WSH scripts. + */ +bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet); + +/** + * Generate a Bitcoin scriptPubKey for the given CTxDestination. Returns a P2PKH + * script for a CKeyID destination, a P2SH script for a CScriptID, and an empty + * script for CNoDestination. + */ +CScript GetScriptForDestination(const CTxDestination& dest); + +#endif // BITCOIN_ADDRESSTYPE_H diff --git a/src/bench/chacha20.cpp b/src/bench/chacha20.cpp index d8bebf9319..e0bb07d8be 100644 --- a/src/bench/chacha20.cpp +++ b/src/bench/chacha20.cpp @@ -14,13 +14,13 @@ static const uint64_t BUFFER_SIZE_LARGE = 1024*1024; static void CHACHA20(benchmark::Bench& bench, size_t buffersize) { - std::vector<uint8_t> key(32,0); - ChaCha20 ctx(key.data()); - ctx.Seek64({0, 0}, 0); - std::vector<uint8_t> in(buffersize,0); - std::vector<uint8_t> out(buffersize,0); + std::vector<std::byte> key(32, {}); + ChaCha20 ctx(key); + ctx.Seek({0, 0}, 0); + std::vector<std::byte> in(buffersize, {}); + std::vector<std::byte> out(buffersize, {}); bench.batch(in.size()).unit("byte").run([&] { - ctx.Crypt(in.data(), out.data(), in.size()); + ctx.Crypt(in, out); }); } diff --git a/src/bench/descriptors.cpp b/src/bench/descriptors.cpp index 5d28d26909..fbef1395fb 100644 --- a/src/bench/descriptors.cpp +++ b/src/bench/descriptors.cpp @@ -6,7 +6,6 @@ #include <key.h> #include <pubkey.h> #include <script/descriptor.h> -#include <script/standard.h> #include <string> #include <utility> diff --git a/src/bench/mempool_eviction.cpp b/src/bench/mempool_eviction.cpp index 735dc92dfb..1a9b013277 100644 --- a/src/bench/mempool_eviction.cpp +++ b/src/bench/mempool_eviction.cpp @@ -13,11 +13,12 @@ static void AddTx(const CTransactionRef& tx, const CAmount& nFee, CTxMemPool& po { int64_t nTime = 0; unsigned int nHeight = 1; + uint64_t sequence = 0; bool spendsCoinbase = false; unsigned int sigOpCost = 4; LockPoints lp; pool.addUnchecked(CTxMemPoolEntry( - tx, nFee, nTime, nHeight, + tx, nFee, nTime, nHeight, sequence, spendsCoinbase, sigOpCost, lp)); } diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp index 826da73800..1f94461d19 100644 --- a/src/bench/mempool_stress.cpp +++ b/src/bench/mempool_stress.cpp @@ -16,10 +16,11 @@ static void AddTx(const CTransactionRef& tx, CTxMemPool& pool) EXCLUSIVE_LOCKS_R { int64_t nTime = 0; unsigned int nHeight = 1; + uint64_t sequence = 0; bool spendsCoinbase = false; unsigned int sigOpCost = 4; LockPoints lp; - pool.addUnchecked(CTxMemPoolEntry(tx, 1000, nTime, nHeight, spendsCoinbase, sigOpCost, lp)); + pool.addUnchecked(CTxMemPoolEntry(tx, 1000, nTime, nHeight, sequence, spendsCoinbase, sigOpCost, lp)); } struct Available { diff --git a/src/bench/rpc_mempool.cpp b/src/bench/rpc_mempool.cpp index 7e274370e0..a55aa0c794 100644 --- a/src/bench/rpc_mempool.cpp +++ b/src/bench/rpc_mempool.cpp @@ -16,7 +16,7 @@ static void AddTx(const CTransactionRef& tx, const CAmount& fee, CTxMemPool& pool) EXCLUSIVE_LOCKS_REQUIRED(cs_main, pool.cs) { LockPoints lp; - pool.addUnchecked(CTxMemPoolEntry(tx, fee, /*time=*/0, /*entry_height=*/1, /*spends_coinbase=*/false, /*sigops_cost=*/4, lp)); + pool.addUnchecked(CTxMemPoolEntry(tx, fee, /*time=*/0, /*entry_height=*/1, /*entry_sequence=*/0, /*spends_coinbase=*/false, /*sigops_cost=*/4, lp)); } static void RpcMempool(benchmark::Bench& bench) diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index 757094167a..11f96b1005 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -8,7 +8,7 @@ #include <script/bitcoinconsensus.h> #endif #include <script/script.h> -#include <script/standard.h> +#include <script/interpreter.h> #include <streams.h> #include <test/util/transaction_utils.h> diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index 580a6badd6..19c4d36126 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -163,7 +163,7 @@ int main(int argc, char* argv[]) << "\t" << "Reindexing: " << std::boolalpha << node::fReindex.load() << std::noboolalpha << std::endl << "\t" << "Snapshot Active: " << std::boolalpha << chainman.IsSnapshotActive() << std::noboolalpha << std::endl << "\t" << "Active Height: " << chainman.ActiveHeight() << std::endl - << "\t" << "Active IBD: " << std::boolalpha << chainman.ActiveChainstate().IsInitialBlockDownload() << std::noboolalpha << std::endl; + << "\t" << "Active IBD: " << std::boolalpha << chainman.IsInitialBlockDownload() << std::noboolalpha << std::endl; CBlockIndex* tip = chainman.ActiveTip(); if (tip) { std::cout << "\t" << tip->ToString() << std::endl; diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index 2f465f1119..985a81f522 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -8,9 +8,11 @@ #include <blockfilter.h> #include <crypto/siphash.h> #include <hash.h> +#include <primitives/block.h> #include <primitives/transaction.h> #include <script/script.h> #include <streams.h> +#include <undo.h> #include <util/golombrice.h> #include <util/string.h> diff --git a/src/blockfilter.h b/src/blockfilter.h index fb5114edb3..8eab4afa76 100644 --- a/src/blockfilter.h +++ b/src/blockfilter.h @@ -5,19 +5,22 @@ #ifndef BITCOIN_BLOCKFILTER_H #define BITCOIN_BLOCKFILTER_H -#include <stdint.h> -#include <string> +#include <cstddef> +#include <cstdint> +#include <ios> #include <set> +#include <string> #include <unordered_set> +#include <utility> #include <vector> #include <attributes.h> -#include <primitives/block.h> -#include <serialize.h> #include <uint256.h> -#include <undo.h> #include <util/bytevectorhash.h> +class CBlock; +class CBlockUndo; + /** * This implements a Golomb-coded set as defined in BIP 158. It is a * compact, probabilistic data structure for testing set membership. diff --git a/src/common/bloom.cpp b/src/common/bloom.cpp index fd3276b5a7..5c3ad882a1 100644 --- a/src/common/bloom.cpp +++ b/src/common/bloom.cpp @@ -8,7 +8,7 @@ #include <primitives/transaction.h> #include <random.h> #include <script/script.h> -#include <script/standard.h> +#include <script/solver.h> #include <span.h> #include <streams.h> #include <util/fastrange.h> diff --git a/src/compressor.cpp b/src/compressor.cpp index 32af8eab49..668e25581c 100644 --- a/src/compressor.cpp +++ b/src/compressor.cpp @@ -6,7 +6,7 @@ #include <compressor.h> #include <pubkey.h> -#include <script/standard.h> +#include <script/script.h> /* * These check for scripts for which a special case with a shorter encoding is defined. diff --git a/src/core_write.cpp b/src/core_write.cpp index 54ca306f60..7cf019a42e 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -11,7 +11,7 @@ #include <key_io.h> #include <script/descriptor.h> #include <script/script.h> -#include <script/standard.h> +#include <script/solver.h> #include <serialize.h> #include <streams.h> #include <undo.h> diff --git a/src/crypto/chacha20.cpp b/src/crypto/chacha20.cpp index 469b280494..a3cc87e81b 100644 --- a/src/crypto/chacha20.cpp +++ b/src/crypto/chacha20.cpp @@ -8,6 +8,7 @@ #include <crypto/common.h> #include <crypto/chacha20.h> #include <support/cleanse.h> +#include <span.h> #include <algorithm> #include <string.h> @@ -22,38 +23,34 @@ constexpr static inline uint32_t rotl32(uint32_t v, int c) { return (v << c) | ( #define REPEAT10(a) do { {a}; {a}; {a}; {a}; {a}; {a}; {a}; {a}; {a}; {a}; } while(0) -void ChaCha20Aligned::SetKey32(const unsigned char* k) +void ChaCha20Aligned::SetKey(Span<const std::byte> key) noexcept { - input[0] = ReadLE32(k + 0); - input[1] = ReadLE32(k + 4); - input[2] = ReadLE32(k + 8); - input[3] = ReadLE32(k + 12); - input[4] = ReadLE32(k + 16); - input[5] = ReadLE32(k + 20); - input[6] = ReadLE32(k + 24); - input[7] = ReadLE32(k + 28); + assert(key.size() == KEYLEN); + input[0] = ReadLE32(UCharCast(key.data() + 0)); + input[1] = ReadLE32(UCharCast(key.data() + 4)); + input[2] = ReadLE32(UCharCast(key.data() + 8)); + input[3] = ReadLE32(UCharCast(key.data() + 12)); + input[4] = ReadLE32(UCharCast(key.data() + 16)); + input[5] = ReadLE32(UCharCast(key.data() + 20)); + input[6] = ReadLE32(UCharCast(key.data() + 24)); + input[7] = ReadLE32(UCharCast(key.data() + 28)); input[8] = 0; input[9] = 0; input[10] = 0; input[11] = 0; } -ChaCha20Aligned::ChaCha20Aligned() -{ - memset(input, 0, sizeof(input)); -} - ChaCha20Aligned::~ChaCha20Aligned() { memory_cleanse(input, sizeof(input)); } -ChaCha20Aligned::ChaCha20Aligned(const unsigned char* key32) +ChaCha20Aligned::ChaCha20Aligned(Span<const std::byte> key) noexcept { - SetKey32(key32); + SetKey(key); } -void ChaCha20Aligned::Seek64(Nonce96 nonce, uint32_t block_counter) +void ChaCha20Aligned::Seek(Nonce96 nonce, uint32_t block_counter) noexcept { input[8] = block_counter; input[9] = nonce.first; @@ -61,8 +58,12 @@ void ChaCha20Aligned::Seek64(Nonce96 nonce, uint32_t block_counter) input[11] = nonce.second >> 32; } -inline void ChaCha20Aligned::Keystream64(unsigned char* c, size_t blocks) +inline void ChaCha20Aligned::Keystream(Span<std::byte> output) noexcept { + unsigned char* c = UCharCast(output.data()); + size_t blocks = output.size() / BLOCKLEN; + assert(blocks * BLOCKLEN == output.size()); + uint32_t x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15; uint32_t j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15; @@ -154,12 +155,18 @@ inline void ChaCha20Aligned::Keystream64(unsigned char* c, size_t blocks) return; } blocks -= 1; - c += 64; + c += BLOCKLEN; } } -inline void ChaCha20Aligned::Crypt64(const unsigned char* m, unsigned char* c, size_t blocks) +inline void ChaCha20Aligned::Crypt(Span<const std::byte> in_bytes, Span<std::byte> out_bytes) noexcept { + assert(in_bytes.size() == out_bytes.size()); + const unsigned char* m = UCharCast(in_bytes.data()); + unsigned char* c = UCharCast(out_bytes.data()); + size_t blocks = out_bytes.size() / BLOCKLEN; + assert(blocks * BLOCKLEN == out_bytes.size()); + uint32_t x0, x1, x2, x3, x4, x5, x6, x7, x8, x9, x10, x11, x12, x13, x14, x15; uint32_t j4, j5, j6, j7, j8, j9, j10, j11, j12, j13, j14, j15; @@ -268,70 +275,75 @@ inline void ChaCha20Aligned::Crypt64(const unsigned char* m, unsigned char* c, s return; } blocks -= 1; - c += 64; - m += 64; + c += BLOCKLEN; + m += BLOCKLEN; } } -void ChaCha20::Keystream(unsigned char* c, size_t bytes) +void ChaCha20::Keystream(Span<std::byte> out) noexcept { - if (!bytes) return; + if (out.empty()) return; if (m_bufleft) { - unsigned reuse = std::min<size_t>(m_bufleft, bytes); - memcpy(c, m_buffer + 64 - m_bufleft, reuse); + unsigned reuse = std::min<size_t>(m_bufleft, out.size()); + std::copy(m_buffer.end() - m_bufleft, m_buffer.end() - m_bufleft + reuse, out.begin()); m_bufleft -= reuse; - bytes -= reuse; - c += reuse; + out = out.subspan(reuse); } - if (bytes >= 64) { - size_t blocks = bytes / 64; - m_aligned.Keystream64(c, blocks); - c += blocks * 64; - bytes -= blocks * 64; + if (out.size() >= m_aligned.BLOCKLEN) { + size_t blocks = out.size() / m_aligned.BLOCKLEN; + m_aligned.Keystream(out.first(blocks * m_aligned.BLOCKLEN)); + out = out.subspan(blocks * m_aligned.BLOCKLEN); } - if (bytes) { - m_aligned.Keystream64(m_buffer, 1); - memcpy(c, m_buffer, bytes); - m_bufleft = 64 - bytes; + if (!out.empty()) { + m_aligned.Keystream(m_buffer); + std::copy(m_buffer.begin(), m_buffer.begin() + out.size(), out.begin()); + m_bufleft = m_aligned.BLOCKLEN - out.size(); } } -void ChaCha20::Crypt(const unsigned char* m, unsigned char* c, size_t bytes) +void ChaCha20::Crypt(Span<const std::byte> input, Span<std::byte> output) noexcept { - if (!bytes) return; + assert(input.size() == output.size()); + + if (!input.size()) return; if (m_bufleft) { - unsigned reuse = std::min<size_t>(m_bufleft, bytes); + unsigned reuse = std::min<size_t>(m_bufleft, input.size()); for (unsigned i = 0; i < reuse; i++) { - c[i] = m[i] ^ m_buffer[64 - m_bufleft + i]; + output[i] = input[i] ^ m_buffer[m_aligned.BLOCKLEN - m_bufleft + i]; } m_bufleft -= reuse; - bytes -= reuse; - c += reuse; - m += reuse; + output = output.subspan(reuse); + input = input.subspan(reuse); } - if (bytes >= 64) { - size_t blocks = bytes / 64; - m_aligned.Crypt64(m, c, blocks); - c += blocks * 64; - m += blocks * 64; - bytes -= blocks * 64; + if (input.size() >= m_aligned.BLOCKLEN) { + size_t blocks = input.size() / m_aligned.BLOCKLEN; + m_aligned.Crypt(input.first(blocks * m_aligned.BLOCKLEN), output.first(blocks * m_aligned.BLOCKLEN)); + output = output.subspan(blocks * m_aligned.BLOCKLEN); + input = input.subspan(blocks * m_aligned.BLOCKLEN); } - if (bytes) { - m_aligned.Keystream64(m_buffer, 1); - for (unsigned i = 0; i < bytes; i++) { - c[i] = m[i] ^ m_buffer[i]; + if (!input.empty()) { + m_aligned.Keystream(m_buffer); + for (unsigned i = 0; i < input.size(); i++) { + output[i] = input[i] ^ m_buffer[i]; } - m_bufleft = 64 - bytes; + m_bufleft = m_aligned.BLOCKLEN - input.size(); } } ChaCha20::~ChaCha20() { - memory_cleanse(m_buffer, sizeof(m_buffer)); + memory_cleanse(m_buffer.data(), m_buffer.size()); +} + +void ChaCha20::SetKey(Span<const std::byte> key) noexcept +{ + m_aligned.SetKey(key); + m_bufleft = 0; + memory_cleanse(m_buffer.data(), m_buffer.size()); } FSChaCha20::FSChaCha20(Span<const std::byte> key, uint32_t rekey_interval) noexcept : - m_chacha20(UCharCast(key.data())), m_rekey_interval(rekey_interval) + m_chacha20(key), m_rekey_interval(rekey_interval) { assert(key.size() == KEYLEN); } @@ -341,20 +353,20 @@ void FSChaCha20::Crypt(Span<const std::byte> input, Span<std::byte> output) noex assert(input.size() == output.size()); // Invoke internal stream cipher for actual encryption/decryption. - m_chacha20.Crypt(UCharCast(input.data()), UCharCast(output.data()), input.size()); + m_chacha20.Crypt(input, output); // Rekey after m_rekey_interval encryptions/decryptions. if (++m_chunk_counter == m_rekey_interval) { // Get new key from the stream cipher. std::byte new_key[KEYLEN]; - m_chacha20.Keystream(UCharCast(new_key), sizeof(new_key)); + m_chacha20.Keystream(new_key); // Update its key. - m_chacha20.SetKey32(UCharCast(new_key)); + m_chacha20.SetKey(new_key); // Wipe the key (a copy remains inside m_chacha20, where it'll be wiped on the next rekey // or on destruction). memory_cleanse(new_key, sizeof(new_key)); // Set the nonce for the new section of output. - m_chacha20.Seek64({0, ++m_rekey_counter}, 0); + m_chacha20.Seek({0, ++m_rekey_counter}, 0); // Reset the chunk counter. m_chunk_counter = 0; } diff --git a/src/crypto/chacha20.h b/src/crypto/chacha20.h index d1b2094e7e..5f0f1ff64b 100644 --- a/src/crypto/chacha20.h +++ b/src/crypto/chacha20.h @@ -28,16 +28,23 @@ private: uint32_t input[12]; public: - ChaCha20Aligned(); + /** Expected key length in constructor and SetKey. */ + static constexpr unsigned KEYLEN{32}; + + /** Block size (inputs/outputs to Keystream / Crypt should be multiples of this). */ + static constexpr unsigned BLOCKLEN{64}; + + /** For safety, disallow initialization without key. */ + ChaCha20Aligned() noexcept = delete; /** Initialize a cipher with specified 32-byte key. */ - ChaCha20Aligned(const unsigned char* key32); + ChaCha20Aligned(Span<const std::byte> key) noexcept; /** Destructor to clean up private memory. */ ~ChaCha20Aligned(); - /** set 32-byte key. */ - void SetKey32(const unsigned char* key32); + /** Set 32-byte key, and seek to nonce 0 and block position 0. */ + void SetKey(Span<const std::byte> key) noexcept; /** Type for 96-bit nonces used by the Set function below. * @@ -51,18 +58,19 @@ public: /** Set the 96-bit nonce and 32-bit block counter. * - * Block_counter selects a position to seek to (to byte 64*block_counter). After 256 GiB, the - * block counter overflows, and nonce.first is incremented. + * Block_counter selects a position to seek to (to byte BLOCKLEN*block_counter). After 256 GiB, + * the block counter overflows, and nonce.first is incremented. */ - void Seek64(Nonce96 nonce, uint32_t block_counter); + void Seek(Nonce96 nonce, uint32_t block_counter) noexcept; - /** outputs the keystream of size <64*blocks> into <c> */ - void Keystream64(unsigned char* c, size_t blocks); + /** outputs the keystream into out, whose length must be a multiple of BLOCKLEN. */ + void Keystream(Span<std::byte> out) noexcept; - /** enciphers the message <input> of length <64*blocks> and write the enciphered representation into <output> - * Used for encryption and decryption (XOR) + /** en/deciphers the message <input> and write the result into <output> + * + * The size of input and output must be equal, and be a multiple of BLOCKLEN. */ - void Crypt64(const unsigned char* input, unsigned char* output, size_t blocks); + void Crypt(Span<const std::byte> input, Span<std::byte> output) noexcept; }; /** Unrestricted ChaCha20 cipher. */ @@ -70,42 +78,43 @@ class ChaCha20 { private: ChaCha20Aligned m_aligned; - unsigned char m_buffer[64] = {0}; + std::array<std::byte, ChaCha20Aligned::BLOCKLEN> m_buffer; unsigned m_bufleft{0}; public: - ChaCha20() = default; + /** Expected key length in constructor and SetKey. */ + static constexpr unsigned KEYLEN = ChaCha20Aligned::KEYLEN; + + /** For safety, disallow initialization without key. */ + ChaCha20() noexcept = delete; /** Initialize a cipher with specified 32-byte key. */ - ChaCha20(const unsigned char* key32) : m_aligned(key32) {} + ChaCha20(Span<const std::byte> key) noexcept : m_aligned(key) {} /** Destructor to clean up private memory. */ ~ChaCha20(); - /** set 32-byte key. */ - void SetKey32(const unsigned char* key32) - { - m_aligned.SetKey32(key32); - m_bufleft = 0; - } + /** Set 32-byte key, and seek to nonce 0 and block position 0. */ + void SetKey(Span<const std::byte> key) noexcept; /** 96-bit nonce type. */ using Nonce96 = ChaCha20Aligned::Nonce96; - /** Set the 96-bit nonce and 32-bit block counter. */ - void Seek64(Nonce96 nonce, uint32_t block_counter) + /** Set the 96-bit nonce and 32-bit block counter. See ChaCha20Aligned::Seek. */ + void Seek(Nonce96 nonce, uint32_t block_counter) noexcept { - m_aligned.Seek64(nonce, block_counter); + m_aligned.Seek(nonce, block_counter); m_bufleft = 0; } - /** outputs the keystream of size <bytes> into <c> */ - void Keystream(unsigned char* c, size_t bytes); - - /** enciphers the message <input> of length <bytes> and write the enciphered representation into <output> - * Used for encryption and decryption (XOR) + /** en/deciphers the message <in_bytes> and write the result into <out_bytes> + * + * The size of in_bytes and out_bytes must be equal. */ - void Crypt(const unsigned char* input, unsigned char* output, size_t bytes); + void Crypt(Span<const std::byte> in_bytes, Span<std::byte> out_bytes) noexcept; + + /** outputs the keystream to out. */ + void Keystream(Span<std::byte> out) noexcept; }; /** Forward-secure ChaCha20 diff --git a/src/crypto/chacha20poly1305.cpp b/src/crypto/chacha20poly1305.cpp index 26161641bb..59671d304c 100644 --- a/src/crypto/chacha20poly1305.cpp +++ b/src/crypto/chacha20poly1305.cpp @@ -13,7 +13,7 @@ #include <assert.h> #include <cstddef> -AEADChaCha20Poly1305::AEADChaCha20Poly1305(Span<const std::byte> key) noexcept : m_chacha20(UCharCast(key.data())) +AEADChaCha20Poly1305::AEADChaCha20Poly1305(Span<const std::byte> key) noexcept : m_chacha20(key) { assert(key.size() == KEYLEN); } @@ -21,7 +21,7 @@ AEADChaCha20Poly1305::AEADChaCha20Poly1305(Span<const std::byte> key) noexcept : void AEADChaCha20Poly1305::SetKey(Span<const std::byte> key) noexcept { assert(key.size() == KEYLEN); - m_chacha20.SetKey32(UCharCast(key.data())); + m_chacha20.SetKey(key); } namespace { @@ -46,8 +46,8 @@ void ComputeTag(ChaCha20& chacha20, Span<const std::byte> aad, Span<const std::b static const std::byte PADDING[16] = {{}}; // Get block of keystream (use a full 64 byte buffer to avoid the need for chacha20's own buffering). - std::byte first_block[64]; - chacha20.Keystream(UCharCast(first_block), sizeof(first_block)); + std::byte first_block[ChaCha20Aligned::BLOCKLEN]; + chacha20.Keystream(first_block); // Use the first 32 bytes of the first keystream block as poly1305 key. Poly1305 poly1305{Span{first_block}.first(Poly1305::KEYLEN)}; @@ -76,12 +76,12 @@ void AEADChaCha20Poly1305::Encrypt(Span<const std::byte> plain1, Span<const std: assert(cipher.size() == plain1.size() + plain2.size() + EXPANSION); // Encrypt using ChaCha20 (starting at block 1). - m_chacha20.Seek64(nonce, 1); - m_chacha20.Crypt(UCharCast(plain1.data()), UCharCast(cipher.data()), plain1.size()); - m_chacha20.Crypt(UCharCast(plain2.data()), UCharCast(cipher.data() + plain1.size()), plain2.size()); + m_chacha20.Seek(nonce, 1); + m_chacha20.Crypt(plain1, cipher.first(plain1.size())); + m_chacha20.Crypt(plain2, cipher.subspan(plain1.size()).first(plain2.size())); // Seek to block 0, and compute tag using key drawn from there. - m_chacha20.Seek64(nonce, 0); + m_chacha20.Seek(nonce, 0); ComputeTag(m_chacha20, aad, cipher.first(cipher.size() - EXPANSION), cipher.last(EXPANSION)); } @@ -90,22 +90,22 @@ bool AEADChaCha20Poly1305::Decrypt(Span<const std::byte> cipher, Span<const std: assert(cipher.size() == plain1.size() + plain2.size() + EXPANSION); // Verify tag (using key drawn from block 0). - m_chacha20.Seek64(nonce, 0); + m_chacha20.Seek(nonce, 0); std::byte expected_tag[EXPANSION]; ComputeTag(m_chacha20, aad, cipher.first(cipher.size() - EXPANSION), expected_tag); if (timingsafe_bcmp(UCharCast(expected_tag), UCharCast(cipher.last(EXPANSION).data()), EXPANSION)) return false; // Decrypt (starting at block 1). - m_chacha20.Crypt(UCharCast(cipher.data()), UCharCast(plain1.data()), plain1.size()); - m_chacha20.Crypt(UCharCast(cipher.data() + plain1.size()), UCharCast(plain2.data()), plain2.size()); + m_chacha20.Crypt(cipher.first(plain1.size()), plain1); + m_chacha20.Crypt(cipher.subspan(plain1.size()).first(plain2.size()), plain2); return true; } void AEADChaCha20Poly1305::Keystream(Nonce96 nonce, Span<std::byte> keystream) noexcept { // Skip the first output block, as it's used for generating the poly1305 key. - m_chacha20.Seek64(nonce, 1); - m_chacha20.Keystream(UCharCast(keystream.data()), keystream.size()); + m_chacha20.Seek(nonce, 1); + m_chacha20.Keystream(keystream); } void FSChaCha20Poly1305::NextPacket() noexcept @@ -113,7 +113,7 @@ void FSChaCha20Poly1305::NextPacket() noexcept if (++m_packet_counter == m_rekey_interval) { // Generate a full block of keystream, to avoid needing the ChaCha20 buffer, even though // we only need KEYLEN (32) bytes. - std::byte one_block[64]; + std::byte one_block[ChaCha20Aligned::BLOCKLEN]; m_aead.Keystream({0xFFFFFFFF, m_rekey_counter}, one_block); // Switch keys. m_aead.SetKey(Span{one_block}.first(KEYLEN)); diff --git a/src/crypto/muhash.cpp b/src/crypto/muhash.cpp index 471ee6af97..9c35b0689d 100644 --- a/src/crypto/muhash.cpp +++ b/src/crypto/muhash.cpp @@ -299,7 +299,8 @@ Num3072 MuHash3072::ToNum3072(Span<const unsigned char> in) { unsigned char tmp[Num3072::BYTE_SIZE]; uint256 hashed_in{(HashWriter{} << in).GetSHA256()}; - ChaCha20Aligned(hashed_in.data()).Keystream64(tmp, Num3072::BYTE_SIZE / 64); + static_assert(sizeof(tmp) % ChaCha20Aligned::BLOCKLEN == 0); + ChaCha20Aligned{MakeByteSpan(hashed_in)}.Keystream(MakeWritableByteSpan(tmp)); Num3072 out{tmp}; return out; diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index cc7d6687b8..b23d66ac1d 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -10,6 +10,7 @@ #include <index/blockfilterindex.h> #include <logging.h> #include <node/blockstorage.h> +#include <undo.h> #include <util/fs_helpers.h> #include <validation.h> diff --git a/src/index/blockfilterindex.h b/src/index/blockfilterindex.h index ce1961c776..10a1cfd2ee 100644 --- a/src/index/blockfilterindex.h +++ b/src/index/blockfilterindex.h @@ -12,6 +12,8 @@ #include <index/base.h> #include <util/hasher.h> +#include <unordered_map> + static const char* const DEFAULT_BLOCKFILTERINDEX = "0"; /** Interval between compact filter checkpoints. See BIP 157. */ diff --git a/src/init.cpp b/src/init.cpp index c2c4dbe459..2db473ec4b 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -65,7 +65,6 @@ #include <rpc/util.h> #include <scheduler.h> #include <script/sigcache.h> -#include <script/standard.h> #include <shutdown.h> #include <sync.h> #include <timedata.h> @@ -491,7 +490,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-listenonion", strprintf("Automatically create Tor onion service (default: %d)", DEFAULT_LISTEN_ONION), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxconnections=<n>", strprintf("Maintain at most <n> connections to peers (default: %u). This limit does not apply to connections manually added via -addnode or the addnode RPC, which have a separate limit of %u.", DEFAULT_MAX_PEER_CONNECTIONS, MAX_ADDNODE_CONNECTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxreceivebuffer=<n>", strprintf("Maximum per-connection receive buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXRECEIVEBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-maxsendbuffer=<n>", strprintf("Maximum per-connection send buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXSENDBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-maxsendbuffer=<n>", strprintf("Maximum per-connection memory usage for the send buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXSENDBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by outbound peers forward or backward by this amount (default: %u seconds).", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxuploadtarget=<n>", strprintf("Tries to keep outbound traffic under the given target per 24h. Limit does not apply to peers with 'download' permission or blocks created within past week. 0 = no limit (default: %s). Optional suffix units [k|K|m|M|g|G|t|T] (default: M). Lowercase is 1000 base while uppercase is 1024 base", DEFAULT_MAX_UPLOAD_TARGET), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-onion=<ip:port>", "Use separate SOCKS5 proxy to reach peers via Tor onion services, set -noonion to disable (default: -proxy)", ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index 8c31112fc9..9987681367 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -5,11 +5,12 @@ #ifndef BITCOIN_INTERFACES_WALLET_H #define BITCOIN_INTERFACES_WALLET_H +#include <addresstype.h> #include <consensus/amount.h> -#include <interfaces/chain.h> // For ChainClient -#include <pubkey.h> // For CKeyID and CScriptID (definitions needed in CTxDestination instantiation) -#include <script/standard.h> // For CTxDestination -#include <support/allocators/secure.h> // For SecureString +#include <interfaces/chain.h> +#include <pubkey.h> +#include <script/script.h> +#include <support/allocators/secure.h> #include <util/fs.h> #include <util/message.h> #include <util/result.h> diff --git a/src/kernel/mempool_entry.h b/src/kernel/mempool_entry.h index 886e1e1b3a..1f175a5ccf 100644 --- a/src/kernel/mempool_entry.h +++ b/src/kernel/mempool_entry.h @@ -78,6 +78,7 @@ private: const int32_t nTxWeight; //!< ... and avoid recomputing tx weight (also used for GetTxSize()) const size_t nUsageSize; //!< ... and total memory usage const int64_t nTime; //!< Local time when entering the mempool + const uint64_t entry_sequence; //!< Sequence number used to determine whether this transaction is too recent for relay const unsigned int entryHeight; //!< Chain height when entering the mempool const bool spendsCoinbase; //!< keep track of transactions that spend a coinbase const int64_t sigOpCost; //!< Total sigop cost @@ -101,7 +102,7 @@ private: public: CTxMemPoolEntry(const CTransactionRef& tx, CAmount fee, - int64_t time, unsigned int entry_height, + int64_t time, unsigned int entry_height, uint64_t entry_sequence, bool spends_coinbase, int64_t sigops_cost, LockPoints lp) : tx{tx}, @@ -109,6 +110,7 @@ public: nTxWeight{GetTransactionWeight(*tx)}, nUsageSize{RecursiveDynamicUsage(tx)}, nTime{time}, + entry_sequence{entry_sequence}, entryHeight{entry_height}, spendsCoinbase{spends_coinbase}, sigOpCost{sigops_cost}, @@ -130,6 +132,7 @@ public: int32_t GetTxWeight() const { return nTxWeight; } std::chrono::seconds GetTime() const { return std::chrono::seconds{nTime}; } unsigned int GetHeight() const { return entryHeight; } + uint64_t GetSequence() const { return entry_sequence; } int64_t GetSigOpCost() const { return sigOpCost; } CAmount GetModifiedFee() const { return m_modified_fee; } size_t DynamicMemoryUsage() const { return nUsageSize; } diff --git a/src/kernel/mempool_options.h b/src/kernel/mempool_options.h index beb5fca5e9..58bb3debbf 100644 --- a/src/kernel/mempool_options.h +++ b/src/kernel/mempool_options.h @@ -8,7 +8,6 @@ #include <policy/feerate.h> #include <policy/policy.h> -#include <script/standard.h> #include <chrono> #include <cstdint> diff --git a/src/kernel/mempool_removal_reason.cpp b/src/kernel/mempool_removal_reason.cpp new file mode 100644 index 0000000000..df27590c7a --- /dev/null +++ b/src/kernel/mempool_removal_reason.cpp @@ -0,0 +1,21 @@ +// Copyright (c) 2016-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit/. + +#include <kernel/mempool_removal_reason.h> + +#include <cassert> +#include <string> + +std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept +{ + switch (r) { + case MemPoolRemovalReason::EXPIRY: return "expiry"; + case MemPoolRemovalReason::SIZELIMIT: return "sizelimit"; + case MemPoolRemovalReason::REORG: return "reorg"; + case MemPoolRemovalReason::BLOCK: return "block"; + case MemPoolRemovalReason::CONFLICT: return "conflict"; + case MemPoolRemovalReason::REPLACED: return "replaced"; + } + assert(false); +} diff --git a/src/kernel/mempool_removal_reason.h b/src/kernel/mempool_removal_reason.h new file mode 100644 index 0000000000..53c2ff1c31 --- /dev/null +++ b/src/kernel/mempool_removal_reason.h @@ -0,0 +1,24 @@ +// Copyright (c) 2016-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://opensource.org/license/mit/. + +#ifndef BITCOIN_KERNEL_MEMPOOL_REMOVAL_REASON_H +#define BITCOIN_KERNEL_MEMPOOL_REMOVAL_REASON_H + +#include <string> + +/** Reason why a transaction was removed from the mempool, + * this is passed to the notification signal. + */ +enum class MemPoolRemovalReason { + EXPIRY, //!< Expired from mempool + SIZELIMIT, //!< Removed in size limiting + REORG, //!< Removed for reorganization + BLOCK, //!< Removed for block + CONFLICT, //!< Removed for conflict with in-block transaction + REPLACED, //!< Removed for replacement +}; + +std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept; + +#endif // BITCOIN_KERNEL_MEMPOOL_REMOVAL_REASON_H diff --git a/src/key_io.cpp b/src/key_io.cpp index 454a96df5e..a061165613 100644 --- a/src/key_io.cpp +++ b/src/key_io.cpp @@ -6,6 +6,8 @@ #include <base58.h> #include <bech32.h> +#include <script/interpreter.h> +#include <script/solver.h> #include <util/strencodings.h> #include <algorithm> diff --git a/src/key_io.h b/src/key_io.h index 07b80c4b85..e387273e54 100644 --- a/src/key_io.h +++ b/src/key_io.h @@ -6,10 +6,10 @@ #ifndef BITCOIN_KEY_IO_H #define BITCOIN_KEY_IO_H +#include <addresstype.h> #include <chainparams.h> #include <key.h> #include <pubkey.h> -#include <script/standard.h> #include <string> diff --git a/src/net.cpp b/src/net.cpp index b51043ba27..e66c0ec7f8 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -19,6 +19,7 @@ #include <crypto/sha256.h> #include <i2p.h> #include <logging.h> +#include <memusage.h> #include <net_permissions.h> #include <netaddress.h> #include <netbase.h> @@ -116,6 +117,14 @@ std::map<CNetAddr, LocalServiceInfo> mapLocalHost GUARDED_BY(g_maplocalhost_mute static bool vfLimited[NET_MAX] GUARDED_BY(g_maplocalhost_mutex) = {}; std::string strSubVersion; +size_t CSerializedNetMsg::GetMemoryUsage() const noexcept +{ + // Don't count the dynamic memory used for the m_type string, by assuming it fits in the + // "small string" optimization area (which stores data inside the object itself, up to some + // size; 15 bytes in modern libstdc++). + return sizeof(*this) + memusage::DynamicUsage(data); +} + void CConnman::AddAddrFetch(const std::string& strDest) { LOCK(m_addr_fetches_mutex); @@ -681,16 +690,15 @@ bool CNode::ReceiveMsgBytes(Span<const uint8_t> msg_bytes, bool& complete) nRecvBytes += msg_bytes.size(); while (msg_bytes.size() > 0) { // absorb network data - int handled = m_deserializer->Read(msg_bytes); - if (handled < 0) { - // Serious header problem, disconnect from the peer. + if (!m_transport->ReceivedBytes(msg_bytes)) { + // Serious transport problem, disconnect from the peer. return false; } - if (m_deserializer->Complete()) { + if (m_transport->ReceivedMessageComplete()) { // decompose a transport agnostic CNetMessage from the deserializer bool reject_message{false}; - CNetMessage msg = m_deserializer->GetMessage(time, reject_message); + CNetMessage msg = m_transport->GetReceivedMessage(time, reject_message); if (reject_message) { // Message deserialization failed. Drop the message but don't disconnect the peer. // store the size of the corrupt message @@ -717,8 +725,18 @@ bool CNode::ReceiveMsgBytes(Span<const uint8_t> msg_bytes, bool& complete) return true; } -int V1TransportDeserializer::readHeader(Span<const uint8_t> msg_bytes) +V1Transport::V1Transport(const NodeId node_id, int nTypeIn, int nVersionIn) noexcept : + m_node_id(node_id), hdrbuf(nTypeIn, nVersionIn), vRecv(nTypeIn, nVersionIn) +{ + assert(std::size(Params().MessageStart()) == std::size(m_magic_bytes)); + std::copy(std::begin(Params().MessageStart()), std::end(Params().MessageStart()), m_magic_bytes); + LOCK(m_recv_mutex); + Reset(); +} + +int V1Transport::readHeader(Span<const uint8_t> msg_bytes) { + AssertLockHeld(m_recv_mutex); // copy data to temporary parsing buffer unsigned int nRemaining = CMessageHeader::HEADER_SIZE - nHdrPos; unsigned int nCopy = std::min<unsigned int>(nRemaining, msg_bytes.size()); @@ -740,7 +758,7 @@ int V1TransportDeserializer::readHeader(Span<const uint8_t> msg_bytes) } // Check start string, network magic - if (memcmp(hdr.pchMessageStart, m_chain_params.MessageStart(), CMessageHeader::MESSAGE_START_SIZE) != 0) { + if (memcmp(hdr.pchMessageStart, m_magic_bytes, CMessageHeader::MESSAGE_START_SIZE) != 0) { LogPrint(BCLog::NET, "Header error: Wrong MessageStart %s received, peer=%d\n", HexStr(hdr.pchMessageStart), m_node_id); return -1; } @@ -757,8 +775,9 @@ int V1TransportDeserializer::readHeader(Span<const uint8_t> msg_bytes) return nCopy; } -int V1TransportDeserializer::readData(Span<const uint8_t> msg_bytes) +int V1Transport::readData(Span<const uint8_t> msg_bytes) { + AssertLockHeld(m_recv_mutex); unsigned int nRemaining = hdr.nMessageSize - nDataPos; unsigned int nCopy = std::min<unsigned int>(nRemaining, msg_bytes.size()); @@ -774,19 +793,22 @@ int V1TransportDeserializer::readData(Span<const uint8_t> msg_bytes) return nCopy; } -const uint256& V1TransportDeserializer::GetMessageHash() const +const uint256& V1Transport::GetMessageHash() const { - assert(Complete()); + AssertLockHeld(m_recv_mutex); + assert(CompleteInternal()); if (data_hash.IsNull()) hasher.Finalize(data_hash); return data_hash; } -CNetMessage V1TransportDeserializer::GetMessage(const std::chrono::microseconds time, bool& reject_message) +CNetMessage V1Transport::GetReceivedMessage(const std::chrono::microseconds time, bool& reject_message) { + AssertLockNotHeld(m_recv_mutex); // Initialize out parameter reject_message = false; // decompose a single CNetMessage from the TransportDeserializer + LOCK(m_recv_mutex); CNetMessage msg(std::move(vRecv)); // store message type string, time, and sizes @@ -819,53 +841,122 @@ CNetMessage V1TransportDeserializer::GetMessage(const std::chrono::microseconds return msg; } -void V1TransportSerializer::prepareForTransport(CSerializedNetMsg& msg, std::vector<unsigned char>& header) const +bool V1Transport::SetMessageToSend(CSerializedNetMsg& msg) noexcept { + AssertLockNotHeld(m_send_mutex); + // Determine whether a new message can be set. + LOCK(m_send_mutex); + if (m_sending_header || m_bytes_sent < m_message_to_send.data.size()) return false; + // create dbl-sha256 checksum uint256 hash = Hash(msg.data); // create header - CMessageHeader hdr(Params().MessageStart(), msg.m_type.c_str(), msg.data.size()); + CMessageHeader hdr(m_magic_bytes, msg.m_type.c_str(), msg.data.size()); memcpy(hdr.pchChecksum, hash.begin(), CMessageHeader::CHECKSUM_SIZE); // serialize header - header.reserve(CMessageHeader::HEADER_SIZE); - CVectorWriter{SER_NETWORK, INIT_PROTO_VERSION, header, 0, hdr}; + m_header_to_send.clear(); + CVectorWriter{SER_NETWORK, INIT_PROTO_VERSION, m_header_to_send, 0, hdr}; + + // update state + m_message_to_send = std::move(msg); + m_sending_header = true; + m_bytes_sent = 0; + return true; } -size_t CConnman::SocketSendData(CNode& node) const +Transport::BytesToSend V1Transport::GetBytesToSend() const noexcept +{ + AssertLockNotHeld(m_send_mutex); + LOCK(m_send_mutex); + if (m_sending_header) { + return {Span{m_header_to_send}.subspan(m_bytes_sent), + // We have more to send after the header if the message has payload. + !m_message_to_send.data.empty(), + m_message_to_send.m_type + }; + } else { + return {Span{m_message_to_send.data}.subspan(m_bytes_sent), + // We never have more to send after this message's payload. + false, + m_message_to_send.m_type + }; + } +} + +void V1Transport::MarkBytesSent(size_t bytes_sent) noexcept +{ + AssertLockNotHeld(m_send_mutex); + LOCK(m_send_mutex); + m_bytes_sent += bytes_sent; + if (m_sending_header && m_bytes_sent == m_header_to_send.size()) { + // We're done sending a message's header. Switch to sending its data bytes. + m_sending_header = false; + m_bytes_sent = 0; + } else if (!m_sending_header && m_bytes_sent == m_message_to_send.data.size()) { + // We're done sending a message's data. Wipe the data vector to reduce memory consumption. + m_message_to_send.data.clear(); + m_message_to_send.data.shrink_to_fit(); + m_bytes_sent = 0; + } +} + +size_t V1Transport::GetSendMemoryUsage() const noexcept +{ + AssertLockNotHeld(m_send_mutex); + LOCK(m_send_mutex); + // Don't count sending-side fields besides m_message_to_send, as they're all small and bounded. + return m_message_to_send.GetMemoryUsage(); +} + +std::pair<size_t, bool> CConnman::SocketSendData(CNode& node) const { auto it = node.vSendMsg.begin(); size_t nSentSize = 0; - - while (it != node.vSendMsg.end()) { - const auto& data = *it; - assert(data.size() > node.nSendOffset); + bool data_left{false}; //!< second return value (whether unsent data remains) + + while (true) { + if (it != node.vSendMsg.end()) { + // If possible, move one message from the send queue to the transport. This fails when + // there is an existing message still being sent. + size_t memusage = it->GetMemoryUsage(); + if (node.m_transport->SetMessageToSend(*it)) { + // Update memory usage of send buffer (as *it will be deleted). + node.m_send_memusage -= memusage; + ++it; + } + } + const auto& [data, more, msg_type] = node.m_transport->GetBytesToSend(); + data_left = !data.empty(); // will be overwritten on next loop if all of data gets sent int nBytes = 0; - { + if (!data.empty()) { LOCK(node.m_sock_mutex); + // There is no socket in case we've already disconnected, or in test cases without + // real connections. In these cases, we bail out immediately and just leave things + // in the send queue and transport. if (!node.m_sock) { break; } int flags = MSG_NOSIGNAL | MSG_DONTWAIT; #ifdef MSG_MORE - if (it + 1 != node.vSendMsg.end()) { + // We have more to send if either the transport itself has more, or if we have more + // messages to send. + if (more || it != node.vSendMsg.end()) { flags |= MSG_MORE; } #endif - nBytes = node.m_sock->Send(reinterpret_cast<const char*>(data.data()) + node.nSendOffset, data.size() - node.nSendOffset, flags); + nBytes = node.m_sock->Send(reinterpret_cast<const char*>(data.data()), data.size(), flags); } if (nBytes > 0) { node.m_last_send = GetTime<std::chrono::seconds>(); node.nSendBytes += nBytes; - node.nSendOffset += nBytes; + // Notify transport that bytes have been processed. + node.m_transport->MarkBytesSent(nBytes); + // Update statistics per message type. + node.AccountForSentBytes(msg_type, nBytes); nSentSize += nBytes; - if (node.nSendOffset == data.size()) { - node.nSendOffset = 0; - node.nSendSize -= data.size(); - node.fPauseSend = node.nSendSize > nSendBufferMaxSize; - it++; - } else { + if ((size_t)nBytes != data.size()) { // could not send full message; stop sending more break; } @@ -878,17 +969,17 @@ size_t CConnman::SocketSendData(CNode& node) const node.CloseSocketDisconnect(); } } - // couldn't send anything at all break; } } + node.fPauseSend = node.m_send_memusage + node.m_transport->GetSendMemoryUsage() > nSendBufferMaxSize; + if (it == node.vSendMsg.end()) { - assert(node.nSendOffset == 0); - assert(node.nSendSize == 0); + assert(node.m_send_memusage == 0); } node.vSendMsg.erase(node.vSendMsg.begin(), it); - return nSentSize; + return {nSentSize, data_left}; } /** Try to find a connection to evict when the node is full. @@ -1226,37 +1317,22 @@ Sock::EventsPerSock CConnman::GenerateWaitSockets(Span<CNode* const> nodes) } for (CNode* pnode : nodes) { - // Implement the following logic: - // * If there is data to send, select() for sending data. As this only - // happens when optimistic write failed, we choose to first drain the - // write buffer in this case before receiving more. This avoids - // needlessly queueing received data, if the remote peer is not themselves - // receiving data. This means properly utilizing TCP flow control signalling. - // * Otherwise, if there is space left in the receive buffer, select() for - // receiving data. - // * Hand off all complete messages to the processor, to be handled without - // blocking here. - bool select_recv = !pnode->fPauseRecv; bool select_send; { LOCK(pnode->cs_vSend); - select_send = !pnode->vSendMsg.empty(); + // Sending is possible if either there are bytes to send right now, or if there will be + // once a potential message from vSendMsg is handed to the transport. + const auto& [to_send, _more, _msg_type] = pnode->m_transport->GetBytesToSend(); + select_send = !to_send.empty() || !pnode->vSendMsg.empty(); } + if (!select_recv && !select_send) continue; LOCK(pnode->m_sock_mutex); - if (!pnode->m_sock) { - continue; - } - - Sock::Event requested{0}; - if (select_send) { - requested = Sock::SEND; - } else if (select_recv) { - requested = Sock::RECV; + if (pnode->m_sock) { + Sock::Event event = (select_send ? Sock::SEND : 0) | (select_recv ? Sock::RECV : 0); + events_per_sock.emplace(pnode->m_sock, Sock::Events{event}); } - - events_per_sock.emplace(pnode->m_sock, Sock::Events{requested}); } return events_per_sock; @@ -1317,6 +1393,24 @@ void CConnman::SocketHandlerConnected(const std::vector<CNode*>& nodes, errorSet = it->second.occurred & Sock::ERR; } } + + if (sendSet) { + // Send data + auto [bytes_sent, data_left] = WITH_LOCK(pnode->cs_vSend, return SocketSendData(*pnode)); + if (bytes_sent) { + RecordBytesSent(bytes_sent); + + // If both receiving and (non-optimistic) sending were possible, we first attempt + // sending. If that succeeds, but does not fully drain the send queue, do not + // attempt to receive. This avoids needlessly queueing data if the remote peer + // is slow at receiving data, by means of TCP flow control. We only do this when + // sending actually succeeded to make sure progress is always made; otherwise a + // deadlock would be possible when both sides have data to send, but neither is + // receiving. + if (data_left) recvSet = false; + } + } + if (recvSet || errorSet) { // typical socket buffer is 8K-64K @@ -1363,12 +1457,6 @@ void CConnman::SocketHandlerConnected(const std::vector<CNode*>& nodes, } } - if (sendSet) { - // Send data - size_t bytes_sent = WITH_LOCK(pnode->cs_vSend, return SocketSendData(*pnode)); - if (bytes_sent) RecordBytesSent(bytes_sent); - } - if (InactivityCheck(*pnode)) pnode->fDisconnect = true; } } @@ -2832,8 +2920,7 @@ CNode::CNode(NodeId idIn, ConnectionType conn_type_in, bool inbound_onion, CNodeOptions&& node_opts) - : m_deserializer{std::make_unique<V1TransportDeserializer>(V1TransportDeserializer(Params(), idIn, SER_NETWORK, INIT_PROTO_VERSION))}, - m_serializer{std::make_unique<V1TransportSerializer>(V1TransportSerializer())}, + : m_transport{std::make_unique<V1Transport>(idIn, SER_NETWORK, INIT_PROTO_VERSION)}, m_permission_flags{node_opts.permission_flags}, m_sock{sock}, m_connected{GetTime<std::chrono::seconds>()}, @@ -2916,26 +3003,24 @@ void CConnman::PushMessage(CNode* pnode, CSerializedNetMsg&& msg) msg.data.data() ); - // make sure we use the appropriate network transport format - std::vector<unsigned char> serializedHeader; - pnode->m_serializer->prepareForTransport(msg, serializedHeader); - size_t nTotalSize = nMessageSize + serializedHeader.size(); - size_t nBytesSent = 0; { LOCK(pnode->cs_vSend); - bool optimisticSend(pnode->vSendMsg.empty()); + const auto& [to_send, _more, _msg_type] = pnode->m_transport->GetBytesToSend(); + const bool queue_was_empty{to_send.empty() && pnode->vSendMsg.empty()}; - //log total amount of bytes per message type - pnode->AccountForSentBytes(msg.m_type, nTotalSize); - pnode->nSendSize += nTotalSize; + // Update memory usage of send buffer. + pnode->m_send_memusage += msg.GetMemoryUsage(); + if (pnode->m_send_memusage + pnode->m_transport->GetSendMemoryUsage() > nSendBufferMaxSize) pnode->fPauseSend = true; + // Move message to vSendMsg queue. + pnode->vSendMsg.push_back(std::move(msg)); - if (pnode->nSendSize > nSendBufferMaxSize) pnode->fPauseSend = true; - pnode->vSendMsg.push_back(std::move(serializedHeader)); - if (nMessageSize) pnode->vSendMsg.push_back(std::move(msg.data)); - - // If write queue empty, attempt "optimistic write" - if (optimisticSend) nBytesSent = SocketSendData(*pnode); + // If there was nothing to send before, attempt "optimistic write": + // because the poll/select loop may pause for SELECT_TIMEOUT_MILLISECONDS before actually + // doing a send, try sending from the calling thread if the queue was empty before. + if (queue_was_empty) { + std::tie(nBytesSent, std::ignore) = SocketSendData(*pnode); + } } if (nBytesSent) RecordBytesSent(nBytesSent); } @@ -122,6 +122,9 @@ struct CSerializedNetMsg { std::vector<unsigned char> data; std::string m_type; + + /** Compute total memory usage of this object (own memory + any dynamic memory). */ + size_t GetMemoryUsage() const noexcept; }; /** @@ -253,42 +256,105 @@ public: } }; -/** The TransportDeserializer takes care of holding and deserializing the - * network receive buffer. It can deserialize the network buffer into a - * transport protocol agnostic CNetMessage (message type & payload) - */ -class TransportDeserializer { +/** The Transport converts one connection's sent messages to wire bytes, and received bytes back. */ +class Transport { public: - // returns true if the current deserialization is complete - virtual bool Complete() const = 0; - // set the serialization context version - virtual void SetVersion(int version) = 0; - /** read and deserialize data, advances msg_bytes data pointer */ - virtual int Read(Span<const uint8_t>& msg_bytes) = 0; - // decomposes a message from the context - virtual CNetMessage GetMessage(std::chrono::microseconds time, bool& reject_message) = 0; - virtual ~TransportDeserializer() {} + virtual ~Transport() {} + + // 1. Receiver side functions, for decoding bytes received on the wire into transport protocol + // agnostic CNetMessage (message type & payload) objects. + + /** Returns true if the current message is complete (so GetReceivedMessage can be called). */ + virtual bool ReceivedMessageComplete() const = 0; + /** Set the deserialization context version for objects returned by GetReceivedMessage. */ + virtual void SetReceiveVersion(int version) = 0; + + /** Feed wire bytes to the transport. + * + * @return false if some bytes were invalid, in which case the transport can't be used anymore. + * + * Consumed bytes are chopped off the front of msg_bytes. + */ + virtual bool ReceivedBytes(Span<const uint8_t>& msg_bytes) = 0; + + /** Retrieve a completed message from transport. + * + * This can only be called when ReceivedMessageComplete() is true. + * + * If reject_message=true is returned the message itself is invalid, but (other than false + * returned by ReceivedBytes) the transport is not in an inconsistent state. + */ + virtual CNetMessage GetReceivedMessage(std::chrono::microseconds time, bool& reject_message) = 0; + + // 2. Sending side functions, for converting messages into bytes to be sent over the wire. + + /** Set the next message to send. + * + * If no message can currently be set (perhaps because the previous one is not yet done being + * sent), returns false, and msg will be unmodified. Otherwise msg is enqueued (and + * possibly moved-from) and true is returned. + */ + virtual bool SetMessageToSend(CSerializedNetMsg& msg) noexcept = 0; + + /** Return type for GetBytesToSend, consisting of: + * - Span<const uint8_t> to_send: span of bytes to be sent over the wire (possibly empty). + * - bool more: whether there will be more bytes to be sent after the ones in to_send are + * all sent (as signaled by MarkBytesSent()). + * - const std::string& m_type: message type on behalf of which this is being sent. + */ + using BytesToSend = std::tuple< + Span<const uint8_t> /*to_send*/, + bool /*more*/, + const std::string& /*m_type*/ + >; + + /** Get bytes to send on the wire. + * + * As a const function, it does not modify the transport's observable state, and is thus safe + * to be called multiple times. + * + * The bytes returned by this function act as a stream which can only be appended to. This + * means that with the exception of MarkBytesSent, operations on the transport can only append + * to what is being returned. + * + * Note that m_type and to_send refer to data that is internal to the transport, and calling + * any non-const function on this object may invalidate them. + */ + virtual BytesToSend GetBytesToSend() const noexcept = 0; + + /** Report how many bytes returned by the last GetBytesToSend() have been sent. + * + * bytes_sent cannot exceed to_send.size() of the last GetBytesToSend() result. + * + * If bytes_sent=0, this call has no effect. + */ + virtual void MarkBytesSent(size_t bytes_sent) noexcept = 0; + + /** Return the memory usage of this transport attributable to buffered data to send. */ + virtual size_t GetSendMemoryUsage() const noexcept = 0; }; -class V1TransportDeserializer final : public TransportDeserializer +class V1Transport final : public Transport { private: - const CChainParams& m_chain_params; + CMessageHeader::MessageStartChars m_magic_bytes; const NodeId m_node_id; // Only for logging - mutable CHash256 hasher; - mutable uint256 data_hash; - bool in_data; // parsing header (false) or data (true) - CDataStream hdrbuf; // partially received header - CMessageHeader hdr; // complete header - CDataStream vRecv; // received message data - unsigned int nHdrPos; - unsigned int nDataPos; - - const uint256& GetMessageHash() const; - int readHeader(Span<const uint8_t> msg_bytes); - int readData(Span<const uint8_t> msg_bytes); - - void Reset() { + mutable Mutex m_recv_mutex; //!< Lock for receive state + mutable CHash256 hasher GUARDED_BY(m_recv_mutex); + mutable uint256 data_hash GUARDED_BY(m_recv_mutex); + bool in_data GUARDED_BY(m_recv_mutex); // parsing header (false) or data (true) + CDataStream hdrbuf GUARDED_BY(m_recv_mutex); // partially received header + CMessageHeader hdr GUARDED_BY(m_recv_mutex); // complete header + CDataStream vRecv GUARDED_BY(m_recv_mutex); // received message data + unsigned int nHdrPos GUARDED_BY(m_recv_mutex); + unsigned int nDataPos GUARDED_BY(m_recv_mutex); + + const uint256& GetMessageHash() const EXCLUSIVE_LOCKS_REQUIRED(m_recv_mutex); + int readHeader(Span<const uint8_t> msg_bytes) EXCLUSIVE_LOCKS_REQUIRED(m_recv_mutex); + int readData(Span<const uint8_t> msg_bytes) EXCLUSIVE_LOCKS_REQUIRED(m_recv_mutex); + + void Reset() EXCLUSIVE_LOCKS_REQUIRED(m_recv_mutex) { + AssertLockHeld(m_recv_mutex); vRecv.clear(); hdrbuf.clear(); hdrbuf.resize(24); @@ -299,52 +365,60 @@ private: hasher.Reset(); } -public: - V1TransportDeserializer(const CChainParams& chain_params, const NodeId node_id, int nTypeIn, int nVersionIn) - : m_chain_params(chain_params), - m_node_id(node_id), - hdrbuf(nTypeIn, nVersionIn), - vRecv(nTypeIn, nVersionIn) + bool CompleteInternal() const noexcept EXCLUSIVE_LOCKS_REQUIRED(m_recv_mutex) { - Reset(); + AssertLockHeld(m_recv_mutex); + if (!in_data) return false; + return hdr.nMessageSize == nDataPos; } - bool Complete() const override + /** Lock for sending state. */ + mutable Mutex m_send_mutex; + /** The header of the message currently being sent. */ + std::vector<uint8_t> m_header_to_send GUARDED_BY(m_send_mutex); + /** The data of the message currently being sent. */ + CSerializedNetMsg m_message_to_send GUARDED_BY(m_send_mutex); + /** Whether we're currently sending header bytes or message bytes. */ + bool m_sending_header GUARDED_BY(m_send_mutex) {false}; + /** How many bytes have been sent so far (from m_header_to_send, or from m_message_to_send.data). */ + size_t m_bytes_sent GUARDED_BY(m_send_mutex) {0}; + +public: + V1Transport(const NodeId node_id, int nTypeIn, int nVersionIn) noexcept; + + bool ReceivedMessageComplete() const override EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex) { - if (!in_data) - return false; - return (hdr.nMessageSize == nDataPos); + AssertLockNotHeld(m_recv_mutex); + return WITH_LOCK(m_recv_mutex, return CompleteInternal()); } - void SetVersion(int nVersionIn) override + + void SetReceiveVersion(int nVersionIn) override EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex) { + AssertLockNotHeld(m_recv_mutex); + LOCK(m_recv_mutex); hdrbuf.SetVersion(nVersionIn); vRecv.SetVersion(nVersionIn); } - int Read(Span<const uint8_t>& msg_bytes) override + + bool ReceivedBytes(Span<const uint8_t>& msg_bytes) override EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex) { + AssertLockNotHeld(m_recv_mutex); + LOCK(m_recv_mutex); int ret = in_data ? readData(msg_bytes) : readHeader(msg_bytes); if (ret < 0) { Reset(); } else { msg_bytes = msg_bytes.subspan(ret); } - return ret; + return ret >= 0; } - CNetMessage GetMessage(std::chrono::microseconds time, bool& reject_message) override; -}; -/** The TransportSerializer prepares messages for the network transport - */ -class TransportSerializer { -public: - // prepare message for transport (header construction, error-correction computation, payload encryption, etc.) - virtual void prepareForTransport(CSerializedNetMsg& msg, std::vector<unsigned char>& header) const = 0; - virtual ~TransportSerializer() {} -}; + CNetMessage GetReceivedMessage(std::chrono::microseconds time, bool& reject_message) override EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex); -class V1TransportSerializer : public TransportSerializer { -public: - void prepareForTransport(CSerializedNetMsg& msg, std::vector<unsigned char>& header) const override; + bool SetMessageToSend(CSerializedNetMsg& msg) noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_send_mutex); + BytesToSend GetBytesToSend() const noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_send_mutex); + void MarkBytesSent(size_t bytes_sent) noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_send_mutex); + size_t GetSendMemoryUsage() const noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_send_mutex); }; struct CNodeOptions @@ -359,8 +433,9 @@ struct CNodeOptions class CNode { public: - const std::unique_ptr<TransportDeserializer> m_deserializer; // Used only by SocketHandler thread - const std::unique_ptr<const TransportSerializer> m_serializer; + /** Transport serializer/deserializer. The receive side functions are only called under cs_vRecv, while + * the sending side functions are only called under cs_vSend. */ + const std::unique_ptr<Transport> m_transport; const NetPermissionFlags m_permission_flags; @@ -374,12 +449,12 @@ public: */ std::shared_ptr<Sock> m_sock GUARDED_BY(m_sock_mutex); - /** Total size of all vSendMsg entries */ - size_t nSendSize GUARDED_BY(cs_vSend){0}; - /** Offset inside the first vSendMsg already sent */ - size_t nSendOffset GUARDED_BY(cs_vSend){0}; + /** Sum of GetMemoryUsage of all vSendMsg entries. */ + size_t m_send_memusage GUARDED_BY(cs_vSend){0}; + /** Total number of bytes sent on the wire to this peer. */ uint64_t nSendBytes GUARDED_BY(cs_vSend){0}; - std::deque<std::vector<unsigned char>> vSendMsg GUARDED_BY(cs_vSend); + /** Messages still to be fed to m_transport->SetMessageToSend. */ + std::deque<CSerializedNetMsg> vSendMsg GUARDED_BY(cs_vSend); Mutex cs_vSend; Mutex m_sock_mutex; Mutex cs_vRecv; @@ -1013,7 +1088,9 @@ private: NodeId GetNewNodeId(); - size_t SocketSendData(CNode& node) const EXCLUSIVE_LOCKS_REQUIRED(node.cs_vSend); + /** (Try to) send data from node's vSendMsg. Returns (bytes_sent, data_left). */ + std::pair<size_t, bool> SocketSendData(CNode& node) const EXCLUSIVE_LOCKS_REQUIRED(node.cs_vSend); + void DumpAddresses(); // Network stats diff --git a/src/net_processing.cpp b/src/net_processing.cpp index e2bbfe3308..8189d6c9f3 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -51,8 +51,6 @@ #include <optional> #include <typeinfo> -/** How long a transaction has to be in the mempool before it can unconditionally be relayed. */ -static constexpr auto UNCONDITIONAL_RELAY_DELAY = 2min; /** Headers download timeout. * Timeout = base + per_header * (expected number of headers) */ static constexpr auto HEADERS_DOWNLOAD_TIMEOUT_BASE = 15min; @@ -149,15 +147,12 @@ static constexpr auto OUTBOUND_INVENTORY_BROADCAST_INTERVAL{2s}; /** Maximum rate of inventory items to send per second. * Limits the impact of low-fee transaction floods. */ static constexpr unsigned int INVENTORY_BROADCAST_PER_SECOND = 7; +/** Target number of tx inventory items to send per transmission. */ +static constexpr unsigned int INVENTORY_BROADCAST_TARGET = INVENTORY_BROADCAST_PER_SECOND * count_seconds(INBOUND_INVENTORY_BROADCAST_INTERVAL); /** Maximum number of inventory items to send per transmission. */ -static constexpr unsigned int INVENTORY_BROADCAST_MAX = INVENTORY_BROADCAST_PER_SECOND * count_seconds(INBOUND_INVENTORY_BROADCAST_INTERVAL); -/** The number of most recently announced transactions a peer can request. */ -static constexpr unsigned int INVENTORY_MAX_RECENT_RELAY = 3500; -/** Verify that INVENTORY_MAX_RECENT_RELAY is enough to cache everything typically - * relayed before unconditional relay from the mempool kicks in. This is only a - * lower bound, and it should be larger to account for higher inv rate to outbound - * peers, and random variations in the broadcast mechanism. */ -static_assert(INVENTORY_MAX_RECENT_RELAY >= INVENTORY_BROADCAST_PER_SECOND * UNCONDITIONAL_RELAY_DELAY / std::chrono::seconds{1}, "INVENTORY_RELAY_MAX too low"); +static constexpr unsigned int INVENTORY_BROADCAST_MAX = 1000; +static_assert(INVENTORY_BROADCAST_MAX >= INVENTORY_BROADCAST_TARGET, "INVENTORY_BROADCAST_MAX too low"); +static_assert(INVENTORY_BROADCAST_MAX <= MAX_PEER_TX_ANNOUNCEMENTS, "INVENTORY_BROADCAST_MAX too high"); /** Average delay between feefilter broadcasts in seconds. */ static constexpr auto AVG_FEEFILTER_BROADCAST_INTERVAL{10min}; /** Maximum feefilter broadcast delay after significant change. */ @@ -273,13 +268,10 @@ struct Peer { /** A bloom filter for which transactions to announce to the peer. See BIP37. */ std::unique_ptr<CBloomFilter> m_bloom_filter PT_GUARDED_BY(m_bloom_filter_mutex) GUARDED_BY(m_bloom_filter_mutex){nullptr}; - /** A rolling bloom filter of all announced tx CInvs to this peer */ - CRollingBloomFilter m_recently_announced_invs GUARDED_BY(NetEventsInterface::g_msgproc_mutex){INVENTORY_MAX_RECENT_RELAY, 0.000001}; - mutable RecursiveMutex m_tx_inventory_mutex; - /** A filter of all the txids and wtxids that the peer has announced to + /** A filter of all the (w)txids that the peer has announced to * us or we have announced to the peer. We use this to avoid announcing - * the same txid/wtxid to a peer that already has the transaction. */ + * the same (w)txid to a peer that already has the transaction. */ CRollingBloomFilter m_tx_inventory_known_filter GUARDED_BY(m_tx_inventory_mutex){50000, 0.000001}; /** Set of transaction ids we still have to announce (txid for * non-wtxid-relay peers, wtxid for wtxid-relay peers). We use the @@ -290,11 +282,12 @@ struct Peer { * permitted if the peer has NetPermissionFlags::Mempool or we advertise * NODE_BLOOM. See BIP35. */ bool m_send_mempool GUARDED_BY(m_tx_inventory_mutex){false}; - /** The last time a BIP35 `mempool` request was serviced. */ - std::atomic<std::chrono::seconds> m_last_mempool_req{0s}; /** The next time after which we will send an `inv` message containing * transaction announcements to this peer. */ std::chrono::microseconds m_next_inv_send_time GUARDED_BY(m_tx_inventory_mutex){0}; + /** The mempool sequence num at which we sent the last `inv` message to this peer. + * Can relay txs with lower sequence numbers than this (see CTxMempool::info_for_relay). */ + uint64_t m_last_inv_sequence GUARDED_BY(NetEventsInterface::g_msgproc_mutex){1}; /** Minimum fee rate with which to filter transaction announcements to this node. See BIP133. */ std::atomic<CAmount> m_fee_filter_received{0}; @@ -907,7 +900,7 @@ private: std::atomic<std::chrono::seconds> m_last_tip_update{0s}; /** Determine whether or not a peer can request a transaction, and return it (or nullptr if not found or not allowed). */ - CTransactionRef FindTxForGetData(const Peer::TxRelay& tx_relay, const GenTxid& gtxid, const std::chrono::seconds mempool_req, const std::chrono::seconds now) + CTransactionRef FindTxForGetData(const Peer::TxRelay& tx_relay, const GenTxid& gtxid) EXCLUSIVE_LOCKS_REQUIRED(!m_most_recent_block_mutex, NetEventsInterface::g_msgproc_mutex); void ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic<bool>& interruptMsgProc) @@ -2017,7 +2010,7 @@ void PeerManagerImpl::BlockChecked(const CBlock& block, const BlockValidationSta // the tip yet so we have no way to check this directly here. Instead we // just check that there are currently no other blocks in flight. else if (state.IsValid() && - !m_chainman.ActiveChainstate().IsInitialBlockDownload() && + !m_chainman.IsInitialBlockDownload() && mapBlocksInFlight.count(hash) == mapBlocksInFlight.size()) { if (it != mapBlockSource.end()) { MaybeSetPeerAsAnnouncingHeaderAndIDs(it->second.first); @@ -2288,22 +2281,14 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& } } -CTransactionRef PeerManagerImpl::FindTxForGetData(const Peer::TxRelay& tx_relay, const GenTxid& gtxid, const std::chrono::seconds mempool_req, const std::chrono::seconds now) +CTransactionRef PeerManagerImpl::FindTxForGetData(const Peer::TxRelay& tx_relay, const GenTxid& gtxid) { - auto txinfo = m_mempool.info(gtxid); + // If a tx was in the mempool prior to the last INV for this peer, permit the request. + auto txinfo = m_mempool.info_for_relay(gtxid, tx_relay.m_last_inv_sequence); if (txinfo.tx) { - // If a TX could have been INVed in reply to a MEMPOOL request, - // or is older than UNCONDITIONAL_RELAY_DELAY, permit the request - // unconditionally. - if ((mempool_req.count() && txinfo.m_time <= mempool_req) || txinfo.m_time <= now - UNCONDITIONAL_RELAY_DELAY) { - return std::move(txinfo.tx); - } + return std::move(txinfo.tx); } - // Otherwise, the transaction might have been announced recently. - bool recent = tx_relay.m_recently_announced_invs.contains(gtxid.GetHash()); - if (recent && txinfo.tx) return std::move(txinfo.tx); - // Or it might be from the most recent block { LOCK(m_most_recent_block_mutex); @@ -2326,10 +2311,6 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic std::vector<CInv> vNotFound; const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); - const auto now{GetTime<std::chrono::seconds>()}; - // Get last mempool request time - const auto mempool_req = tx_relay != nullptr ? tx_relay->m_last_mempool_req.load() : std::chrono::seconds::min(); - // Process as many TX items from the front of the getdata queue as // possible, since they're common and it's efficient to batch process // them. @@ -2347,33 +2328,12 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic continue; } - CTransactionRef tx = FindTxForGetData(*tx_relay, ToGenTxid(inv), mempool_req, now); + CTransactionRef tx = FindTxForGetData(*tx_relay, ToGenTxid(inv)); if (tx) { // WTX and WITNESS_TX imply we serialize with witness int nSendFlags = (inv.IsMsgTx() ? SERIALIZE_TRANSACTION_NO_WITNESS : 0); m_connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *tx)); m_mempool.RemoveUnbroadcastTx(tx->GetHash()); - // As we're going to send tx, make sure its unconfirmed parents are made requestable. - std::vector<uint256> parent_ids_to_add; - { - LOCK(m_mempool.cs); - auto tx_iter = m_mempool.GetIter(tx->GetHash()); - if (tx_iter) { - const CTxMemPoolEntry::Parents& parents = (*tx_iter)->GetMemPoolParentsConst(); - parent_ids_to_add.reserve(parents.size()); - for (const CTxMemPoolEntry& parent : parents) { - if (parent.GetTime() > now - UNCONDITIONAL_RELAY_DELAY) { - parent_ids_to_add.push_back(parent.GetTx().GetHash()); - } - } - } - } - for (const uint256& parent_txid : parent_ids_to_add) { - // Relaying a transaction with a recent but unconfirmed parent. - if (WITH_LOCK(tx_relay->m_tx_inventory_mutex, return !tx_relay->m_tx_inventory_known_filter.contains(parent_txid))) { - tx_relay->m_recently_announced_invs.insert(parent_txid); - } - } } else { vNotFound.push_back(inv); } @@ -2769,7 +2729,7 @@ void PeerManagerImpl::UpdatePeerStateForReceivedHeaders(CNode& pfrom, Peer& peer // If we're in IBD, we want outbound peers that will serve us a useful // chain. Disconnect peers that are on chains with insufficient work. - if (m_chainman.ActiveChainstate().IsInitialBlockDownload() && !may_have_more_headers) { + if (m_chainman.IsInitialBlockDownload() && !may_have_more_headers) { // If the peer has no more headers to give us, then we know we have // their tip. if (nodestate->pindexBestKnownBlock && nodestate->pindexBestKnownBlock->nChainWork < m_chainman.MinimumChainWork()) { @@ -3848,7 +3808,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, LogPrint(BCLog::NET, "got inv: %s %s peer=%d\n", inv.ToString(), fAlreadyHave ? "have" : "new", pfrom.GetId()); AddKnownTx(*peer, inv.hash); - if (!fAlreadyHave && !m_chainman.ActiveChainstate().IsInitialBlockDownload()) { + if (!fAlreadyHave && !m_chainman.IsInitialBlockDownload()) { AddTxAnnouncement(pfrom, gtxid, current_time); } } else { @@ -4120,7 +4080,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Stop processing the transaction early if we are still in IBD since we don't // have enough information to validate it yet. Sending unsolicited transactions // is not considered a protocol violation, so don't punish the peer. - if (m_chainman.ActiveChainstate().IsInitialBlockDownload()) return; + if (m_chainman.IsInitialBlockDownload()) return; CTransactionRef ptx; vRecv >> ptx; @@ -4131,14 +4091,6 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, const uint256& hash = peer->m_wtxid_relay ? wtxid : txid; AddKnownTx(*peer, hash); - if (peer->m_wtxid_relay && txid != wtxid) { - // Insert txid into m_tx_inventory_known_filter, even for - // wtxidrelay peers. This prevents re-adding of - // unconfirmed parents to the recently_announced - // filter, when a child tx is requested. See - // ProcessGetData(). - AddKnownTx(*peer, txid); - } LOCK(cs_main); @@ -4332,7 +4284,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, const CBlockIndex* prev_block = m_chainman.m_blockman.LookupBlockIndex(cmpctblock.header.hashPrevBlock); if (!prev_block) { // Doesn't connect (or is genesis), instead of DoSing in AcceptBlockHeader, request deeper headers - if (!m_chainman.ActiveChainstate().IsInitialBlockDownload()) { + if (!m_chainman.IsInitialBlockDownload()) { MaybeSendGetHeaders(pfrom, GetLocator(m_chainman.m_best_header), *peer); } return; @@ -5276,7 +5228,7 @@ void PeerManagerImpl::MaybeSendAddr(CNode& node, Peer& peer, std::chrono::micros LOCK(peer.m_addr_send_times_mutex); // Periodically advertise our local address to the peer. - if (fListen && !m_chainman.ActiveChainstate().IsInitialBlockDownload() && + if (fListen && !m_chainman.IsInitialBlockDownload() && peer.m_next_local_addr_send < current_time) { // If we've sent before, clear the bloom filter for the peer, so that our // self-announcement will actually go out. @@ -5371,7 +5323,7 @@ void PeerManagerImpl::MaybeSendFeefilter(CNode& pto, Peer& peer, std::chrono::mi CAmount currentFilter = m_mempool.GetMinFee().GetFeePerK(); static FeeFilterRounder g_filter_rounder{CFeeRate{DEFAULT_MIN_RELAY_TX_FEE}}; - if (m_chainman.ActiveChainstate().IsInitialBlockDownload()) { + if (m_chainman.IsInitialBlockDownload()) { // Received tx-inv messages are discarded when the active // chainstate is in IBD, so tell the peer to not send them. currentFilter = MAX_MONEY; @@ -5684,7 +5636,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) std::vector<CInv> vInv; { LOCK(peer->m_block_inv_mutex); - vInv.reserve(std::max<size_t>(peer->m_blocks_for_inv_relay.size(), INVENTORY_BROADCAST_MAX)); + vInv.reserve(std::max<size_t>(peer->m_blocks_for_inv_relay.size(), INVENTORY_BROADCAST_TARGET)); // Add blocks for (const uint256& hash : peer->m_blocks_for_inv_relay) { @@ -5736,14 +5688,12 @@ bool PeerManagerImpl::SendMessages(CNode* pto) if (!tx_relay->m_bloom_filter->IsRelevantAndUpdate(*txinfo.tx)) continue; } tx_relay->m_tx_inventory_known_filter.insert(hash); - // Responses to MEMPOOL requests bypass the m_recently_announced_invs filter. vInv.push_back(inv); if (vInv.size() == MAX_INV_SZ) { m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); vInv.clear(); } } - tx_relay->m_last_mempool_req = std::chrono::duration_cast<std::chrono::seconds>(current_time); } // Determine transactions to relay @@ -5763,8 +5713,8 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // especially since we have many peers and some will draw much shorter delays. unsigned int nRelayedTransactions = 0; LOCK(tx_relay->m_bloom_filter_mutex); - size_t broadcast_max{INVENTORY_BROADCAST_MAX + (tx_relay->m_tx_inventory_to_send.size()/1000)*5}; - broadcast_max = std::min<size_t>(1000, broadcast_max); + size_t broadcast_max{INVENTORY_BROADCAST_TARGET + (tx_relay->m_tx_inventory_to_send.size()/1000)*5}; + broadcast_max = std::min<size_t>(INVENTORY_BROADCAST_MAX, broadcast_max); while (!vInvTx.empty() && nRelayedTransactions < broadcast_max) { // Fetch the top element from the heap std::pop_heap(vInvTx.begin(), vInvTx.end(), compareInvMempoolOrder); @@ -5783,14 +5733,12 @@ bool PeerManagerImpl::SendMessages(CNode* pto) if (!txinfo.tx) { continue; } - auto txid = txinfo.tx->GetHash(); // Peer told you to not send transactions at that feerate? Don't bother sending it. if (txinfo.fee < filterrate.GetFee(txinfo.vsize)) { continue; } if (tx_relay->m_bloom_filter && !tx_relay->m_bloom_filter->IsRelevantAndUpdate(*txinfo.tx)) continue; // Send - tx_relay->m_recently_announced_invs.insert(hash); vInv.push_back(inv); nRelayedTransactions++; if (vInv.size() == MAX_INV_SZ) { @@ -5798,15 +5746,11 @@ bool PeerManagerImpl::SendMessages(CNode* pto) vInv.clear(); } tx_relay->m_tx_inventory_known_filter.insert(hash); - if (hash != txid) { - // Insert txid into m_tx_inventory_known_filter, even for - // wtxidrelay peers. This prevents re-adding of - // unconfirmed parents to the recently_announced - // filter, when a child tx is requested. See - // ProcessGetData(). - tx_relay->m_tx_inventory_known_filter.insert(txid); - } } + + // Ensure we'll respond to GETDATA requests for anything we've just announced + LOCK(m_mempool.cs); + tx_relay->m_last_inv_sequence = m_mempool.GetSequence(); } } if (!vInv.empty()) @@ -5883,7 +5827,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) // Message: getdata (blocks) // std::vector<CInv> vGetData; - if (CanServeBlocks(*peer) && ((sync_blocks_and_headers_from_peer && !IsLimitedPeer(*peer)) || !m_chainman.ActiveChainstate().IsInitialBlockDownload()) && state.vBlocksInFlight.size() < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { + if (CanServeBlocks(*peer) && ((sync_blocks_and_headers_from_peer && !IsLimitedPeer(*peer)) || !m_chainman.IsInitialBlockDownload()) && state.vBlocksInFlight.size() < MAX_BLOCKS_IN_TRANSIT_PER_PEER) { std::vector<const CBlockIndex*> vToDownload; NodeId staller = -1; FindNextBlocksToDownload(*peer, MAX_BLOCKS_IN_TRANSIT_PER_PEER - state.vBlocksInFlight.size(), vToDownload, staller); diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index 42e021fcc9..a6d84555c0 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -298,8 +298,9 @@ public: { return GuessVerificationProgress(chainman().GetParams().TxData(), WITH_LOCK(::cs_main, return chainman().ActiveChain().Tip())); } - bool isInitialBlockDownload() override { - return chainman().ActiveChainstate().IsInitialBlockDownload(); + bool isInitialBlockDownload() override + { + return chainman().IsInitialBlockDownload(); } bool isLoadingBlocks() override { return chainman().m_blockman.LoadingBlocks(); } void setNetworkActive(bool active) override @@ -677,7 +678,7 @@ public: { if (!m_node.mempool) return true; LockPoints lp; - CTxMemPoolEntry entry(tx, 0, 0, 0, false, 0, lp); + CTxMemPoolEntry entry(tx, 0, 0, 0, 0, false, 0, lp); const CTxMemPool::Limits& limits{m_node.mempool->m_limits}; LOCK(m_node.mempool->cs); return m_node.mempool->CalculateMemPoolAncestors(entry, limits).has_value(); @@ -720,7 +721,7 @@ public: bool isReadyToBroadcast() override { return !chainman().m_blockman.LoadingBlocks() && !isInitialBlockDownload(); } bool isInitialBlockDownload() override { - return chainman().ActiveChainstate().IsInitialBlockDownload(); + return chainman().IsInitialBlockDownload(); } bool shutdownRequested() override { return ShutdownRequested(); } void initMessage(const std::string& message) override { ::uiInterface.InitMessage(message); } diff --git a/src/node/mempool_args.cpp b/src/node/mempool_args.cpp index 5381902263..7d8244980d 100644 --- a/src/node/mempool_args.cpp +++ b/src/node/mempool_args.cpp @@ -13,7 +13,6 @@ #include <logging.h> #include <policy/feerate.h> #include <policy/policy.h> -#include <script/standard.h> #include <tinyformat.h> #include <util/error.h> #include <util/moneystr.h> diff --git a/src/node/miner.h b/src/node/miner.h index 70de9e1db0..4173521585 100644 --- a/src/node/miner.h +++ b/src/node/miner.h @@ -21,10 +21,11 @@ #include <boost/multi_index_container.hpp> class ArgsManager; -class ChainstateManager; class CBlockIndex; class CChainParams; class CScript; +class Chainstate; +class ChainstateManager; namespace Consensus { struct Params; }; diff --git a/src/outputtype.cpp b/src/outputtype.cpp index 9a3870d8dc..566e5ec55a 100644 --- a/src/outputtype.cpp +++ b/src/outputtype.cpp @@ -9,7 +9,6 @@ #include <script/script.h> #include <script/sign.h> #include <script/signingprovider.h> -#include <script/standard.h> #include <util/vector.h> #include <assert.h> diff --git a/src/outputtype.h b/src/outputtype.h index 7c50f445fc..a2d5942320 100644 --- a/src/outputtype.h +++ b/src/outputtype.h @@ -6,8 +6,8 @@ #ifndef BITCOIN_OUTPUTTYPE_H #define BITCOIN_OUTPUTTYPE_H +#include <addresstype.h> #include <script/signingprovider.h> -#include <script/standard.h> #include <array> #include <optional> diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index c8f2df781b..553c88fddc 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -548,14 +548,12 @@ CBlockPolicyEstimator::CBlockPolicyEstimator(const fs::path& estimation_filepath AutoFile est_file{fsbridge::fopen(m_estimation_filepath, "rb")}; - // Whenever the fee estimation file is not present return early if (est_file.IsNull()) { LogPrintf("%s is not found. Continue anyway.\n", fs::PathToString(m_estimation_filepath)); return; } std::chrono::hours file_age = GetFeeEstimatorFileAge(); - // fee estimate file must not be too old to avoid wrong fee estimates. if (file_age > MAX_FILE_AGE && !read_stale_estimates) { LogPrintf("Fee estimation file %s too old (age=%lld > %lld hours) and will not be used to avoid serving stale estimates.\n", fs::PathToString(m_estimation_filepath), Ticks<std::chrono::hours>(file_age), Ticks<std::chrono::hours>(MAX_FILE_AGE)); return; diff --git a/src/policy/fees.h b/src/policy/fees.h index 52761f03ca..8ed13482e9 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -25,8 +25,9 @@ // How often to flush fee estimates to fee_estimates.dat. static constexpr std::chrono::hours FEE_FLUSH_INTERVAL{1}; -/** fee_estimates.dat that are more than 60 hours (2.5 days) will not be read, - * as the estimates in the file are stale. +/** fee_estimates.dat that are more than 60 hours (2.5 days) old will not be read, + * as fee estimates are based on historical data and may be inaccurate if + * network activity has changed. */ static constexpr std::chrono::hours MAX_FILE_AGE{60}; diff --git a/src/policy/policy.cpp b/src/policy/policy.cpp index 41b5b2d0f1..d08ec4fb7f 100644 --- a/src/policy/policy.cpp +++ b/src/policy/policy.cpp @@ -15,7 +15,7 @@ #include <primitives/transaction.h> #include <script/interpreter.h> #include <script/script.h> -#include <script/standard.h> +#include <script/solver.h> #include <serialize.h> #include <span.h> diff --git a/src/policy/policy.h b/src/policy/policy.h index 9135cae91c..d1c8148800 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -10,7 +10,7 @@ #include <consensus/consensus.h> #include <primitives/transaction.h> #include <script/interpreter.h> -#include <script/standard.h> +#include <script/solver.h> #include <cstdint> #include <string> @@ -63,34 +63,54 @@ static constexpr unsigned int DEFAULT_ANCESTOR_SIZE_LIMIT_KVB{101}; static constexpr unsigned int DEFAULT_DESCENDANT_LIMIT{25}; /** Default for -limitdescendantsize, maximum kilobytes of in-mempool descendants */ static constexpr unsigned int DEFAULT_DESCENDANT_SIZE_LIMIT_KVB{101}; +/** Default for -datacarrier */ +static const bool DEFAULT_ACCEPT_DATACARRIER = true; +/** + * Default setting for -datacarriersize. 80 bytes of data, +1 for OP_RETURN, + * +2 for the pushdata opcodes. + */ +static const unsigned int MAX_OP_RETURN_RELAY = 83; /** * An extra transaction can be added to a package, as long as it only has one * ancestor and is no larger than this. Not really any reason to make this * configurable as it doesn't materially change DoS parameters. */ static constexpr unsigned int EXTRA_DESCENDANT_TX_SIZE_LIMIT{10000}; + + +/** + * Mandatory script verification flags that all new transactions must comply with for + * them to be valid. Failing one of these tests may trigger a DoS ban; + * see CheckInputScripts() for details. + * + * Note that this does not affect consensus validity; see GetBlockScriptFlags() + * for that. + */ +static constexpr unsigned int MANDATORY_SCRIPT_VERIFY_FLAGS{SCRIPT_VERIFY_P2SH | + SCRIPT_VERIFY_DERSIG | + SCRIPT_VERIFY_NULLDUMMY | + SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY | + SCRIPT_VERIFY_CHECKSEQUENCEVERIFY | + SCRIPT_VERIFY_WITNESS | + SCRIPT_VERIFY_TAPROOT}; + /** * Standard script verification flags that standard transactions will comply - * with. However scripts violating these flags may still be present in valid - * blocks and we must accept those blocks. + * with. However we do not ban/disconnect nodes that forward txs violating + * the additional (non-mandatory) rules here, to improve forwards and + * backwards compatability. */ static constexpr unsigned int STANDARD_SCRIPT_VERIFY_FLAGS{MANDATORY_SCRIPT_VERIFY_FLAGS | - SCRIPT_VERIFY_DERSIG | SCRIPT_VERIFY_STRICTENC | SCRIPT_VERIFY_MINIMALDATA | - SCRIPT_VERIFY_NULLDUMMY | SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_NOPS | SCRIPT_VERIFY_CLEANSTACK | SCRIPT_VERIFY_MINIMALIF | SCRIPT_VERIFY_NULLFAIL | - SCRIPT_VERIFY_CHECKLOCKTIMEVERIFY | - SCRIPT_VERIFY_CHECKSEQUENCEVERIFY | SCRIPT_VERIFY_LOW_S | - SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_WITNESS_PROGRAM | SCRIPT_VERIFY_WITNESS_PUBKEYTYPE | SCRIPT_VERIFY_CONST_SCRIPTCODE | - SCRIPT_VERIFY_TAPROOT | SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_TAPROOT_VERSION | SCRIPT_VERIFY_DISCOURAGE_OP_SUCCESS | SCRIPT_VERIFY_DISCOURAGE_UPGRADABLE_PUBKEYTYPE}; diff --git a/src/psbt.cpp b/src/psbt.cpp index 009ed966ed..7ec9b9c136 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -5,6 +5,7 @@ #include <psbt.h> #include <policy/policy.h> +#include <script/signingprovider.h> #include <util/check.h> #include <util/strencodings.h> diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index 8d8328aad8..eb9c65caf2 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -10,6 +10,7 @@ #include <qt/qvalidatedlineedit.h> #include <qt/sendcoinsrecipient.h> +#include <addresstype.h> #include <base58.h> #include <chainparams.h> #include <common/args.h> @@ -20,7 +21,6 @@ #include <primitives/transaction.h> #include <protocol.h> #include <script/script.h> -#include <script/standard.h> #include <util/chaintype.h> #include <util/exception.h> #include <util/fs.h> diff --git a/src/qt/test/wallettests.cpp b/src/qt/test/wallettests.cpp index 5f789e400e..f5b86f44a6 100644 --- a/src/qt/test/wallettests.cpp +++ b/src/qt/test/wallettests.cpp @@ -24,6 +24,7 @@ #include <qt/transactiontablemodel.h> #include <qt/transactionview.h> #include <qt/walletmodel.h> +#include <script/solver.h> #include <test/util/setup_common.h> #include <validation.h> #include <wallet/test/util.h> diff --git a/src/qt/walletmodel.h b/src/qt/walletmodel.h index 4f75d41404..68218b0c1b 100644 --- a/src/qt/walletmodel.h +++ b/src/qt/walletmodel.h @@ -10,7 +10,6 @@ #endif #include <key.h> -#include <script/standard.h> #include <qt/walletmodeltransaction.h> diff --git a/src/random.cpp b/src/random.cpp index 5ff6f573b8..51b8b3ad9d 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -6,6 +6,7 @@ #include <random.h> #include <compat/cpuid.h> +#include <crypto/chacha20.h> #include <crypto/sha256.h> #include <crypto/sha512.h> #include <logging.h> @@ -16,6 +17,7 @@ #include <sync.h> #include <util/time.h> +#include <array> #include <cmath> #include <cstdlib> #include <thread> @@ -577,7 +579,7 @@ uint256 GetRandHash() noexcept void FastRandomContext::RandomSeed() { uint256 seed = GetRandHash(); - rng.SetKey32(seed.begin()); + rng.SetKey(MakeByteSpan(seed)); requires_seed = false; } @@ -585,18 +587,15 @@ uint256 FastRandomContext::rand256() noexcept { if (requires_seed) RandomSeed(); uint256 ret; - rng.Keystream(ret.data(), ret.size()); + rng.Keystream(MakeWritableByteSpan(ret)); return ret; } template <typename B> std::vector<B> FastRandomContext::randbytes(size_t len) { - if (requires_seed) RandomSeed(); std::vector<B> ret(len); - if (len > 0) { - rng.Keystream(UCharCast(ret.data()), len); - } + fillrand(MakeWritableByteSpan(ret)); return ret; } template std::vector<unsigned char> FastRandomContext::randbytes(size_t); @@ -605,13 +604,10 @@ template std::vector<std::byte> FastRandomContext::randbytes(size_t); void FastRandomContext::fillrand(Span<std::byte> output) { if (requires_seed) RandomSeed(); - rng.Keystream(UCharCast(output.data()), output.size()); + rng.Keystream(output); } -FastRandomContext::FastRandomContext(const uint256& seed) noexcept : requires_seed(false), bitbuf_size(0) -{ - rng.SetKey32(seed.begin()); -} +FastRandomContext::FastRandomContext(const uint256& seed) noexcept : requires_seed(false), rng(MakeByteSpan(seed)), bitbuf_size(0) {} bool Random_SanityCheck() { @@ -659,13 +655,13 @@ bool Random_SanityCheck() return true; } -FastRandomContext::FastRandomContext(bool fDeterministic) noexcept : requires_seed(!fDeterministic), bitbuf_size(0) +static constexpr std::array<std::byte, ChaCha20::KEYLEN> ZERO_KEY{}; + +FastRandomContext::FastRandomContext(bool fDeterministic) noexcept : requires_seed(!fDeterministic), rng(ZERO_KEY), bitbuf_size(0) { - if (!fDeterministic) { - return; - } - uint256 seed; - rng.SetKey32(seed.begin()); + // Note that despite always initializing with ZERO_KEY, requires_seed is set to true if not + // fDeterministic. That means the rng will be reinitialized with a secure random key upon first + // use. } FastRandomContext& FastRandomContext::operator=(FastRandomContext&& from) noexcept diff --git a/src/random.h b/src/random.h index 3b15477ae9..76bae5838d 100644 --- a/src/random.h +++ b/src/random.h @@ -175,9 +175,9 @@ public: uint64_t rand64() noexcept { if (requires_seed) RandomSeed(); - unsigned char buf[8]; - rng.Keystream(buf, 8); - return ReadLE64(buf); + std::array<std::byte, 8> buf; + rng.Keystream(buf); + return ReadLE64(UCharCast(buf.data())); } /** Generate a random (bits)-bit integer. */ diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 717a119b56..f4d88e4209 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -1260,7 +1260,7 @@ RPCHelpMan getblockchaininfo() obj.pushKV("time", tip.GetBlockTime()); obj.pushKV("mediantime", tip.GetMedianTimePast()); obj.pushKV("verificationprogress", GuessVerificationProgress(chainman.GetParams().TxData(), &tip)); - obj.pushKV("initialblockdownload", active_chainstate.IsInitialBlockDownload()); + obj.pushKV("initialblockdownload", chainman.IsInitialBlockDownload()); obj.pushKV("chainwork", tip.nChainWork.GetHex()); obj.pushKV("size_on_disk", chainman.m_blockman.CalculateCurrentUsage()); obj.pushKV("pruned", chainman.m_blockman.IsPruneMode()); @@ -2312,7 +2312,7 @@ static RPCHelpMan scanblocks() { {"filter_false_positives", RPCArg::Type::BOOL, RPCArg::Default{false}, "Filter false positives (slower and may fail on pruned nodes). Otherwise they may occur at a rate of 1/M"}, }, - RPCArgOptions{.oneline_description="\"options\""}}, + RPCArgOptions{.oneline_description="options"}}, }, { scan_result_status_none, diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 90ee2a48af..377e9de0e8 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -17,7 +17,6 @@ #include <rpc/server.h> #include <rpc/server_util.h> #include <rpc/util.h> -#include <script/standard.h> #include <txmempool.h> #include <univalue.h> #include <util/fs.h> @@ -745,7 +744,7 @@ static RPCHelpMan importmempool() "Whether to apply the unbroadcast set metadata from the mempool file.\n" "Warning: Importing untrusted metadata may lead to unexpected issues and undesirable behavior."}, }, - RPCArgOptions{.oneline_description = "\"options\""}}, + RPCArgOptions{.oneline_description = "options"}}, }, RPCResult{RPCResult::Type::OBJ, "", "", std::vector<RPCResult>{}}, RPCExamples{HelpExampleCli("importmempool", "/path/to/mempool.dat") + HelpExampleRpc("importmempool", "/path/to/mempool.dat")}, @@ -753,9 +752,10 @@ static RPCHelpMan importmempool() const NodeContext& node{EnsureAnyNodeContext(request.context)}; CTxMemPool& mempool{EnsureMemPool(node)}; - Chainstate& chainstate = EnsureChainman(node).ActiveChainstate(); + ChainstateManager& chainman = EnsureChainman(node); + Chainstate& chainstate = chainman.ActiveChainstate(); - if (chainstate.IsInitialBlockDownload()) { + if (chainman.IsInitialBlockDownload()) { throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, "Can only import the mempool after the block download and sync is done."); } diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 1f9b264626..76170c3201 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -110,7 +110,7 @@ static RPCHelpMan getnetworkhashps() { ChainstateManager& chainman = EnsureAnyChainman(request.context); LOCK(cs_main); - return GetNetworkHashPS(!request.params[0].isNull() ? request.params[0].getInt<int>() : 120, !request.params[1].isNull() ? request.params[1].getInt<int>() : -1, chainman.ActiveChain()); + return GetNetworkHashPS(self.Arg<int>(0), self.Arg<int>(1), chainman.ActiveChain()); }, }; } @@ -217,12 +217,12 @@ static RPCHelpMan generatetodescriptor() "\nGenerate 11 blocks to mydesc\n" + HelpExampleCli("generatetodescriptor", "11 \"mydesc\"")}, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - const int num_blocks{request.params[0].getInt<int>()}; - const uint64_t max_tries{request.params[2].isNull() ? DEFAULT_MAX_TRIES : request.params[2].getInt<int>()}; + const auto num_blocks{self.Arg<int>(0)}; + const auto max_tries{self.Arg<uint64_t>(2)}; CScript coinbase_script; std::string error; - if (!getScriptFromDescriptor(request.params[1].get_str(), coinbase_script, error)) { + if (!getScriptFromDescriptor(self.Arg<std::string>(1), coinbase_script, error)) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, error); } @@ -468,9 +468,10 @@ static RPCHelpMan prioritisetransaction() LOCK(cs_main); uint256 hash(ParseHashV(request.params[0], "txid")); + const auto dummy{self.MaybeArg<double>(1)}; CAmount nAmount = request.params[2].getInt<int64_t>(); - if (!(request.params[1].isNull() || request.params[1].get_real() == 0)) { + if (dummy && *dummy != 0) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Priority is no longer supported, dummy argument to prioritisetransaction must be 0."); } @@ -554,7 +555,7 @@ static RPCHelpMan getblocktemplate() " https://github.com/bitcoin/bips/blob/master/bip-0009.mediawiki#getblocktemplate_changes\n" " https://github.com/bitcoin/bips/blob/master/bip-0145.mediawiki\n", { - {"template_request", RPCArg::Type::OBJ, RPCArg::Default{UniValue::VOBJ}, "Format of the template", + {"template_request", RPCArg::Type::OBJ, RPCArg::Optional::NO, "Format of the template", { {"mode", RPCArg::Type::STR, /* treat as named arg */ RPCArg::Optional::OMITTED, "This must be set to \"template\", \"proposal\" (see BIP 23), or omitted"}, {"capabilities", RPCArg::Type::ARR, /* treat as named arg */ RPCArg::Optional::OMITTED, "A list of strings", @@ -569,7 +570,7 @@ static RPCHelpMan getblocktemplate() {"longpollid", RPCArg::Type::STR, RPCArg::Optional::OMITTED, "delay processing request until the result would vary significantly from the \"longpollid\" of a prior template"}, {"data", RPCArg::Type::STR_HEX, RPCArg::Optional::OMITTED, "proposed block data to check, encoded in hexadecimal; valid only for mode=\"proposal\""}, }, - RPCArgOptions{.oneline_description="\"template_request\""}}, + }, }, { RPCResult{"If the proposal was accepted with mode=='proposal'", RPCResult::Type::NONE, "", ""}, @@ -706,7 +707,7 @@ static RPCHelpMan getblocktemplate() throw JSONRPCError(RPC_CLIENT_NOT_CONNECTED, PACKAGE_NAME " is not connected!"); } - if (active_chainstate.IsInitialBlockDownload()) { + if (chainman.IsInitialBlockDownload()) { throw JSONRPCError(RPC_CLIENT_IN_INITIAL_DOWNLOAD, PACKAGE_NAME " is in initial sync and waiting for blocks..."); } } diff --git a/src/rpc/output_script.cpp b/src/rpc/output_script.cpp index 990ec3ab0c..4dd424fa14 100644 --- a/src/rpc/output_script.cpp +++ b/src/rpc/output_script.cpp @@ -13,7 +13,6 @@ #include <script/descriptor.h> #include <script/script.h> #include <script/signingprovider.h> -#include <script/standard.h> #include <tinyformat.h> #include <univalue.h> #include <util/check.h> diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index eb0200ccf5..fa5dd281a1 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -30,7 +30,7 @@ #include <script/script.h> #include <script/sign.h> #include <script/signingprovider.h> -#include <script/standard.h> +#include <script/solver.h> #include <uint256.h> #include <undo.h> #include <util/bip32.h> @@ -147,8 +147,9 @@ static std::vector<RPCArg> CreateTxDoc() }, }, }, - {"outputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "The outputs (key-value pairs), where none of the keys are duplicated.\n" - "That is, each address can only appear once and there can only be one 'data' object.\n" + {"outputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "The outputs specified as key-value pairs.\n" + "Each key may only appear once, i.e. there can only be one 'data' output, and no address may be duplicated.\n" + "At least one output of either type must be specified.\n" "For compatibility reasons, a dictionary, which holds the key-value pairs directly, is also\n" " accepted as second parameter.", { diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index faae840d40..45b7d89a7b 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -12,6 +12,7 @@ #include <rpc/util.h> #include <script/descriptor.h> #include <script/signingprovider.h> +#include <script/solver.h> #include <tinyformat.h> #include <util/check.h> #include <util/result.h> @@ -608,7 +609,10 @@ UniValue RPCHelpMan::HandleRequest(const JSONRPCRequest& request) const if (!arg_mismatch.empty()) { throw JSONRPCError(RPC_TYPE_ERROR, strprintf("Wrong type passed:\n%s", arg_mismatch.write(4))); } + CHECK_NONFATAL(m_req == nullptr); + m_req = &request; UniValue ret = m_fun(*this, request); + m_req = nullptr; if (gArgs.GetBoolArg("-rpcdoccheck", DEFAULT_RPC_DOC_CHECK)) { UniValue mismatch{UniValue::VARR}; for (const auto& res : m_results.m_results) { @@ -634,6 +638,49 @@ UniValue RPCHelpMan::HandleRequest(const JSONRPCRequest& request) const return ret; } +using CheckFn = void(const RPCArg&); +static const UniValue* DetailMaybeArg(CheckFn* check, const std::vector<RPCArg>& params, const JSONRPCRequest* req, size_t i) +{ + CHECK_NONFATAL(i < params.size()); + const UniValue& arg{CHECK_NONFATAL(req)->params[i]}; + const RPCArg& param{params.at(i)}; + if (check) check(param); + + if (!arg.isNull()) return &arg; + if (!std::holds_alternative<RPCArg::Default>(param.m_fallback)) return nullptr; + return &std::get<RPCArg::Default>(param.m_fallback); +} + +static void CheckRequiredOrDefault(const RPCArg& param) +{ + // Must use `Arg<Type>(i)` to get the argument or its default value. + const bool required{ + std::holds_alternative<RPCArg::Optional>(param.m_fallback) && RPCArg::Optional::NO == std::get<RPCArg::Optional>(param.m_fallback), + }; + CHECK_NONFATAL(required || std::holds_alternative<RPCArg::Default>(param.m_fallback)); +} + +#define TMPL_INST(check_param, ret_type, return_code) \ + template <> \ + ret_type RPCHelpMan::ArgValue<ret_type>(size_t i) const \ + { \ + const UniValue* maybe_arg{ \ + DetailMaybeArg(check_param, m_args, m_req, i), \ + }; \ + return return_code \ + } \ + void force_semicolon(ret_type) + +// Optional arg (without default). Can also be called on required args, if needed. +TMPL_INST(nullptr, std::optional<double>, maybe_arg ? std::optional{maybe_arg->get_real()} : std::nullopt;); +TMPL_INST(nullptr, std::optional<bool>, maybe_arg ? std::optional{maybe_arg->get_bool()} : std::nullopt;); +TMPL_INST(nullptr, const std::string*, maybe_arg ? &maybe_arg->get_str() : nullptr;); + +// Required arg or optional arg with default value. +TMPL_INST(CheckRequiredOrDefault, int, CHECK_NONFATAL(maybe_arg)->getInt<int>();); +TMPL_INST(CheckRequiredOrDefault, uint64_t, CHECK_NONFATAL(maybe_arg)->getInt<uint64_t>();); +TMPL_INST(CheckRequiredOrDefault, const std::string&, CHECK_NONFATAL(maybe_arg)->get_str();); + bool RPCHelpMan::IsValidNumArgs(size_t num_args) const { size_t num_required_args = 0; @@ -1137,7 +1184,16 @@ std::string RPCArg::ToStringObj(const bool oneline) const std::string RPCArg::ToString(const bool oneline) const { - if (oneline && !m_opts.oneline_description.empty()) return m_opts.oneline_description; + if (oneline && !m_opts.oneline_description.empty()) { + if (m_opts.oneline_description[0] == '\"' && m_type != Type::STR_HEX && m_type != Type::STR && gArgs.GetBoolArg("-rpcdoccheck", DEFAULT_RPC_DOC_CHECK)) { + throw std::runtime_error{ + strprintf("Internal bug detected: non-string RPC arg \"%s\" quotes oneline_description:\n%s\n%s %s\nPlease report this issue here: %s\n", + m_names, m_opts.oneline_description, + PACKAGE_NAME, FormatFullVersion(), + PACKAGE_BUGREPORT)}; + } + return m_opts.oneline_description; + } switch (m_type) { case Type::STR_HEX: diff --git a/src/rpc/util.h b/src/rpc/util.h index 02d26f1ab7..392540ffad 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -5,6 +5,8 @@ #ifndef BITCOIN_RPC_UTIL_H #define BITCOIN_RPC_UTIL_H +#include <addresstype.h> +#include <consensus/amount.h> #include <node/transaction.h> #include <outputtype.h> #include <protocol.h> @@ -13,14 +15,29 @@ #include <rpc/request.h> #include <script/script.h> #include <script/sign.h> -#include <script/standard.h> +#include <uint256.h> #include <univalue.h> #include <util/check.h> +#include <cstddef> +#include <cstdint> +#include <functional> +#include <initializer_list> +#include <map> +#include <optional> #include <string> +#include <type_traits> +#include <utility> #include <variant> #include <vector> +class JSONRPCRequest; +enum ServiceFlags : uint64_t; +enum class OutputType; +enum class TransactionError; +struct FlatSigningProvider; +struct bilingual_str; + static constexpr bool DEFAULT_RPC_DOC_CHECK{ #ifdef RPC_DOC_CHECK true @@ -383,6 +400,44 @@ public: RPCHelpMan(std::string name, std::string description, std::vector<RPCArg> args, RPCResults results, RPCExamples examples, RPCMethodImpl fun); UniValue HandleRequest(const JSONRPCRequest& request) const; + /** + * Helper to get a request argument. + * This function only works during m_fun(), i.e. it should only be used in + * RPC method implementations. The helper internally checks whether the + * user-passed argument isNull() and parses (from JSON) and returns the + * user-passed argument, or the default value derived from the RPCArg + * documention, or a falsy value if no default was given. + * + * Use Arg<Type>(i) to get the argument or its default value. Otherwise, + * use MaybeArg<Type>(i) to get the optional argument or a falsy value. + * + * The Type passed to this helper must match the corresponding + * RPCArg::Type. + */ + template <typename R> + auto Arg(size_t i) const + { + // Return argument (required or with default value). + if constexpr (std::is_integral_v<R> || std::is_floating_point_v<R>) { + // Return numbers by value. + return ArgValue<R>(i); + } else { + // Return everything else by reference. + return ArgValue<const R&>(i); + } + } + template <typename R> + auto MaybeArg(size_t i) const + { + // Return optional argument (without default). + if constexpr (std::is_integral_v<R> || std::is_floating_point_v<R>) { + // Return numbers by value, wrapped in optional. + return ArgValue<std::optional<R>>(i); + } else { + // Return other types by pointer. + return ArgValue<const R*>(i); + } + } std::string ToString() const; /** Return the named args that need to be converted from string to another JSON type */ UniValue GetArgMap() const; @@ -399,6 +454,9 @@ private: const std::vector<RPCArg> m_args; const RPCResults m_results; const RPCExamples m_examples; + mutable const JSONRPCRequest* m_req{nullptr}; // A pointer to the request for the duration of m_fun() + template <typename R> + R ArgValue(size_t i) const; }; /** diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 09ded5fc61..436ea9c093 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -9,7 +9,8 @@ #include <pubkey.h> #include <script/miniscript.h> #include <script/script.h> -#include <script/standard.h> +#include <script/signingprovider.h> +#include <script/solver.h> #include <uint256.h> #include <common/args.h> diff --git a/src/script/miniscript.cpp b/src/script/miniscript.cpp index 3937638cf8..19556a9775 100644 --- a/src/script/miniscript.cpp +++ b/src/script/miniscript.cpp @@ -5,7 +5,6 @@ #include <string> #include <vector> #include <script/script.h> -#include <script/standard.h> #include <script/miniscript.h> #include <assert.h> diff --git a/src/script/script.cpp b/src/script/script.cpp index 79d19b9085..1594d3cc79 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -5,10 +5,13 @@ #include <script/script.h> +#include <hash.h> #include <util/strencodings.h> #include <string> +CScriptID::CScriptID(const CScript& in) : BaseHash(Hash160(in)) {} + std::string GetOpName(opcodetype opcode) { switch (opcode) diff --git a/src/script/script.h b/src/script/script.h index 374ae1642e..902f756afc 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -10,6 +10,8 @@ #include <crypto/common.h> #include <prevector.h> #include <serialize.h> +#include <uint256.h> +#include <util/hash_type.h> #include <assert.h> #include <climits> @@ -575,6 +577,15 @@ struct CScriptWitness std::string ToString() const; }; +/** A reference to a CScript: the Hash160 of its serialization */ +class CScriptID : public BaseHash<uint160> +{ +public: + CScriptID() : BaseHash() {} + explicit CScriptID(const CScript& in); + explicit CScriptID(const uint160& in) : BaseHash(in) {} +}; + /** Test for OP_SUCCESSx opcodes as defined by BIP342. */ bool IsOpSuccess(const opcodetype& opcode); diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 85589fe86b..92b7ad50b5 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -11,8 +11,9 @@ #include <primitives/transaction.h> #include <script/keyorigin.h> #include <script/miniscript.h> +#include <script/script.h> #include <script/signingprovider.h> -#include <script/standard.h> +#include <script/solver.h> #include <uint256.h> #include <util/translation.h> #include <util/vector.h> diff --git a/src/script/sign.h b/src/script/sign.h index f46bc55992..4d7dade44e 100644 --- a/src/script/sign.h +++ b/src/script/sign.h @@ -12,7 +12,7 @@ #include <pubkey.h> #include <script/interpreter.h> #include <script/keyorigin.h> -#include <script/standard.h> +#include <script/signingprovider.h> #include <uint256.h> class CKey; diff --git a/src/script/signingprovider.cpp b/src/script/signingprovider.cpp index ef055573b9..f3a69e5d21 100644 --- a/src/script/signingprovider.cpp +++ b/src/script/signingprovider.cpp @@ -4,8 +4,8 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <script/keyorigin.h> +#include <script/interpreter.h> #include <script/signingprovider.h> -#include <script/standard.h> #include <logging.h> @@ -205,7 +205,7 @@ CKeyID GetKeyForDestination(const SigningProvider& store, const CTxDestination& } if (auto script_hash = std::get_if<ScriptHash>(&dest)) { CScript script; - CScriptID script_id(*script_hash); + CScriptID script_id = ToScriptID(*script_hash); CTxDestination inner_dest; if (store.GetCScript(script_id, script) && ExtractDestination(script, inner_dest)) { if (auto inner_witness_id = std::get_if<WitnessV0KeyHash>(&inner_dest)) { @@ -225,3 +225,297 @@ CKeyID GetKeyForDestination(const SigningProvider& store, const CTxDestination& } return CKeyID(); } +/*static*/ TaprootBuilder::NodeInfo TaprootBuilder::Combine(NodeInfo&& a, NodeInfo&& b) +{ + NodeInfo ret; + /* Iterate over all tracked leaves in a, add b's hash to their Merkle branch, and move them to ret. */ + for (auto& leaf : a.leaves) { + leaf.merkle_branch.push_back(b.hash); + ret.leaves.emplace_back(std::move(leaf)); + } + /* Iterate over all tracked leaves in b, add a's hash to their Merkle branch, and move them to ret. */ + for (auto& leaf : b.leaves) { + leaf.merkle_branch.push_back(a.hash); + ret.leaves.emplace_back(std::move(leaf)); + } + ret.hash = ComputeTapbranchHash(a.hash, b.hash); + return ret; +} + +void TaprootSpendData::Merge(TaprootSpendData other) +{ + // TODO: figure out how to better deal with conflicting information + // being merged. + if (internal_key.IsNull() && !other.internal_key.IsNull()) { + internal_key = other.internal_key; + } + if (merkle_root.IsNull() && !other.merkle_root.IsNull()) { + merkle_root = other.merkle_root; + } + for (auto& [key, control_blocks] : other.scripts) { + scripts[key].merge(std::move(control_blocks)); + } +} + +void TaprootBuilder::Insert(TaprootBuilder::NodeInfo&& node, int depth) +{ + assert(depth >= 0 && (size_t)depth <= TAPROOT_CONTROL_MAX_NODE_COUNT); + /* We cannot insert a leaf at a lower depth while a deeper branch is unfinished. Doing + * so would mean the Add() invocations do not correspond to a DFS traversal of a + * binary tree. */ + if ((size_t)depth + 1 < m_branch.size()) { + m_valid = false; + return; + } + /* As long as an entry in the branch exists at the specified depth, combine it and propagate up. + * The 'node' variable is overwritten here with the newly combined node. */ + while (m_valid && m_branch.size() > (size_t)depth && m_branch[depth].has_value()) { + node = Combine(std::move(node), std::move(*m_branch[depth])); + m_branch.pop_back(); + if (depth == 0) m_valid = false; /* Can't propagate further up than the root */ + --depth; + } + if (m_valid) { + /* Make sure the branch is big enough to place the new node. */ + if (m_branch.size() <= (size_t)depth) m_branch.resize((size_t)depth + 1); + assert(!m_branch[depth].has_value()); + m_branch[depth] = std::move(node); + } +} + +/*static*/ bool TaprootBuilder::ValidDepths(const std::vector<int>& depths) +{ + std::vector<bool> branch; + for (int depth : depths) { + // This inner loop corresponds to effectively the same logic on branch + // as what Insert() performs on the m_branch variable. Instead of + // storing a NodeInfo object, just remember whether or not there is one + // at that depth. + if (depth < 0 || (size_t)depth > TAPROOT_CONTROL_MAX_NODE_COUNT) return false; + if ((size_t)depth + 1 < branch.size()) return false; + while (branch.size() > (size_t)depth && branch[depth]) { + branch.pop_back(); + if (depth == 0) return false; + --depth; + } + if (branch.size() <= (size_t)depth) branch.resize((size_t)depth + 1); + assert(!branch[depth]); + branch[depth] = true; + } + // And this check corresponds to the IsComplete() check on m_branch. + return branch.size() == 0 || (branch.size() == 1 && branch[0]); +} + +TaprootBuilder& TaprootBuilder::Add(int depth, Span<const unsigned char> script, int leaf_version, bool track) +{ + assert((leaf_version & ~TAPROOT_LEAF_MASK) == 0); + if (!IsValid()) return *this; + /* Construct NodeInfo object with leaf hash and (if track is true) also leaf information. */ + NodeInfo node; + node.hash = ComputeTapleafHash(leaf_version, script); + if (track) node.leaves.emplace_back(LeafInfo{std::vector<unsigned char>(script.begin(), script.end()), leaf_version, {}}); + /* Insert into the branch. */ + Insert(std::move(node), depth); + return *this; +} + +TaprootBuilder& TaprootBuilder::AddOmitted(int depth, const uint256& hash) +{ + if (!IsValid()) return *this; + /* Construct NodeInfo object with the hash directly, and insert it into the branch. */ + NodeInfo node; + node.hash = hash; + Insert(std::move(node), depth); + return *this; +} + +TaprootBuilder& TaprootBuilder::Finalize(const XOnlyPubKey& internal_key) +{ + /* Can only call this function when IsComplete() is true. */ + assert(IsComplete()); + m_internal_key = internal_key; + auto ret = m_internal_key.CreateTapTweak(m_branch.size() == 0 ? nullptr : &m_branch[0]->hash); + assert(ret.has_value()); + std::tie(m_output_key, m_parity) = *ret; + return *this; +} + +WitnessV1Taproot TaprootBuilder::GetOutput() { return WitnessV1Taproot{m_output_key}; } + +TaprootSpendData TaprootBuilder::GetSpendData() const +{ + assert(IsComplete()); + assert(m_output_key.IsFullyValid()); + TaprootSpendData spd; + spd.merkle_root = m_branch.size() == 0 ? uint256() : m_branch[0]->hash; + spd.internal_key = m_internal_key; + if (m_branch.size()) { + // If any script paths exist, they have been combined into the root m_branch[0] + // by now. Compute the control block for each of its tracked leaves, and put them in + // spd.scripts. + for (const auto& leaf : m_branch[0]->leaves) { + std::vector<unsigned char> control_block; + control_block.resize(TAPROOT_CONTROL_BASE_SIZE + TAPROOT_CONTROL_NODE_SIZE * leaf.merkle_branch.size()); + control_block[0] = leaf.leaf_version | (m_parity ? 1 : 0); + std::copy(m_internal_key.begin(), m_internal_key.end(), control_block.begin() + 1); + if (leaf.merkle_branch.size()) { + std::copy(leaf.merkle_branch[0].begin(), + leaf.merkle_branch[0].begin() + TAPROOT_CONTROL_NODE_SIZE * leaf.merkle_branch.size(), + control_block.begin() + TAPROOT_CONTROL_BASE_SIZE); + } + spd.scripts[{leaf.script, leaf.leaf_version}].insert(std::move(control_block)); + } + } + return spd; +} + +std::optional<std::vector<std::tuple<int, std::vector<unsigned char>, int>>> InferTaprootTree(const TaprootSpendData& spenddata, const XOnlyPubKey& output) +{ + // Verify that the output matches the assumed Merkle root and internal key. + auto tweak = spenddata.internal_key.CreateTapTweak(spenddata.merkle_root.IsNull() ? nullptr : &spenddata.merkle_root); + if (!tweak || tweak->first != output) return std::nullopt; + // If the Merkle root is 0, the tree is empty, and we're done. + std::vector<std::tuple<int, std::vector<unsigned char>, int>> ret; + if (spenddata.merkle_root.IsNull()) return ret; + + /** Data structure to represent the nodes of the tree we're going to build. */ + struct TreeNode { + /** Hash of this node, if known; 0 otherwise. */ + uint256 hash; + /** The left and right subtrees (note that their order is irrelevant). */ + std::unique_ptr<TreeNode> sub[2]; + /** If this is known to be a leaf node, a pointer to the (script, leaf_ver) pair. + * nullptr otherwise. */ + const std::pair<std::vector<unsigned char>, int>* leaf = nullptr; + /** Whether or not this node has been explored (is known to be a leaf, or known to have children). */ + bool explored = false; + /** Whether or not this node is an inner node (unknown until explored = true). */ + bool inner; + /** Whether or not we have produced output for this subtree. */ + bool done = false; + }; + + // Build tree from the provided branches. + TreeNode root; + root.hash = spenddata.merkle_root; + for (const auto& [key, control_blocks] : spenddata.scripts) { + const auto& [script, leaf_ver] = key; + for (const auto& control : control_blocks) { + // Skip script records with nonsensical leaf version. + if (leaf_ver < 0 || leaf_ver >= 0x100 || leaf_ver & 1) continue; + // Skip script records with invalid control block sizes. + if (control.size() < TAPROOT_CONTROL_BASE_SIZE || control.size() > TAPROOT_CONTROL_MAX_SIZE || + ((control.size() - TAPROOT_CONTROL_BASE_SIZE) % TAPROOT_CONTROL_NODE_SIZE) != 0) continue; + // Skip script records that don't match the control block. + if ((control[0] & TAPROOT_LEAF_MASK) != leaf_ver) continue; + // Skip script records that don't match the provided Merkle root. + const uint256 leaf_hash = ComputeTapleafHash(leaf_ver, script); + const uint256 merkle_root = ComputeTaprootMerkleRoot(control, leaf_hash); + if (merkle_root != spenddata.merkle_root) continue; + + TreeNode* node = &root; + size_t levels = (control.size() - TAPROOT_CONTROL_BASE_SIZE) / TAPROOT_CONTROL_NODE_SIZE; + for (size_t depth = 0; depth < levels; ++depth) { + // Can't descend into a node which we already know is a leaf. + if (node->explored && !node->inner) return std::nullopt; + + // Extract partner hash from Merkle branch in control block. + uint256 hash; + std::copy(control.begin() + TAPROOT_CONTROL_BASE_SIZE + (levels - 1 - depth) * TAPROOT_CONTROL_NODE_SIZE, + control.begin() + TAPROOT_CONTROL_BASE_SIZE + (levels - depth) * TAPROOT_CONTROL_NODE_SIZE, + hash.begin()); + + if (node->sub[0]) { + // Descend into the existing left or right branch. + bool desc = false; + for (int i = 0; i < 2; ++i) { + if (node->sub[i]->hash == hash || (node->sub[i]->hash.IsNull() && node->sub[1-i]->hash != hash)) { + node->sub[i]->hash = hash; + node = &*node->sub[1-i]; + desc = true; + break; + } + } + if (!desc) return std::nullopt; // This probably requires a hash collision to hit. + } else { + // We're in an unexplored node. Create subtrees and descend. + node->explored = true; + node->inner = true; + node->sub[0] = std::make_unique<TreeNode>(); + node->sub[1] = std::make_unique<TreeNode>(); + node->sub[1]->hash = hash; + node = &*node->sub[0]; + } + } + // Cannot turn a known inner node into a leaf. + if (node->sub[0]) return std::nullopt; + node->explored = true; + node->inner = false; + node->leaf = &key; + node->hash = leaf_hash; + } + } + + // Recursive processing to turn the tree into flattened output. Use an explicit stack here to avoid + // overflowing the call stack (the tree may be 128 levels deep). + std::vector<TreeNode*> stack{&root}; + while (!stack.empty()) { + TreeNode& node = *stack.back(); + if (!node.explored) { + // Unexplored node, which means the tree is incomplete. + return std::nullopt; + } else if (!node.inner) { + // Leaf node; produce output. + ret.emplace_back(stack.size() - 1, node.leaf->first, node.leaf->second); + node.done = true; + stack.pop_back(); + } else if (node.sub[0]->done && !node.sub[1]->done && !node.sub[1]->explored && !node.sub[1]->hash.IsNull() && + ComputeTapbranchHash(node.sub[1]->hash, node.sub[1]->hash) == node.hash) { + // Whenever there are nodes with two identical subtrees under it, we run into a problem: + // the control blocks for the leaves underneath those will be identical as well, and thus + // they will all be matched to the same path in the tree. The result is that at the location + // where the duplicate occurred, the left child will contain a normal tree that can be explored + // and processed, but the right one will remain unexplored. + // + // This situation can be detected, by encountering an inner node with unexplored right subtree + // with known hash, and H_TapBranch(hash, hash) is equal to the parent node (this node)'s hash. + // + // To deal with this, simply process the left tree a second time (set its done flag to false; + // noting that the done flag of its children have already been set to false after processing + // those). To avoid ending up in an infinite loop, set the done flag of the right (unexplored) + // subtree to true. + node.sub[0]->done = false; + node.sub[1]->done = true; + } else if (node.sub[0]->done && node.sub[1]->done) { + // An internal node which we're finished with. + node.sub[0]->done = false; + node.sub[1]->done = false; + node.done = true; + stack.pop_back(); + } else if (!node.sub[0]->done) { + // An internal node whose left branch hasn't been processed yet. Do so first. + stack.push_back(&*node.sub[0]); + } else if (!node.sub[1]->done) { + // An internal node whose right branch hasn't been processed yet. Do so first. + stack.push_back(&*node.sub[1]); + } + } + + return ret; +} + +std::vector<std::tuple<uint8_t, uint8_t, std::vector<unsigned char>>> TaprootBuilder::GetTreeTuples() const +{ + assert(IsComplete()); + std::vector<std::tuple<uint8_t, uint8_t, std::vector<unsigned char>>> tuples; + if (m_branch.size()) { + const auto& leaves = m_branch[0]->leaves; + for (const auto& leaf : leaves) { + assert(leaf.merkle_branch.size() <= TAPROOT_CONTROL_MAX_NODE_COUNT); + uint8_t depth = (uint8_t)leaf.merkle_branch.size(); + uint8_t leaf_ver = (uint8_t)leaf.leaf_version; + tuples.push_back(std::make_tuple(depth, leaf_ver, leaf.script)); + } + } + return tuples; +} diff --git a/src/script/signingprovider.h b/src/script/signingprovider.h index a5bbcff6a0..712e2e73d1 100644 --- a/src/script/signingprovider.h +++ b/src/script/signingprovider.h @@ -6,14 +6,146 @@ #ifndef BITCOIN_SCRIPT_SIGNINGPROVIDER_H #define BITCOIN_SCRIPT_SIGNINGPROVIDER_H +#include <addresstype.h> #include <attributes.h> #include <key.h> #include <pubkey.h> #include <script/keyorigin.h> #include <script/script.h> -#include <script/standard.h> #include <sync.h> +struct ShortestVectorFirstComparator +{ + bool operator()(const std::vector<unsigned char>& a, const std::vector<unsigned char>& b) const + { + if (a.size() < b.size()) return true; + if (a.size() > b.size()) return false; + return a < b; + } +}; + +struct TaprootSpendData +{ + /** The BIP341 internal key. */ + XOnlyPubKey internal_key; + /** The Merkle root of the script tree (0 if no scripts). */ + uint256 merkle_root; + /** Map from (script, leaf_version) to (sets of) control blocks. + * More than one control block for a given script is only possible if it + * appears in multiple branches of the tree. We keep them all so that + * inference can reconstruct the full tree. Within each set, the control + * blocks are sorted by size, so that the signing logic can easily + * prefer the cheapest one. */ + std::map<std::pair<std::vector<unsigned char>, int>, std::set<std::vector<unsigned char>, ShortestVectorFirstComparator>> scripts; + /** Merge other TaprootSpendData (for the same scriptPubKey) into this. */ + void Merge(TaprootSpendData other); +}; + +/** Utility class to construct Taproot outputs from internal key and script tree. */ +class TaprootBuilder +{ +private: + /** Information about a tracked leaf in the Merkle tree. */ + struct LeafInfo + { + std::vector<unsigned char> script; //!< The script. + int leaf_version; //!< The leaf version for that script. + std::vector<uint256> merkle_branch; //!< The hashing partners above this leaf. + }; + + /** Information associated with a node in the Merkle tree. */ + struct NodeInfo + { + /** Merkle hash of this node. */ + uint256 hash; + /** Tracked leaves underneath this node (either from the node itself, or its children). + * The merkle_branch field of each is the partners to get to *this* node. */ + std::vector<LeafInfo> leaves; + }; + /** Whether the builder is in a valid state so far. */ + bool m_valid = true; + + /** The current state of the builder. + * + * For each level in the tree, one NodeInfo object may be present. m_branch[0] + * is information about the root; further values are for deeper subtrees being + * explored. + * + * For every right branch taken to reach the position we're currently + * working in, there will be a (non-nullopt) entry in m_branch corresponding + * to the left branch at that level. + * + * For example, imagine this tree: - N0 - + * / \ + * N1 N2 + * / \ / \ + * A B C N3 + * / \ + * D E + * + * Initially, m_branch is empty. After processing leaf A, it would become + * {nullopt, nullopt, A}. When processing leaf B, an entry at level 2 already + * exists, and it would thus be combined with it to produce a level 1 one, + * resulting in {nullopt, N1}. Adding C and D takes us to {nullopt, N1, C} + * and {nullopt, N1, C, D} respectively. When E is processed, it is combined + * with D, and then C, and then N1, to produce the root, resulting in {N0}. + * + * This structure allows processing with just O(log n) overhead if the leaves + * are computed on the fly. + * + * As an invariant, there can never be nullopt entries at the end. There can + * also not be more than 128 entries (as that would mean more than 128 levels + * in the tree). The depth of newly added entries will always be at least + * equal to the current size of m_branch (otherwise it does not correspond + * to a depth-first traversal of a tree). m_branch is only empty if no entries + * have ever be processed. m_branch having length 1 corresponds to being done. + */ + std::vector<std::optional<NodeInfo>> m_branch; + + XOnlyPubKey m_internal_key; //!< The internal key, set when finalizing. + XOnlyPubKey m_output_key; //!< The output key, computed when finalizing. + bool m_parity; //!< The tweak parity, computed when finalizing. + + /** Combine information about a parent Merkle tree node from its child nodes. */ + static NodeInfo Combine(NodeInfo&& a, NodeInfo&& b); + /** Insert information about a node at a certain depth, and propagate information up. */ + void Insert(NodeInfo&& node, int depth); + +public: + /** Add a new script at a certain depth in the tree. Add() operations must be called + * in depth-first traversal order of binary tree. If track is true, it will be included in + * the GetSpendData() output. */ + TaprootBuilder& Add(int depth, Span<const unsigned char> script, int leaf_version, bool track = true); + /** Like Add(), but for a Merkle node with a given hash to the tree. */ + TaprootBuilder& AddOmitted(int depth, const uint256& hash); + /** Finalize the construction. Can only be called when IsComplete() is true. + internal_key.IsFullyValid() must be true. */ + TaprootBuilder& Finalize(const XOnlyPubKey& internal_key); + + /** Return true if so far all input was valid. */ + bool IsValid() const { return m_valid; } + /** Return whether there were either no leaves, or the leaves form a Huffman tree. */ + bool IsComplete() const { return m_valid && (m_branch.size() == 0 || (m_branch.size() == 1 && m_branch[0].has_value())); } + /** Compute scriptPubKey (after Finalize()). */ + WitnessV1Taproot GetOutput(); + /** Check if a list of depths is legal (will lead to IsComplete()). */ + static bool ValidDepths(const std::vector<int>& depths); + /** Compute spending data (after Finalize()). */ + TaprootSpendData GetSpendData() const; + /** Returns a vector of tuples representing the depth, leaf version, and script */ + std::vector<std::tuple<uint8_t, uint8_t, std::vector<unsigned char>>> GetTreeTuples() const; + /** Returns true if there are any tapscripts */ + bool HasScripts() const { return !m_branch.empty(); } +}; + +/** Given a TaprootSpendData and the output key, reconstruct its script tree. + * + * If the output doesn't match the spenddata, or if the data in spenddata is incomplete, + * std::nullopt is returned. Otherwise, a vector of (depth, script, leaf_ver) tuples is + * returned, corresponding to a depth-first traversal of the script tree. + */ +std::optional<std::vector<std::tuple<int, std::vector<unsigned char>, int>>> InferTaprootTree(const TaprootSpendData& spenddata, const XOnlyPubKey& output); + /** An interface to be implemented by keystores that support signing. */ class SigningProvider { diff --git a/src/script/solver.cpp b/src/script/solver.cpp new file mode 100644 index 0000000000..3dfa9cd6ba --- /dev/null +++ b/src/script/solver.cpp @@ -0,0 +1,224 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <pubkey.h> +#include <script/interpreter.h> +#include <script/script.h> +#include <script/solver.h> +#include <span.h> + +#include <algorithm> +#include <cassert> +#include <string> + +typedef std::vector<unsigned char> valtype; + +std::string GetTxnOutputType(TxoutType t) +{ + switch (t) { + case TxoutType::NONSTANDARD: return "nonstandard"; + case TxoutType::PUBKEY: return "pubkey"; + case TxoutType::PUBKEYHASH: return "pubkeyhash"; + case TxoutType::SCRIPTHASH: return "scripthash"; + case TxoutType::MULTISIG: return "multisig"; + case TxoutType::NULL_DATA: return "nulldata"; + case TxoutType::WITNESS_V0_KEYHASH: return "witness_v0_keyhash"; + case TxoutType::WITNESS_V0_SCRIPTHASH: return "witness_v0_scripthash"; + case TxoutType::WITNESS_V1_TAPROOT: return "witness_v1_taproot"; + case TxoutType::WITNESS_UNKNOWN: return "witness_unknown"; + } // no default case, so the compiler can warn about missing cases + assert(false); +} + +static bool MatchPayToPubkey(const CScript& script, valtype& pubkey) +{ + if (script.size() == CPubKey::SIZE + 2 && script[0] == CPubKey::SIZE && script.back() == OP_CHECKSIG) { + pubkey = valtype(script.begin() + 1, script.begin() + CPubKey::SIZE + 1); + return CPubKey::ValidSize(pubkey); + } + if (script.size() == CPubKey::COMPRESSED_SIZE + 2 && script[0] == CPubKey::COMPRESSED_SIZE && script.back() == OP_CHECKSIG) { + pubkey = valtype(script.begin() + 1, script.begin() + CPubKey::COMPRESSED_SIZE + 1); + return CPubKey::ValidSize(pubkey); + } + return false; +} + +static bool MatchPayToPubkeyHash(const CScript& script, valtype& pubkeyhash) +{ + if (script.size() == 25 && script[0] == OP_DUP && script[1] == OP_HASH160 && script[2] == 20 && script[23] == OP_EQUALVERIFY && script[24] == OP_CHECKSIG) { + pubkeyhash = valtype(script.begin () + 3, script.begin() + 23); + return true; + } + return false; +} + +/** Test for "small positive integer" script opcodes - OP_1 through OP_16. */ +static constexpr bool IsSmallInteger(opcodetype opcode) +{ + return opcode >= OP_1 && opcode <= OP_16; +} + +/** Retrieve a minimally-encoded number in range [min,max] from an (opcode, data) pair, + * whether it's OP_n or through a push. */ +static std::optional<int> GetScriptNumber(opcodetype opcode, valtype data, int min, int max) +{ + int count; + if (IsSmallInteger(opcode)) { + count = CScript::DecodeOP_N(opcode); + } else if (IsPushdataOp(opcode)) { + if (!CheckMinimalPush(data, opcode)) return {}; + try { + count = CScriptNum(data, /* fRequireMinimal = */ true).getint(); + } catch (const scriptnum_error&) { + return {}; + } + } else { + return {}; + } + if (count < min || count > max) return {}; + return count; +} + +static bool MatchMultisig(const CScript& script, int& required_sigs, std::vector<valtype>& pubkeys) +{ + opcodetype opcode; + valtype data; + + CScript::const_iterator it = script.begin(); + if (script.size() < 1 || script.back() != OP_CHECKMULTISIG) return false; + + if (!script.GetOp(it, opcode, data)) return false; + auto req_sigs = GetScriptNumber(opcode, data, 1, MAX_PUBKEYS_PER_MULTISIG); + if (!req_sigs) return false; + required_sigs = *req_sigs; + while (script.GetOp(it, opcode, data) && CPubKey::ValidSize(data)) { + pubkeys.emplace_back(std::move(data)); + } + auto num_keys = GetScriptNumber(opcode, data, required_sigs, MAX_PUBKEYS_PER_MULTISIG); + if (!num_keys) return false; + if (pubkeys.size() != static_cast<unsigned long>(*num_keys)) return false; + + return (it + 1 == script.end()); +} + +std::optional<std::pair<int, std::vector<Span<const unsigned char>>>> MatchMultiA(const CScript& script) +{ + std::vector<Span<const unsigned char>> keyspans; + + // Redundant, but very fast and selective test. + if (script.size() == 0 || script[0] != 32 || script.back() != OP_NUMEQUAL) return {}; + + // Parse keys + auto it = script.begin(); + while (script.end() - it >= 34) { + if (*it != 32) return {}; + ++it; + keyspans.emplace_back(&*it, 32); + it += 32; + if (*it != (keyspans.size() == 1 ? OP_CHECKSIG : OP_CHECKSIGADD)) return {}; + ++it; + } + if (keyspans.size() == 0 || keyspans.size() > MAX_PUBKEYS_PER_MULTI_A) return {}; + + // Parse threshold. + opcodetype opcode; + std::vector<unsigned char> data; + if (!script.GetOp(it, opcode, data)) return {}; + if (it == script.end()) return {}; + if (*it != OP_NUMEQUAL) return {}; + ++it; + if (it != script.end()) return {}; + auto threshold = GetScriptNumber(opcode, data, 1, (int)keyspans.size()); + if (!threshold) return {}; + + // Construct result. + return std::pair{*threshold, std::move(keyspans)}; +} + +TxoutType Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned char>>& vSolutionsRet) +{ + vSolutionsRet.clear(); + + // Shortcut for pay-to-script-hash, which are more constrained than the other types: + // it is always OP_HASH160 20 [20 byte hash] OP_EQUAL + if (scriptPubKey.IsPayToScriptHash()) + { + std::vector<unsigned char> hashBytes(scriptPubKey.begin()+2, scriptPubKey.begin()+22); + vSolutionsRet.push_back(hashBytes); + return TxoutType::SCRIPTHASH; + } + + int witnessversion; + std::vector<unsigned char> witnessprogram; + if (scriptPubKey.IsWitnessProgram(witnessversion, witnessprogram)) { + if (witnessversion == 0 && witnessprogram.size() == WITNESS_V0_KEYHASH_SIZE) { + vSolutionsRet.push_back(std::move(witnessprogram)); + return TxoutType::WITNESS_V0_KEYHASH; + } + if (witnessversion == 0 && witnessprogram.size() == WITNESS_V0_SCRIPTHASH_SIZE) { + vSolutionsRet.push_back(std::move(witnessprogram)); + return TxoutType::WITNESS_V0_SCRIPTHASH; + } + if (witnessversion == 1 && witnessprogram.size() == WITNESS_V1_TAPROOT_SIZE) { + vSolutionsRet.push_back(std::move(witnessprogram)); + return TxoutType::WITNESS_V1_TAPROOT; + } + if (witnessversion != 0) { + vSolutionsRet.push_back(std::vector<unsigned char>{(unsigned char)witnessversion}); + vSolutionsRet.push_back(std::move(witnessprogram)); + return TxoutType::WITNESS_UNKNOWN; + } + return TxoutType::NONSTANDARD; + } + + // Provably prunable, data-carrying output + // + // So long as script passes the IsUnspendable() test and all but the first + // byte passes the IsPushOnly() test we don't care what exactly is in the + // script. + if (scriptPubKey.size() >= 1 && scriptPubKey[0] == OP_RETURN && scriptPubKey.IsPushOnly(scriptPubKey.begin()+1)) { + return TxoutType::NULL_DATA; + } + + std::vector<unsigned char> data; + if (MatchPayToPubkey(scriptPubKey, data)) { + vSolutionsRet.push_back(std::move(data)); + return TxoutType::PUBKEY; + } + + if (MatchPayToPubkeyHash(scriptPubKey, data)) { + vSolutionsRet.push_back(std::move(data)); + return TxoutType::PUBKEYHASH; + } + + int required; + std::vector<std::vector<unsigned char>> keys; + if (MatchMultisig(scriptPubKey, required, keys)) { + vSolutionsRet.push_back({static_cast<unsigned char>(required)}); // safe as required is in range 1..20 + vSolutionsRet.insert(vSolutionsRet.end(), keys.begin(), keys.end()); + vSolutionsRet.push_back({static_cast<unsigned char>(keys.size())}); // safe as size is in range 1..20 + return TxoutType::MULTISIG; + } + + vSolutionsRet.clear(); + return TxoutType::NONSTANDARD; +} + +CScript GetScriptForRawPubKey(const CPubKey& pubKey) +{ + return CScript() << std::vector<unsigned char>(pubKey.begin(), pubKey.end()) << OP_CHECKSIG; +} + +CScript GetScriptForMultisig(int nRequired, const std::vector<CPubKey>& keys) +{ + CScript script; + + script << nRequired; + for (const CPubKey& key : keys) + script << ToByteVector(key); + script << keys.size() << OP_CHECKMULTISIG; + + return script; +} diff --git a/src/script/solver.h b/src/script/solver.h new file mode 100644 index 0000000000..dc8f4c357d --- /dev/null +++ b/src/script/solver.h @@ -0,0 +1,66 @@ +// Copyright (c) 2009-2010 Satoshi Nakamoto +// Copyright (c) 2009-2022 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +// The Solver functions are used by policy and the wallet, but not consensus. + +#ifndef BITCOIN_SCRIPT_SOLVER_H +#define BITCOIN_SCRIPT_SOLVER_H + +#include <attributes.h> +#include <script/script.h> + +#include <string> +#include <optional> +#include <utility> +#include <vector> + +class CPubKey; +template <typename C> class Span; + +enum class TxoutType { + NONSTANDARD, + // 'standard' transaction types: + PUBKEY, + PUBKEYHASH, + SCRIPTHASH, + MULTISIG, + NULL_DATA, //!< unspendable OP_RETURN script that carries data + WITNESS_V0_SCRIPTHASH, + WITNESS_V0_KEYHASH, + WITNESS_V1_TAPROOT, + WITNESS_UNKNOWN, //!< Only for Witness versions not already defined above +}; + +/** Get the name of a TxoutType as a string */ +std::string GetTxnOutputType(TxoutType t); + +constexpr bool IsPushdataOp(opcodetype opcode) +{ + return opcode > OP_FALSE && opcode <= OP_PUSHDATA4; +} + +/** + * Parse a scriptPubKey and identify script type for standard scripts. If + * successful, returns script type and parsed pubkeys or hashes, depending on + * the type. For example, for a P2SH script, vSolutionsRet will contain the + * script hash, for P2PKH it will contain the key hash, etc. + * + * @param[in] scriptPubKey Script to parse + * @param[out] vSolutionsRet Vector of parsed pubkeys and hashes + * @return The script type. TxoutType::NONSTANDARD represents a failed solve. + */ +TxoutType Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned char>>& vSolutionsRet); + +/** Generate a P2PK script for the given pubkey. */ +CScript GetScriptForRawPubKey(const CPubKey& pubkey); + +/** Determine if script is a "multi_a" script. Returns (threshold, keyspans) if so, and nullopt otherwise. + * The keyspans refer to bytes in the passed script. */ +std::optional<std::pair<int, std::vector<Span<const unsigned char>>>> MatchMultiA(const CScript& script LIFETIMEBOUND); + +/** Generate a multisig script. */ +CScript GetScriptForMultisig(int nRequired, const std::vector<CPubKey>& keys); + +#endif // BITCOIN_SCRIPT_SOLVER_H diff --git a/src/script/standard.cpp b/src/script/standard.cpp deleted file mode 100644 index 7c4a05b6e6..0000000000 --- a/src/script/standard.cpp +++ /dev/null @@ -1,653 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <script/standard.h> - -#include <crypto/sha256.h> -#include <hash.h> -#include <pubkey.h> -#include <script/interpreter.h> -#include <script/script.h> -#include <util/strencodings.h> - -#include <string> - -typedef std::vector<unsigned char> valtype; - -CScriptID::CScriptID(const CScript& in) : BaseHash(Hash160(in)) {} -CScriptID::CScriptID(const ScriptHash& in) : BaseHash(static_cast<uint160>(in)) {} - -ScriptHash::ScriptHash(const CScript& in) : BaseHash(Hash160(in)) {} -ScriptHash::ScriptHash(const CScriptID& in) : BaseHash(static_cast<uint160>(in)) {} - -PKHash::PKHash(const CPubKey& pubkey) : BaseHash(pubkey.GetID()) {} -PKHash::PKHash(const CKeyID& pubkey_id) : BaseHash(pubkey_id) {} - -WitnessV0KeyHash::WitnessV0KeyHash(const CPubKey& pubkey) : BaseHash(pubkey.GetID()) {} -WitnessV0KeyHash::WitnessV0KeyHash(const PKHash& pubkey_hash) : BaseHash(static_cast<uint160>(pubkey_hash)) {} - -CKeyID ToKeyID(const PKHash& key_hash) -{ - return CKeyID{static_cast<uint160>(key_hash)}; -} - -CKeyID ToKeyID(const WitnessV0KeyHash& key_hash) -{ - return CKeyID{static_cast<uint160>(key_hash)}; -} - -WitnessV0ScriptHash::WitnessV0ScriptHash(const CScript& in) -{ - CSHA256().Write(in.data(), in.size()).Finalize(begin()); -} - -std::string GetTxnOutputType(TxoutType t) -{ - switch (t) { - case TxoutType::NONSTANDARD: return "nonstandard"; - case TxoutType::PUBKEY: return "pubkey"; - case TxoutType::PUBKEYHASH: return "pubkeyhash"; - case TxoutType::SCRIPTHASH: return "scripthash"; - case TxoutType::MULTISIG: return "multisig"; - case TxoutType::NULL_DATA: return "nulldata"; - case TxoutType::WITNESS_V0_KEYHASH: return "witness_v0_keyhash"; - case TxoutType::WITNESS_V0_SCRIPTHASH: return "witness_v0_scripthash"; - case TxoutType::WITNESS_V1_TAPROOT: return "witness_v1_taproot"; - case TxoutType::WITNESS_UNKNOWN: return "witness_unknown"; - } // no default case, so the compiler can warn about missing cases - assert(false); -} - -static bool MatchPayToPubkey(const CScript& script, valtype& pubkey) -{ - if (script.size() == CPubKey::SIZE + 2 && script[0] == CPubKey::SIZE && script.back() == OP_CHECKSIG) { - pubkey = valtype(script.begin() + 1, script.begin() + CPubKey::SIZE + 1); - return CPubKey::ValidSize(pubkey); - } - if (script.size() == CPubKey::COMPRESSED_SIZE + 2 && script[0] == CPubKey::COMPRESSED_SIZE && script.back() == OP_CHECKSIG) { - pubkey = valtype(script.begin() + 1, script.begin() + CPubKey::COMPRESSED_SIZE + 1); - return CPubKey::ValidSize(pubkey); - } - return false; -} - -static bool MatchPayToPubkeyHash(const CScript& script, valtype& pubkeyhash) -{ - if (script.size() == 25 && script[0] == OP_DUP && script[1] == OP_HASH160 && script[2] == 20 && script[23] == OP_EQUALVERIFY && script[24] == OP_CHECKSIG) { - pubkeyhash = valtype(script.begin () + 3, script.begin() + 23); - return true; - } - return false; -} - -/** Test for "small positive integer" script opcodes - OP_1 through OP_16. */ -static constexpr bool IsSmallInteger(opcodetype opcode) -{ - return opcode >= OP_1 && opcode <= OP_16; -} - -/** Retrieve a minimally-encoded number in range [min,max] from an (opcode, data) pair, - * whether it's OP_n or through a push. */ -static std::optional<int> GetScriptNumber(opcodetype opcode, valtype data, int min, int max) -{ - int count; - if (IsSmallInteger(opcode)) { - count = CScript::DecodeOP_N(opcode); - } else if (IsPushdataOp(opcode)) { - if (!CheckMinimalPush(data, opcode)) return {}; - try { - count = CScriptNum(data, /* fRequireMinimal = */ true).getint(); - } catch (const scriptnum_error&) { - return {}; - } - } else { - return {}; - } - if (count < min || count > max) return {}; - return count; -} - -static bool MatchMultisig(const CScript& script, int& required_sigs, std::vector<valtype>& pubkeys) -{ - opcodetype opcode; - valtype data; - - CScript::const_iterator it = script.begin(); - if (script.size() < 1 || script.back() != OP_CHECKMULTISIG) return false; - - if (!script.GetOp(it, opcode, data)) return false; - auto req_sigs = GetScriptNumber(opcode, data, 1, MAX_PUBKEYS_PER_MULTISIG); - if (!req_sigs) return false; - required_sigs = *req_sigs; - while (script.GetOp(it, opcode, data) && CPubKey::ValidSize(data)) { - pubkeys.emplace_back(std::move(data)); - } - auto num_keys = GetScriptNumber(opcode, data, required_sigs, MAX_PUBKEYS_PER_MULTISIG); - if (!num_keys) return false; - if (pubkeys.size() != static_cast<unsigned long>(*num_keys)) return false; - - return (it + 1 == script.end()); -} - -std::optional<std::pair<int, std::vector<Span<const unsigned char>>>> MatchMultiA(const CScript& script) -{ - std::vector<Span<const unsigned char>> keyspans; - - // Redundant, but very fast and selective test. - if (script.size() == 0 || script[0] != 32 || script.back() != OP_NUMEQUAL) return {}; - - // Parse keys - auto it = script.begin(); - while (script.end() - it >= 34) { - if (*it != 32) return {}; - ++it; - keyspans.emplace_back(&*it, 32); - it += 32; - if (*it != (keyspans.size() == 1 ? OP_CHECKSIG : OP_CHECKSIGADD)) return {}; - ++it; - } - if (keyspans.size() == 0 || keyspans.size() > MAX_PUBKEYS_PER_MULTI_A) return {}; - - // Parse threshold. - opcodetype opcode; - std::vector<unsigned char> data; - if (!script.GetOp(it, opcode, data)) return {}; - if (it == script.end()) return {}; - if (*it != OP_NUMEQUAL) return {}; - ++it; - if (it != script.end()) return {}; - auto threshold = GetScriptNumber(opcode, data, 1, (int)keyspans.size()); - if (!threshold) return {}; - - // Construct result. - return std::pair{*threshold, std::move(keyspans)}; -} - -TxoutType Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned char>>& vSolutionsRet) -{ - vSolutionsRet.clear(); - - // Shortcut for pay-to-script-hash, which are more constrained than the other types: - // it is always OP_HASH160 20 [20 byte hash] OP_EQUAL - if (scriptPubKey.IsPayToScriptHash()) - { - std::vector<unsigned char> hashBytes(scriptPubKey.begin()+2, scriptPubKey.begin()+22); - vSolutionsRet.push_back(hashBytes); - return TxoutType::SCRIPTHASH; - } - - int witnessversion; - std::vector<unsigned char> witnessprogram; - if (scriptPubKey.IsWitnessProgram(witnessversion, witnessprogram)) { - if (witnessversion == 0 && witnessprogram.size() == WITNESS_V0_KEYHASH_SIZE) { - vSolutionsRet.push_back(std::move(witnessprogram)); - return TxoutType::WITNESS_V0_KEYHASH; - } - if (witnessversion == 0 && witnessprogram.size() == WITNESS_V0_SCRIPTHASH_SIZE) { - vSolutionsRet.push_back(std::move(witnessprogram)); - return TxoutType::WITNESS_V0_SCRIPTHASH; - } - if (witnessversion == 1 && witnessprogram.size() == WITNESS_V1_TAPROOT_SIZE) { - vSolutionsRet.push_back(std::move(witnessprogram)); - return TxoutType::WITNESS_V1_TAPROOT; - } - if (witnessversion != 0) { - vSolutionsRet.push_back(std::vector<unsigned char>{(unsigned char)witnessversion}); - vSolutionsRet.push_back(std::move(witnessprogram)); - return TxoutType::WITNESS_UNKNOWN; - } - return TxoutType::NONSTANDARD; - } - - // Provably prunable, data-carrying output - // - // So long as script passes the IsUnspendable() test and all but the first - // byte passes the IsPushOnly() test we don't care what exactly is in the - // script. - if (scriptPubKey.size() >= 1 && scriptPubKey[0] == OP_RETURN && scriptPubKey.IsPushOnly(scriptPubKey.begin()+1)) { - return TxoutType::NULL_DATA; - } - - std::vector<unsigned char> data; - if (MatchPayToPubkey(scriptPubKey, data)) { - vSolutionsRet.push_back(std::move(data)); - return TxoutType::PUBKEY; - } - - if (MatchPayToPubkeyHash(scriptPubKey, data)) { - vSolutionsRet.push_back(std::move(data)); - return TxoutType::PUBKEYHASH; - } - - int required; - std::vector<std::vector<unsigned char>> keys; - if (MatchMultisig(scriptPubKey, required, keys)) { - vSolutionsRet.push_back({static_cast<unsigned char>(required)}); // safe as required is in range 1..20 - vSolutionsRet.insert(vSolutionsRet.end(), keys.begin(), keys.end()); - vSolutionsRet.push_back({static_cast<unsigned char>(keys.size())}); // safe as size is in range 1..20 - return TxoutType::MULTISIG; - } - - vSolutionsRet.clear(); - return TxoutType::NONSTANDARD; -} - -bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet) -{ - std::vector<valtype> vSolutions; - TxoutType whichType = Solver(scriptPubKey, vSolutions); - - switch (whichType) { - case TxoutType::PUBKEY: { - CPubKey pubKey(vSolutions[0]); - if (!pubKey.IsValid()) - return false; - - addressRet = PKHash(pubKey); - return true; - } - case TxoutType::PUBKEYHASH: { - addressRet = PKHash(uint160(vSolutions[0])); - return true; - } - case TxoutType::SCRIPTHASH: { - addressRet = ScriptHash(uint160(vSolutions[0])); - return true; - } - case TxoutType::WITNESS_V0_KEYHASH: { - WitnessV0KeyHash hash; - std::copy(vSolutions[0].begin(), vSolutions[0].end(), hash.begin()); - addressRet = hash; - return true; - } - case TxoutType::WITNESS_V0_SCRIPTHASH: { - WitnessV0ScriptHash hash; - std::copy(vSolutions[0].begin(), vSolutions[0].end(), hash.begin()); - addressRet = hash; - return true; - } - case TxoutType::WITNESS_V1_TAPROOT: { - WitnessV1Taproot tap; - std::copy(vSolutions[0].begin(), vSolutions[0].end(), tap.begin()); - addressRet = tap; - return true; - } - case TxoutType::WITNESS_UNKNOWN: { - WitnessUnknown unk; - unk.version = vSolutions[0][0]; - std::copy(vSolutions[1].begin(), vSolutions[1].end(), unk.program); - unk.length = vSolutions[1].size(); - addressRet = unk; - return true; - } - case TxoutType::MULTISIG: - case TxoutType::NULL_DATA: - case TxoutType::NONSTANDARD: - return false; - } // no default case, so the compiler can warn about missing cases - assert(false); -} - -namespace { -class CScriptVisitor -{ -public: - CScript operator()(const CNoDestination& dest) const - { - return CScript(); - } - - CScript operator()(const PKHash& keyID) const - { - return CScript() << OP_DUP << OP_HASH160 << ToByteVector(keyID) << OP_EQUALVERIFY << OP_CHECKSIG; - } - - CScript operator()(const ScriptHash& scriptID) const - { - return CScript() << OP_HASH160 << ToByteVector(scriptID) << OP_EQUAL; - } - - CScript operator()(const WitnessV0KeyHash& id) const - { - return CScript() << OP_0 << ToByteVector(id); - } - - CScript operator()(const WitnessV0ScriptHash& id) const - { - return CScript() << OP_0 << ToByteVector(id); - } - - CScript operator()(const WitnessV1Taproot& tap) const - { - return CScript() << OP_1 << ToByteVector(tap); - } - - CScript operator()(const WitnessUnknown& id) const - { - return CScript() << CScript::EncodeOP_N(id.version) << std::vector<unsigned char>(id.program, id.program + id.length); - } -}; -} // namespace - -CScript GetScriptForDestination(const CTxDestination& dest) -{ - return std::visit(CScriptVisitor(), dest); -} - -CScript GetScriptForRawPubKey(const CPubKey& pubKey) -{ - return CScript() << std::vector<unsigned char>(pubKey.begin(), pubKey.end()) << OP_CHECKSIG; -} - -CScript GetScriptForMultisig(int nRequired, const std::vector<CPubKey>& keys) -{ - CScript script; - - script << nRequired; - for (const CPubKey& key : keys) - script << ToByteVector(key); - script << keys.size() << OP_CHECKMULTISIG; - - return script; -} - -bool IsValidDestination(const CTxDestination& dest) { - return dest.index() != 0; -} - -/*static*/ TaprootBuilder::NodeInfo TaprootBuilder::Combine(NodeInfo&& a, NodeInfo&& b) -{ - NodeInfo ret; - /* Iterate over all tracked leaves in a, add b's hash to their Merkle branch, and move them to ret. */ - for (auto& leaf : a.leaves) { - leaf.merkle_branch.push_back(b.hash); - ret.leaves.emplace_back(std::move(leaf)); - } - /* Iterate over all tracked leaves in b, add a's hash to their Merkle branch, and move them to ret. */ - for (auto& leaf : b.leaves) { - leaf.merkle_branch.push_back(a.hash); - ret.leaves.emplace_back(std::move(leaf)); - } - ret.hash = ComputeTapbranchHash(a.hash, b.hash); - return ret; -} - -void TaprootSpendData::Merge(TaprootSpendData other) -{ - // TODO: figure out how to better deal with conflicting information - // being merged. - if (internal_key.IsNull() && !other.internal_key.IsNull()) { - internal_key = other.internal_key; - } - if (merkle_root.IsNull() && !other.merkle_root.IsNull()) { - merkle_root = other.merkle_root; - } - for (auto& [key, control_blocks] : other.scripts) { - scripts[key].merge(std::move(control_blocks)); - } -} - -void TaprootBuilder::Insert(TaprootBuilder::NodeInfo&& node, int depth) -{ - assert(depth >= 0 && (size_t)depth <= TAPROOT_CONTROL_MAX_NODE_COUNT); - /* We cannot insert a leaf at a lower depth while a deeper branch is unfinished. Doing - * so would mean the Add() invocations do not correspond to a DFS traversal of a - * binary tree. */ - if ((size_t)depth + 1 < m_branch.size()) { - m_valid = false; - return; - } - /* As long as an entry in the branch exists at the specified depth, combine it and propagate up. - * The 'node' variable is overwritten here with the newly combined node. */ - while (m_valid && m_branch.size() > (size_t)depth && m_branch[depth].has_value()) { - node = Combine(std::move(node), std::move(*m_branch[depth])); - m_branch.pop_back(); - if (depth == 0) m_valid = false; /* Can't propagate further up than the root */ - --depth; - } - if (m_valid) { - /* Make sure the branch is big enough to place the new node. */ - if (m_branch.size() <= (size_t)depth) m_branch.resize((size_t)depth + 1); - assert(!m_branch[depth].has_value()); - m_branch[depth] = std::move(node); - } -} - -/*static*/ bool TaprootBuilder::ValidDepths(const std::vector<int>& depths) -{ - std::vector<bool> branch; - for (int depth : depths) { - // This inner loop corresponds to effectively the same logic on branch - // as what Insert() performs on the m_branch variable. Instead of - // storing a NodeInfo object, just remember whether or not there is one - // at that depth. - if (depth < 0 || (size_t)depth > TAPROOT_CONTROL_MAX_NODE_COUNT) return false; - if ((size_t)depth + 1 < branch.size()) return false; - while (branch.size() > (size_t)depth && branch[depth]) { - branch.pop_back(); - if (depth == 0) return false; - --depth; - } - if (branch.size() <= (size_t)depth) branch.resize((size_t)depth + 1); - assert(!branch[depth]); - branch[depth] = true; - } - // And this check corresponds to the IsComplete() check on m_branch. - return branch.size() == 0 || (branch.size() == 1 && branch[0]); -} - -TaprootBuilder& TaprootBuilder::Add(int depth, Span<const unsigned char> script, int leaf_version, bool track) -{ - assert((leaf_version & ~TAPROOT_LEAF_MASK) == 0); - if (!IsValid()) return *this; - /* Construct NodeInfo object with leaf hash and (if track is true) also leaf information. */ - NodeInfo node; - node.hash = ComputeTapleafHash(leaf_version, script); - if (track) node.leaves.emplace_back(LeafInfo{std::vector<unsigned char>(script.begin(), script.end()), leaf_version, {}}); - /* Insert into the branch. */ - Insert(std::move(node), depth); - return *this; -} - -TaprootBuilder& TaprootBuilder::AddOmitted(int depth, const uint256& hash) -{ - if (!IsValid()) return *this; - /* Construct NodeInfo object with the hash directly, and insert it into the branch. */ - NodeInfo node; - node.hash = hash; - Insert(std::move(node), depth); - return *this; -} - -TaprootBuilder& TaprootBuilder::Finalize(const XOnlyPubKey& internal_key) -{ - /* Can only call this function when IsComplete() is true. */ - assert(IsComplete()); - m_internal_key = internal_key; - auto ret = m_internal_key.CreateTapTweak(m_branch.size() == 0 ? nullptr : &m_branch[0]->hash); - assert(ret.has_value()); - std::tie(m_output_key, m_parity) = *ret; - return *this; -} - -WitnessV1Taproot TaprootBuilder::GetOutput() { return WitnessV1Taproot{m_output_key}; } - -TaprootSpendData TaprootBuilder::GetSpendData() const -{ - assert(IsComplete()); - assert(m_output_key.IsFullyValid()); - TaprootSpendData spd; - spd.merkle_root = m_branch.size() == 0 ? uint256() : m_branch[0]->hash; - spd.internal_key = m_internal_key; - if (m_branch.size()) { - // If any script paths exist, they have been combined into the root m_branch[0] - // by now. Compute the control block for each of its tracked leaves, and put them in - // spd.scripts. - for (const auto& leaf : m_branch[0]->leaves) { - std::vector<unsigned char> control_block; - control_block.resize(TAPROOT_CONTROL_BASE_SIZE + TAPROOT_CONTROL_NODE_SIZE * leaf.merkle_branch.size()); - control_block[0] = leaf.leaf_version | (m_parity ? 1 : 0); - std::copy(m_internal_key.begin(), m_internal_key.end(), control_block.begin() + 1); - if (leaf.merkle_branch.size()) { - std::copy(leaf.merkle_branch[0].begin(), - leaf.merkle_branch[0].begin() + TAPROOT_CONTROL_NODE_SIZE * leaf.merkle_branch.size(), - control_block.begin() + TAPROOT_CONTROL_BASE_SIZE); - } - spd.scripts[{leaf.script, leaf.leaf_version}].insert(std::move(control_block)); - } - } - return spd; -} - -std::optional<std::vector<std::tuple<int, std::vector<unsigned char>, int>>> InferTaprootTree(const TaprootSpendData& spenddata, const XOnlyPubKey& output) -{ - // Verify that the output matches the assumed Merkle root and internal key. - auto tweak = spenddata.internal_key.CreateTapTweak(spenddata.merkle_root.IsNull() ? nullptr : &spenddata.merkle_root); - if (!tweak || tweak->first != output) return std::nullopt; - // If the Merkle root is 0, the tree is empty, and we're done. - std::vector<std::tuple<int, std::vector<unsigned char>, int>> ret; - if (spenddata.merkle_root.IsNull()) return ret; - - /** Data structure to represent the nodes of the tree we're going to build. */ - struct TreeNode { - /** Hash of this node, if known; 0 otherwise. */ - uint256 hash; - /** The left and right subtrees (note that their order is irrelevant). */ - std::unique_ptr<TreeNode> sub[2]; - /** If this is known to be a leaf node, a pointer to the (script, leaf_ver) pair. - * nullptr otherwise. */ - const std::pair<std::vector<unsigned char>, int>* leaf = nullptr; - /** Whether or not this node has been explored (is known to be a leaf, or known to have children). */ - bool explored = false; - /** Whether or not this node is an inner node (unknown until explored = true). */ - bool inner; - /** Whether or not we have produced output for this subtree. */ - bool done = false; - }; - - // Build tree from the provided branches. - TreeNode root; - root.hash = spenddata.merkle_root; - for (const auto& [key, control_blocks] : spenddata.scripts) { - const auto& [script, leaf_ver] = key; - for (const auto& control : control_blocks) { - // Skip script records with nonsensical leaf version. - if (leaf_ver < 0 || leaf_ver >= 0x100 || leaf_ver & 1) continue; - // Skip script records with invalid control block sizes. - if (control.size() < TAPROOT_CONTROL_BASE_SIZE || control.size() > TAPROOT_CONTROL_MAX_SIZE || - ((control.size() - TAPROOT_CONTROL_BASE_SIZE) % TAPROOT_CONTROL_NODE_SIZE) != 0) continue; - // Skip script records that don't match the control block. - if ((control[0] & TAPROOT_LEAF_MASK) != leaf_ver) continue; - // Skip script records that don't match the provided Merkle root. - const uint256 leaf_hash = ComputeTapleafHash(leaf_ver, script); - const uint256 merkle_root = ComputeTaprootMerkleRoot(control, leaf_hash); - if (merkle_root != spenddata.merkle_root) continue; - - TreeNode* node = &root; - size_t levels = (control.size() - TAPROOT_CONTROL_BASE_SIZE) / TAPROOT_CONTROL_NODE_SIZE; - for (size_t depth = 0; depth < levels; ++depth) { - // Can't descend into a node which we already know is a leaf. - if (node->explored && !node->inner) return std::nullopt; - - // Extract partner hash from Merkle branch in control block. - uint256 hash; - std::copy(control.begin() + TAPROOT_CONTROL_BASE_SIZE + (levels - 1 - depth) * TAPROOT_CONTROL_NODE_SIZE, - control.begin() + TAPROOT_CONTROL_BASE_SIZE + (levels - depth) * TAPROOT_CONTROL_NODE_SIZE, - hash.begin()); - - if (node->sub[0]) { - // Descend into the existing left or right branch. - bool desc = false; - for (int i = 0; i < 2; ++i) { - if (node->sub[i]->hash == hash || (node->sub[i]->hash.IsNull() && node->sub[1-i]->hash != hash)) { - node->sub[i]->hash = hash; - node = &*node->sub[1-i]; - desc = true; - break; - } - } - if (!desc) return std::nullopt; // This probably requires a hash collision to hit. - } else { - // We're in an unexplored node. Create subtrees and descend. - node->explored = true; - node->inner = true; - node->sub[0] = std::make_unique<TreeNode>(); - node->sub[1] = std::make_unique<TreeNode>(); - node->sub[1]->hash = hash; - node = &*node->sub[0]; - } - } - // Cannot turn a known inner node into a leaf. - if (node->sub[0]) return std::nullopt; - node->explored = true; - node->inner = false; - node->leaf = &key; - node->hash = leaf_hash; - } - } - - // Recursive processing to turn the tree into flattened output. Use an explicit stack here to avoid - // overflowing the call stack (the tree may be 128 levels deep). - std::vector<TreeNode*> stack{&root}; - while (!stack.empty()) { - TreeNode& node = *stack.back(); - if (!node.explored) { - // Unexplored node, which means the tree is incomplete. - return std::nullopt; - } else if (!node.inner) { - // Leaf node; produce output. - ret.emplace_back(stack.size() - 1, node.leaf->first, node.leaf->second); - node.done = true; - stack.pop_back(); - } else if (node.sub[0]->done && !node.sub[1]->done && !node.sub[1]->explored && !node.sub[1]->hash.IsNull() && - ComputeTapbranchHash(node.sub[1]->hash, node.sub[1]->hash) == node.hash) { - // Whenever there are nodes with two identical subtrees under it, we run into a problem: - // the control blocks for the leaves underneath those will be identical as well, and thus - // they will all be matched to the same path in the tree. The result is that at the location - // where the duplicate occurred, the left child will contain a normal tree that can be explored - // and processed, but the right one will remain unexplored. - // - // This situation can be detected, by encountering an inner node with unexplored right subtree - // with known hash, and H_TapBranch(hash, hash) is equal to the parent node (this node)'s hash. - // - // To deal with this, simply process the left tree a second time (set its done flag to false; - // noting that the done flag of its children have already been set to false after processing - // those). To avoid ending up in an infinite loop, set the done flag of the right (unexplored) - // subtree to true. - node.sub[0]->done = false; - node.sub[1]->done = true; - } else if (node.sub[0]->done && node.sub[1]->done) { - // An internal node which we're finished with. - node.sub[0]->done = false; - node.sub[1]->done = false; - node.done = true; - stack.pop_back(); - } else if (!node.sub[0]->done) { - // An internal node whose left branch hasn't been processed yet. Do so first. - stack.push_back(&*node.sub[0]); - } else if (!node.sub[1]->done) { - // An internal node whose right branch hasn't been processed yet. Do so first. - stack.push_back(&*node.sub[1]); - } - } - - return ret; -} - -std::vector<std::tuple<uint8_t, uint8_t, std::vector<unsigned char>>> TaprootBuilder::GetTreeTuples() const -{ - assert(IsComplete()); - std::vector<std::tuple<uint8_t, uint8_t, std::vector<unsigned char>>> tuples; - if (m_branch.size()) { - const auto& leaves = m_branch[0]->leaves; - for (const auto& leaf : leaves) { - assert(leaf.merkle_branch.size() <= TAPROOT_CONTROL_MAX_NODE_COUNT); - uint8_t depth = (uint8_t)leaf.merkle_branch.size(); - uint8_t leaf_ver = (uint8_t)leaf.leaf_version; - tuples.push_back(std::make_tuple(depth, leaf_ver, leaf.script)); - } - } - return tuples; -} diff --git a/src/script/standard.h b/src/script/standard.h deleted file mode 100644 index 18cf5c8c88..0000000000 --- a/src/script/standard.h +++ /dev/null @@ -1,330 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_SCRIPT_STANDARD_H -#define BITCOIN_SCRIPT_STANDARD_H - -#include <attributes.h> -#include <pubkey.h> -#include <script/interpreter.h> -#include <uint256.h> -#include <util/hash_type.h> - -#include <map> -#include <string> -#include <variant> - -static const bool DEFAULT_ACCEPT_DATACARRIER = true; - -class CKeyID; -class CScript; -struct ScriptHash; - -/** A reference to a CScript: the Hash160 of its serialization (see script.h) */ -class CScriptID : public BaseHash<uint160> -{ -public: - CScriptID() : BaseHash() {} - explicit CScriptID(const CScript& in); - explicit CScriptID(const uint160& in) : BaseHash(in) {} - explicit CScriptID(const ScriptHash& in); -}; - -/** - * Default setting for -datacarriersize. 80 bytes of data, +1 for OP_RETURN, - * +2 for the pushdata opcodes. - */ -static const unsigned int MAX_OP_RETURN_RELAY = 83; - -/** - * Mandatory script verification flags that all new blocks must comply with for - * them to be valid. (but old blocks may not comply with) Currently just P2SH, - * but in the future other flags may be added. - * - * Failing one of these tests may trigger a DoS ban - see CheckInputScripts() for - * details. - */ -static const unsigned int MANDATORY_SCRIPT_VERIFY_FLAGS = SCRIPT_VERIFY_P2SH; - -enum class TxoutType { - NONSTANDARD, - // 'standard' transaction types: - PUBKEY, - PUBKEYHASH, - SCRIPTHASH, - MULTISIG, - NULL_DATA, //!< unspendable OP_RETURN script that carries data - WITNESS_V0_SCRIPTHASH, - WITNESS_V0_KEYHASH, - WITNESS_V1_TAPROOT, - WITNESS_UNKNOWN, //!< Only for Witness versions not already defined above -}; - -class CNoDestination { -public: - friend bool operator==(const CNoDestination &a, const CNoDestination &b) { return true; } - friend bool operator<(const CNoDestination &a, const CNoDestination &b) { return true; } -}; - -struct PKHash : public BaseHash<uint160> -{ - PKHash() : BaseHash() {} - explicit PKHash(const uint160& hash) : BaseHash(hash) {} - explicit PKHash(const CPubKey& pubkey); - explicit PKHash(const CKeyID& pubkey_id); -}; -CKeyID ToKeyID(const PKHash& key_hash); - -struct WitnessV0KeyHash; -struct ScriptHash : public BaseHash<uint160> -{ - ScriptHash() : BaseHash() {} - // These don't do what you'd expect. - // Use ScriptHash(GetScriptForDestination(...)) instead. - explicit ScriptHash(const WitnessV0KeyHash& hash) = delete; - explicit ScriptHash(const PKHash& hash) = delete; - - explicit ScriptHash(const uint160& hash) : BaseHash(hash) {} - explicit ScriptHash(const CScript& script); - explicit ScriptHash(const CScriptID& script); -}; - -struct WitnessV0ScriptHash : public BaseHash<uint256> -{ - WitnessV0ScriptHash() : BaseHash() {} - explicit WitnessV0ScriptHash(const uint256& hash) : BaseHash(hash) {} - explicit WitnessV0ScriptHash(const CScript& script); -}; - -struct WitnessV0KeyHash : public BaseHash<uint160> -{ - WitnessV0KeyHash() : BaseHash() {} - explicit WitnessV0KeyHash(const uint160& hash) : BaseHash(hash) {} - explicit WitnessV0KeyHash(const CPubKey& pubkey); - explicit WitnessV0KeyHash(const PKHash& pubkey_hash); -}; -CKeyID ToKeyID(const WitnessV0KeyHash& key_hash); - -struct WitnessV1Taproot : public XOnlyPubKey -{ - WitnessV1Taproot() : XOnlyPubKey() {} - explicit WitnessV1Taproot(const XOnlyPubKey& xpk) : XOnlyPubKey(xpk) {} -}; - -//! CTxDestination subtype to encode any future Witness version -struct WitnessUnknown -{ - unsigned int version; - unsigned int length; - unsigned char program[40]; - - friend bool operator==(const WitnessUnknown& w1, const WitnessUnknown& w2) { - if (w1.version != w2.version) return false; - if (w1.length != w2.length) return false; - return std::equal(w1.program, w1.program + w1.length, w2.program); - } - - friend bool operator<(const WitnessUnknown& w1, const WitnessUnknown& w2) { - if (w1.version < w2.version) return true; - if (w1.version > w2.version) return false; - if (w1.length < w2.length) return true; - if (w1.length > w2.length) return false; - return std::lexicographical_compare(w1.program, w1.program + w1.length, w2.program, w2.program + w2.length); - } -}; - -/** - * A txout script template with a specific destination. It is either: - * * CNoDestination: no destination set - * * PKHash: TxoutType::PUBKEYHASH destination (P2PKH) - * * ScriptHash: TxoutType::SCRIPTHASH destination (P2SH) - * * WitnessV0ScriptHash: TxoutType::WITNESS_V0_SCRIPTHASH destination (P2WSH) - * * WitnessV0KeyHash: TxoutType::WITNESS_V0_KEYHASH destination (P2WPKH) - * * WitnessV1Taproot: TxoutType::WITNESS_V1_TAPROOT destination (P2TR) - * * WitnessUnknown: TxoutType::WITNESS_UNKNOWN destination (P2W???) - * A CTxDestination is the internal data type encoded in a bitcoin address - */ -using CTxDestination = std::variant<CNoDestination, PKHash, ScriptHash, WitnessV0ScriptHash, WitnessV0KeyHash, WitnessV1Taproot, WitnessUnknown>; - -/** Check whether a CTxDestination is a CNoDestination. */ -bool IsValidDestination(const CTxDestination& dest); - -/** Get the name of a TxoutType as a string */ -std::string GetTxnOutputType(TxoutType t); - -constexpr bool IsPushdataOp(opcodetype opcode) -{ - return opcode > OP_FALSE && opcode <= OP_PUSHDATA4; -} - -/** - * Parse a scriptPubKey and identify script type for standard scripts. If - * successful, returns script type and parsed pubkeys or hashes, depending on - * the type. For example, for a P2SH script, vSolutionsRet will contain the - * script hash, for P2PKH it will contain the key hash, etc. - * - * @param[in] scriptPubKey Script to parse - * @param[out] vSolutionsRet Vector of parsed pubkeys and hashes - * @return The script type. TxoutType::NONSTANDARD represents a failed solve. - */ -TxoutType Solver(const CScript& scriptPubKey, std::vector<std::vector<unsigned char>>& vSolutionsRet); - -/** - * Parse a standard scriptPubKey for the destination address. Assigns result to - * the addressRet parameter and returns true if successful. Currently only works for P2PK, - * P2PKH, P2SH, P2WPKH, and P2WSH scripts. - */ -bool ExtractDestination(const CScript& scriptPubKey, CTxDestination& addressRet); - -/** - * Generate a Bitcoin scriptPubKey for the given CTxDestination. Returns a P2PKH - * script for a CKeyID destination, a P2SH script for a CScriptID, and an empty - * script for CNoDestination. - */ -CScript GetScriptForDestination(const CTxDestination& dest); - -/** Generate a P2PK script for the given pubkey. */ -CScript GetScriptForRawPubKey(const CPubKey& pubkey); - -/** Determine if script is a "multi_a" script. Returns (threshold, keyspans) if so, and nullopt otherwise. - * The keyspans refer to bytes in the passed script. */ -std::optional<std::pair<int, std::vector<Span<const unsigned char>>>> MatchMultiA(const CScript& script LIFETIMEBOUND); - -/** Generate a multisig script. */ -CScript GetScriptForMultisig(int nRequired, const std::vector<CPubKey>& keys); - -struct ShortestVectorFirstComparator -{ - bool operator()(const std::vector<unsigned char>& a, const std::vector<unsigned char>& b) const - { - if (a.size() < b.size()) return true; - if (a.size() > b.size()) return false; - return a < b; - } -}; - -struct TaprootSpendData -{ - /** The BIP341 internal key. */ - XOnlyPubKey internal_key; - /** The Merkle root of the script tree (0 if no scripts). */ - uint256 merkle_root; - /** Map from (script, leaf_version) to (sets of) control blocks. - * More than one control block for a given script is only possible if it - * appears in multiple branches of the tree. We keep them all so that - * inference can reconstruct the full tree. Within each set, the control - * blocks are sorted by size, so that the signing logic can easily - * prefer the cheapest one. */ - std::map<std::pair<std::vector<unsigned char>, int>, std::set<std::vector<unsigned char>, ShortestVectorFirstComparator>> scripts; - /** Merge other TaprootSpendData (for the same scriptPubKey) into this. */ - void Merge(TaprootSpendData other); -}; - -/** Utility class to construct Taproot outputs from internal key and script tree. */ -class TaprootBuilder -{ -private: - /** Information about a tracked leaf in the Merkle tree. */ - struct LeafInfo - { - std::vector<unsigned char> script; //!< The script. - int leaf_version; //!< The leaf version for that script. - std::vector<uint256> merkle_branch; //!< The hashing partners above this leaf. - }; - - /** Information associated with a node in the Merkle tree. */ - struct NodeInfo - { - /** Merkle hash of this node. */ - uint256 hash; - /** Tracked leaves underneath this node (either from the node itself, or its children). - * The merkle_branch field of each is the partners to get to *this* node. */ - std::vector<LeafInfo> leaves; - }; - /** Whether the builder is in a valid state so far. */ - bool m_valid = true; - - /** The current state of the builder. - * - * For each level in the tree, one NodeInfo object may be present. m_branch[0] - * is information about the root; further values are for deeper subtrees being - * explored. - * - * For every right branch taken to reach the position we're currently - * working in, there will be a (non-nullopt) entry in m_branch corresponding - * to the left branch at that level. - * - * For example, imagine this tree: - N0 - - * / \ - * N1 N2 - * / \ / \ - * A B C N3 - * / \ - * D E - * - * Initially, m_branch is empty. After processing leaf A, it would become - * {nullopt, nullopt, A}. When processing leaf B, an entry at level 2 already - * exists, and it would thus be combined with it to produce a level 1 one, - * resulting in {nullopt, N1}. Adding C and D takes us to {nullopt, N1, C} - * and {nullopt, N1, C, D} respectively. When E is processed, it is combined - * with D, and then C, and then N1, to produce the root, resulting in {N0}. - * - * This structure allows processing with just O(log n) overhead if the leaves - * are computed on the fly. - * - * As an invariant, there can never be nullopt entries at the end. There can - * also not be more than 128 entries (as that would mean more than 128 levels - * in the tree). The depth of newly added entries will always be at least - * equal to the current size of m_branch (otherwise it does not correspond - * to a depth-first traversal of a tree). m_branch is only empty if no entries - * have ever be processed. m_branch having length 1 corresponds to being done. - */ - std::vector<std::optional<NodeInfo>> m_branch; - - XOnlyPubKey m_internal_key; //!< The internal key, set when finalizing. - XOnlyPubKey m_output_key; //!< The output key, computed when finalizing. - bool m_parity; //!< The tweak parity, computed when finalizing. - - /** Combine information about a parent Merkle tree node from its child nodes. */ - static NodeInfo Combine(NodeInfo&& a, NodeInfo&& b); - /** Insert information about a node at a certain depth, and propagate information up. */ - void Insert(NodeInfo&& node, int depth); - -public: - /** Add a new script at a certain depth in the tree. Add() operations must be called - * in depth-first traversal order of binary tree. If track is true, it will be included in - * the GetSpendData() output. */ - TaprootBuilder& Add(int depth, Span<const unsigned char> script, int leaf_version, bool track = true); - /** Like Add(), but for a Merkle node with a given hash to the tree. */ - TaprootBuilder& AddOmitted(int depth, const uint256& hash); - /** Finalize the construction. Can only be called when IsComplete() is true. - internal_key.IsFullyValid() must be true. */ - TaprootBuilder& Finalize(const XOnlyPubKey& internal_key); - - /** Return true if so far all input was valid. */ - bool IsValid() const { return m_valid; } - /** Return whether there were either no leaves, or the leaves form a Huffman tree. */ - bool IsComplete() const { return m_valid && (m_branch.size() == 0 || (m_branch.size() == 1 && m_branch[0].has_value())); } - /** Compute scriptPubKey (after Finalize()). */ - WitnessV1Taproot GetOutput(); - /** Check if a list of depths is legal (will lead to IsComplete()). */ - static bool ValidDepths(const std::vector<int>& depths); - /** Compute spending data (after Finalize()). */ - TaprootSpendData GetSpendData() const; - /** Returns a vector of tuples representing the depth, leaf version, and script */ - std::vector<std::tuple<uint8_t, uint8_t, std::vector<unsigned char>>> GetTreeTuples() const; - /** Returns true if there are any tapscripts */ - bool HasScripts() const { return !m_branch.empty(); } -}; - -/** Given a TaprootSpendData and the output key, reconstruct its script tree. - * - * If the output doesn't match the spenddata, or if the data in spenddata is incomplete, - * std::nullopt is returned. Otherwise, a vector of (depth, script, leaf_ver) tuples is - * returned, corresponding to a depth-first traversal of the script tree. - */ -std::optional<std::vector<std::tuple<int, std::vector<unsigned char>, int>>> InferTaprootTree(const TaprootSpendData& spenddata, const XOnlyPubKey& output); - -#endif // BITCOIN_SCRIPT_STANDARD_H diff --git a/src/signet.cpp b/src/signet.cpp index b73d82bb2e..21b289b637 100644 --- a/src/signet.cpp +++ b/src/signet.cpp @@ -18,7 +18,6 @@ #include <primitives/block.h> #include <primitives/transaction.h> #include <script/interpreter.h> -#include <script/standard.h> #include <span.h> #include <streams.h> #include <uint256.h> diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index 9bd5c7c2b6..97ea5cfbf3 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <addresstype.h> #include <blockfilter.h> #include <chainparams.h> #include <consensus/merkle.h> @@ -10,7 +11,6 @@ #include <interfaces/chain.h> #include <node/miner.h> #include <pow.h> -#include <script/standard.h> #include <test/util/blockfilter.h> #include <test/util/index.h> #include <test/util/setup_common.h> diff --git a/src/test/blockfilter_tests.cpp b/src/test/blockfilter_tests.cpp index dfeac6ca42..b372f25ea9 100644 --- a/src/test/blockfilter_tests.cpp +++ b/src/test/blockfilter_tests.cpp @@ -7,8 +7,10 @@ #include <blockfilter.h> #include <core_io.h> +#include <primitives/block.h> #include <serialize.h> #include <streams.h> +#include <undo.h> #include <univalue.h> #include <util/strencodings.h> diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp index f52c692649..553bb31ba1 100644 --- a/src/test/blockmanager_tests.cpp +++ b/src/test/blockmanager_tests.cpp @@ -6,6 +6,7 @@ #include <node/blockstorage.h> #include <node/context.h> #include <node/kernel_notifications.h> +#include <script/solver.h> #include <util/chaintype.h> #include <validation.h> diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 853dc6dc1e..12dc4d1ccc 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -2,9 +2,9 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <addresstype.h> #include <clientversion.h> #include <coins.h> -#include <script/standard.h> #include <streams.h> #include <test/util/poolresourcetester.h> #include <test/util/random.h> diff --git a/src/test/compress_tests.cpp b/src/test/compress_tests.cpp index de99b91c7f..b56629ae40 100644 --- a/src/test/compress_tests.cpp +++ b/src/test/compress_tests.cpp @@ -3,7 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <compressor.h> -#include <script/standard.h> +#include <script/script.h> #include <test/util/setup_common.h> #include <stdint.h> diff --git a/src/test/crypto_tests.cpp b/src/test/crypto_tests.cpp index 6fbe74a680..0a6378adf4 100644 --- a/src/test/crypto_tests.cpp +++ b/src/test/crypto_tests.cpp @@ -133,27 +133,27 @@ static void TestAES256CBC(const std::string &hexkey, const std::string &hexiv, b static void TestChaCha20(const std::string &hex_message, const std::string &hexkey, ChaCha20::Nonce96 nonce, uint32_t seek, const std::string& hexout) { - std::vector<unsigned char> key = ParseHex(hexkey); + auto key = ParseHex<std::byte>(hexkey); assert(key.size() == 32); - std::vector<unsigned char> m = ParseHex(hex_message); - ChaCha20 rng(key.data()); - rng.Seek64(nonce, seek); - std::vector<unsigned char> outres; + auto m = ParseHex<std::byte>(hex_message); + ChaCha20 rng{key}; + rng.Seek(nonce, seek); + std::vector<std::byte> outres; outres.resize(hexout.size() / 2); assert(hex_message.empty() || m.size() * 2 == hexout.size()); // perform the ChaCha20 round(s), if message is provided it will output the encrypted ciphertext otherwise the keystream if (!hex_message.empty()) { - rng.Crypt(m.data(), outres.data(), outres.size()); + rng.Crypt(m, outres); } else { - rng.Keystream(outres.data(), outres.size()); + rng.Keystream(outres); } BOOST_CHECK_EQUAL(hexout, HexStr(outres)); if (!hex_message.empty()) { // Manually XOR with the keystream and compare the output - rng.Seek64(nonce, seek); - std::vector<unsigned char> only_keystream(outres.size()); - rng.Keystream(only_keystream.data(), only_keystream.size()); + rng.Seek(nonce, seek); + std::vector<std::byte> only_keystream(outres.size()); + rng.Keystream(only_keystream); for (size_t i = 0; i != m.size(); i++) { outres[i] = m[i] ^ only_keystream[i]; } @@ -167,14 +167,14 @@ static void TestChaCha20(const std::string &hex_message, const std::string &hexk lens[1] = InsecureRandRange(hexout.size() / 2U + 1U - lens[0]); lens[2] = hexout.size() / 2U - lens[0] - lens[1]; - rng.Seek64(nonce, seek); - outres.assign(hexout.size() / 2U, 0); + rng.Seek(nonce, seek); + outres.assign(hexout.size() / 2U, {}); size_t pos = 0; for (int j = 0; j < 3; ++j) { if (!hex_message.empty()) { - rng.Crypt(m.data() + pos, outres.data() + pos, lens[j]); + rng.Crypt(Span{m}.subspan(pos, lens[j]), Span{outres}.subspan(pos, lens[j])); } else { - rng.Keystream(outres.data() + pos, lens[j]); + rng.Keystream(Span{outres}.subspan(pos, lens[j])); } pos += lens[j]; } @@ -190,7 +190,7 @@ static void TestFSChaCha20(const std::string& hex_plaintext, const std::string& auto plaintext = ParseHex<std::byte>(hex_plaintext); auto fsc20 = FSChaCha20{key, rekey_interval}; - auto c20 = ChaCha20{UCharCast(key.data())}; + auto c20 = ChaCha20{key}; std::vector<std::byte> fsc20_output; fsc20_output.resize(plaintext.size()); @@ -200,23 +200,23 @@ static void TestFSChaCha20(const std::string& hex_plaintext, const std::string& for (size_t i = 0; i < rekey_interval; i++) { fsc20.Crypt(plaintext, fsc20_output); - c20.Crypt(UCharCast(plaintext.data()), UCharCast(c20_output.data()), plaintext.size()); + c20.Crypt(plaintext, c20_output); BOOST_CHECK(c20_output == fsc20_output); } // At the rotation interval, the outputs will no longer match fsc20.Crypt(plaintext, fsc20_output); auto c20_copy = c20; - c20.Crypt(UCharCast(plaintext.data()), UCharCast(c20_output.data()), plaintext.size()); + c20.Crypt(plaintext, c20_output); BOOST_CHECK(c20_output != fsc20_output); std::byte new_key[FSChaCha20::KEYLEN]; - c20_copy.Keystream(UCharCast(new_key), sizeof(new_key)); - c20.SetKey32(UCharCast(new_key)); - c20.Seek64({0, 1}, 0); + c20_copy.Keystream(new_key); + c20.SetKey(new_key); + c20.Seek({0, 1}, 0); // Outputs should match again after simulating key rotation - c20.Crypt(UCharCast(plaintext.data()), UCharCast(c20_output.data()), plaintext.size()); + c20.Crypt(plaintext, c20_output); BOOST_CHECK(c20_output == fsc20_output); BOOST_CHECK_EQUAL(HexStr(fsc20_output), ciphertext_after_rotation); @@ -226,10 +226,9 @@ static void TestPoly1305(const std::string &hexmessage, const std::string &hexke { auto key = ParseHex<std::byte>(hexkey); auto m = ParseHex<std::byte>(hexmessage); - auto tag = ParseHex<std::byte>(hextag); std::vector<std::byte> tagres(Poly1305::TAGLEN); Poly1305{key}.Update(m).Finalize(tagres); - BOOST_CHECK(tag == tagres); + BOOST_CHECK_EQUAL(HexStr(tagres), hextag); // Test incremental interface for (int splits = 0; splits < 10; ++splits) { @@ -243,7 +242,7 @@ static void TestPoly1305(const std::string &hexmessage, const std::string &hexke } tagres.assign(Poly1305::TAGLEN, std::byte{}); poly1305.Update(data).Finalize(tagres); - BOOST_CHECK(tag == tagres); + BOOST_CHECK_EQUAL(HexStr(tagres), hextag); } } } @@ -823,20 +822,20 @@ BOOST_AUTO_TEST_CASE(chacha20_testvector) BOOST_AUTO_TEST_CASE(chacha20_midblock) { - auto key = ParseHex("0000000000000000000000000000000000000000000000000000000000000000"); - ChaCha20 c20{key.data()}; + auto key = ParseHex<std::byte>("0000000000000000000000000000000000000000000000000000000000000000"); + ChaCha20 c20{key}; // get one block of keystream - unsigned char block[64]; - c20.Keystream(block, sizeof(block)); - unsigned char b1[5], b2[7], b3[52]; - c20 = ChaCha20{key.data()}; - c20.Keystream(b1, 5); - c20.Keystream(b2, 7); - c20.Keystream(b3, 52); - - BOOST_CHECK_EQUAL(0, memcmp(b1, block, 5)); - BOOST_CHECK_EQUAL(0, memcmp(b2, block + 5, 7)); - BOOST_CHECK_EQUAL(0, memcmp(b3, block + 12, 52)); + std::byte block[64]; + c20.Keystream(block); + std::byte b1[5], b2[7], b3[52]; + c20 = ChaCha20{key}; + c20.Keystream(b1); + c20.Keystream(b2); + c20.Keystream(b3); + + BOOST_CHECK(Span{block}.first(5) == Span{b1}); + BOOST_CHECK(Span{block}.subspan(5, 7) == Span{b2}); + BOOST_CHECK(Span{block}.last(52) == Span{b3}); } BOOST_AUTO_TEST_CASE(poly1305_testvector) @@ -922,15 +921,15 @@ BOOST_AUTO_TEST_CASE(poly1305_testvector) auto total_key = ParseHex<std::byte>("01020304050607fffefdfcfbfaf9ffffffffffffffffffffffffffff00000000"); Poly1305 total_ctx(total_key); for (unsigned i = 0; i < 256; ++i) { - std::vector<std::byte> key(32, std::byte{(uint8_t)i}); - std::vector<std::byte> msg(i, std::byte{(uint8_t)i}); + std::vector<std::byte> key(32, std::byte{uint8_t(i)}); + std::vector<std::byte> msg(i, std::byte{uint8_t(i)}); std::array<std::byte, Poly1305::TAGLEN> tag; Poly1305{key}.Update(msg).Finalize(tag); total_ctx.Update(tag); } std::vector<std::byte> total_tag(Poly1305::TAGLEN); total_ctx.Finalize(total_tag); - BOOST_CHECK(total_tag == ParseHex<std::byte>("64afe2e8d6ad7bbdd287f97c44623d39")); + BOOST_CHECK_EQUAL(HexStr(total_tag), "64afe2e8d6ad7bbdd287f97c44623d39"); } // Tests with sparse messages and random keys. diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index b740a51574..7f5d587cf6 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -12,7 +12,6 @@ #include <pubkey.h> #include <script/sign.h> #include <script/signingprovider.h> -#include <script/standard.h> #include <serialize.h> #include <test/util/net.h> #include <test/util/random.h> @@ -87,9 +86,10 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) { LOCK(dummyNode1.cs_vSend); - BOOST_CHECK(dummyNode1.vSendMsg.size() > 0); - dummyNode1.vSendMsg.clear(); + const auto& [to_send, _more, _msg_type] = dummyNode1.m_transport->GetBytesToSend(); + BOOST_CHECK(!to_send.empty()); } + connman.FlushSendBuffer(dummyNode1); int64_t nStartTime = GetTime(); // Wait 21 minutes @@ -97,7 +97,8 @@ BOOST_AUTO_TEST_CASE(outbound_slow_chain_eviction) BOOST_CHECK(peerman.SendMessages(&dummyNode1)); // should result in getheaders { LOCK(dummyNode1.cs_vSend); - BOOST_CHECK(dummyNode1.vSendMsg.size() > 0); + const auto& [to_send, _more, _msg_type] = dummyNode1.m_transport->GetBytesToSend(); + BOOST_CHECK(!to_send.empty()); } // Wait 3 more minutes SetMockTime(nStartTime+24*60); diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index 5cec0b07c9..829afab8da 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -5,7 +5,6 @@ #include <pubkey.h> #include <script/descriptor.h> #include <script/sign.h> -#include <script/standard.h> #include <test/util/setup_common.h> #include <util/strencodings.h> diff --git a/src/test/fuzz/crypto_chacha20.cpp b/src/test/fuzz/crypto_chacha20.cpp index 76370b4e57..50c77bf699 100644 --- a/src/test/fuzz/crypto_chacha20.cpp +++ b/src/test/fuzz/crypto_chacha20.cpp @@ -17,20 +17,18 @@ FUZZ_TARGET(crypto_chacha20) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - ChaCha20 chacha20; - if (fuzzed_data_provider.ConsumeBool()) { - const std::vector<unsigned char> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, 32); - chacha20 = ChaCha20{key.data()}; - } + const auto key = ConsumeFixedLengthByteVector<std::byte>(fuzzed_data_provider, ChaCha20::KEYLEN); + ChaCha20 chacha20{key}; + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { CallOneOf( fuzzed_data_provider, [&] { - std::vector<unsigned char> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, 32); - chacha20.SetKey32(key.data()); + auto key = ConsumeFixedLengthByteVector<std::byte>(fuzzed_data_provider, ChaCha20::KEYLEN); + chacha20.SetKey(key); }, [&] { - chacha20.Seek64( + chacha20.Seek( { fuzzed_data_provider.ConsumeIntegral<uint32_t>(), fuzzed_data_provider.ConsumeIntegral<uint64_t>() @@ -38,12 +36,12 @@ FUZZ_TARGET(crypto_chacha20) }, [&] { std::vector<uint8_t> output(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); - chacha20.Keystream(output.data(), output.size()); + chacha20.Keystream(MakeWritableByteSpan(output)); }, [&] { - std::vector<uint8_t> output(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); - const std::vector<uint8_t> input = ConsumeFixedLengthByteVector(fuzzed_data_provider, output.size()); - chacha20.Crypt(input.data(), output.data(), input.size()); + std::vector<std::byte> output(fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096)); + const auto input = ConsumeFixedLengthByteVector<std::byte>(fuzzed_data_provider, output.size()); + chacha20.Crypt(input, output); }); } } @@ -62,9 +60,7 @@ template<bool UseCrypt> void ChaCha20SplitFuzz(FuzzedDataProvider& provider) { // Determine key, iv, start position, length. - unsigned char key[32] = {0}; - auto key_bytes = provider.ConsumeBytes<unsigned char>(32); - std::copy(key_bytes.begin(), key_bytes.end(), key); + auto key_bytes = ConsumeFixedLengthByteVector<std::byte>(provider, ChaCha20::KEYLEN); uint64_t iv = provider.ConsumeIntegral<uint64_t>(); uint32_t iv_prefix = provider.ConsumeIntegral<uint32_t>(); uint64_t total_bytes = provider.ConsumeIntegralInRange<uint64_t>(0, 1000000); @@ -72,13 +68,13 @@ void ChaCha20SplitFuzz(FuzzedDataProvider& provider) uint32_t seek = provider.ConsumeIntegralInRange<uint32_t>(0, ~(uint32_t)(total_bytes >> 6)); // Initialize two ChaCha20 ciphers, with the same key/iv/position. - ChaCha20 crypt1(key); - ChaCha20 crypt2(key); - crypt1.Seek64({iv_prefix, iv}, seek); - crypt2.Seek64({iv_prefix, iv}, seek); + ChaCha20 crypt1(key_bytes); + ChaCha20 crypt2(key_bytes); + crypt1.Seek({iv_prefix, iv}, seek); + crypt2.Seek({iv_prefix, iv}, seek); // Construct vectors with data. - std::vector<unsigned char> data1, data2; + std::vector<std::byte> data1, data2; data1.resize(total_bytes); data2.resize(total_bytes); @@ -90,14 +86,14 @@ void ChaCha20SplitFuzz(FuzzedDataProvider& provider) uint64_t bytes = 0; while (bytes < (total_bytes & ~uint64_t{7})) { uint64_t val = rng(); - WriteLE64(data1.data() + bytes, val); - WriteLE64(data2.data() + bytes, val); + WriteLE64(UCharCast(data1.data() + bytes), val); + WriteLE64(UCharCast(data2.data() + bytes), val); bytes += 8; } if (bytes < total_bytes) { - unsigned char valbytes[8]; + std::byte valbytes[8]; uint64_t val = rng(); - WriteLE64(valbytes, val); + WriteLE64(UCharCast(valbytes), val); std::copy(valbytes, valbytes + (total_bytes - bytes), data1.data() + bytes); std::copy(valbytes, valbytes + (total_bytes - bytes), data2.data() + bytes); } @@ -108,9 +104,9 @@ void ChaCha20SplitFuzz(FuzzedDataProvider& provider) // Encrypt data1, the whole array at once. if constexpr (UseCrypt) { - crypt1.Crypt(data1.data(), data1.data(), total_bytes); + crypt1.Crypt(data1, data1); } else { - crypt1.Keystream(data1.data(), total_bytes); + crypt1.Keystream(data1); } // Encrypt data2, in at most 256 chunks. @@ -127,9 +123,9 @@ void ChaCha20SplitFuzz(FuzzedDataProvider& provider) // This tests that Keystream() has the same behavior as Crypt() applied // to 0x00 input bytes. if (UseCrypt || provider.ConsumeBool()) { - crypt2.Crypt(data2.data() + bytes2, data2.data() + bytes2, now); + crypt2.Crypt(Span{data2}.subspan(bytes2, now), Span{data2}.subspan(bytes2, now)); } else { - crypt2.Keystream(data2.data() + bytes2, now); + crypt2.Keystream(Span{data2}.subspan(bytes2, now)); } bytes2 += now; if (is_last) break; diff --git a/src/test/fuzz/crypto_diff_fuzz_chacha20.cpp b/src/test/fuzz/crypto_diff_fuzz_chacha20.cpp index 285ea2dfe0..ab7fa513ec 100644 --- a/src/test/fuzz/crypto_diff_fuzz_chacha20.cpp +++ b/src/test/fuzz/crypto_diff_fuzz_chacha20.cpp @@ -267,22 +267,15 @@ void ECRYPT_keystream_bytes(ECRYPT_ctx* x, u8* stream, u32 bytes) FUZZ_TARGET(crypto_diff_fuzz_chacha20) { - static const unsigned char ZEROKEY[32] = {0}; FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - ChaCha20 chacha20; ECRYPT_ctx ctx; - if (fuzzed_data_provider.ConsumeBool()) { - const std::vector<unsigned char> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, 32); - chacha20 = ChaCha20{key.data()}; - ECRYPT_keysetup(&ctx, key.data(), key.size() * 8, 0); - } else { - // The default ChaCha20 constructor is equivalent to using the all-0 key. - ECRYPT_keysetup(&ctx, ZEROKEY, 256, 0); - } + const std::vector<unsigned char> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, 32); + ChaCha20 chacha20{MakeByteSpan(key)}; + ECRYPT_keysetup(&ctx, key.data(), key.size() * 8, 0); - // ECRYPT_keysetup() doesn't set the counter and nonce to 0 while SetKey32() does + // ECRYPT_keysetup() doesn't set the counter and nonce to 0 while SetKey() does static const uint8_t iv[8] = {0, 0, 0, 0, 0, 0, 0, 0}; ChaCha20::Nonce96 nonce{0, 0}; uint32_t counter{0}; @@ -293,11 +286,11 @@ FUZZ_TARGET(crypto_diff_fuzz_chacha20) fuzzed_data_provider, [&] { const std::vector<unsigned char> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, 32); - chacha20.SetKey32(key.data()); + chacha20.SetKey(MakeByteSpan(key)); nonce = {0, 0}; counter = 0; ECRYPT_keysetup(&ctx, key.data(), key.size() * 8, 0); - // ECRYPT_keysetup() doesn't set the counter and nonce to 0 while SetKey32() does + // ECRYPT_keysetup() doesn't set the counter and nonce to 0 while SetKey() does uint8_t iv[8] = {0, 0, 0, 0, 0, 0, 0, 0}; ECRYPT_ivsetup(&ctx, iv); }, @@ -306,7 +299,7 @@ FUZZ_TARGET(crypto_diff_fuzz_chacha20) uint64_t iv = fuzzed_data_provider.ConsumeIntegral<uint64_t>(); nonce = {iv_prefix, iv}; counter = fuzzed_data_provider.ConsumeIntegral<uint32_t>(); - chacha20.Seek64(nonce, counter); + chacha20.Seek(nonce, counter); ctx.input[12] = counter; ctx.input[13] = iv_prefix; ctx.input[14] = iv; @@ -315,7 +308,7 @@ FUZZ_TARGET(crypto_diff_fuzz_chacha20) [&] { uint32_t integralInRange = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096); std::vector<uint8_t> output(integralInRange); - chacha20.Keystream(output.data(), output.size()); + chacha20.Keystream(MakeWritableByteSpan(output)); std::vector<uint8_t> djb_output(integralInRange); ECRYPT_keystream_bytes(&ctx, djb_output.data(), djb_output.size()); assert(output == djb_output); @@ -324,7 +317,7 @@ FUZZ_TARGET(crypto_diff_fuzz_chacha20) counter += (integralInRange + 63) >> 6; if (counter < old_counter) ++nonce.first; if (integralInRange & 63) { - chacha20.Seek64(nonce, counter); + chacha20.Seek(nonce, counter); } assert(counter == ctx.input[12]); }, @@ -332,7 +325,7 @@ FUZZ_TARGET(crypto_diff_fuzz_chacha20) uint32_t integralInRange = fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096); std::vector<uint8_t> output(integralInRange); const std::vector<uint8_t> input = ConsumeFixedLengthByteVector(fuzzed_data_provider, output.size()); - chacha20.Crypt(input.data(), output.data(), input.size()); + chacha20.Crypt(MakeByteSpan(input), MakeWritableByteSpan(output)); std::vector<uint8_t> djb_output(integralInRange); ECRYPT_encrypt_bytes(&ctx, input.data(), djb_output.data(), input.size()); assert(output == djb_output); @@ -341,7 +334,7 @@ FUZZ_TARGET(crypto_diff_fuzz_chacha20) counter += (integralInRange + 63) >> 6; if (counter < old_counter) ++nonce.first; if (integralInRange & 63) { - chacha20.Seek64(nonce, counter); + chacha20.Seek(nonce, counter); } assert(counter == ctx.input[12]); }); diff --git a/src/test/fuzz/crypto_poly1305.cpp b/src/test/fuzz/crypto_poly1305.cpp index f49729a34b..6ce6648f56 100644 --- a/src/test/fuzz/crypto_poly1305.cpp +++ b/src/test/fuzz/crypto_poly1305.cpp @@ -14,14 +14,13 @@ FUZZ_TARGET(crypto_poly1305) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - const std::vector<uint8_t> key = ConsumeFixedLengthByteVector(fuzzed_data_provider, Poly1305::KEYLEN); - const std::vector<uint8_t> in = ConsumeRandomLengthByteVector(fuzzed_data_provider); + const auto key = ConsumeFixedLengthByteVector<std::byte>(fuzzed_data_provider, Poly1305::KEYLEN); + const auto in = ConsumeRandomLengthByteVector<std::byte>(fuzzed_data_provider); std::vector<std::byte> tag_out(Poly1305::TAGLEN); - Poly1305{MakeByteSpan(key)}.Update(MakeByteSpan(in)).Finalize(tag_out); + Poly1305{key}.Update(in).Finalize(tag_out); } - FUZZ_TARGET(crypto_poly1305_split) { FuzzedDataProvider provider{buffer.data(), buffer.size()}; @@ -36,10 +35,10 @@ FUZZ_TARGET(crypto_poly1305_split) // Process input in pieces. LIMITED_WHILE(provider.remaining_bytes(), 100) { - auto in = provider.ConsumeRandomLengthString(); - poly_split.Update(MakeByteSpan(in)); + auto in = ConsumeRandomLengthByteVector<std::byte>(provider); + poly_split.Update(in); // Update total_input to match what was processed. - total_input.insert(total_input.end(), MakeByteSpan(in).begin(), MakeByteSpan(in).end()); + total_input.insert(total_input.end(), in.begin(), in.end()); } // Process entire input at once. diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index 91521bc7f4..849618d748 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -19,7 +19,7 @@ #include <pow.h> #include <protocol.h> #include <pubkey.h> -#include <script/standard.h> +#include <script/script.h> #include <serialize.h> #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> diff --git a/src/test/fuzz/key.cpp b/src/test/fuzz/key.cpp index 8faeb9e04f..a5a579d982 100644 --- a/src/test/fuzz/key.cpp +++ b/src/test/fuzz/key.cpp @@ -13,7 +13,7 @@ #include <script/script.h> #include <script/sign.h> #include <script/signingprovider.h> -#include <script/standard.h> +#include <script/solver.h> #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> diff --git a/src/test/fuzz/p2p_transport_serialization.cpp b/src/test/fuzz/p2p_transport_serialization.cpp index 78350a600e..2fa5de5008 100644 --- a/src/test/fuzz/p2p_transport_serialization.cpp +++ b/src/test/fuzz/p2p_transport_serialization.cpp @@ -9,6 +9,8 @@ #include <protocol.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/util/xoroshiro128plusplus.h> #include <util/chaintype.h> #include <cassert> @@ -17,16 +19,25 @@ #include <optional> #include <vector> +namespace { + +std::vector<std::string> g_all_messages; + void initialize_p2p_transport_serialization() { SelectParams(ChainType::REGTEST); + g_all_messages = getAllNetMessageTypes(); + std::sort(g_all_messages.begin(), g_all_messages.end()); } +} // namespace + FUZZ_TARGET(p2p_transport_serialization, .init = initialize_p2p_transport_serialization) { - // Construct deserializer, with a dummy NodeId - V1TransportDeserializer deserializer{Params(), NodeId{0}, SER_NETWORK, INIT_PROTO_VERSION}; - V1TransportSerializer serializer{}; + // Construct transports for both sides, with dummy NodeIds. + V1Transport recv_transport{NodeId{0}, SER_NETWORK, INIT_PROTO_VERSION}; + V1Transport send_transport{NodeId{1}, SER_NETWORK, INIT_PROTO_VERSION}; + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; auto checksum_assist = fuzzed_data_provider.ConsumeBool(); @@ -63,14 +74,13 @@ FUZZ_TARGET(p2p_transport_serialization, .init = initialize_p2p_transport_serial mutable_msg_bytes.insert(mutable_msg_bytes.end(), payload_bytes.begin(), payload_bytes.end()); Span<const uint8_t> msg_bytes{mutable_msg_bytes}; while (msg_bytes.size() > 0) { - const int handled = deserializer.Read(msg_bytes); - if (handled < 0) { + if (!recv_transport.ReceivedBytes(msg_bytes)) { break; } - if (deserializer.Complete()) { + if (recv_transport.ReceivedMessageComplete()) { const std::chrono::microseconds m_time{std::numeric_limits<int64_t>::max()}; bool reject_message{false}; - CNetMessage msg = deserializer.GetMessage(m_time, reject_message); + CNetMessage msg = recv_transport.GetReceivedMessage(m_time, reject_message); assert(msg.m_type.size() <= CMessageHeader::COMMAND_SIZE); assert(msg.m_raw_message_size <= mutable_msg_bytes.size()); assert(msg.m_raw_message_size == CMessageHeader::HEADER_SIZE + msg.m_message_size); @@ -78,7 +88,247 @@ FUZZ_TARGET(p2p_transport_serialization, .init = initialize_p2p_transport_serial std::vector<unsigned char> header; auto msg2 = CNetMsgMaker{msg.m_recv.GetVersion()}.Make(msg.m_type, Span{msg.m_recv}); - serializer.prepareForTransport(msg2, header); + bool queued = send_transport.SetMessageToSend(msg2); + assert(queued); + std::optional<bool> known_more; + while (true) { + const auto& [to_send, more, _msg_type] = send_transport.GetBytesToSend(); + if (known_more) assert(!to_send.empty() == *known_more); + if (to_send.empty()) break; + send_transport.MarkBytesSent(to_send.size()); + known_more = more; + } } } } + +namespace { + +template<typename R> +void SimulationTest(Transport& initiator, Transport& responder, R& rng, FuzzedDataProvider& provider) +{ + // Simulation test with two Transport objects, which send messages to each other, with + // sending and receiving fragmented into multiple pieces that may be interleaved. It primarily + // verifies that the sending and receiving side are compatible with each other, plus a few + // sanity checks. It does not attempt to introduce errors in the communicated data. + + // Put the transports in an array for by-index access. + const std::array<Transport*, 2> transports = {&initiator, &responder}; + + // Two vectors representing in-flight bytes. inflight[i] is from transport[i] to transport[!i]. + std::array<std::vector<uint8_t>, 2> in_flight; + + // Two queues with expected messages. expected[i] is expected to arrive in transport[!i]. + std::array<std::deque<CSerializedNetMsg>, 2> expected; + + // Vectors with bytes last returned by GetBytesToSend() on transport[i]. + std::array<std::vector<uint8_t>, 2> to_send; + + // Last returned 'more' values (if still relevant) by transport[i]->GetBytesToSend(). + std::array<std::optional<bool>, 2> last_more; + + // Whether more bytes to be sent are expected on transport[i]. + std::array<std::optional<bool>, 2> expect_more; + + // Function to consume a message type. + auto msg_type_fn = [&]() { + uint8_t v = provider.ConsumeIntegral<uint8_t>(); + if (v == 0xFF) { + // If v is 0xFF, construct a valid (but possibly unknown) message type from the fuzz + // data. + std::string ret; + while (ret.size() < CMessageHeader::COMMAND_SIZE) { + char c = provider.ConsumeIntegral<char>(); + // Match the allowed characters in CMessageHeader::IsCommandValid(). Any other + // character is interpreted as end. + if (c < ' ' || c > 0x7E) break; + ret += c; + } + return ret; + } else { + // Otherwise, use it as index into the list of known messages. + return g_all_messages[v % g_all_messages.size()]; + } + }; + + // Function to construct a CSerializedNetMsg to send. + auto make_msg_fn = [&](bool first) { + CSerializedNetMsg msg; + if (first) { + // Always send a "version" message as first one. + msg.m_type = "version"; + } else { + msg.m_type = msg_type_fn(); + } + // Determine size of message to send (limited to 75 kB for performance reasons). + size_t size = provider.ConsumeIntegralInRange<uint32_t>(0, 75000); + // Get payload of message from RNG. + msg.data.resize(size); + for (auto& v : msg.data) v = uint8_t(rng()); + // Return. + return msg; + }; + + // The next message to be sent (initially version messages, but will be replaced once sent). + std::array<CSerializedNetMsg, 2> next_msg = { + make_msg_fn(/*first=*/true), + make_msg_fn(/*first=*/true) + }; + + // Wrapper around transport[i]->GetBytesToSend() that performs sanity checks. + auto bytes_to_send_fn = [&](int side) -> Transport::BytesToSend { + const auto& [bytes, more, msg_type] = transports[side]->GetBytesToSend(); + // Compare with expected more. + if (expect_more[side].has_value()) assert(!bytes.empty() == *expect_more[side]); + // Compare with previously reported output. + assert(to_send[side].size() <= bytes.size()); + assert(to_send[side] == Span{bytes}.first(to_send[side].size())); + to_send[side].resize(bytes.size()); + std::copy(bytes.begin(), bytes.end(), to_send[side].begin()); + // Remember 'more' result. + last_more[side] = {more}; + // Return. + return {bytes, more, msg_type}; + }; + + // Function to make side send a new message. + auto new_msg_fn = [&](int side) { + // Don't do anything if there are too many unreceived messages already. + if (expected[side].size() >= 16) return; + // Try to send (a copy of) the message in next_msg[side]. + CSerializedNetMsg msg = next_msg[side].Copy(); + bool queued = transports[side]->SetMessageToSend(msg); + // Update expected more data. + expect_more[side] = std::nullopt; + // Verify consistency of GetBytesToSend after SetMessageToSend + bytes_to_send_fn(/*side=*/side); + if (queued) { + // Remember that this message is now expected by the receiver. + expected[side].emplace_back(std::move(next_msg[side])); + // Construct a new next message to send. + next_msg[side] = make_msg_fn(/*first=*/false); + } + }; + + // Function to make side send out bytes (if any). + auto send_fn = [&](int side, bool everything = false) { + const auto& [bytes, more, msg_type] = bytes_to_send_fn(/*side=*/side); + // Don't do anything if no bytes to send. + if (bytes.empty()) return false; + size_t send_now = everything ? bytes.size() : provider.ConsumeIntegralInRange<size_t>(0, bytes.size()); + if (send_now == 0) return false; + // Add bytes to the in-flight queue, and mark those bytes as consumed. + in_flight[side].insert(in_flight[side].end(), bytes.begin(), bytes.begin() + send_now); + transports[side]->MarkBytesSent(send_now); + // If all to-be-sent bytes were sent, move last_more data to expect_more data. + if (send_now == bytes.size()) { + expect_more[side] = last_more[side]; + } + // Remove the bytes from the last reported to-be-sent vector. + assert(to_send[side].size() >= send_now); + to_send[side].erase(to_send[side].begin(), to_send[side].begin() + send_now); + // Verify that GetBytesToSend gives a result consistent with earlier. + bytes_to_send_fn(/*side=*/side); + // Return whether anything was sent. + return send_now > 0; + }; + + // Function to make !side receive bytes (if any). + auto recv_fn = [&](int side, bool everything = false) { + // Don't do anything if no bytes in flight. + if (in_flight[side].empty()) return false; + // Decide span to receive + size_t to_recv_len = in_flight[side].size(); + if (!everything) to_recv_len = provider.ConsumeIntegralInRange<size_t>(0, to_recv_len); + Span<const uint8_t> to_recv = Span{in_flight[side]}.first(to_recv_len); + // Process those bytes + while (!to_recv.empty()) { + size_t old_len = to_recv.size(); + bool ret = transports[!side]->ReceivedBytes(to_recv); + // Bytes must always be accepted, as this test does not introduce any errors in + // communication. + assert(ret); + // Clear cached expected 'more' information: if certainly no more data was to be sent + // before, receiving bytes makes this uncertain. + if (expect_more[!side] == false) expect_more[!side] = std::nullopt; + // Verify consistency of GetBytesToSend after ReceivedBytes + bytes_to_send_fn(/*side=*/!side); + bool progress = to_recv.size() < old_len; + if (transports[!side]->ReceivedMessageComplete()) { + bool reject{false}; + auto received = transports[!side]->GetReceivedMessage({}, reject); + // Receiving must succeed. + assert(!reject); + // There must be a corresponding expected message. + assert(!expected[side].empty()); + // The m_message_size field must be correct. + assert(received.m_message_size == received.m_recv.size()); + // The m_type must match what is expected. + assert(received.m_type == expected[side].front().m_type); + // The data must match what is expected. + assert(MakeByteSpan(received.m_recv) == MakeByteSpan(expected[side].front().data)); + expected[side].pop_front(); + progress = true; + } + // Progress must be made (by processing incoming bytes and/or returning complete + // messages) until all received bytes are processed. + assert(progress); + } + // Remove the processed bytes from the in_flight buffer. + in_flight[side].erase(in_flight[side].begin(), in_flight[side].begin() + to_recv_len); + // Return whether anything was received. + return to_recv_len > 0; + }; + + // Main loop, interleaving new messages, sends, and receives. + LIMITED_WHILE(provider.remaining_bytes(), 1000) { + CallOneOf(provider, + // (Try to) give the next message to the transport. + [&] { new_msg_fn(/*side=*/0); }, + [&] { new_msg_fn(/*side=*/1); }, + // (Try to) send some bytes from the transport to the network. + [&] { send_fn(/*side=*/0); }, + [&] { send_fn(/*side=*/1); }, + // (Try to) receive bytes from the network, converting to messages. + [&] { recv_fn(/*side=*/0); }, + [&] { recv_fn(/*side=*/1); } + ); + } + + // When we're done, perform sends and receives of existing messages to flush anything already + // in flight. + while (true) { + bool any = false; + if (send_fn(/*side=*/0, /*everything=*/true)) any = true; + if (send_fn(/*side=*/1, /*everything=*/true)) any = true; + if (recv_fn(/*side=*/0, /*everything=*/true)) any = true; + if (recv_fn(/*side=*/1, /*everything=*/true)) any = true; + if (!any) break; + } + + // Make sure nothing is left in flight. + assert(in_flight[0].empty()); + assert(in_flight[1].empty()); + + // Make sure all expected messages were received. + assert(expected[0].empty()); + assert(expected[1].empty()); +} + +std::unique_ptr<Transport> MakeV1Transport(NodeId nodeid) noexcept +{ + return std::make_unique<V1Transport>(nodeid, SER_NETWORK, INIT_PROTO_VERSION); +} + +} // namespace + +FUZZ_TARGET(p2p_transport_bidirectional, .init = initialize_p2p_transport_serialization) +{ + // Test with two V1 transports talking to each other. + FuzzedDataProvider provider{buffer.data(), buffer.size()}; + XoRoShiRo128PlusPlus rng(provider.ConsumeIntegral<uint64_t>()); + auto t1 = MakeV1Transport(NodeId{0}); + auto t2 = MakeV1Transport(NodeId{1}); + if (!t1 || !t2) return; + SimulationTest(*t1, *t2, rng, provider); +} diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index 6ed83feddf..d38d1bb40e 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -63,9 +63,9 @@ FUZZ_TARGET(process_message, .init = initialize_process_message) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); ConnmanTestMsg& connman = *static_cast<ConnmanTestMsg*>(g_setup->m_node.connman.get()); - TestChainState& chainstate = *static_cast<TestChainState*>(&g_setup->m_node.chainman->ActiveChainstate()); + auto& chainman = static_cast<TestChainstateManager&>(*g_setup->m_node.chainman); SetMockTime(1610000000); // any time to successfully reset ibd - chainstate.ResetIbd(); + chainman.ResetIbd(); LOCK(NetEventsInterface::g_msgproc_mutex); diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index 58e023956c..4cb388c20b 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -38,9 +38,9 @@ FUZZ_TARGET(process_messages, .init = initialize_process_messages) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); ConnmanTestMsg& connman = *static_cast<ConnmanTestMsg*>(g_setup->m_node.connman.get()); - TestChainState& chainstate = *static_cast<TestChainState*>(&g_setup->m_node.chainman->ActiveChainstate()); + auto& chainman = static_cast<TestChainstateManager&>(*g_setup->m_node.chainman); SetMockTime(1610000000); // any time to successfully reset ibd - chainstate.ResetIbd(); + chainman.ResetIbd(); LOCK(NetEventsInterface::g_msgproc_mutex); @@ -67,7 +67,8 @@ FUZZ_TARGET(process_messages, .init = initialize_process_messages) CNode& random_node = *PickValue(fuzzed_data_provider, peers); - (void)connman.ReceiveMsgFrom(random_node, net_msg); + connman.FlushSendBuffer(random_node); + (void)connman.ReceiveMsgFrom(random_node, std::move(net_msg)); random_node.fPauseSend = false; try { diff --git a/src/test/fuzz/script.cpp b/src/test/fuzz/script.cpp index 1cb7d01906..acc82f55f6 100644 --- a/src/test/fuzz/script.cpp +++ b/src/test/fuzz/script.cpp @@ -16,7 +16,7 @@ #include <script/script_error.h> #include <script/sign.h> #include <script/signingprovider.h> -#include <script/standard.h> +#include <script/solver.h> #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 5f2aff08da..5d27d2a180 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_TEST_FUZZ_UTIL_H #define BITCOIN_TEST_FUZZ_UTIL_H +#include <addresstype.h> #include <arith_uint256.h> #include <coins.h> #include <compat/compat.h> @@ -13,7 +14,6 @@ #include <merkleblock.h> #include <primitives/transaction.h> #include <script/script.h> -#include <script/standard.h> #include <serialize.h> #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> @@ -53,12 +53,16 @@ auto& PickValue(FuzzedDataProvider& fuzzed_data_provider, Collection& col) return *it; } -[[nodiscard]] inline std::vector<uint8_t> ConsumeRandomLengthByteVector(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length = std::nullopt) noexcept +template<typename B = uint8_t> +[[nodiscard]] inline std::vector<B> ConsumeRandomLengthByteVector(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length = std::nullopt) noexcept { + static_assert(sizeof(B) == 1); const std::string s = max_length ? fuzzed_data_provider.ConsumeRandomLengthString(*max_length) : fuzzed_data_provider.ConsumeRandomLengthString(); - return {s.begin(), s.end()}; + std::vector<B> ret(s.size()); + std::copy(s.begin(), s.end(), reinterpret_cast<char*>(ret.data())); + return ret; } [[nodiscard]] inline std::vector<bool> ConsumeRandomLengthBitVector(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length = std::nullopt) noexcept @@ -209,14 +213,13 @@ inline void SetFuzzedErrNo(FuzzedDataProvider& fuzzed_data_provider) noexcept * Returns a byte vector of specified size regardless of the number of remaining bytes available * from the fuzzer. Pads with zero value bytes if needed to achieve the specified size. */ -[[nodiscard]] inline std::vector<uint8_t> ConsumeFixedLengthByteVector(FuzzedDataProvider& fuzzed_data_provider, const size_t length) noexcept +template<typename B = uint8_t> +[[nodiscard]] inline std::vector<B> ConsumeFixedLengthByteVector(FuzzedDataProvider& fuzzed_data_provider, const size_t length) noexcept { - std::vector<uint8_t> result(length); - const std::vector<uint8_t> random_bytes = fuzzed_data_provider.ConsumeBytes<uint8_t>(length); - if (!random_bytes.empty()) { - std::memcpy(result.data(), random_bytes.data(), random_bytes.size()); - } - return result; + static_assert(sizeof(B) == 1); + auto random_bytes = fuzzed_data_provider.ConsumeBytes<B>(length); + random_bytes.resize(length); + return random_bytes; } class FuzzedFileProvider diff --git a/src/test/fuzz/util/mempool.cpp b/src/test/fuzz/util/mempool.cpp index 4baca5ec77..8e7499a860 100644 --- a/src/test/fuzz/util/mempool.cpp +++ b/src/test/fuzz/util/mempool.cpp @@ -23,8 +23,9 @@ CTxMemPoolEntry ConsumeTxMemPoolEntry(FuzzedDataProvider& fuzzed_data_provider, const CAmount fee{ConsumeMoney(fuzzed_data_provider, /*max=*/std::numeric_limits<CAmount>::max() / CAmount{100'000})}; assert(MoneyRange(fee)); const int64_t time = fuzzed_data_provider.ConsumeIntegral<int64_t>(); + const uint64_t entry_sequence{fuzzed_data_provider.ConsumeIntegral<uint64_t>()}; const unsigned int entry_height = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); const bool spends_coinbase = fuzzed_data_provider.ConsumeBool(); const unsigned int sig_op_cost = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, MAX_BLOCK_SIGOPS_COST); - return CTxMemPoolEntry{MakeTransactionRef(tx), fee, time, entry_height, spends_coinbase, sig_op_cost, {}}; + return CTxMemPoolEntry{MakeTransactionRef(tx), fee, time, entry_height, entry_sequence, spends_coinbase, sig_op_cost, {}}; } diff --git a/src/test/interfaces_tests.cpp b/src/test/interfaces_tests.cpp index 68377e197f..29f0aea3b1 100644 --- a/src/test/interfaces_tests.cpp +++ b/src/test/interfaces_tests.cpp @@ -5,8 +5,8 @@ #include <chainparams.h> #include <consensus/validation.h> #include <interfaces/chain.h> -#include <script/standard.h> #include <test/util/setup_common.h> +#include <script/solver.h> #include <validation.h> #include <boost/test/unit_test.hpp> diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index 94e3f27930..b4c7cac223 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <addresstype.h> #include <coins.h> #include <common/system.h> #include <consensus/consensus.h> @@ -9,7 +10,6 @@ #include <consensus/tx_verify.h> #include <node/miner.h> #include <policy/policy.h> -#include <script/standard.h> #include <test/util/random.h> #include <test/util/txmempool.h> #include <timedata.h> diff --git a/src/test/miniminer_tests.cpp b/src/test/miniminer_tests.cpp index 1ee9e0c066..da724f8d7b 100644 --- a/src/test/miniminer_tests.cpp +++ b/src/test/miniminer_tests.cpp @@ -2,6 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <node/mini_miner.h> +#include <random.h> #include <txmempool.h> #include <util/time.h> diff --git a/src/test/miniscript_tests.cpp b/src/test/miniscript_tests.cpp index 9c811db3e9..0cb58450e8 100644 --- a/src/test/miniscript_tests.cpp +++ b/src/test/miniscript_tests.cpp @@ -10,6 +10,7 @@ #include <test/util/setup_common.h> #include <boost/test/unit_test.hpp> +#include <addresstype.h> #include <core_io.h> #include <hash.h> #include <pubkey.h> @@ -18,7 +19,6 @@ #include <crypto/sha256.h> #include <script/interpreter.h> #include <script/miniscript.h> -#include <script/standard.h> #include <script/script_error.h> namespace { diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index ead604598e..ae342a6278 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -841,11 +841,10 @@ BOOST_AUTO_TEST_CASE(initial_advertise_from_version_message) const int64_t time{0}; const CNetMsgMaker msg_maker{PROTOCOL_VERSION}; - // Force Chainstate::IsInitialBlockDownload() to return false. + // Force ChainstateManager::IsInitialBlockDownload() to return false. // Otherwise PushAddress() isn't called by PeerManager::ProcessMessage(). - TestChainState& chainstate = - *static_cast<TestChainState*>(&m_node.chainman->ActiveChainstate()); - chainstate.JumpOutOfIbd(); + auto& chainman = static_cast<TestChainstateManager&>(*m_node.chainman); + chainman.JumpOutOfIbd(); m_node.peerman->InitializeNode(peer, NODE_NETWORK); @@ -895,7 +894,7 @@ BOOST_AUTO_TEST_CASE(initial_advertise_from_version_message) BOOST_CHECK(sent); CaptureMessage = CaptureMessageOrig; - chainstate.ResetIbd(); + chainman.ResetIbd(); m_node.args->ForceSetArg("-capturemessages", "0"); m_node.args->ForceSetArg("-bind", ""); // PeerManager::ProcessMessage() calls AddTimeData() which changes the internal state diff --git a/src/test/orphanage_tests.cpp b/src/test/orphanage_tests.cpp index a2c4774338..af53737fec 100644 --- a/src/test/orphanage_tests.cpp +++ b/src/test/orphanage_tests.cpp @@ -6,7 +6,6 @@ #include <pubkey.h> #include <script/sign.h> #include <script/signingprovider.h> -#include <script/standard.h> #include <test/util/random.h> #include <test/util/setup_common.h> #include <txorphanage.h> diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index 884e3d0634..1a205728d6 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -8,7 +8,7 @@ #include <key_io.h> #include <script/script.h> #include <script/signingprovider.h> -#include <script/standard.h> +#include <script/solver.h> #include <test/util/setup_common.h> #include <util/strencodings.h> diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 411924496c..d63bfb9603 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -14,6 +14,7 @@ #include <script/sigcache.h> #include <script/sign.h> #include <script/signingprovider.h> +#include <script/solver.h> #include <streams.h> #include <test/util/json.h> #include <test/util/random.h> diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index a17be54419..ec9490d745 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -2,13 +2,15 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <addresstype.h> #include <coins.h> #include <consensus/consensus.h> #include <consensus/tx_verify.h> #include <key.h> #include <pubkey.h> +#include <script/interpreter.h> #include <script/script.h> -#include <script/standard.h> +#include <script/solver.h> #include <test/util/setup_common.h> #include <uint256.h> diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index 34dd2c3e9f..a4c0db8aea 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -19,7 +19,7 @@ #include <script/script_error.h> #include <script/sign.h> #include <script/signingprovider.h> -#include <script/standard.h> +#include <script/solver.h> #include <streams.h> #include <test/util/json.h> #include <test/util/random.h> diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index cb80dbed69..9fa59bab57 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -2,10 +2,10 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <addresstype.h> #include <chainparams.h> #include <index/txindex.h> #include <interfaces/chain.h> -#include <script/standard.h> #include <test/util/index.h> #include <test/util/setup_common.h> #include <validation.h> diff --git a/src/test/txpackage_tests.cpp b/src/test/txpackage_tests.cpp index c08d2748a6..df5f8b4cce 100644 --- a/src/test/txpackage_tests.cpp +++ b/src/test/txpackage_tests.cpp @@ -8,7 +8,6 @@ #include <policy/policy.h> #include <primitives/transaction.h> #include <script/script.h> -#include <script/standard.h> #include <test/util/random.h> #include <test/util/setup_common.h> #include <validation.h> diff --git a/src/test/txvalidation_tests.cpp b/src/test/txvalidation_tests.cpp index f06bef3589..ecf0889711 100644 --- a/src/test/txvalidation_tests.cpp +++ b/src/test/txvalidation_tests.cpp @@ -8,7 +8,6 @@ #include <policy/policy.h> #include <primitives/transaction.h> #include <script/script.h> -#include <script/standard.h> #include <test/util/setup_common.h> #include <validation.h> diff --git a/src/test/txvalidationcache_tests.cpp b/src/test/txvalidationcache_tests.cpp index 8f8628169f..c1f6226982 100644 --- a/src/test/txvalidationcache_tests.cpp +++ b/src/test/txvalidationcache_tests.cpp @@ -6,7 +6,6 @@ #include <key.h> #include <script/sign.h> #include <script/signingprovider.h> -#include <script/standard.h> #include <test/util/setup_common.h> #include <txmempool.h> #include <util/chaintype.h> diff --git a/src/test/util/blockfilter.cpp b/src/test/util/blockfilter.cpp index a806844e34..19f3d51d5e 100644 --- a/src/test/util/blockfilter.cpp +++ b/src/test/util/blockfilter.cpp @@ -6,6 +6,8 @@ #include <chainparams.h> #include <node/blockstorage.h> +#include <primitives/block.h> +#include <undo.h> #include <validation.h> using node::BlockManager; diff --git a/src/test/util/mining.cpp b/src/test/util/mining.cpp index 51f4b89512..08d1b4c902 100644 --- a/src/test/util/mining.cpp +++ b/src/test/util/mining.cpp @@ -11,7 +11,6 @@ #include <node/context.h> #include <pow.h> #include <primitives/transaction.h> -#include <script/standard.h> #include <test/util/script.h> #include <util/check.h> #include <validation.h> diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index 3f72384b3b..8015db3e80 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -25,6 +25,7 @@ void ConnmanTestMsg::Handshake(CNode& node, const CNetMsgMaker mm{0}; peerman.InitializeNode(node, local_services); + FlushSendBuffer(node); // Drop the version message added by InitializeNode. CSerializedNetMsg msg_version{ mm.Make(NetMsgType::VERSION, @@ -41,10 +42,11 @@ void ConnmanTestMsg::Handshake(CNode& node, relay_txs), }; - (void)connman.ReceiveMsgFrom(node, msg_version); + (void)connman.ReceiveMsgFrom(node, std::move(msg_version)); node.fPauseSend = false; connman.ProcessMessagesOnce(node); peerman.SendMessages(&node); + FlushSendBuffer(node); // Drop the verack message added by SendMessages. if (node.fDisconnect) return; assert(node.nVersion == version); assert(node.GetCommonVersion() == std::min(version, PROTOCOL_VERSION)); @@ -54,7 +56,7 @@ void ConnmanTestMsg::Handshake(CNode& node, assert(statestats.their_services == remote_services); if (successfully_connected) { CSerializedNetMsg msg_verack{mm.Make(NetMsgType::VERACK)}; - (void)connman.ReceiveMsgFrom(node, msg_verack); + (void)connman.ReceiveMsgFrom(node, std::move(msg_verack)); node.fPauseSend = false; connman.ProcessMessagesOnce(node); peerman.SendMessages(&node); @@ -70,14 +72,29 @@ void ConnmanTestMsg::NodeReceiveMsgBytes(CNode& node, Span<const uint8_t> msg_by } } -bool ConnmanTestMsg::ReceiveMsgFrom(CNode& node, CSerializedNetMsg& ser_msg) const +void ConnmanTestMsg::FlushSendBuffer(CNode& node) const { - std::vector<uint8_t> ser_msg_header; - node.m_serializer->prepareForTransport(ser_msg, ser_msg_header); + LOCK(node.cs_vSend); + node.vSendMsg.clear(); + node.m_send_memusage = 0; + while (true) { + const auto& [to_send, _more, _msg_type] = node.m_transport->GetBytesToSend(); + if (to_send.empty()) break; + node.m_transport->MarkBytesSent(to_send.size()); + } +} - bool complete; - NodeReceiveMsgBytes(node, ser_msg_header, complete); - NodeReceiveMsgBytes(node, ser_msg.data, complete); +bool ConnmanTestMsg::ReceiveMsgFrom(CNode& node, CSerializedNetMsg&& ser_msg) const +{ + bool queued = node.m_transport->SetMessageToSend(ser_msg); + assert(queued); + bool complete{false}; + while (true) { + const auto& [to_send, _more, _msg_type] = node.m_transport->GetBytesToSend(); + if (to_send.empty()) break; + NodeReceiveMsgBytes(node, to_send, complete); + node.m_transport->MarkBytesSent(to_send.size()); + } return complete; } diff --git a/src/test/util/net.h b/src/test/util/net.h index b2f6ebb163..1684da777a 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -54,7 +54,8 @@ struct ConnmanTestMsg : public CConnman { void NodeReceiveMsgBytes(CNode& node, Span<const uint8_t> msg_bytes, bool& complete) const; - bool ReceiveMsgFrom(CNode& node, CSerializedNetMsg& ser_msg) const; + bool ReceiveMsgFrom(CNode& node, CSerializedNetMsg&& ser_msg) const; + void FlushSendBuffer(CNode& node) const; }; constexpr ServiceFlags ALL_SERVICE_FLAGS[]{ diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 08ef890ec4..ecae743d14 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -424,7 +424,7 @@ std::vector<CTransactionRef> TestChain100Setup::PopulateMempool(FastRandomContex LOCK2(cs_main, m_node.mempool->cs); LockPoints lp; m_node.mempool->addUnchecked(CTxMemPoolEntry(ptx, /*fee=*/(total_in - num_outputs * amount_per_output), - /*time=*/0, /*entry_height=*/1, + /*time=*/0, /*entry_height=*/1, /*entry_sequence=*/0, /*spends_coinbase=*/false, /*sigops_cost=*/4, lp)); } --num_transactions; @@ -454,7 +454,7 @@ void TestChain100Setup::MockMempoolMinFee(const CFeeRate& target_feerate) const auto tx_fee = target_feerate.GetFee(GetVirtualTransactionSize(*tx)) - m_node.mempool->m_incremental_relay_feerate.GetFee(GetVirtualTransactionSize(*tx)); m_node.mempool->addUnchecked(CTxMemPoolEntry(tx, /*fee=*/tx_fee, - /*time=*/0, /*entry_height=*/1, + /*time=*/0, /*entry_height=*/1, /*entry_sequence=*/0, /*spends_coinbase=*/true, /*sigops_cost=*/1, lp)); m_node.mempool->TrimToSize(0); assert(m_node.mempool->GetMinFee() == target_feerate); diff --git a/src/test/util/txmempool.cpp b/src/test/util/txmempool.cpp index 4797d9c310..c945f35d79 100644 --- a/src/test/util/txmempool.cpp +++ b/src/test/util/txmempool.cpp @@ -34,5 +34,5 @@ CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CMutableTransaction& tx) co CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CTransactionRef& tx) const { - return CTxMemPoolEntry{tx, nFee, TicksSinceEpoch<std::chrono::seconds>(time), nHeight, spendsCoinbase, sigOpCost, lp}; + return CTxMemPoolEntry{tx, nFee, TicksSinceEpoch<std::chrono::seconds>(time), nHeight, m_sequence, spendsCoinbase, sigOpCost, lp}; } diff --git a/src/test/util/txmempool.h b/src/test/util/txmempool.h index 2fe7d69693..4b0daf0d42 100644 --- a/src/test/util/txmempool.h +++ b/src/test/util/txmempool.h @@ -19,6 +19,7 @@ struct TestMemPoolEntryHelper { CAmount nFee{0}; NodeSeconds time{}; unsigned int nHeight{1}; + uint64_t m_sequence{0}; bool spendsCoinbase{false}; unsigned int sigOpCost{4}; LockPoints lp; @@ -30,6 +31,7 @@ struct TestMemPoolEntryHelper { TestMemPoolEntryHelper& Fee(CAmount _fee) { nFee = _fee; return *this; } TestMemPoolEntryHelper& Time(NodeSeconds tp) { time = tp; return *this; } TestMemPoolEntryHelper& Height(unsigned int _height) { nHeight = _height; return *this; } + TestMemPoolEntryHelper& Sequence(uint64_t _seq) { m_sequence = _seq; return *this; } TestMemPoolEntryHelper& SpendsCoinbase(bool _flag) { spendsCoinbase = _flag; return *this; } TestMemPoolEntryHelper& SigOpsCost(unsigned int _sigopsCost) { sigOpCost = _sigopsCost; return *this; } }; diff --git a/src/test/util/validation.cpp b/src/test/util/validation.cpp index c0d7a532dc..2d5562ae66 100644 --- a/src/test/util/validation.cpp +++ b/src/test/util/validation.cpp @@ -9,13 +9,13 @@ #include <validation.h> #include <validationinterface.h> -void TestChainState::ResetIbd() +void TestChainstateManager::ResetIbd() { m_cached_finished_ibd = false; assert(IsInitialBlockDownload()); } -void TestChainState::JumpOutOfIbd() +void TestChainstateManager::JumpOutOfIbd() { Assert(IsInitialBlockDownload()); m_cached_finished_ibd = true; diff --git a/src/test/util/validation.h b/src/test/util/validation.h index 7a511a2b79..64654f3fb6 100644 --- a/src/test/util/validation.h +++ b/src/test/util/validation.h @@ -9,7 +9,7 @@ class CValidationInterface; -struct TestChainState : public Chainstate { +struct TestChainstateManager : public ChainstateManager { /** Reset the ibd cache to its initial state */ void ResetIbd(); /** Toggle IsInitialBlockDownload from true to false */ diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index e08f2c98c2..d1463634cc 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -10,7 +10,6 @@ #include <node/miner.h> #include <pow.h> #include <random.h> -#include <script/standard.h> #include <test/util/random.h> #include <test/util/script.h> #include <test/util/setup_common.h> diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 160a807f69..9e359eeee4 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -13,6 +13,7 @@ #include <test/util/logging.h> #include <test/util/random.h> #include <test/util/setup_common.h> +#include <test/util/validation.h> #include <timedata.h> #include <uint256.h> #include <validation.h> @@ -143,14 +144,21 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_rebalance_caches, TestChain100Setup) c2.InitCoinsDB( /*cache_size_bytes=*/1 << 23, /*in_memory=*/true, /*should_wipe=*/false); + // Reset IBD state so IsInitialBlockDownload() returns true and causes + // MaybeRebalancesCaches() to prioritize the snapshot chainstate, giving it + // more cache space than the snapshot chainstate. Calling ResetIbd() is + // necessary because m_cached_finished_ibd is already latched to true before + // the test starts due to the test setup. After ResetIbd() is called. + // IsInitialBlockDownload will return true because at this point the active + // chainstate has a null chain tip. + static_cast<TestChainstateManager&>(manager).ResetIbd(); + { LOCK(::cs_main); c2.InitCoinsCache(1 << 23); manager.MaybeRebalanceCaches(); } - // Since both chainstates are considered to be in initial block download, - // the snapshot chainstate should take priority. BOOST_CHECK_CLOSE(c1.m_coinstip_cache_size_bytes, max_cache * 0.05, 1); BOOST_CHECK_CLOSE(c1.m_coinsdb_cache_size_bytes, max_cache * 0.05, 1); BOOST_CHECK_CLOSE(c2.m_coinstip_cache_size_bytes, max_cache * 0.95, 1); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 79b2b4ec94..226dd9f353 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -853,6 +853,17 @@ TxMempoolInfo CTxMemPool::info(const GenTxid& gtxid) const return GetInfo(i); } +TxMempoolInfo CTxMemPool::info_for_relay(const GenTxid& gtxid, uint64_t last_sequence) const +{ + LOCK(cs); + indexed_transaction_set::const_iterator i = (gtxid.IsWtxid() ? get_iter_from_wtxid(gtxid.GetHash()) : mapTx.find(gtxid.GetHash())); + if (i != mapTx.end() && i->GetSequence() < last_sequence) { + return GetInfo(i); + } else { + return TxMempoolInfo(); + } +} + void CTxMemPool::PrioritiseTransaction(const uint256& hash, const CAmount& nFeeDelta) { { @@ -1186,19 +1197,6 @@ void CTxMemPool::SetLoadTried(bool load_tried) m_load_tried = load_tried; } -std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept -{ - switch (r) { - case MemPoolRemovalReason::EXPIRY: return "expiry"; - case MemPoolRemovalReason::SIZELIMIT: return "sizelimit"; - case MemPoolRemovalReason::REORG: return "reorg"; - case MemPoolRemovalReason::BLOCK: return "block"; - case MemPoolRemovalReason::CONFLICT: return "conflict"; - case MemPoolRemovalReason::REPLACED: return "replaced"; - } - assert(false); -} - std::vector<CTxMemPool::txiter> CTxMemPool::GatherClusters(const std::vector<uint256>& txids) const { AssertLockHeld(cs); diff --git a/src/txmempool.h b/src/txmempool.h index fa1dbbf4b5..869612a4a2 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -6,27 +6,17 @@ #ifndef BITCOIN_TXMEMPOOL_H #define BITCOIN_TXMEMPOOL_H -#include <atomic> -#include <map> -#include <optional> -#include <set> -#include <string> -#include <string_view> -#include <utility> -#include <vector> - -#include <kernel/mempool_limits.h> -#include <kernel/mempool_options.h> - #include <coins.h> #include <consensus/amount.h> #include <indirectmap.h> #include <kernel/cs_main.h> -#include <kernel/mempool_entry.h> +#include <kernel/mempool_entry.h> // IWYU pragma: export +#include <kernel/mempool_limits.h> // IWYU pragma: export +#include <kernel/mempool_options.h> // IWYU pragma: export +#include <kernel/mempool_removal_reason.h> // IWYU pragma: export #include <policy/feerate.h> #include <policy/packages.h> #include <primitives/transaction.h> -#include <random.h> #include <sync.h> #include <util/epochguard.h> #include <util/hasher.h> @@ -40,9 +30,16 @@ #include <boost/multi_index/tag.hpp> #include <boost/multi_index_container.hpp> -class CBlockIndex; +#include <atomic> +#include <map> +#include <optional> +#include <set> +#include <string> +#include <string_view> +#include <utility> +#include <vector> + class CChain; -class Chainstate; /** Fake height value used in Coin to signify they are only in the memory pool (since 0.8) */ static const uint32_t MEMPOOL_HEIGHT = 0x7FFFFFFF; @@ -228,20 +225,6 @@ struct TxMempoolInfo int64_t nFeeDelta; }; -/** Reason why a transaction was removed from the mempool, - * this is passed to the notification signal. - */ -enum class MemPoolRemovalReason { - EXPIRY, //!< Expired from mempool - SIZELIMIT, //!< Removed in size limiting - REORG, //!< Removed for reorganization - BLOCK, //!< Removed for block - CONFLICT, //!< Removed for conflict with in-block transaction - REPLACED, //!< Removed for replacement -}; - -std::string RemovalReasonToString(const MemPoolRemovalReason& r) noexcept; - /** * CTxMemPool stores valid-according-to-the-current-best-chain transactions * that may be included in the next block. @@ -708,6 +691,10 @@ public: return mapTx.project<0>(mapTx.get<index_by_wtxid>().find(wtxid)); } TxMempoolInfo info(const GenTxid& gtxid) const; + + /** Returns info for a transaction if its entry_sequence < last_sequence */ + TxMempoolInfo info_for_relay(const GenTxid& gtxid, uint64_t last_sequence) const; + std::vector<TxMempoolInfo> infoAll() const; size_t DynamicMemoryUsage() const; diff --git a/src/util/message.cpp b/src/util/message.cpp index f8ea8247d5..ec845aeffb 100644 --- a/src/util/message.cpp +++ b/src/util/message.cpp @@ -7,7 +7,6 @@ #include <key.h> #include <key_io.h> #include <pubkey.h> -#include <script/standard.h> #include <uint256.h> #include <util/message.h> #include <util/strencodings.h> diff --git a/src/validation.cpp b/src/validation.cpp index cec6d13181..ed9889d9dd 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -275,8 +275,9 @@ static void LimitMempoolSize(CTxMemPool& pool, CCoinsViewCache& coins_cache) static bool IsCurrentForFeeEstimation(Chainstate& active_chainstate) EXCLUSIVE_LOCKS_REQUIRED(cs_main) { AssertLockHeld(cs_main); - if (active_chainstate.IsInitialBlockDownload()) + if (active_chainstate.m_chainman.IsInitialBlockDownload()) { return false; + } if (active_chainstate.m_chain.Tip()->GetBlockTime() < count_seconds(GetTime<std::chrono::seconds>() - MAX_FEE_ESTIMATION_TIP_AGE)) return false; if (active_chainstate.m_chain.Height() < active_chainstate.m_chainman.m_best_header->nHeight - 1) { @@ -833,7 +834,10 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) } } - entry.reset(new CTxMemPoolEntry(ptx, ws.m_base_fees, nAcceptTime, m_active_chainstate.m_chain.Height(), + // Set entry_sequence to 0 when bypass_limits is used; this allows txs from a block + // reorg to be marked earlier than any child txs that were already in the mempool. + const uint64_t entry_sequence = bypass_limits ? 0 : m_pool.GetSequence(); + entry.reset(new CTxMemPoolEntry(ptx, ws.m_base_fees, nAcceptTime, m_active_chainstate.m_chain.Height(), entry_sequence, fSpendsCoinbase, nSigOpsCost, lock_points.value())); ws.m_vsize = entry->GetTxSize(); @@ -1619,7 +1623,7 @@ void Chainstate::InitCoinsCache(size_t cache_size_bytes) // `const` so that `CValidationInterface` clients (which are given a `const Chainstate*`) // can call it. // -bool Chainstate::IsInitialBlockDownload() const +bool ChainstateManager::IsInitialBlockDownload() const { // Optimization: pre-test latch before taking the lock. if (m_cached_finished_ibd.load(std::memory_order_relaxed)) @@ -1628,15 +1632,17 @@ bool Chainstate::IsInitialBlockDownload() const LOCK(cs_main); if (m_cached_finished_ibd.load(std::memory_order_relaxed)) return false; - if (m_chainman.m_blockman.LoadingBlocks()) { + if (m_blockman.LoadingBlocks()) { return true; } - if (m_chain.Tip() == nullptr) + CChain& chain{ActiveChain()}; + if (chain.Tip() == nullptr) { return true; - if (m_chain.Tip()->nChainWork < m_chainman.MinimumChainWork()) { + } + if (chain.Tip()->nChainWork < MinimumChainWork()) { return true; } - if (m_chain.Tip()->Time() < Now<NodeSeconds>() - m_chainman.m_options.max_tip_age) { + if (chain.Tip()->Time() < Now<NodeSeconds>() - m_options.max_tip_age) { return true; } LogPrintf("Leaving InitialBlockDownload (latching to false)\n"); @@ -1650,7 +1656,7 @@ void Chainstate::CheckForkWarningConditions() // Before we get past initial download, we cannot reliably alert about forks // (we assume we don't get stuck on a fork before finishing our initial sync) - if (IsInitialBlockDownload()) { + if (m_chainman.IsInitialBlockDownload()) { return; } @@ -2471,7 +2477,7 @@ bool Chainstate::FlushStateToDisk( } else { LOG_TIME_MILLIS_WITH_CATEGORY("find files to prune", BCLog::BENCH); - m_blockman.FindFilesToPrune(setFilesToPrune, m_chainman.GetParams().PruneAfterHeight(), m_chain.Height(), last_prune, IsInitialBlockDownload()); + m_blockman.FindFilesToPrune(setFilesToPrune, m_chainman.GetParams().PruneAfterHeight(), m_chain.Height(), last_prune, m_chainman.IsInitialBlockDownload()); m_blockman.m_check_for_pruning = false; } if (!setFilesToPrune.empty()) { @@ -2640,7 +2646,7 @@ void Chainstate::UpdateTip(const CBlockIndex* pindexNew) } bilingual_str warning_messages; - if (!this->IsInitialBlockDownload()) { + if (!m_chainman.IsInitialBlockDownload()) { const CBlockIndex* pindex = pindexNew; for (int bit = 0; bit < VERSIONBITS_NUM_BITS; bit++) { WarningBitsConditionChecker checker(m_chainman, bit); @@ -3052,24 +3058,25 @@ static SynchronizationState GetSynchronizationState(bool init) return SynchronizationState::INIT_DOWNLOAD; } -static bool NotifyHeaderTip(Chainstate& chainstate) LOCKS_EXCLUDED(cs_main) { +static bool NotifyHeaderTip(ChainstateManager& chainman) LOCKS_EXCLUDED(cs_main) +{ bool fNotify = false; bool fInitialBlockDownload = false; static CBlockIndex* pindexHeaderOld = nullptr; CBlockIndex* pindexHeader = nullptr; { LOCK(cs_main); - pindexHeader = chainstate.m_chainman.m_best_header; + pindexHeader = chainman.m_best_header; if (pindexHeader != pindexHeaderOld) { fNotify = true; - fInitialBlockDownload = chainstate.IsInitialBlockDownload(); + fInitialBlockDownload = chainman.IsInitialBlockDownload(); pindexHeaderOld = pindexHeader; } } // Send block tip changed notifications without cs_main if (fNotify) { - chainstate.m_chainman.GetNotifications().headerTip(GetSynchronizationState(fInitialBlockDownload), pindexHeader->nHeight, pindexHeader->nTime, false); + chainman.GetNotifications().headerTip(GetSynchronizationState(fInitialBlockDownload), pindexHeader->nHeight, pindexHeader->nTime, false); } return fNotify; } @@ -3168,7 +3175,7 @@ bool Chainstate::ActivateBestChain(BlockValidationState& state, std::shared_ptr< if (!blocks_connected) return true; const CBlockIndex* pindexFork = m_chain.FindFork(starting_tip); - bool fInitialDownload = IsInitialBlockDownload(); + bool fInitialDownload = m_chainman.IsInitialBlockDownload(); // Notify external listeners about the new tip. // Enqueue while holding cs_main to ensure that UpdatedBlockTip is called in the order in which blocks are connected @@ -3386,7 +3393,7 @@ bool Chainstate::InvalidateBlock(BlockValidationState& state, CBlockIndex* pinde // parameter indicating the source of the tip change so hooks can // distinguish user-initiated invalidateblock changes from other // changes. - (void)m_chainman.GetNotifications().blockTip(GetSynchronizationState(IsInitialBlockDownload()), *to_mark_failed->pprev); + (void)m_chainman.GetNotifications().blockTip(GetSynchronizationState(m_chainman.IsInitialBlockDownload()), *to_mark_failed->pprev); } return true; } @@ -3871,7 +3878,7 @@ bool ChainstateManager::AcceptBlockHeader(const CBlockHeader& block, BlockValida const auto msg = strprintf( "Saw new header hash=%s height=%d", hash.ToString(), pindex->nHeight); - if (ActiveChainstate().IsInitialBlockDownload()) { + if (IsInitialBlockDownload()) { LogPrintLevel(BCLog::VALIDATION, BCLog::Level::Debug, "%s\n", msg); } else { LogPrintf("%s\n", msg); @@ -3899,8 +3906,8 @@ bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& } } } - if (NotifyHeaderTip(ActiveChainstate())) { - if (ActiveChainstate().IsInitialBlockDownload() && ppindex && *ppindex) { + if (NotifyHeaderTip(*this)) { + if (IsInitialBlockDownload() && ppindex && *ppindex) { const CBlockIndex& last_accepted{**ppindex}; const int64_t blocks_left{(GetTime() - last_accepted.GetBlockTime()) / GetConsensus().nPowTargetSpacing}; const double progress{100.0 * last_accepted.nHeight / (last_accepted.nHeight + blocks_left)}; @@ -3913,7 +3920,6 @@ bool ChainstateManager::ProcessNewBlockHeaders(const std::vector<CBlockHeader>& void ChainstateManager::ReportHeadersPresync(const arith_uint256& work, int64_t height, int64_t timestamp) { AssertLockNotHeld(cs_main); - const auto& chainstate = ActiveChainstate(); { LOCK(cs_main); // Don't report headers presync progress if we already have a post-minchainwork header chain. @@ -3926,7 +3932,7 @@ void ChainstateManager::ReportHeadersPresync(const arith_uint256& work, int64_t if (now < m_last_presync_update + std::chrono::milliseconds{250}) return; m_last_presync_update = now; } - bool initial_download = chainstate.IsInitialBlockDownload(); + bool initial_download = IsInitialBlockDownload(); GetNotifications().headerTip(GetSynchronizationState(initial_download), height, timestamp, /*presync=*/true); if (initial_download) { const int64_t blocks_left{(GetTime() - timestamp) / GetConsensus().nPowTargetSpacing}; @@ -3999,7 +4005,7 @@ bool ChainstateManager::AcceptBlock(const std::shared_ptr<const CBlock>& pblock, // Header is valid/has work, merkle tree and segwit merkle tree are good...RELAY NOW // (but if it does not build on our best tip, let the SendMessages loop relay it) - if (!ActiveChainstate().IsInitialBlockDownload() && ActiveTip() == pindex->pprev) + if (!IsInitialBlockDownload() && ActiveTip() == pindex->pprev) GetMainSignals().NewPoWValidBlock(pindex, pblock); // Write block to history file @@ -4058,7 +4064,7 @@ bool ChainstateManager::ProcessNewBlock(const std::shared_ptr<const CBlock>& blo } } - NotifyHeaderTip(ActiveChainstate()); + NotifyHeaderTip(*this); BlockValidationState state; // Only used to report errors, not invalidity - ignore it if (!ActiveChainstate().ActivateBestChain(state, block)) { @@ -4626,7 +4632,7 @@ void ChainstateManager::LoadExternalBlockFile( } } - NotifyHeaderTip(ActiveChainstate()); + NotifyHeaderTip(*this); if (!blocks_with_unknown_parent) continue; @@ -4652,7 +4658,7 @@ void ChainstateManager::LoadExternalBlockFile( } range.first++; blocks_with_unknown_parent->erase(it); - NotifyHeaderTip(ActiveChainstate()); + NotifyHeaderTip(*this); } } } catch (const std::exception& e) { @@ -5575,7 +5581,7 @@ void ChainstateManager::MaybeRebalanceCaches() // If both chainstates exist, determine who needs more cache based on IBD status. // // Note: shrink caches first so that we don't inadvertently overwhelm available memory. - if (m_snapshot_chainstate->IsInitialBlockDownload()) { + if (IsInitialBlockDownload()) { m_ibd_chainstate->ResizeCoinsCaches( m_total_coinstip_cache * 0.05, m_total_coinsdb_cache * 0.05); m_snapshot_chainstate->ResizeCoinsCaches( diff --git a/src/validation.h b/src/validation.h index 4c9f807f5d..ba427afc64 100644 --- a/src/validation.h +++ b/src/validation.h @@ -241,7 +241,8 @@ struct PackageMempoolAcceptResult * @param[in] tx The transaction to submit for mempool acceptance. * @param[in] accept_time The timestamp for adding the transaction to the mempool. * It is also used to determine when the entry expires. - * @param[in] bypass_limits When true, don't enforce mempool fee and capacity limits. + * @param[in] bypass_limits When true, don't enforce mempool fee and capacity limits, + * and set entry_sequence to zero. * @param[in] test_accept When true, run validation checks but don't submit to mempool. * * @returns a MempoolAcceptResult indicating whether the transaction was accepted/rejected with reason. @@ -472,14 +473,6 @@ protected: */ Mutex m_chainstate_mutex; - /** - * Whether this chainstate is undergoing initial block download. - * - * Mutable because we need to be able to mark IsInitialBlockDownload() - * const, which latches this for caching purposes. - */ - mutable std::atomic<bool> m_cached_finished_ibd{false}; - //! Optional mempool that is kept in sync with the chain. //! Only the active chainstate has a mempool. CTxMemPool* m_mempool; @@ -706,9 +699,6 @@ public: void ClearBlockIndexCandidates() EXCLUSIVE_LOCKS_REQUIRED(::cs_main); - /** Check whether we are doing an initial block download (synchronizing from disk or network) */ - bool IsInitialBlockDownload() const; - /** Find the last common block of this chain and a locator. */ const CBlockIndex* FindForkInGlobalIndex(const CBlockLocator& locator) const EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -951,6 +941,15 @@ public: node::BlockManager m_blockman; /** + * Whether initial block download has ended and IsInitialBlockDownload + * should return false from now on. + * + * Mutable because we need to be able to mark IsInitialBlockDownload() + * const, which latches this for caching purposes. + */ + mutable std::atomic<bool> m_cached_finished_ibd{false}; + + /** * Every received block is assigned a unique and increasing identifier, so we * know which one to give priority in case of a fork. */ @@ -1066,6 +1065,9 @@ public: return m_snapshot_chainstate && m_ibd_chainstate && m_ibd_chainstate->m_disabled; } + /** Check whether we are doing an initial block download (synchronizing from disk or network) */ + bool IsInitialBlockDownload() const; + /** * Import blocks from an external file * diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index 7ff8fee5bc..71593e236f 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -11,7 +11,6 @@ #include <primitives/transaction.h> #include <script/keyorigin.h> #include <script/signingprovider.h> -#include <script/standard.h> #include <algorithm> #include <map> diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index cd438cfe2f..5f2aee6923 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -11,7 +11,6 @@ #include <policy/fees.h> #include <primitives/transaction.h> #include <rpc/server.h> -#include <script/standard.h> #include <support/allocators/secure.h> #include <sync.h> #include <uint256.h> diff --git a/src/wallet/rpc/addresses.cpp b/src/wallet/rpc/addresses.cpp index a8ef0a5731..c1b99a4f97 100644 --- a/src/wallet/rpc/addresses.cpp +++ b/src/wallet/rpc/addresses.cpp @@ -5,6 +5,8 @@ #include <core_io.h> #include <key_io.h> #include <rpc/util.h> +#include <script/script.h> +#include <script/solver.h> #include <util/bip32.h> #include <util/translation.h> #include <wallet/receive.h> @@ -440,10 +442,9 @@ public: UniValue operator()(const ScriptHash& scripthash) const { - CScriptID scriptID(scripthash); UniValue obj(UniValue::VOBJ); CScript subscript; - if (provider && provider->GetCScript(scriptID, subscript)) { + if (provider && provider->GetCScript(ToScriptID(scripthash), subscript)) { ProcessSubScript(subscript, obj); } return obj; diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index af8043f158..d100b69d3c 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -12,7 +12,7 @@ #include <rpc/util.h> #include <script/descriptor.h> #include <script/script.h> -#include <script/standard.h> +#include <script/solver.h> #include <sync.h> #include <uint256.h> #include <util/bip32.h> @@ -1297,12 +1297,12 @@ RPCHelpMan importmulti() }, }, }, - RPCArgOptions{.oneline_description="\"requests\""}}, + RPCArgOptions{.oneline_description="requests"}}, {"options", RPCArg::Type::OBJ_NAMED_PARAMS, RPCArg::Optional::OMITTED, "", { {"rescan", RPCArg::Type::BOOL, RPCArg::Default{true}, "Scan the chain and mempool for wallet transactions after all imports."}, }, - RPCArgOptions{.oneline_description="\"options\""}}, + RPCArgOptions{.oneline_description="options"}}, }, RPCResult{ RPCResult::Type::ARR, "", "Response is an array with the same size as the input that has the execution result", @@ -1617,7 +1617,7 @@ RPCHelpMan importdescriptors() }, }, }, - RPCArgOptions{.oneline_description="\"requests\""}}, + RPCArgOptions{.oneline_description="requests"}}, }, RPCResult{ RPCResult::Type::ARR, "", "Response is an array with the same size as the input that has the execution result", diff --git a/src/wallet/rpc/coins.cpp b/src/wallet/rpc/coins.cpp index 22f0f0b83c..fdc6ee055d 100644 --- a/src/wallet/rpc/coins.cpp +++ b/src/wallet/rpc/coins.cpp @@ -6,6 +6,7 @@ #include <hash.h> #include <key_io.h> #include <rpc/util.h> +#include <script/script.h> #include <util/moneystr.h> #include <wallet/coincontrol.h> #include <wallet/receive.h> @@ -193,8 +194,8 @@ RPCHelpMan getbalance() LOCK(pwallet->cs_wallet); - const UniValue& dummy_value = request.params[0]; - if (!dummy_value.isNull() && dummy_value.get_str() != "*") { + const auto dummy_value{self.MaybeArg<std::string>(0)}; + if (dummy_value && *dummy_value != "*") { throw JSONRPCError(RPC_METHOD_DEPRECATED, "dummy first argument must be excluded or set to \"*\"."); } @@ -672,7 +673,7 @@ RPCHelpMan listunspent() std::unique_ptr<SigningProvider> provider = pwallet->GetSolvingProvider(scriptPubKey); if (provider) { if (scriptPubKey.IsPayToScriptHash()) { - const CScriptID& hash = CScriptID(std::get<ScriptHash>(address)); + const CScriptID hash = ToScriptID(std::get<ScriptHash>(address)); CScript redeemScript; if (provider->GetCScript(hash, redeemScript)) { entry.pushKV("redeemScript", HexStr(redeemScript)); diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index b695d4bed3..0c2be26ddf 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -8,6 +8,7 @@ #include <policy/policy.h> #include <rpc/rawtransaction_util.h> #include <rpc/util.h> +#include <script/script.h> #include <util/fees.h> #include <util/rbf.h> #include <util/translation.h> @@ -1013,9 +1014,9 @@ static RPCHelpMan bumpfee_helper(std::string method_name) "are replaceable).\n"}, {"estimate_mode", RPCArg::Type::STR, RPCArg::Default{"unset"}, "The fee estimate mode, must be one of (case insensitive):\n" "\"" + FeeModes("\"\n\"") + "\""}, - {"outputs", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "New outputs (key-value pairs) which will replace\n" - "the original ones, if provided. Each address can only appear once and there can\n" - "only be one \"data\" object.\n" + {"outputs", RPCArg::Type::ARR, RPCArg::Default{UniValue::VARR}, "The outputs specified as key-value pairs.\n" + "Each key may only appear once, i.e. there can only be one 'data' output, and no address may be duplicated.\n" + "At least one output of either type must be specified.\n" "Cannot be provided if 'reduce_output' is specified.", OutputsDoc(), RPCArgOptions{.skip_type_check = true}}, @@ -1187,8 +1188,9 @@ RPCHelpMan send() "\nEXPERIMENTAL warning: this call may be changed in future releases.\n" "\nSend a transaction.\n", { - {"outputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "The outputs (key-value pairs), where none of the keys are duplicated.\n" - "That is, each address can only appear once and there can only be one 'data' object.\n" + {"outputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "The outputs specified as key-value pairs.\n" + "Each key may only appear once, i.e. there can only be one 'data' output, and no address may be duplicated.\n" + "At least one output of either type must be specified.\n" "For convenience, a dictionary, which holds the key-value pairs directly, is also accepted.", OutputsDoc(), RPCArgOptions{.skip_type_check = true}}, @@ -1637,8 +1639,9 @@ RPCHelpMan walletcreatefundedpsbt() }, }, }, - {"outputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "The outputs (key-value pairs), where none of the keys are duplicated.\n" - "That is, each address can only appear once and there can only be one 'data' object.\n" + {"outputs", RPCArg::Type::ARR, RPCArg::Optional::NO, "The outputs specified as key-value pairs.\n" + "Each key may only appear once, i.e. there can only be one 'data' output, and no address may be duplicated.\n" + "At least one output of either type must be specified.\n" "For compatibility reasons, a dictionary, which holds the key-value pairs directly, is also\n" "accepted as second parameter.", OutputsDoc(), diff --git a/src/wallet/rpc/wallet.cpp b/src/wallet/rpc/wallet.cpp index fb4b642bba..3774e6a3ef 100644 --- a/src/wallet/rpc/wallet.cpp +++ b/src/wallet/rpc/wallet.cpp @@ -478,7 +478,7 @@ static RPCHelpMan unloadwallet() // Release the "main" shared pointer and prevent further notifications. // Note that any attempt to load the same wallet would fail until the wallet // is destroyed (see CheckUniqueFileid). - std::optional<bool> load_on_start = request.params[1].isNull() ? std::nullopt : std::optional<bool>(request.params[1].get_bool()); + std::optional<bool> load_on_start{self.MaybeArg<bool>(1)}; if (!RemoveWallet(context, wallet, load_on_start, warnings)) { throw JSONRPCError(RPC_MISC_ERROR, "Requested wallet already unloaded"); } diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index 5b110b4d14..1f510d1c58 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -7,7 +7,9 @@ #include <logging.h> #include <outputtype.h> #include <script/descriptor.h> +#include <script/script.h> #include <script/sign.h> +#include <script/solver.h> #include <util/bip32.h> #include <util/strencodings.h> #include <util/string.h> diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index bf35c776ae..ec7b017720 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -5,11 +5,12 @@ #ifndef BITCOIN_WALLET_SCRIPTPUBKEYMAN_H #define BITCOIN_WALLET_SCRIPTPUBKEYMAN_H +#include <addresstype.h> #include <logging.h> #include <psbt.h> #include <script/descriptor.h> +#include <script/script.h> #include <script/signingprovider.h> -#include <script/standard.h> #include <util/error.h> #include <util/message.h> #include <util/result.h> @@ -249,9 +250,10 @@ public: virtual std::unordered_set<CScript, SaltedSipHasher> GetScriptPubKeys() const { return {}; }; /** Prepends the wallet name in logging output to ease debugging in multi-wallet use cases */ - template<typename... Params> - void WalletLogPrintf(std::string fmt, Params... parameters) const { - LogPrintf(("%s " + fmt).c_str(), m_storage.GetDisplayName(), parameters...); + template <typename... Params> + void WalletLogPrintf(const char* fmt, Params... parameters) const + { + LogPrintf(("%s " + std::string{fmt}).c_str(), m_storage.GetDisplayName(), parameters...); }; /** Watch-only address added */ diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index a3728223fb..c0ee00e097 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -11,7 +11,9 @@ #include <numeric> #include <policy/policy.h> #include <primitives/transaction.h> +#include <script/script.h> #include <script/signingprovider.h> +#include <script/solver.h> #include <util/check.h> #include <util/fees.h> #include <util/moneystr.h> diff --git a/src/wallet/test/ismine_tests.cpp b/src/wallet/test/ismine_tests.cpp index fd0718fbb9..95d5c1b9ce 100644 --- a/src/wallet/test/ismine_tests.cpp +++ b/src/wallet/test/ismine_tests.cpp @@ -6,7 +6,8 @@ #include <key_io.h> #include <node/context.h> #include <script/script.h> -#include <script/standard.h> +#include <script/solver.h> +#include <script/signingprovider.h> #include <test/util/setup_common.h> #include <wallet/types.h> #include <wallet/wallet.h> diff --git a/src/wallet/test/scriptpubkeyman_tests.cpp b/src/wallet/test/scriptpubkeyman_tests.cpp index d4997c418a..15bff04221 100644 --- a/src/wallet/test/scriptpubkeyman_tests.cpp +++ b/src/wallet/test/scriptpubkeyman_tests.cpp @@ -3,8 +3,8 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <key.h> -#include <script/standard.h> #include <test/util/setup_common.h> +#include <script/solver.h> #include <wallet/scriptpubkeyman.h> #include <wallet/wallet.h> #include <wallet/test/util.h> diff --git a/src/wallet/test/spend_tests.cpp b/src/wallet/test/spend_tests.cpp index 6e87d1cb49..b5ea275bcb 100644 --- a/src/wallet/test/spend_tests.cpp +++ b/src/wallet/test/spend_tests.cpp @@ -4,6 +4,7 @@ #include <consensus/amount.h> #include <policy/fees.h> +#include <script/solver.h> #include <validation.h> #include <wallet/coincontrol.h> #include <wallet/spend.h> diff --git a/src/wallet/test/util.cpp b/src/wallet/test/util.cpp index 069ab25f26..ad8613d515 100644 --- a/src/wallet/test/util.cpp +++ b/src/wallet/test/util.cpp @@ -9,6 +9,7 @@ #include <key_io.h> #include <streams.h> #include <test/util/setup_common.h> +#include <validationinterface.h> #include <wallet/context.h> #include <wallet/wallet.h> #include <wallet/walletdb.h> diff --git a/src/wallet/test/util.h b/src/wallet/test/util.h index 2a1fe639de..8bd238648f 100644 --- a/src/wallet/test/util.h +++ b/src/wallet/test/util.h @@ -5,7 +5,7 @@ #ifndef BITCOIN_WALLET_TEST_UTIL_H #define BITCOIN_WALLET_TEST_UTIL_H -#include <script/standard.h> +#include <addresstype.h> #include <wallet/db.h> #include <memory> diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index dd914f159c..5c297d76e4 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -9,16 +9,19 @@ #include <stdint.h> #include <vector> +#include <addresstype.h> #include <interfaces/chain.h> #include <key_io.h> #include <node/blockstorage.h> #include <policy/policy.h> #include <rpc/server.h> +#include <script/solver.h> #include <test/util/logging.h> #include <test/util/random.h> #include <test/util/setup_common.h> #include <util/translation.h> #include <validation.h> +#include <validationinterface.h> #include <wallet/coincontrol.h> #include <wallet/context.h> #include <wallet/receive.h> diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 6b2755ea53..e77dd676e0 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -5,50 +5,82 @@ #include <wallet/wallet.h> +#if defined(HAVE_CONFIG_H) +#include <config/bitcoin-config.h> +#endif +#include <addresstype.h> #include <blockfilter.h> #include <chain.h> +#include <coins.h> #include <common/args.h> +#include <common/settings.h> +#include <common/system.h> #include <consensus/amount.h> #include <consensus/consensus.h> #include <consensus/validation.h> #include <external_signer.h> #include <interfaces/chain.h> +#include <interfaces/handler.h> #include <interfaces/wallet.h> +#include <kernel/mempool_removal_reason.h> #include <key.h> #include <key_io.h> +#include <logging.h> #include <outputtype.h> -#include <policy/fees.h> -#include <policy/policy.h> +#include <policy/feerate.h> #include <primitives/block.h> #include <primitives/transaction.h> #include <psbt.h> +#include <pubkey.h> #include <random.h> #include <script/descriptor.h> +#include <script/interpreter.h> #include <script/script.h> +#include <script/sign.h> #include <script/signingprovider.h> +#include <script/solver.h> +#include <serialize.h> +#include <span.h> +#include <streams.h> +#include <support/allocators/secure.h> +#include <support/allocators/zeroafterfree.h> #include <support/cleanse.h> -#include <txmempool.h> -#include <util/bip32.h> +#include <sync.h> +#include <tinyformat.h> +#include <uint256.h> +#include <univalue.h> #include <util/check.h> #include <util/error.h> -#include <util/fees.h> #include <util/fs.h> #include <util/fs_helpers.h> +#include <util/message.h> #include <util/moneystr.h> -#include <util/rbf.h> +#include <util/result.h> #include <util/string.h> +#include <util/time.h> #include <util/translation.h> #include <wallet/coincontrol.h> #include <wallet/context.h> +#include <wallet/crypter.h> +#include <wallet/db.h> #include <wallet/external_signer_scriptpubkeyman.h> -#include <wallet/fees.h> #include <wallet/scriptpubkeyman.h> - -#include <univalue.h> +#include <wallet/transaction.h> +#include <wallet/types.h> +#include <wallet/walletdb.h> +#include <wallet/walletutil.h> #include <algorithm> -#include <assert.h> +#include <cassert> +#include <condition_variable> +#include <exception> #include <optional> +#include <stdexcept> +#include <thread> +#include <tuple> +#include <variant> + +struct KeyOriginInfo; using interfaces::FoundBlock; @@ -2319,7 +2351,7 @@ OutputType CWallet::TransactionChangeType(const std::optional<OutputType>& chang void CWallet::CommitTransaction(CTransactionRef tx, mapValue_t mapValue, std::vector<std::pair<std::string, std::string>> orderForm) { LOCK(cs_wallet); - WalletLogPrintf("CommitTransaction:\n%s", tx->ToString()); + WalletLogPrintf("CommitTransaction:\n%s", tx->ToString()); // NOLINT(bitcoin-unterminated-logprintf) // Add tx to wallet, because if it has change it's also ours, // otherwise just for transaction history. diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index cbd5008366..da58c9e876 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -6,50 +6,76 @@ #ifndef BITCOIN_WALLET_WALLET_H #define BITCOIN_WALLET_WALLET_H +#include <addresstype.h> #include <consensus/amount.h> #include <interfaces/chain.h> #include <interfaces/handler.h> -#include <interfaces/wallet.h> +#include <kernel/cs_main.h> #include <logging.h> #include <outputtype.h> #include <policy/feerate.h> -#include <psbt.h> +#include <primitives/transaction.h> +#include <script/interpreter.h> +#include <script/script.h> +#include <support/allocators/secure.h> +#include <sync.h> #include <tinyformat.h> +#include <uint256.h> #include <util/fs.h> #include <util/hasher.h> -#include <util/message.h> #include <util/result.h> -#include <util/strencodings.h> #include <util/string.h> #include <util/time.h> #include <util/ui_change_type.h> -#include <validationinterface.h> #include <wallet/crypter.h> +#include <wallet/db.h> #include <wallet/scriptpubkeyman.h> #include <wallet/transaction.h> -#include <wallet/walletdb.h> +#include <wallet/types.h> #include <wallet/walletutil.h> -#include <algorithm> #include <atomic> +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <functional> +#include <limits> #include <map> #include <memory> #include <optional> #include <set> -#include <stdexcept> -#include <stdint.h> #include <string> -#include <utility> #include <unordered_map> +#include <utility> #include <vector> #include <boost/signals2/signal.hpp> +class CKey; +class CKeyID; +class CPubKey; +class Coin; +class SigningProvider; +enum class MemPoolRemovalReason; +enum class SigningResult; +enum class TransactionError; +namespace interfaces { +class Wallet; +} +namespace wallet { +class CWallet; +class WalletBatch; +enum class DBErrors : int; +} // namespace wallet +struct CBlockLocator; +struct CExtKey; +struct FlatSigningProvider; +struct KeyOriginInfo; +struct PartiallySignedTransaction; +struct SignatureData; using LoadWalletFn = std::function<void(std::unique_ptr<interfaces::Wallet> wallet)>; -class CScript; -enum class FeeEstimateMode; struct bilingual_str; namespace wallet { @@ -118,8 +144,6 @@ constexpr CAmount HIGH_MAX_TX_FEE{100 * HIGH_TX_FEE_PER_KB}; static constexpr size_t DUMMY_NESTED_P2WPKH_INPUT_SIZE = 91; class CCoinControl; -class CWalletTx; -class ReserveDestination; //! Default for -addresstype constexpr OutputType DEFAULT_ADDRESS_TYPE{OutputType::BECH32}; @@ -890,9 +914,10 @@ public: }; /** Prepends the wallet name in logging output to ease debugging in multi-wallet use cases */ - template<typename... Params> - void WalletLogPrintf(std::string fmt, Params... parameters) const { - LogPrintf(("%s " + fmt).c_str(), GetDisplayName(), parameters...); + template <typename... Params> + void WalletLogPrintf(const char* fmt, Params... parameters) const + { + LogPrintf(("%s " + std::string{fmt}).c_str(), GetDisplayName(), parameters...); }; /** Upgrade the wallet */ diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 8212c04464..92eca46f05 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -8,6 +8,7 @@ #include <common/system.h> #include <key_io.h> #include <protocol.h> +#include <script/script.h> #include <serialize.h> #include <sync.h> #include <util/bip32.h> diff --git a/src/wallet/walletdb.h b/src/wallet/walletdb.h index 8f7c2f030c..dad0b18a78 100644 --- a/src/wallet/walletdb.h +++ b/src/wallet/walletdb.h @@ -7,7 +7,6 @@ #define BITCOIN_WALLET_WALLETDB_H #include <script/sign.h> -#include <script/standard.h> #include <wallet/db.h> #include <wallet/walletutil.h> #include <key.h> |