diff options
Diffstat (limited to 'src')
420 files changed, 7549 insertions, 5727 deletions
diff --git a/src/.clang-format b/src/.clang-format index 791b3b8f9f..f20e5ee2d4 100644 --- a/src/.clang-format +++ b/src/.clang-format @@ -43,5 +43,7 @@ SpacesInAngles: false SpacesInContainerLiterals: true SpacesInCStyleCastParentheses: false SpacesInParentheses: false -Standard: c++17 +BreakBeforeConceptDeclarations: Always +RequiresExpressionIndentation: OuterScope +Standard: c++20 UseTab: Never diff --git a/src/.clang-tidy b/src/.clang-tidy index 0318bd01ce..bfaa5ab8e7 100644 --- a/src/.clang-tidy +++ b/src/.clang-tidy @@ -2,7 +2,9 @@ Checks: ' -*, bitcoin-*, bugprone-argument-comment, +bugprone-string-constructor, bugprone-use-after-move, +bugprone-lambda-function-name, misc-unused-using-decls, modernize-use-default-member-init, modernize-use-emplace, diff --git a/src/Makefile.am b/src/Makefile.am index 8905c0ad1c..b6f0daaaba 100644 --- a/src/Makefile.am +++ b/src/Makefile.am @@ -230,6 +230,7 @@ BITCOIN_CORE_H = \ node/mini_miner.h \ node/minisketchwrapper.h \ node/peerman_args.h \ + node/protocol_version.h \ node/psbt.h \ node/transaction.h \ node/txreconciliation.h \ @@ -270,7 +271,6 @@ BITCOIN_CORE_H = \ script/sign.h \ script/signingprovider.h \ script/solver.h \ - shutdown.h \ signet.h \ streams.h \ support/allocators/pool.h \ @@ -303,7 +303,6 @@ BITCOIN_CORE_H = \ util/fees.h \ util/fs.h \ util/fs_helpers.h \ - util/getuniquepath.h \ util/golombrice.h \ util/hash_type.h \ util/hasher.h \ @@ -328,6 +327,7 @@ BITCOIN_CORE_H = \ util/time.h \ util/tokenpipe.h \ util/trace.h \ + util/transaction_identifier.h \ util/translation.h \ util/types.h \ util/ui_change_type.h \ @@ -403,6 +403,7 @@ libbitcoin_node_a_SOURCES = \ kernel/coinstats.cpp \ kernel/context.cpp \ kernel/cs_main.cpp \ + kernel/disconnected_transactions.cpp \ kernel/mempool_persist.cpp \ kernel/mempool_removal_reason.cpp \ mapport.cpp \ @@ -456,7 +457,6 @@ libbitcoin_node_a_SOURCES = \ rpc/signmessage.cpp \ rpc/txoutproof.cpp \ script/sigcache.cpp \ - shutdown.cpp \ signet.cpp \ timedata.cpp \ torcontrol.cpp \ @@ -656,8 +656,7 @@ libbitcoin_consensus_a_SOURCES = \ uint256.cpp \ uint256.h \ util/strencodings.cpp \ - util/strencodings.h \ - version.h + util/strencodings.h # # common # @@ -739,7 +738,6 @@ libbitcoin_util_a_SOURCES = \ util/fees.cpp \ util/fs.cpp \ util/fs_helpers.cpp \ - util/getuniquepath.cpp \ util/hasher.cpp \ util/sock.cpp \ util/syserror.cpp \ @@ -943,6 +941,7 @@ libbitcoinkernel_la_SOURCES = \ kernel/coinstats.cpp \ kernel/context.cpp \ kernel/cs_main.cpp \ + kernel/disconnected_transactions.cpp \ kernel/mempool_persist.cpp \ kernel/mempool_removal_reason.cpp \ key.cpp \ @@ -951,7 +950,6 @@ libbitcoinkernel_la_SOURCES = \ node/chainstate.cpp \ node/utxo_snapshot.cpp \ policy/feerate.cpp \ - policy/fees.cpp \ policy/packages.cpp \ policy/policy.cpp \ policy/rbf.cpp \ @@ -982,7 +980,6 @@ libbitcoinkernel_la_SOURCES = \ util/exception.cpp \ util/fs.cpp \ util/fs_helpers.cpp \ - util/getuniquepath.cpp \ util/hasher.cpp \ util/moneystr.cpp \ util/rbf.cpp \ diff --git a/src/Makefile.bench.include b/src/Makefile.bench.include index 28b779a5a8..9e5366f0b4 100644 --- a/src/Makefile.bench.include +++ b/src/Makefile.bench.include @@ -46,6 +46,7 @@ bench_bench_bitcoin_SOURCES = \ bench/poly1305.cpp \ bench/pool.cpp \ bench/prevector.cpp \ + bench/readblock.cpp \ bench/rollingbloom.cpp \ bench/rpc_blockchain.cpp \ bench/rpc_mempool.cpp \ @@ -84,6 +85,7 @@ endif if ENABLE_WALLET bench_bench_bitcoin_SOURCES += bench/coin_selection.cpp bench_bench_bitcoin_SOURCES += bench/wallet_balance.cpp +bench_bench_bitcoin_SOURCES += bench/wallet_create.cpp bench_bench_bitcoin_SOURCES += bench/wallet_loading.cpp bench_bench_bitcoin_SOURCES += bench/wallet_create_tx.cpp bench_bench_bitcoin_LDADD += $(BDB_LIBS) $(SQLITE_LIBS) diff --git a/src/Makefile.test.include b/src/Makefile.test.include index b610dabd07..416a11b0c0 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -92,6 +92,7 @@ BITCOIN_TESTS =\ test/dbwrapper_tests.cpp \ test/denialofservice_tests.cpp \ test/descriptor_tests.cpp \ + test/disconnected_transactions.cpp \ test/flatfile_tests.cpp \ test/fs_tests.cpp \ test/getarg_tests.cpp \ @@ -111,6 +112,7 @@ BITCOIN_TESTS =\ test/miniscript_tests.cpp \ test/minisketch_tests.cpp \ test/multisig_tests.cpp \ + test/net_peer_connection_tests.cpp \ test/net_peer_eviction_tests.cpp \ test/net_tests.cpp \ test/netbase_tests.cpp \ @@ -144,6 +146,7 @@ BITCOIN_TESTS =\ test/sigopcount_tests.cpp \ test/skiplist_tests.cpp \ test/sock_tests.cpp \ + test/span_tests.cpp \ test/streams_tests.cpp \ test/sync_tests.cpp \ test/system_tests.cpp \ @@ -202,7 +205,8 @@ FUZZ_WALLET_SRC = \ if USE_SQLITE FUZZ_WALLET_SRC += \ - wallet/test/fuzz/notifications.cpp + wallet/test/fuzz/notifications.cpp \ + wallet/test/fuzz/scriptpubkeyman.cpp endif # USE_SQLITE BITCOIN_TEST_SUITE += \ diff --git a/src/Makefile.test_fuzz.include b/src/Makefile.test_fuzz.include index aa9c052750..b4337991e4 100644 --- a/src/Makefile.test_fuzz.include +++ b/src/Makefile.test_fuzz.include @@ -11,6 +11,7 @@ TEST_FUZZ_H = \ test/fuzz/fuzz.h \ test/fuzz/FuzzedDataProvider.h \ test/fuzz/util.h \ + test/fuzz/util/descriptor.h \ test/fuzz/util/mempool.h \ test/fuzz/util/net.h @@ -19,6 +20,7 @@ libtest_fuzz_a_CXXFLAGS = $(AM_CXXFLAGS) $(PIE_FLAGS) libtest_fuzz_a_SOURCES = \ test/fuzz/fuzz.cpp \ test/fuzz/util.cpp \ + test/fuzz/util/descriptor.cpp \ test/fuzz/util/mempool.cpp \ test/fuzz/util/net.cpp \ $(TEST_FUZZ_H) diff --git a/src/addrdb.cpp b/src/addrdb.cpp index 8b85b77e2b..8cf932bcb6 100644 --- a/src/addrdb.cpp +++ b/src/addrdb.cpp @@ -209,8 +209,7 @@ util::Result<std::unique_ptr<AddrMan>> LoadAddrman(const NetGroupManager& netgro return util::Error{strprintf(_("Invalid or corrupt peers.dat (%s). If you believe this is a bug, please report it to %s. As a workaround, you can move the file (%s) out of the way (rename, move, or delete) to have a new one created on the next start."), e.what(), PACKAGE_BUGREPORT, fs::quoted(fs::PathToString(path_addr)))}; } - return {std::move(addrman)}; // std::move should be unnecessary but is temporarily needed to work around clang bug - // (https://github.com/bitcoin/bitcoin/pull/25977#issuecomment-1561270092) + return addrman; } void DumpAnchors(const fs::path& anchors_db_path, const std::vector<CAddress>& anchors) diff --git a/src/addresstype.h b/src/addresstype.h index d3422c6813..0152858bad 100644 --- a/src/addresstype.h +++ b/src/addresstype.h @@ -5,23 +5,26 @@ #ifndef BITCOIN_ADDRESSTYPE_H #define BITCOIN_ADDRESSTYPE_H +#include <attributes.h> #include <pubkey.h> #include <script/script.h> #include <uint256.h> #include <util/hash_type.h> -#include <variant> #include <algorithm> +#include <variant> +#include <vector> -class CNoDestination { +class CNoDestination +{ private: CScript m_script; public: CNoDestination() = default; - CNoDestination(const CScript& script) : m_script(script) {} + explicit CNoDestination(const CScript& script) : m_script(script) {} - const CScript& GetScript() const { return m_script; } + const CScript& GetScript() const LIFETIMEBOUND { return m_script; } friend bool operator==(const CNoDestination& a, const CNoDestination& b) { return a.GetScript() == b.GetScript(); } friend bool operator<(const CNoDestination& a, const CNoDestination& b) { return a.GetScript() < b.GetScript(); } @@ -32,7 +35,7 @@ private: CPubKey m_pubkey; public: - PubKeyDestination(const CPubKey& pubkey) : m_pubkey(pubkey) {} + explicit PubKeyDestination(const CPubKey& pubkey) : m_pubkey(pubkey) {} const CPubKey& GetPubKey() const LIFETIMEBOUND { return m_pubkey; } diff --git a/src/addrman.cpp b/src/addrman.cpp index b001365ab3..a8206de6ee 100644 --- a/src/addrman.cpp +++ b/src/addrman.cpp @@ -610,8 +610,9 @@ bool AddrManImpl::AddSingle(const CAddress& addr, const CNetAddr& source, std::c ClearNew(nUBucket, nUBucketPos); pinfo->nRefCount++; vvNew[nUBucket][nUBucketPos] = nId; - LogPrint(BCLog::ADDRMAN, "Added %s mapped to AS%i to new[%i][%i]\n", - addr.ToStringAddrPort(), m_netgroupman.GetMappedAS(addr), nUBucket, nUBucketPos); + const auto mapped_as{m_netgroupman.GetMappedAS(addr)}; + LogPrint(BCLog::ADDRMAN, "Added %s%s to new[%i][%i]\n", + addr.ToStringAddrPort(), (mapped_as ? strprintf(" mapped to AS%i", mapped_as) : ""), nUBucket, nUBucketPos); } else { if (pinfo->nRefCount == 0) { Delete(nId); @@ -669,8 +670,9 @@ bool AddrManImpl::Good_(const CService& addr, bool test_before_evict, NodeSecond } else { // move nId to the tried tables MakeTried(info, nId); - LogPrint(BCLog::ADDRMAN, "Moved %s mapped to AS%i to tried[%i][%i]\n", - addr.ToStringAddrPort(), m_netgroupman.GetMappedAS(addr), tried_bucket, tried_bucket_pos); + const auto mapped_as{m_netgroupman.GetMappedAS(addr)}; + LogPrint(BCLog::ADDRMAN, "Moved %s%s to tried[%i][%i]\n", + addr.ToStringAddrPort(), (mapped_as ? strprintf(" mapped to AS%i", mapped_as) : ""), tried_bucket, tried_bucket_pos); return true; } } @@ -800,7 +802,7 @@ int AddrManImpl::GetEntry(bool use_tried, size_t bucket, size_t position) const return -1; } -std::vector<CAddress> AddrManImpl::GetAddr_(size_t max_addresses, size_t max_pct, std::optional<Network> network) const +std::vector<CAddress> AddrManImpl::GetAddr_(size_t max_addresses, size_t max_pct, std::optional<Network> network, const bool filtered) const { AssertLockHeld(cs); @@ -830,7 +832,7 @@ std::vector<CAddress> AddrManImpl::GetAddr_(size_t max_addresses, size_t max_pct if (network != std::nullopt && ai.GetNetClass() != network) continue; // Filter for quality - if (ai.IsTerrible(now)) continue; + if (ai.IsTerrible(now) && filtered) continue; addresses.push_back(ai); } @@ -1214,11 +1216,11 @@ std::pair<CAddress, NodeSeconds> AddrManImpl::Select(bool new_only, std::optiona return addrRet; } -std::vector<CAddress> AddrManImpl::GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const +std::vector<CAddress> AddrManImpl::GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network, const bool filtered) const { LOCK(cs); Check(); - auto addresses = GetAddr_(max_addresses, max_pct, network); + auto addresses = GetAddr_(max_addresses, max_pct, network, filtered); Check(); return addresses; } @@ -1317,9 +1319,9 @@ std::pair<CAddress, NodeSeconds> AddrMan::Select(bool new_only, std::optional<Ne return m_impl->Select(new_only, network); } -std::vector<CAddress> AddrMan::GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const +std::vector<CAddress> AddrMan::GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network, const bool filtered) const { - return m_impl->GetAddr(max_addresses, max_pct, network); + return m_impl->GetAddr(max_addresses, max_pct, network, filtered); } std::vector<std::pair<AddrInfo, AddressPosition>> AddrMan::GetEntries(bool use_tried) const diff --git a/src/addrman.h b/src/addrman.h index 4d44c943ac..ef9c766eff 100644 --- a/src/addrman.h +++ b/src/addrman.h @@ -164,15 +164,16 @@ public: * @param[in] max_addresses Maximum number of addresses to return (0 = all). * @param[in] max_pct Maximum percentage of addresses to return (0 = all). * @param[in] network Select only addresses of this network (nullopt = all). + * @param[in] filtered Select only addresses that are considered good quality (false = all). * * @return A vector of randomly selected addresses from vRandom. */ - std::vector<CAddress> GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const; + std::vector<CAddress> GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network, const bool filtered = true) const; /** * Returns an information-location pair for all addresses in the selected addrman table. * If an address appears multiple times in the new table, an information-location pair - * is returned for each occurence. Addresses only ever appear once in the tried table. + * is returned for each occurrence. Addresses only ever appear once in the tried table. * * @param[in] from_tried Selects which table to return entries from. * diff --git a/src/addrman_impl.h b/src/addrman_impl.h index 512f085a21..867c894d01 100644 --- a/src/addrman_impl.h +++ b/src/addrman_impl.h @@ -129,7 +129,7 @@ public: std::pair<CAddress, NodeSeconds> Select(bool new_only, std::optional<Network> network) const EXCLUSIVE_LOCKS_REQUIRED(!cs); - std::vector<CAddress> GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network) const + std::vector<CAddress> GetAddr(size_t max_addresses, size_t max_pct, std::optional<Network> network, const bool filtered = true) const EXCLUSIVE_LOCKS_REQUIRED(!cs); std::vector<std::pair<AddrInfo, AddressPosition>> GetEntries(bool from_tried) const @@ -261,7 +261,7 @@ private: * */ int GetEntry(bool use_tried, size_t bucket, size_t position) const EXCLUSIVE_LOCKS_REQUIRED(cs); - std::vector<CAddress> GetAddr_(size_t max_addresses, size_t max_pct, std::optional<Network> network) const EXCLUSIVE_LOCKS_REQUIRED(cs); + std::vector<CAddress> GetAddr_(size_t max_addresses, size_t max_pct, std::optional<Network> network, const bool filtered = true) const EXCLUSIVE_LOCKS_REQUIRED(cs); std::vector<std::pair<AddrInfo, AddressPosition>> GetEntries_(bool from_tried) const EXCLUSIVE_LOCKS_REQUIRED(cs); diff --git a/src/arith_uint256.cpp b/src/arith_uint256.cpp index 3776cfb6de..0d5b3d5b0e 100644 --- a/src/arith_uint256.cpp +++ b/src/arith_uint256.cpp @@ -8,14 +8,7 @@ #include <uint256.h> #include <crypto/common.h> - -template <unsigned int BITS> -base_uint<BITS>::base_uint(const std::string& str) -{ - static_assert(BITS/32 > 0 && BITS%32 == 0, "Template parameter BITS must be a positive multiple of 32."); - - SetHex(str); -} +#include <cassert> template <unsigned int BITS> base_uint<BITS>& base_uint<BITS>::operator<<=(unsigned int shift) @@ -154,22 +147,6 @@ std::string base_uint<BITS>::GetHex() const } template <unsigned int BITS> -void base_uint<BITS>::SetHex(const char* psz) -{ - base_blob<BITS> b; - b.SetHex(psz); - for (int x = 0; x < this->WIDTH; ++x) { - this->pn[x] = ReadLE32(b.begin() + x*4); - } -} - -template <unsigned int BITS> -void base_uint<BITS>::SetHex(const std::string& str) -{ - SetHex(str.c_str()); -} - -template <unsigned int BITS> std::string base_uint<BITS>::ToString() const { return GetHex(); diff --git a/src/arith_uint256.h b/src/arith_uint256.h index c710fe9471..ba36cebbdc 100644 --- a/src/arith_uint256.h +++ b/src/arith_uint256.h @@ -6,10 +6,10 @@ #ifndef BITCOIN_ARITH_UINT256_H #define BITCOIN_ARITH_UINT256_H +#include <cstdint> #include <cstring> #include <limits> #include <stdexcept> -#include <stdint.h> #include <string> class uint256; @@ -56,8 +56,6 @@ public: pn[i] = 0; } - explicit base_uint(const std::string& str); - base_uint operator~() const { base_uint ret; @@ -219,8 +217,6 @@ public: friend inline bool operator!=(const base_uint& a, uint64_t b) { return !a.EqualTo(b); } std::string GetHex() const; - void SetHex(const char* psz); - void SetHex(const std::string& str); std::string ToString() const; unsigned int size() const @@ -247,7 +243,6 @@ public: arith_uint256() {} arith_uint256(const base_uint<256>& b) : base_uint<256>(b) {} arith_uint256(uint64_t b) : base_uint<256>(b) {} - explicit arith_uint256(const std::string& str) : base_uint<256>(str) {} /** * The "compact" format is a representation of a whole diff --git a/src/banman.cpp b/src/banman.cpp index a96b7e3c53..9f668d76a3 100644 --- a/src/banman.cpp +++ b/src/banman.cpp @@ -28,7 +28,7 @@ BanMan::~BanMan() void BanMan::LoadBanlist() { - LOCK(m_cs_banned); + LOCK(m_banned_mutex); if (m_client_interface) m_client_interface->InitMessage(_("Loading banlist…").translated); @@ -52,16 +52,17 @@ void BanMan::DumpBanlist() banmap_t banmap; { - LOCK(m_cs_banned); + LOCK(m_banned_mutex); SweepBanned(); - if (!BannedSetIsDirty()) return; + if (!m_is_dirty) return; banmap = m_banned; - SetBannedSetDirty(false); + m_is_dirty = false; } const auto start{SteadyClock::now()}; if (!m_ban_db.Write(banmap)) { - SetBannedSetDirty(true); + LOCK(m_banned_mutex); + m_is_dirty = true; } LogPrint(BCLog::NET, "Flushed %d banned node addresses/subnets to disk %dms\n", banmap.size(), @@ -71,7 +72,7 @@ void BanMan::DumpBanlist() void BanMan::ClearBanned() { { - LOCK(m_cs_banned); + LOCK(m_banned_mutex); m_banned.clear(); m_is_dirty = true; } @@ -81,14 +82,14 @@ void BanMan::ClearBanned() bool BanMan::IsDiscouraged(const CNetAddr& net_addr) { - LOCK(m_cs_banned); + LOCK(m_banned_mutex); return m_discouraged.contains(net_addr.GetAddrBytes()); } bool BanMan::IsBanned(const CNetAddr& net_addr) { auto current_time = GetTime(); - LOCK(m_cs_banned); + LOCK(m_banned_mutex); for (const auto& it : m_banned) { CSubNet sub_net = it.first; CBanEntry ban_entry = it.second; @@ -103,7 +104,7 @@ bool BanMan::IsBanned(const CNetAddr& net_addr) bool BanMan::IsBanned(const CSubNet& sub_net) { auto current_time = GetTime(); - LOCK(m_cs_banned); + LOCK(m_banned_mutex); banmap_t::iterator i = m_banned.find(sub_net); if (i != m_banned.end()) { CBanEntry ban_entry = (*i).second; @@ -122,7 +123,7 @@ void BanMan::Ban(const CNetAddr& net_addr, int64_t ban_time_offset, bool since_u void BanMan::Discourage(const CNetAddr& net_addr) { - LOCK(m_cs_banned); + LOCK(m_banned_mutex); m_discouraged.insert(net_addr.GetAddrBytes()); } @@ -139,7 +140,7 @@ void BanMan::Ban(const CSubNet& sub_net, int64_t ban_time_offset, bool since_uni ban_entry.nBanUntil = (normalized_since_unix_epoch ? 0 : GetTime()) + normalized_ban_time_offset; { - LOCK(m_cs_banned); + LOCK(m_banned_mutex); if (m_banned[sub_net].nBanUntil < ban_entry.nBanUntil) { m_banned[sub_net] = ban_entry; m_is_dirty = true; @@ -161,7 +162,7 @@ bool BanMan::Unban(const CNetAddr& net_addr) bool BanMan::Unban(const CSubNet& sub_net) { { - LOCK(m_cs_banned); + LOCK(m_banned_mutex); if (m_banned.erase(sub_net) == 0) return false; m_is_dirty = true; } @@ -172,7 +173,7 @@ bool BanMan::Unban(const CSubNet& sub_net) void BanMan::GetBanned(banmap_t& banmap) { - LOCK(m_cs_banned); + LOCK(m_banned_mutex); // Sweep the banlist so expired bans are not returned SweepBanned(); banmap = m_banned; //create a thread safe copy @@ -180,7 +181,7 @@ void BanMan::GetBanned(banmap_t& banmap) void BanMan::SweepBanned() { - AssertLockHeld(m_cs_banned); + AssertLockHeld(m_banned_mutex); int64_t now = GetTime(); bool notify_ui = false; @@ -203,15 +204,3 @@ void BanMan::SweepBanned() m_client_interface->BannedListChanged(); } } - -bool BanMan::BannedSetIsDirty() -{ - LOCK(m_cs_banned); - return m_is_dirty; -} - -void BanMan::SetBannedSetDirty(bool dirty) -{ - LOCK(m_cs_banned); //reuse m_banned lock for the m_is_dirty flag - m_is_dirty = dirty; -} diff --git a/src/banman.h b/src/banman.h index 5a5f5677b0..c6df7ec3c3 100644 --- a/src/banman.h +++ b/src/banman.h @@ -60,40 +60,37 @@ class BanMan public: ~BanMan(); BanMan(fs::path ban_file, CClientUIInterface* client_interface, int64_t default_ban_time); - void Ban(const CNetAddr& net_addr, int64_t ban_time_offset = 0, bool since_unix_epoch = false); - void Ban(const CSubNet& sub_net, int64_t ban_time_offset = 0, bool since_unix_epoch = false); - void Discourage(const CNetAddr& net_addr); - void ClearBanned(); + void Ban(const CNetAddr& net_addr, int64_t ban_time_offset = 0, bool since_unix_epoch = false) EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); + void Ban(const CSubNet& sub_net, int64_t ban_time_offset = 0, bool since_unix_epoch = false) EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); + void Discourage(const CNetAddr& net_addr) EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); + void ClearBanned() EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); //! Return whether net_addr is banned - bool IsBanned(const CNetAddr& net_addr); + bool IsBanned(const CNetAddr& net_addr) EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); //! Return whether sub_net is exactly banned - bool IsBanned(const CSubNet& sub_net); + bool IsBanned(const CSubNet& sub_net) EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); //! Return whether net_addr is discouraged. - bool IsDiscouraged(const CNetAddr& net_addr); + bool IsDiscouraged(const CNetAddr& net_addr) EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); - bool Unban(const CNetAddr& net_addr); - bool Unban(const CSubNet& sub_net); - void GetBanned(banmap_t& banmap); - void DumpBanlist(); + bool Unban(const CNetAddr& net_addr) EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); + bool Unban(const CSubNet& sub_net) EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); + void GetBanned(banmap_t& banmap) EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); + void DumpBanlist() EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); private: - void LoadBanlist() EXCLUSIVE_LOCKS_REQUIRED(!m_cs_banned); - bool BannedSetIsDirty(); - //!set the "dirty" flag for the banlist - void SetBannedSetDirty(bool dirty = true); + void LoadBanlist() EXCLUSIVE_LOCKS_REQUIRED(!m_banned_mutex); //!clean unused entries (if bantime has expired) - void SweepBanned() EXCLUSIVE_LOCKS_REQUIRED(m_cs_banned); + void SweepBanned() EXCLUSIVE_LOCKS_REQUIRED(m_banned_mutex); - RecursiveMutex m_cs_banned; - banmap_t m_banned GUARDED_BY(m_cs_banned); - bool m_is_dirty GUARDED_BY(m_cs_banned){false}; + Mutex m_banned_mutex; + banmap_t m_banned GUARDED_BY(m_banned_mutex); + bool m_is_dirty GUARDED_BY(m_banned_mutex){false}; CClientUIInterface* m_client_interface = nullptr; CBanDB m_ban_db; const int64_t m_default_ban_time; - CRollingBloomFilter m_discouraged GUARDED_BY(m_cs_banned) {50000, 0.000001}; + CRollingBloomFilter m_discouraged GUARDED_BY(m_banned_mutex) {50000, 0.000001}; }; #endif // BITCOIN_BANMAN_H diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 269ac847a5..b81878006c 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -18,13 +18,13 @@ static void DeserializeBlockTest(benchmark::Bench& bench) { - CDataStream stream(benchmark::data::block413567, SER_NETWORK, PROTOCOL_VERSION); + DataStream stream(benchmark::data::block413567); std::byte a{0}; stream.write({&a, 1}); // Prevent compaction bench.unit("block").run([&] { CBlock block; - stream >> block; + stream >> TX_WITH_WITNESS(block); bool rewound = stream.Rewind(benchmark::data::block413567.size()); assert(rewound); }); @@ -32,7 +32,7 @@ static void DeserializeBlockTest(benchmark::Bench& bench) static void DeserializeAndCheckBlockTest(benchmark::Bench& bench) { - CDataStream stream(benchmark::data::block413567, SER_NETWORK, PROTOCOL_VERSION); + DataStream stream(benchmark::data::block413567); std::byte a{0}; stream.write({&a, 1}); // Prevent compaction @@ -41,7 +41,7 @@ static void DeserializeAndCheckBlockTest(benchmark::Bench& bench) bench.unit("block").run([&] { CBlock block; // Note that CBlock caches its checked state, so we need to recreate it here - stream >> block; + stream >> TX_WITH_WITNESS(block); bool rewound = stream.Rewind(benchmark::data::block413567.size()); assert(rewound); diff --git a/src/bench/checkqueue.cpp b/src/bench/checkqueue.cpp index 70e0b86eba..114dd9d39c 100644 --- a/src/bench/checkqueue.cpp +++ b/src/bench/checkqueue.cpp @@ -37,10 +37,11 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) return true; } }; - CCheckQueue<PrevectorJob> queue {QUEUE_BATCH_SIZE}; + // The main thread should be counted to prevent thread oversubscription, and // to decrease the variance of benchmark results. - queue.StartWorkerThreads(GetNumCores() - 1); + int worker_threads_num{GetNumCores() - 1}; + CCheckQueue<PrevectorJob> queue{QUEUE_BATCH_SIZE, worker_threads_num}; // create all the data once, then submit copies in the benchmark. FastRandomContext insecure_rand(true); @@ -61,7 +62,6 @@ static void CCheckQueueSpeedPrevectorJob(benchmark::Bench& bench) // it is done explicitly here for clarity control.Wait(); }); - queue.StopWorkerThreads(); ECC_Stop(); } BENCHMARK(CCheckQueueSpeedPrevectorJob, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/disconnected_transactions.cpp b/src/bench/disconnected_transactions.cpp index 264c0aa1e8..f48175d472 100644 --- a/src/bench/disconnected_transactions.cpp +++ b/src/bench/disconnected_transactions.cpp @@ -28,7 +28,7 @@ struct ReorgTxns { static BlockTxns CreateRandomTransactions(size_t num_txns) { // Ensure every transaction has a different txid by having each one spend the previous one. - static uint256 prevout_hash{uint256::ZERO}; + static Txid prevout_hash{}; BlockTxns txns; txns.reserve(num_txns); @@ -73,7 +73,7 @@ static ReorgTxns CreateBlocks(size_t num_not_shared) static void Reorg(const ReorgTxns& reorg) { - DisconnectedBlockTransactions disconnectpool{MAX_DISCONNECTED_TX_POOL_SIZE * 1000}; + DisconnectedBlockTransactions disconnectpool{MAX_DISCONNECTED_TX_POOL_BYTES}; // Disconnect block const auto evicted = disconnectpool.AddTransactionsFromBlock(reorg.disconnected_txns); assert(evicted.empty()); diff --git a/src/bench/duplicate_inputs.cpp b/src/bench/duplicate_inputs.cpp index a1c8d4d942..b56054ae89 100644 --- a/src/bench/duplicate_inputs.cpp +++ b/src/bench/duplicate_inputs.cpp @@ -47,7 +47,7 @@ static void DuplicateInputs(benchmark::Bench& bench) uint64_t n_inputs = (((MAX_BLOCK_SERIALIZED_SIZE / WITNESS_SCALE_FACTOR) - (CTransaction(coinbaseTx).GetTotalSize() + CTransaction(naughtyTx).GetTotalSize())) / 41) - 100; for (uint64_t x = 0; x < (n_inputs - 1); ++x) { - naughtyTx.vin.emplace_back(GetRandHash(), 0, CScript(), 0); + naughtyTx.vin.emplace_back(Txid::FromUint256(GetRandHash()), 0, CScript(), 0); } naughtyTx.vin.emplace_back(naughtyTx.vin.back()); diff --git a/src/bench/ellswift.cpp b/src/bench/ellswift.cpp index f0348421b3..82e8dec982 100644 --- a/src/bench/ellswift.cpp +++ b/src/bench/ellswift.cpp @@ -11,9 +11,7 @@ static void EllSwiftCreate(benchmark::Bench& bench) { ECC_Start(); - CKey key; - key.MakeNewKey(true); - + CKey key = GenerateRandomKey(); uint256 entropy = GetRandHash(); bench.batch(1).unit("pubkey").run([&] { diff --git a/src/bench/load_external.cpp b/src/bench/load_external.cpp index 3b100d97b0..fba1233901 100644 --- a/src/bench/load_external.cpp +++ b/src/bench/load_external.cpp @@ -55,7 +55,7 @@ static void LoadExternalBlockFile(benchmark::Bench& bench) bench.run([&] { // "rb" is "binary, O_RDONLY", positioned to the start of the file. // The file will be closed by LoadExternalBlockFile(). - CAutoFile file{fsbridge::fopen(blkfile, "rb"), CLIENT_VERSION}; + AutoFile file{fsbridge::fopen(blkfile, "rb")}; testing_setup->m_node.chainman->LoadExternalBlockFile(file, &pos, &blocks_with_unknown_parent); }); fs::remove(blkfile); diff --git a/src/bench/mempool_stress.cpp b/src/bench/mempool_stress.cpp index 993db66653..fe3e204fb3 100644 --- a/src/bench/mempool_stress.cpp +++ b/src/bench/mempool_stress.cpp @@ -56,7 +56,7 @@ static std::vector<CTransactionRef> CreateOrderedCoins(FastRandomContext& det_ra for (size_t ancestor = 0; ancestor < n_ancestors && !available_coins.empty(); ++ancestor){ size_t idx = det_rand.randrange(available_coins.size()); Available coin = available_coins[idx]; - uint256 hash = coin.ref->GetHash(); + Txid hash = coin.ref->GetHash(); // biased towards taking min_ancestors parents, but maybe more size_t n_to_take = det_rand.randrange(2) == 0 ? min_ancestors : diff --git a/src/bench/nanobench.h b/src/bench/nanobench.h index 8b3dc6c71c..127240d3c7 100644 --- a/src/bench/nanobench.h +++ b/src/bench/nanobench.h @@ -33,7 +33,7 @@ // see https://semver.org/ #define ANKERL_NANOBENCH_VERSION_MAJOR 4 // incompatible API changes #define ANKERL_NANOBENCH_VERSION_MINOR 3 // backwards-compatible changes -#define ANKERL_NANOBENCH_VERSION_PATCH 10 // backwards-compatible bug fixes +#define ANKERL_NANOBENCH_VERSION_PATCH 11 // backwards-compatible bug fixes /////////////////////////////////////////////////////////////////////////////////////////////////// // public facing api - as minimal as possible @@ -120,6 +120,10 @@ # define ANKERL_NANOBENCH_IS_TRIVIALLY_COPYABLE(...) std::is_trivially_copyable<__VA_ARGS__>::value #endif +// noexcept may be missing for std::string. +// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58265 +#define ANKERL_NANOBENCH_PRIVATE_NOEXCEPT_STRING_MOVE() std::is_nothrow_move_assignable<std::string>::value + // declarations /////////////////////////////////////////////////////////////////////////////////// namespace ankerl { @@ -404,7 +408,7 @@ struct Config { Config(); ~Config(); Config& operator=(Config const& other); - Config& operator=(Config&& other) noexcept; + Config& operator=(Config&& other) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)); Config(Config const& other); Config(Config&& other) noexcept; }; @@ -430,7 +434,7 @@ public: ~Result(); Result& operator=(Result const& other); - Result& operator=(Result&& other) noexcept; + Result& operator=(Result&& other) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)); Result(Result const& other); Result(Result&& other) noexcept; @@ -596,7 +600,7 @@ public: * * @return Vector containing the full state: */ - std::vector<uint64_t> state() const; + ANKERL_NANOBENCH(NODISCARD) std::vector<uint64_t> state() const; private: static constexpr uint64_t rotl(uint64_t x, unsigned k) noexcept; @@ -628,7 +632,7 @@ public: Bench(); Bench(Bench&& other) noexcept; - Bench& operator=(Bench&& other) noexcept; + Bench& operator=(Bench&& other) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)); Bench(Bench const& other); Bench& operator=(Bench const& other); ~Bench() noexcept; @@ -818,7 +822,7 @@ public: * Default is zero, so we are fully relying on clockResolutionMultiple(). In most cases this is exactly what you want. If you see * that the evaluation is unreliable with a high `err%`, you can increase either minEpochTime() or minEpochIterations(). * - * @see maxEpochTim), minEpochIterations + * @see maxEpochTime, minEpochIterations * * @param t Minimum time each epoch should take. */ @@ -1030,7 +1034,7 @@ void doNotOptimizeAway(T const& val); // These assembly magic is directly from what Google Benchmark is doing. I have previously used what facebook's folly was doing, but // this seemed to have compilation problems in some cases. Google Benchmark seemed to be the most well tested anyways. -// see https://github.com/google/benchmark/blob/master/include/benchmark/benchmark.h#L307 +// see https://github.com/google/benchmark/blob/v1.7.1/include/benchmark/benchmark.h#L443-L446 template <typename T> void doNotOptimizeAway(T const& val) { // NOLINTNEXTLINE(hicpp-no-assembler) @@ -1781,7 +1785,7 @@ bool isEndlessRunning(std::string const& name); bool isWarningsEnabled(); template <typename T> -T parseFile(std::string const& filename); +T parseFile(std::string const& filename, bool* fail); void gatherStabilityInformation(std::vector<std::string>& warnings, std::vector<std::string>& recommendations); void printStabilityInformationOnce(std::ostream* outStream); @@ -1839,7 +1843,7 @@ class Number { public: Number(int width, int precision, double value); Number(int width, int precision, int64_t value); - std::string to_s() const; + ANKERL_NANOBENCH(NODISCARD) std::string to_s() const; private: friend std::ostream& operator<<(std::ostream& os, Number const& n); @@ -1857,11 +1861,11 @@ std::ostream& operator<<(std::ostream& os, Number const& n); class MarkDownColumn { public: - MarkDownColumn(int w, int prec, std::string tit, std::string suff, double val); - std::string title() const; - std::string separator() const; - std::string invalid() const; - std::string value() const; + MarkDownColumn(int w, int prec, std::string tit, std::string suff, double val) noexcept; + ANKERL_NANOBENCH(NODISCARD) std::string title() const; + ANKERL_NANOBENCH(NODISCARD) std::string separator() const; + ANKERL_NANOBENCH(NODISCARD) std::string invalid() const; + ANKERL_NANOBENCH(NODISCARD) std::string value() const; private: int mWidth; @@ -1976,9 +1980,9 @@ PerformanceCounters& performanceCounters() { } // Windows version of doNotOptimizeAway -// see https://github.com/google/benchmark/blob/master/include/benchmark/benchmark.h#L307 -// see https://github.com/facebook/folly/blob/master/folly/Benchmark.h#L280 -// see https://docs.microsoft.com/en-us/cpp/preprocessor/optimize +// see https://github.com/google/benchmark/blob/v1.7.1/include/benchmark/benchmark.h#L514 +// see https://github.com/facebook/folly/blob/v2023.01.30.00/folly/lang/Hint-inl.h#L54-L58 +// see https://learn.microsoft.com/en-us/cpp/preprocessor/optimize # if defined(_MSC_VER) # pragma optimize("", off) void doNotOptimizeAwaySink(void const*) {} @@ -1986,10 +1990,13 @@ void doNotOptimizeAwaySink(void const*) {} # endif template <typename T> -T parseFile(std::string const& filename) { +T parseFile(std::string const& filename, bool* fail) { std::ifstream fin(filename); // NOLINT(misc-const-correctness) T num{}; fin >> num; + if (fail != nullptr) { + *fail = fin.fail(); + } return num; } @@ -2032,16 +2039,15 @@ void gatherStabilityInformation(std::vector<std::string>& warnings, std::vector< if (nprocs <= 0) { warnings.emplace_back("couldn't figure out number of processors - no governor, turbo check possible"); } else { - // check frequency scaling for (long id = 0; id < nprocs; ++id) { auto idStr = detail::fmt::to_s(static_cast<uint64_t>(id)); auto sysCpu = "/sys/devices/system/cpu/cpu" + idStr; - auto minFreq = parseFile<int64_t>(sysCpu + "/cpufreq/scaling_min_freq"); - auto maxFreq = parseFile<int64_t>(sysCpu + "/cpufreq/scaling_max_freq"); + auto minFreq = parseFile<int64_t>(sysCpu + "/cpufreq/scaling_min_freq", nullptr); + auto maxFreq = parseFile<int64_t>(sysCpu + "/cpufreq/scaling_max_freq", nullptr); if (minFreq != maxFreq) { - auto minMHz = static_cast<double>(minFreq) / 1000.0; - auto maxMHz = static_cast<double>(maxFreq) / 1000.0; + auto minMHz = d(minFreq) / 1000.0; + auto maxMHz = d(maxFreq) / 1000.0; warnings.emplace_back("CPU frequency scaling enabled: CPU " + idStr + " between " + detail::fmt::Number(1, 1, minMHz).to_s() + " and " + detail::fmt::Number(1, 1, maxMHz).to_s() + " MHz"); @@ -2050,13 +2056,15 @@ void gatherStabilityInformation(std::vector<std::string>& warnings, std::vector< } } - auto currentGovernor = parseFile<std::string>("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"); - if ("performance" != currentGovernor) { + auto fail = false; + auto currentGovernor = parseFile<std::string>("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor", &fail); + if (!fail && "performance" != currentGovernor) { warnings.emplace_back("CPU governor is '" + currentGovernor + "' but should be 'performance'"); recommendPyPerf = true; } - if (0 == parseFile<int>("/sys/devices/system/cpu/intel_pstate/no_turbo")) { + auto noTurbo = parseFile<int>("/sys/devices/system/cpu/intel_pstate/no_turbo", &fail); + if (!fail && noTurbo == 0) { warnings.emplace_back("Turbo is enabled, CPU frequency will fluctuate"); recommendPyPerf = true; } @@ -2250,10 +2258,9 @@ struct IterationLogic::Impl { mNumIters = 0; } - ANKERL_NANOBENCH_LOG(mBench.name() << ": " << detail::fmt::Number(20, 3, static_cast<double>(elapsed.count())) << " elapsed, " - << detail::fmt::Number(20, 3, static_cast<double>(mTargetRuntimePerEpoch.count())) - << " target. oldIters=" << oldIters << ", mNumIters=" << mNumIters - << ", mState=" << static_cast<int>(mState)); + ANKERL_NANOBENCH_LOG(mBench.name() << ": " << detail::fmt::Number(20, 3, d(elapsed.count())) << " elapsed, " + << detail::fmt::Number(20, 3, d(mTargetRuntimePerEpoch.count())) << " target. oldIters=" + << oldIters << ", mNumIters=" << mNumIters << ", mState=" << static_cast<int>(mState)); } // NOLINTNEXTLINE(readability-function-cognitive-complexity) @@ -2357,7 +2364,7 @@ struct IterationLogic::Impl { } os << fmt::MarkDownCode(mBench.name()); if (showUnstable) { - auto avgIters = static_cast<double>(mTotalNumIters) / static_cast<double>(mBench.epochs()); + auto avgIters = d(mTotalNumIters) / d(mBench.epochs()); // NOLINTNEXTLINE(bugprone-incorrect-roundings) auto suggestedIters = static_cast<uint64_t>(avgIters * 10 + 0.5); @@ -2435,7 +2442,7 @@ public: bool monitor(perf_sw_ids swId, Target target); bool monitor(perf_hw_id hwId, Target target); - bool hasError() const noexcept { + ANKERL_NANOBENCH(NODISCARD) bool hasError() const noexcept { return mHasError; } @@ -2691,16 +2698,23 @@ PerformanceCounters::PerformanceCounters() , mVal() , mHas() { - mHas.pageFaults = mPc->monitor(PERF_COUNT_SW_PAGE_FAULTS, LinuxPerformanceCounters::Target(&mVal.pageFaults, true, false)); + // HW events mHas.cpuCycles = mPc->monitor(PERF_COUNT_HW_REF_CPU_CYCLES, LinuxPerformanceCounters::Target(&mVal.cpuCycles, true, false)); - mHas.contextSwitches = - mPc->monitor(PERF_COUNT_SW_CONTEXT_SWITCHES, LinuxPerformanceCounters::Target(&mVal.contextSwitches, true, false)); + if (!mHas.cpuCycles) { + // Fallback to cycles counter, reference cycles not available in many systems. + mHas.cpuCycles = mPc->monitor(PERF_COUNT_HW_CPU_CYCLES, LinuxPerformanceCounters::Target(&mVal.cpuCycles, true, false)); + } mHas.instructions = mPc->monitor(PERF_COUNT_HW_INSTRUCTIONS, LinuxPerformanceCounters::Target(&mVal.instructions, true, true)); mHas.branchInstructions = mPc->monitor(PERF_COUNT_HW_BRANCH_INSTRUCTIONS, LinuxPerformanceCounters::Target(&mVal.branchInstructions, true, false)); mHas.branchMisses = mPc->monitor(PERF_COUNT_HW_BRANCH_MISSES, LinuxPerformanceCounters::Target(&mVal.branchMisses, true, false)); // mHas.branchMisses = false; + // SW events + mHas.pageFaults = mPc->monitor(PERF_COUNT_SW_PAGE_FAULTS, LinuxPerformanceCounters::Target(&mVal.pageFaults, true, false)); + mHas.contextSwitches = + mPc->monitor(PERF_COUNT_SW_CONTEXT_SWITCHES, LinuxPerformanceCounters::Target(&mVal.contextSwitches, true, false)); + mPc->start(); mPc->calibrate([] { auto before = ankerl::nanobench::Clock::now(); @@ -2789,7 +2803,7 @@ void StreamStateRestorer::restore() { Number::Number(int width, int precision, int64_t value) : mWidth(width) , mPrecision(precision) - , mValue(static_cast<double>(value)) {} + , mValue(d(value)) {} Number::Number(int width, int precision, double value) : mWidth(width) @@ -2823,7 +2837,7 @@ std::ostream& operator<<(std::ostream& os, Number const& n) { return n.write(os); } -MarkDownColumn::MarkDownColumn(int w, int prec, std::string tit, std::string suff, double val) +MarkDownColumn::MarkDownColumn(int w, int prec, std::string tit, std::string suff, double val) noexcept : mWidth(w) , mPrecision(prec) , mTitle(std::move(tit)) @@ -2884,14 +2898,14 @@ std::ostream& operator<<(std::ostream& os, MarkDownCode const& mdCode) { Config::Config() = default; Config::~Config() = default; Config& Config::operator=(Config const&) = default; -Config& Config::operator=(Config&&) noexcept = default; +Config& Config::operator=(Config&&) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)) = default; Config::Config(Config const&) = default; Config::Config(Config&&) noexcept = default; // provide implementation here so it's only generated once Result::~Result() = default; Result& Result::operator=(Result const&) = default; -Result& Result::operator=(Result&&) noexcept = default; +Result& Result::operator=(Result&&) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)) = default; Result::Result(Result const&) = default; Result::Result(Result&&) noexcept = default; @@ -2992,7 +3006,7 @@ double Result::medianAbsolutePercentError(Measure m) const { auto data = mNameToMeasurements[detail::u(m)]; // calculates MdAPE which is the median of percentage error - // see https://www.spiderfinancial.com/support/documentation/numxl/reference-manual/forecasting-performance/mdape + // see https://support.numxl.com/hc/en-us/articles/115001223503-MdAPE-Median-Absolute-Percentage-Error auto med = calcMedian(data); // transform the data to absolute error @@ -3106,7 +3120,7 @@ Bench::Bench() { } Bench::Bench(Bench&&) noexcept = default; -Bench& Bench::operator=(Bench&&) noexcept = default; +Bench& Bench::operator=(Bench&&) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)) = default; Bench::Bench(Bench const&) = default; Bench& Bench::operator=(Bench const&) = default; Bench::~Bench() noexcept = default; @@ -3423,7 +3437,7 @@ BigO::BigO(std::string bigOName, RangeMeasure const& rangeMeasure) sumMeasure += rm.second; } - auto n = static_cast<double>(rangeMeasure.size()); + auto n = detail::d(rangeMeasure.size()); auto mean = sumMeasure / n; mNormalizedRootMeanSquare = std::sqrt(err / n) / mean; } diff --git a/src/bench/pool.cpp b/src/bench/pool.cpp index b3e54d85a2..b2a5f8debf 100644 --- a/src/bench/pool.cpp +++ b/src/bench/pool.cpp @@ -37,8 +37,7 @@ static void PoolAllocator_StdUnorderedMapWithPoolResource(benchmark::Bench& benc std::hash<uint64_t>, std::equal_to<uint64_t>, PoolAllocator<std::pair<const uint64_t, uint64_t>, - sizeof(std::pair<const uint64_t, uint64_t>) + 4 * sizeof(void*), - alignof(void*)>>; + sizeof(std::pair<const uint64_t, uint64_t>) + 4 * sizeof(void*)>>; // make sure the resource supports large enough pools to hold the node. We do this by adding the size of a few pointers to it. auto pool_resource = Map::allocator_type::ResourceType(); diff --git a/src/bench/readblock.cpp b/src/bench/readblock.cpp new file mode 100644 index 0000000000..0545c6b017 --- /dev/null +++ b/src/bench/readblock.cpp @@ -0,0 +1,53 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <bench/data.h> + +#include <consensus/validation.h> +#include <node/blockstorage.h> +#include <streams.h> +#include <test/util/setup_common.h> +#include <util/chaintype.h> +#include <validation.h> + +static FlatFilePos WriteBlockToDisk(ChainstateManager& chainman) +{ + DataStream stream{benchmark::data::block413567}; + CBlock block; + stream >> TX_WITH_WITNESS(block); + + return chainman.m_blockman.SaveBlockToDisk(block, 0, nullptr); +} + +static void ReadBlockFromDiskTest(benchmark::Bench& bench) +{ + const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN)}; + ChainstateManager& chainman{*testing_setup->m_node.chainman}; + + CBlock block; + const auto pos{WriteBlockToDisk(chainman)}; + + bench.run([&] { + const auto success{chainman.m_blockman.ReadBlockFromDisk(block, pos)}; + assert(success); + }); +} + +static void ReadRawBlockFromDiskTest(benchmark::Bench& bench) +{ + const auto testing_setup{MakeNoLogFileContext<const TestingSetup>(ChainType::MAIN)}; + ChainstateManager& chainman{*testing_setup->m_node.chainman}; + + std::vector<uint8_t> block_data; + const auto pos{WriteBlockToDisk(chainman)}; + + bench.run([&] { + const auto success{chainman.m_blockman.ReadRawBlockFromDisk(block_data, pos)}; + assert(success); + }); +} + +BENCHMARK(ReadBlockFromDiskTest, benchmark::PriorityLevel::HIGH); +BENCHMARK(ReadRawBlockFromDiskTest, benchmark::PriorityLevel::HIGH); diff --git a/src/bench/rpc_blockchain.cpp b/src/bench/rpc_blockchain.cpp index a9b197b190..713853e8c5 100644 --- a/src/bench/rpc_blockchain.cpp +++ b/src/bench/rpc_blockchain.cpp @@ -23,11 +23,11 @@ struct TestBlockAndIndex { TestBlockAndIndex() { - CDataStream stream(benchmark::data::block413567, SER_NETWORK, PROTOCOL_VERSION); + DataStream stream{benchmark::data::block413567}; std::byte a{0}; stream.write({&a, 1}); // Prevent compaction - stream >> block; + stream >> TX_WITH_WITNESS(block); blockHash = block.GetHash(); blockindex.phashBlock = &blockHash; @@ -41,7 +41,7 @@ static void BlockToJsonVerbose(benchmark::Bench& bench) { TestBlockAndIndex data; bench.run([&] { - auto univalue = blockToJSON(data.testing_setup->m_node.chainman->m_blockman, data.block, &data.blockindex, &data.blockindex, TxVerbosity::SHOW_DETAILS_AND_PREVOUT); + auto univalue = blockToJSON(data.testing_setup->m_node.chainman->m_blockman, data.block, data.blockindex, data.blockindex, TxVerbosity::SHOW_DETAILS_AND_PREVOUT); ankerl::nanobench::doNotOptimizeAway(univalue); }); } @@ -51,7 +51,7 @@ BENCHMARK(BlockToJsonVerbose, benchmark::PriorityLevel::HIGH); static void BlockToJsonVerboseWrite(benchmark::Bench& bench) { TestBlockAndIndex data; - auto univalue = blockToJSON(data.testing_setup->m_node.chainman->m_blockman, data.block, &data.blockindex, &data.blockindex, TxVerbosity::SHOW_DETAILS_AND_PREVOUT); + auto univalue = blockToJSON(data.testing_setup->m_node.chainman->m_blockman, data.block, data.blockindex, data.blockindex, TxVerbosity::SHOW_DETAILS_AND_PREVOUT); bench.run([&] { auto str = univalue.write(); ankerl::nanobench::doNotOptimizeAway(str); diff --git a/src/bench/streams_findbyte.cpp b/src/bench/streams_findbyte.cpp index 4aaccb2af8..5098262e9a 100644 --- a/src/bench/streams_findbyte.cpp +++ b/src/bench/streams_findbyte.cpp @@ -14,7 +14,7 @@ static void FindByte(benchmark::Bench& bench) { // Setup - CAutoFile file{fsbridge::fopen("streams_tmp", "w+b"), 0}; + AutoFile file{fsbridge::fopen("streams_tmp", "w+b")}; const size_t file_size = 200; uint8_t data[file_size] = {0}; data[file_size-1] = 1; diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index 11f96b1005..3682b33518 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -61,8 +61,8 @@ static void VerifyScriptBench(benchmark::Bench& bench) assert(success); #if defined(HAVE_CONSENSUS_LIB) - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << txSpend; + DataStream stream; + stream << TX_WITH_WITNESS(txSpend); int csuccess = bitcoinconsensus_verify_script_with_amount( txCredit.vout[0].scriptPubKey.data(), txCredit.vout[0].scriptPubKey.size(), diff --git a/src/bench/wallet_create.cpp b/src/bench/wallet_create.cpp new file mode 100644 index 0000000000..993c4c4b3f --- /dev/null +++ b/src/bench/wallet_create.cpp @@ -0,0 +1,55 @@ +// Copyright (c) 2023-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or https://www.opensource.org/licenses/mit-license.php. + +#include <bench/bench.h> +#include <node/context.h> +#include <random.h> +#include <test/util/setup_common.h> +#include <wallet/context.h> +#include <wallet/wallet.h> + +namespace wallet { +static void WalletCreate(benchmark::Bench& bench, bool encrypted) +{ + auto test_setup = MakeNoLogFileContext<TestingSetup>(); + FastRandomContext random; + + WalletContext context; + context.args = &test_setup->m_args; + context.chain = test_setup->m_node.chain.get(); + + DatabaseOptions options; + options.require_format = DatabaseFormat::SQLITE; + options.require_create = true; + options.create_flags = WALLET_FLAG_DESCRIPTORS; + + if (encrypted) { + options.create_passphrase = random.rand256().ToString(); + } + + DatabaseStatus status; + bilingual_str error_string; + std::vector<bilingual_str> warnings; + + fs::path wallet_path = test_setup->m_path_root / strprintf("test_wallet_%d", random.rand32()).c_str(); + bench.run([&] { + auto wallet = CreateWallet(context, wallet_path.utf8string(), /*load_on_start=*/std::nullopt, options, status, error_string, warnings); + assert(status == DatabaseStatus::SUCCESS); + assert(wallet != nullptr); + + // Cleanup + wallet.reset(); + fs::remove_all(wallet_path); + }); +} + +static void WalletCreatePlain(benchmark::Bench& bench) { WalletCreate(bench, /*encrypted=*/false); } +static void WalletCreateEncrypted(benchmark::Bench& bench) { WalletCreate(bench, /*encrypted=*/true); } + +#ifdef USE_SQLITE +BENCHMARK(WalletCreatePlain, benchmark::PriorityLevel::LOW); +BENCHMARK(WalletCreateEncrypted, benchmark::PriorityLevel::LOW); +#endif + +} // namespace wallet diff --git a/src/bench/wallet_create_tx.cpp b/src/bench/wallet_create_tx.cpp index 160534b63c..84e02d2a26 100644 --- a/src/bench/wallet_create_tx.cpp +++ b/src/bench/wallet_create_tx.cpp @@ -16,7 +16,6 @@ using wallet::CWallet; using wallet::CreateMockableWalletDatabase; -using wallet::DBErrors; using wallet::WALLET_FLAG_DESCRIPTORS; struct TipBlock @@ -90,17 +89,17 @@ static void WalletCreateTx(benchmark::Bench& bench, const OutputType output_type LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet.SetupDescriptorScriptPubKeyMans(); - if (wallet.LoadWallet() != DBErrors::LOAD_OK) assert(false); } // Generate destinations - CScript dest = GetScriptForDestination(getNewDestination(wallet, output_type)); + const auto dest{getNewDestination(wallet, output_type)}; // Generate chain; each coinbase will have two outputs to fill-up the wallet const auto& params = Params(); + const CScript coinbase_out{GetScriptForDestination(dest)}; unsigned int chain_size = 5000; // 5k blocks means 10k UTXO for the wallet (minus 200 due COINBASE_MATURITY) for (unsigned int i = 0; i < chain_size; ++i) { - generateFakeBlock(params, test_setup->m_node, wallet, dest); + generateFakeBlock(params, test_setup->m_node, wallet, coinbase_out); } // Check available balance @@ -130,7 +129,7 @@ static void WalletCreateTx(benchmark::Bench& bench, const OutputType output_type bench.epochIterations(5).run([&] { LOCK(wallet.cs_wallet); - const auto& tx_res = CreateTransaction(wallet, recipients, -1, coin_control); + const auto& tx_res = CreateTransaction(wallet, recipients, /*change_pos=*/std::nullopt, coin_control); assert(tx_res); }); } @@ -145,7 +144,6 @@ static void AvailableCoins(benchmark::Bench& bench, const std::vector<OutputType LOCK(wallet.cs_wallet); wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); wallet.SetupDescriptorScriptPubKeyMans(); - if (wallet.LoadWallet() != DBErrors::LOAD_OK) assert(false); } // Generate destinations @@ -185,4 +183,4 @@ static void WalletAvailableCoins(benchmark::Bench& bench) { AvailableCoins(bench BENCHMARK(WalletCreateTxUseOnlyPresetInputs, benchmark::PriorityLevel::LOW) BENCHMARK(WalletCreateTxUsePresetInputsAndCoinSelection, benchmark::PriorityLevel::LOW) -BENCHMARK(WalletAvailableCoins, benchmark::PriorityLevel::LOW);
\ No newline at end of file +BENCHMARK(WalletAvailableCoins, benchmark::PriorityLevel::LOW); diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index fc83a4ad3a..ee8b0a44c5 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -26,13 +26,13 @@ #include <scheduler.h> #include <script/sigcache.h> #include <util/chaintype.h> +#include <util/fs.h> #include <util/thread.h> #include <validation.h> #include <validationinterface.h> #include <cassert> #include <cstdint> -#include <filesystem> #include <functional> #include <iosfwd> #include <memory> @@ -50,8 +50,8 @@ int main(int argc, char* argv[]) << " BREAK IN FUTURE VERSIONS. DO NOT USE ON YOUR ACTUAL DATADIR." << std::endl; return 1; } - std::filesystem::path abs_datadir = std::filesystem::absolute(argv[1]); - std::filesystem::create_directories(abs_datadir); + fs::path abs_datadir{fs::absolute(argv[1])}; + fs::create_directories(abs_datadir); // SETUP: Context @@ -125,14 +125,14 @@ int main(int argc, char* argv[]) .blocks_dir = abs_datadir / "blocks", .notifications = chainman_opts.notifications, }; - ChainstateManager chainman{kernel_context.interrupt, chainman_opts, blockman_opts}; + util::SignalInterrupt interrupt; + ChainstateManager chainman{interrupt, chainman_opts, blockman_opts}; node::CacheSizes cache_sizes; cache_sizes.block_tree_db = 2 << 20; cache_sizes.coins_db = 2 << 22; cache_sizes.coins = (450 << 20) - (2 << 20) - (2 << 22); node::ChainstateLoadOptions options; - options.check_interrupt = [] { return false; }; auto [status, error] = node::LoadChainstate(chainman, cache_sizes, options); if (status != node::ChainstateLoadStatus::SUCCESS) { std::cerr << "Failed to load Chain state from your datadir." << std::endl; @@ -290,7 +290,6 @@ epilogue: // dereferencing and UB. scheduler.stop(); if (chainman.m_thread_load.joinable()) chainman.m_thread_load.join(); - StopScriptCheckWorkerThreads(); GetMainSignals().FlushBackgroundCallbacks(); { diff --git a/src/bitcoin-tx.cpp b/src/bitcoin-tx.cpp index 103d8885db..320624c419 100644 --- a/src/bitcoin-tx.cpp +++ b/src/bitcoin-tx.cpp @@ -66,7 +66,9 @@ static void SetupBitcoinTxArgs(ArgsManager &argsman) argsman.AddArg("outscript=VALUE:SCRIPT[:FLAGS]", "Add raw script output to TX. " "Optionally add the \"W\" flag to produce a pay-to-witness-script-hash output. " "Optionally add the \"S\" flag to wrap the output in a pay-to-script-hash.", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); - argsman.AddArg("replaceable(=N)", "Set RBF opt-in sequence number for input N (if not provided, opt-in all available inputs)", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); + argsman.AddArg("replaceable(=N)", "Sets Replace-By-Fee (RBF) opt-in sequence number for input N. " + "If N is not provided, the command attempts to opt-in all available inputs for RBF. " + "If the transaction has no inputs, this option is ignored.", ArgsManager::ALLOW_ANY, OptionsCategory::COMMANDS); argsman.AddArg("sign=SIGHASH-FLAGS", "Add zero or more signatures to transaction. " "This command requires JSON registers:" "prevtxs=JSON object, " @@ -223,8 +225,8 @@ static void MutateTxLocktime(CMutableTransaction& tx, const std::string& cmdVal) static void MutateTxRBFOptIn(CMutableTransaction& tx, const std::string& strInIdx) { // parse requested index - int64_t inIdx; - if (!ParseInt64(strInIdx, &inIdx) || inIdx < 0 || inIdx >= static_cast<int64_t>(tx.vin.size())) { + int64_t inIdx = -1; + if (strInIdx != "" && (!ParseInt64(strInIdx, &inIdx) || inIdx < 0 || inIdx >= static_cast<int64_t>(tx.vin.size()))) { throw std::runtime_error("Invalid TX input index '" + strInIdx + "'"); } @@ -280,7 +282,7 @@ static void MutateTxAddInput(CMutableTransaction& tx, const std::string& strInpu } // append to transaction input list - CTxIn txin(txid, vout, CScript(), nSequenceIn); + CTxIn txin(Txid::FromUint256(txid), vout, CScript(), nSequenceIn); tx.vin.push_back(txin); } @@ -629,7 +631,7 @@ static void MutateTxSign(CMutableTransaction& tx, const std::string& flagStr) if (nOut < 0) throw std::runtime_error("vout cannot be negative"); - COutPoint out(txid, nOut); + COutPoint out(Txid::FromUint256(txid), nOut); std::vector<unsigned char> pkData(ParseHexUV(prevOut["scriptPubKey"], "scriptPubKey")); CScript scriptPubKey(pkData.begin(), pkData.end()); diff --git a/src/bitcoin-util.cpp b/src/bitcoin-util.cpp index 8b68d04b2b..96387e8c71 100644 --- a/src/bitcoin-util.cpp +++ b/src/bitcoin-util.cpp @@ -19,7 +19,6 @@ #include <util/exception.h> #include <util/strencodings.h> #include <util/translation.h> -#include <version.h> #include <atomic> #include <cstdio> diff --git a/src/bitcoind.cpp b/src/bitcoind.cpp index e2224befef..4f0a816388 100644 --- a/src/bitcoind.cpp +++ b/src/bitcoind.cpp @@ -20,7 +20,6 @@ #include <node/context.h> #include <node/interface_ui.h> #include <noui.h> -#include <shutdown.h> #include <util/check.h> #include <util/exception.h> #include <util/strencodings.h> @@ -272,9 +271,7 @@ MAIN_FUNCTION if (ProcessInitCommands(args)) return EXIT_SUCCESS; // Start application - if (AppInit(node)) { - WaitForShutdown(); - } else { + if (!AppInit(node) || !Assert(node.shutdown)->wait()) { node.exit_status = EXIT_FAILURE; } Interrupt(node); diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 211b4740be..1e940e8f03 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -107,12 +107,12 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c std::vector<bool> have_txn(txn_available.size()); { LOCK(pool->cs); - for (size_t i = 0; i < pool->vTxHashes.size(); i++) { - uint64_t shortid = cmpctblock.GetShortID(pool->vTxHashes[i].first); + for (const auto& tx : pool->txns_randomized) { + uint64_t shortid = cmpctblock.GetShortID(tx->GetWitnessHash()); std::unordered_map<uint64_t, uint16_t>::iterator idit = shorttxids.find(shortid); if (idit != shorttxids.end()) { if (!have_txn[idit->second]) { - txn_available[idit->second] = pool->vTxHashes[i].second->GetSharedTx(); + txn_available[idit->second] = tx; have_txn[idit->second] = true; mempool_count++; } else { @@ -164,7 +164,7 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c break; } - LogPrint(BCLog::CMPCTBLOCK, "Initialized PartiallyDownloadedBlock for block %s using a cmpctblock of size %lu\n", cmpctblock.header.GetHash().ToString(), GetSerializeSize(cmpctblock, PROTOCOL_VERSION)); + LogPrint(BCLog::CMPCTBLOCK, "Initialized PartiallyDownloadedBlock for block %s using a cmpctblock of size %lu\n", cmpctblock.header.GetHash().ToString(), GetSerializeSize(cmpctblock)); return READ_STATUS_OK; } diff --git a/src/blockencodings.h b/src/blockencodings.h index afdfa426f1..fb0f734ff8 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -65,7 +65,7 @@ public: SERIALIZE_METHODS(BlockTransactions, obj) { - READWRITE(obj.blockhash, Using<VectorFormatter<TransactionCompression>>(obj.txn)); + READWRITE(obj.blockhash, TX_WITH_WITNESS(Using<VectorFormatter<TransactionCompression>>(obj.txn))); } }; @@ -76,7 +76,7 @@ struct PrefilledTransaction { uint16_t index; CTransactionRef tx; - SERIALIZE_METHODS(PrefilledTransaction, obj) { READWRITE(COMPACTSIZE(obj.index), Using<TransactionCompression>(obj.tx)); } + SERIALIZE_METHODS(PrefilledTransaction, obj) { READWRITE(COMPACTSIZE(obj.index), TX_WITH_WITNESS(Using<TransactionCompression>(obj.tx))); } }; typedef enum ReadStatus_t diff --git a/src/blockfilter.cpp b/src/blockfilter.cpp index dd3824fb1c..e045b88513 100644 --- a/src/blockfilter.cpp +++ b/src/blockfilter.cpp @@ -16,9 +16,6 @@ #include <util/golombrice.h> #include <util/string.h> -/// Protocol version used to serialize parameters in GCS filter encoding. -static constexpr int GCS_SER_VERSION = 0; - static const std::map<BlockFilterType, std::string> g_filter_types = { {BlockFilterType::BASIC, "basic"}, }; @@ -49,7 +46,7 @@ GCSFilter::GCSFilter(const Params& params) GCSFilter::GCSFilter(const Params& params, std::vector<unsigned char> encoded_filter, bool skip_decode_check) : m_params(params), m_encoded(std::move(encoded_filter)) { - SpanReader stream{GCS_SER_VERSION, m_encoded}; + SpanReader stream{m_encoded}; uint64_t N = ReadCompactSize(stream); m_N = static_cast<uint32_t>(N); @@ -62,7 +59,7 @@ GCSFilter::GCSFilter(const Params& params, std::vector<unsigned char> encoded_fi // Verify that the encoded filter contains exactly N elements. If it has too much or too little // data, a std::ios_base::failure exception will be raised. - BitStreamReader<SpanReader> bitreader{stream}; + BitStreamReader bitreader{stream}; for (uint64_t i = 0; i < m_N; ++i) { GolombRiceDecode(bitreader, m_params.m_P); } @@ -81,7 +78,7 @@ GCSFilter::GCSFilter(const Params& params, const ElementSet& elements) } m_F = static_cast<uint64_t>(m_N) * static_cast<uint64_t>(m_params.m_M); - CVectorWriter stream(GCS_SER_VERSION, m_encoded, 0); + VectorWriter stream{m_encoded, 0}; WriteCompactSize(stream, m_N); @@ -89,7 +86,7 @@ GCSFilter::GCSFilter(const Params& params, const ElementSet& elements) return; } - BitStreamWriter<CVectorWriter> bitwriter(stream); + BitStreamWriter bitwriter{stream}; uint64_t last_value = 0; for (uint64_t value : BuildHashedSet(elements)) { @@ -103,13 +100,13 @@ GCSFilter::GCSFilter(const Params& params, const ElementSet& elements) bool GCSFilter::MatchInternal(const uint64_t* element_hashes, size_t size) const { - SpanReader stream{GCS_SER_VERSION, m_encoded}; + SpanReader stream{m_encoded}; // Seek forward by size of N uint64_t N = ReadCompactSize(stream); assert(N == m_N); - BitStreamReader<SpanReader> bitreader{stream}; + BitStreamReader bitreader{stream}; uint64_t value = 0; size_t hashes_index = 0; diff --git a/src/checkqueue.h b/src/checkqueue.h index a3299fb3fe..a1de000714 100644 --- a/src/checkqueue.h +++ b/src/checkqueue.h @@ -13,9 +13,6 @@ #include <iterator> #include <vector> -template <typename T> -class CCheckQueueControl; - /** * Queue for verifications that have to be performed. * The verifications are represented by a type T, which must provide an @@ -130,22 +127,11 @@ public: Mutex m_control_mutex; //! Create a new check queue - explicit CCheckQueue(unsigned int nBatchSizeIn) - : nBatchSize(nBatchSizeIn) + explicit CCheckQueue(unsigned int batch_size, int worker_threads_num) + : nBatchSize(batch_size) { - } - - //! Create a pool of new worker threads. - void StartWorkerThreads(const int threads_num) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) - { - { - LOCK(m_mutex); - nIdle = 0; - nTotal = 0; - fAllOk = true; - } - assert(m_worker_threads.empty()); - for (int n = 0; n < threads_num; ++n) { + m_worker_threads.reserve(worker_threads_num); + for (int n = 0; n < worker_threads_num; ++n) { m_worker_threads.emplace_back([this, n]() { util::ThreadRename(strprintf("scriptch.%i", n)); Loop(false /* worker thread */); @@ -153,6 +139,13 @@ public: } } + // Since this class manages its own resources, which is a thread + // pool `m_worker_threads`, copy and move operations are not appropriate. + CCheckQueue(const CCheckQueue&) = delete; + CCheckQueue& operator=(const CCheckQueue&) = delete; + CCheckQueue(CCheckQueue&&) = delete; + CCheckQueue& operator=(CCheckQueue&&) = delete; + //! Wait until execution finishes, and return whether all evaluations were successful. bool Wait() EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) { @@ -179,24 +172,16 @@ public: } } - //! Stop all of the worker threads. - void StopWorkerThreads() EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) + ~CCheckQueue() { WITH_LOCK(m_mutex, m_request_stop = true); m_worker_cv.notify_all(); for (std::thread& t : m_worker_threads) { t.join(); } - m_worker_threads.clear(); - WITH_LOCK(m_mutex, m_request_stop = false); } bool HasThreads() const { return !m_worker_threads.empty(); } - - ~CCheckQueue() - { - assert(m_worker_threads.empty()); - } }; /** diff --git a/src/coins.cpp b/src/coins.cpp index b44d920ee1..b62653e0de 100644 --- a/src/coins.cpp +++ b/src/coins.cpp @@ -8,7 +8,6 @@ #include <logging.h> #include <random.h> #include <util/trace.h> -#include <version.h> bool CCoinsView::GetCoin(const COutPoint &outpoint, Coin &coin) const { return false; } uint256 CCoinsView::GetBestBlock() const { return uint256(); } @@ -117,7 +116,7 @@ void CCoinsViewCache::EmplaceCoinInternalDANGER(COutPoint&& outpoint, Coin&& coi void AddCoins(CCoinsViewCache& cache, const CTransaction &tx, int nHeight, bool check_for_overwrite) { bool fCoinbase = tx.IsCoinBase(); - const uint256& txid = tx.GetHash(); + const Txid& txid = tx.GetHash(); for (size_t i = 0; i < tx.vout.size(); ++i) { bool overwrite = check_for_overwrite ? cache.HaveCoin(COutPoint(txid, i)) : fCoinbase; // Coinbase transactions can always be overwritten, in order to correctly @@ -339,10 +338,10 @@ void CCoinsViewCache::SanityCheck() const assert(recomputed_usage == cachedCoinsUsage); } -static const size_t MIN_TRANSACTION_OUTPUT_WEIGHT = WITNESS_SCALE_FACTOR * ::GetSerializeSize(CTxOut(), PROTOCOL_VERSION); +static const size_t MIN_TRANSACTION_OUTPUT_WEIGHT = WITNESS_SCALE_FACTOR * ::GetSerializeSize(CTxOut()); static const size_t MAX_OUTPUTS_PER_BLOCK = MAX_BLOCK_WEIGHT / MIN_TRANSACTION_OUTPUT_WEIGHT; -const Coin& AccessByTxid(const CCoinsViewCache& view, const uint256& txid) +const Coin& AccessByTxid(const CCoinsViewCache& view, const Txid& txid) { COutPoint iter(txid, 0); while (iter.n < MAX_OUTPUTS_PER_BLOCK) { diff --git a/src/coins.h b/src/coins.h index a6cbb03133..c798cc38ba 100644 --- a/src/coins.h +++ b/src/coins.h @@ -145,8 +145,7 @@ using CCoinsMap = std::unordered_map<COutPoint, SaltedOutpointHasher, std::equal_to<COutPoint>, PoolAllocator<std::pair<const COutPoint, CCoinsCacheEntry>, - sizeof(std::pair<const COutPoint, CCoinsCacheEntry>) + sizeof(void*) * 4, - alignof(void*)>>; + sizeof(std::pair<const COutPoint, CCoinsCacheEntry>) + sizeof(void*) * 4>>; using CCoinsMapMemoryResource = CCoinsMap::allocator_type::ResourceType; @@ -364,7 +363,7 @@ void AddCoins(CCoinsViewCache& cache, const CTransaction& tx, int nHeight, bool //! This function can be quite expensive because in the event of a transaction //! which is not found in the cache, it can cause up to MAX_OUTPUTS_PER_BLOCK //! lookups to database, so it should be used with care. -const Coin& AccessByTxid(const CCoinsViewCache& cache, const uint256& txid); +const Coin& AccessByTxid(const CCoinsViewCache& cache, const Txid& txid); /** * This is a minimally invasive approach to shutdown on LevelDB read errors from the diff --git a/src/common/args.cpp b/src/common/args.cpp index ca04175696..a9108e5916 100644 --- a/src/common/args.cpp +++ b/src/common/args.cpp @@ -28,7 +28,6 @@ #include <cstdint> #include <cstdlib> #include <cstring> -#include <filesystem> #include <map> #include <optional> #include <stdexcept> @@ -278,7 +277,7 @@ fs::path ArgsManager::GetPathArg(std::string arg, const fs::path& default_value) return result.has_filename() ? result : result.parent_path(); } -const fs::path& ArgsManager::GetBlocksDirPath() const +fs::path ArgsManager::GetBlocksDirPath() const { LOCK(cs_args); fs::path& path = m_cached_blocks_path; @@ -303,7 +302,7 @@ const fs::path& ArgsManager::GetBlocksDirPath() const return path; } -const fs::path& ArgsManager::GetDataDir(bool net_specific) const +fs::path ArgsManager::GetDataDir(bool net_specific) const { LOCK(cs_args); fs::path& path = net_specific ? m_cached_network_datadir_path : m_cached_datadir_path; @@ -720,6 +719,13 @@ fs::path ArgsManager::GetConfigFilePath() const return *Assert(m_config_path); } +void ArgsManager::SetConfigFilePath(fs::path path) +{ + LOCK(cs_args); + assert(!m_config_path); + m_config_path = path; +} + ChainType ArgsManager::GetChainType() const { std::variant<ChainType, std::string> arg = GetChainArg(); diff --git a/src/common/args.h b/src/common/args.h index ae3ed02bc7..6451b194d1 100644 --- a/src/common/args.h +++ b/src/common/args.h @@ -180,6 +180,7 @@ protected: * Return config file path (read-only) */ fs::path GetConfigFilePath() const; + void SetConfigFilePath(fs::path); [[nodiscard]] bool ReadConfigFiles(std::string& error, bool ignore_invalid_keys = false); /** @@ -214,21 +215,21 @@ protected: * * @return Blocks path which is network specific */ - const fs::path& GetBlocksDirPath() const; + fs::path GetBlocksDirPath() const; /** * Get data directory path * * @return Absolute path on success, otherwise an empty path when a non-directory path would be returned */ - const fs::path& GetDataDirBase() const { return GetDataDir(false); } + fs::path GetDataDirBase() const { return GetDataDir(false); } /** * Get data directory path with appended network identifier * * @return Absolute path on success, otherwise an empty path when a non-directory path would be returned */ - const fs::path& GetDataDirNet() const { return GetDataDir(true); } + fs::path GetDataDirNet() const { return GetDataDir(true); } /** * Clear cached directory paths @@ -419,7 +420,7 @@ private: * @param net_specific Append network identifier to the returned path * @return Absolute path on success, otherwise an empty path when a non-directory path would be returned */ - const fs::path& GetDataDir(bool net_specific) const; + fs::path GetDataDir(bool net_specific) const; /** * Return -regtest/-signet/-testnet/-chain= setting as a ChainType enum if a diff --git a/src/common/bloom.cpp b/src/common/bloom.cpp index 5c3ad882a1..ca6af90b76 100644 --- a/src/common/bloom.cpp +++ b/src/common/bloom.cpp @@ -98,8 +98,8 @@ bool CBloomFilter::IsRelevantAndUpdate(const CTransaction& tx) // for finding tx when they appear in a block if (vData.empty()) // zero-size = "match-all" filter return true; - const uint256& hash = tx.GetHash(); - if (contains(hash)) + const Txid& hash = tx.GetHash(); + if (contains(hash.ToUint256())) fFound = true; for (unsigned int i = 0; i < tx.vout.size(); i++) diff --git a/src/common/system.cpp b/src/common/system.cpp index 1d1c5fa56a..ba42c6df50 100644 --- a/src/common/system.cpp +++ b/src/common/system.cpp @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2022 The Bitcoin Core developers +// Copyright (c) 2009-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -12,6 +12,7 @@ #ifndef WIN32 #include <sys/stat.h> #else +#include <compat/compat.h> #include <codecvt> #endif diff --git a/src/common/system.h b/src/common/system.h index 40206aaa01..83280d46ee 100644 --- a/src/common/system.h +++ b/src/common/system.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2022 The Bitcoin Core developers +// Copyright (c) 2009-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -10,18 +10,14 @@ #include <config/bitcoin-config.h> #endif -#include <compat/assumptions.h> -#include <compat/compat.h> - -#include <set> -#include <stdint.h> +#include <cstdint> #include <string> // Application startup time (used for uptime calculation) int64_t GetStartupTime(); void SetupEnvironment(); -bool SetupNetworking(); +[[nodiscard]] bool SetupNetworking(); #ifndef WIN32 std::string ShellEscape(const std::string& arg); #endif diff --git a/src/compat/assumptions.h b/src/compat/assumptions.h index 4488db0886..7b66ab1b15 100644 --- a/src/compat/assumptions.h +++ b/src/compat/assumptions.h @@ -11,20 +11,6 @@ #include <cstddef> #include <limits> -// Assumption: We assume that the macro NDEBUG is not defined. -// Example(s): We use assert(...) extensively with the assumption of it never -// being a noop at runtime. -#if defined(NDEBUG) -# error "Bitcoin cannot be compiled without assertions." -#endif - -// Assumption: We assume a C++17 (ISO/IEC 14882:2017) compiler (minimum requirement). -// Example(s): We assume the presence of C++17 features everywhere :-) -// ISO Standard C++17 [cpp.predefined]p1: -// "The name __cplusplus is defined to the value 201703L when compiling a C++ -// translation unit." -static_assert(__cplusplus >= 201703L, "C++17 standard assumed"); - // Assumption: We assume the floating-point types to fulfill the requirements of // IEC 559 (IEEE 754) standard. // Example(s): Floating-point division by zero in ConnectBlock, CreateTransaction diff --git a/src/consensus/tx_check.cpp b/src/consensus/tx_check.cpp index f949655909..b3fee1e8b1 100644 --- a/src/consensus/tx_check.cpp +++ b/src/consensus/tx_check.cpp @@ -16,8 +16,9 @@ bool CheckTransaction(const CTransaction& tx, TxValidationState& state) if (tx.vout.empty()) return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-vout-empty"); // Size limits (this doesn't take the witness into account, as that hasn't been checked for malleability) - if (::GetSerializeSize(tx, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT) + if (::GetSerializeSize(TX_NO_WITNESS(tx)) * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT) { return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-oversize"); + } // Check for negative or overflow output values (see CVE-2010-5139) CAmount nValueOut = 0; diff --git a/src/consensus/validation.h b/src/consensus/validation.h index d5bf08cd61..1556c7888f 100644 --- a/src/consensus/validation.h +++ b/src/consensus/validation.h @@ -7,7 +7,6 @@ #define BITCOIN_CONSENSUS_VALIDATION_H #include <string> -#include <version.h> #include <consensus/consensus.h> #include <primitives/transaction.h> #include <primitives/block.h> @@ -54,6 +53,8 @@ enum class TxValidationResult { TX_CONFLICT, TX_MEMPOOL_POLICY, //!< violated mempool's fee/size/descendant/RBF/etc limits TX_NO_MEMPOOL, //!< this node does not have a mempool so can't validate the transaction + TX_RECONSIDERABLE, //!< fails some policy, but might be acceptable if submitted in a (different) package + TX_UNKNOWN, //!< transaction was not validated because package failed }; /** A "reason" why a block was invalid, suitable for determining whether the @@ -147,16 +148,16 @@ class BlockValidationState : public ValidationState<BlockValidationResult> {}; // weight = (stripped_size * 3) + total_size. static inline int32_t GetTransactionWeight(const CTransaction& tx) { - return ::GetSerializeSize(tx, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(tx, PROTOCOL_VERSION); + return ::GetSerializeSize(TX_NO_WITNESS(tx)) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(TX_WITH_WITNESS(tx)); } static inline int64_t GetBlockWeight(const CBlock& block) { - return ::GetSerializeSize(block, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(block, PROTOCOL_VERSION); + return ::GetSerializeSize(TX_NO_WITNESS(block)) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(TX_WITH_WITNESS(block)); } static inline int64_t GetTransactionInputWeight(const CTxIn& txin) { // scriptWitness size is added here because witnesses and txins are split up in segwit serialization. - return ::GetSerializeSize(txin, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(txin, PROTOCOL_VERSION) + ::GetSerializeSize(txin.scriptWitness.stack, PROTOCOL_VERSION); + return ::GetSerializeSize(TX_NO_WITNESS(txin)) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(TX_WITH_WITNESS(txin)) + ::GetSerializeSize(txin.scriptWitness.stack); } /** Compute at which vout of the block's coinbase transaction the witness commitment occurs, or -1 if not found */ diff --git a/src/core_io.h b/src/core_io.h index 1f5ecbaea6..4405f5c8f8 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -51,9 +51,9 @@ bool ParseHashStr(const std::string& strHex, uint256& result); // core_write.cpp UniValue ValueFromAmount(const CAmount amount); std::string FormatScript(const CScript& script); -std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags = 0); +std::string EncodeHexTx(const CTransaction& tx); std::string SighashToStr(unsigned char sighash_type); void ScriptToUniv(const CScript& script, UniValue& out, bool include_hex = true, bool include_address = false, const SigningProvider* provider = nullptr); -void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex = true, int serialize_flags = 0, const CTxUndo* txundo = nullptr, TxVerbosity verbosity = TxVerbosity::SHOW_DETAILS); +void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex = true, const CTxUndo* txundo = nullptr, TxVerbosity verbosity = TxVerbosity::SHOW_DETAILS); #endif // BITCOIN_CORE_IO_H diff --git a/src/core_read.cpp b/src/core_read.cpp index dfabf3a0c2..e32e46d1b9 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -12,7 +12,6 @@ #include <streams.h> #include <util/result.h> #include <util/strencodings.h> -#include <version.h> #include <algorithm> #include <string> @@ -142,9 +141,9 @@ static bool DecodeTx(CMutableTransaction& tx, const std::vector<unsigned char>& // Try decoding with extended serialization support, and remember if the result successfully // consumes the entire input. if (try_witness) { - CDataStream ssData(tx_data, SER_NETWORK, PROTOCOL_VERSION); + DataStream ssData(tx_data); try { - ssData >> tx_extended; + ssData >> TX_WITH_WITNESS(tx_extended); if (ssData.empty()) ok_extended = true; } catch (const std::exception&) { // Fall through. @@ -160,9 +159,9 @@ static bool DecodeTx(CMutableTransaction& tx, const std::vector<unsigned char>& // Try decoding with legacy serialization, and remember if the result successfully consumes the entire input. if (try_no_witness) { - CDataStream ssData(tx_data, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS); + DataStream ssData(tx_data); try { - ssData >> tx_legacy; + ssData >> TX_NO_WITNESS(tx_legacy); if (ssData.empty()) ok_legacy = true; } catch (const std::exception&) { // Fall through. @@ -222,9 +221,9 @@ bool DecodeHexBlk(CBlock& block, const std::string& strHexBlk) return false; std::vector<unsigned char> blockData(ParseHex(strHexBlk)); - CDataStream ssBlock(blockData, SER_NETWORK, PROTOCOL_VERSION); + DataStream ssBlock(blockData); try { - ssBlock >> block; + ssBlock >> TX_WITH_WITNESS(block); } catch (const std::exception&) { return false; diff --git a/src/core_write.cpp b/src/core_write.cpp index 7cf019a42e..63f2c36b5a 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -140,10 +140,10 @@ std::string ScriptToAsmStr(const CScript& script, const bool fAttemptSighashDeco return str; } -std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags) +std::string EncodeHexTx(const CTransaction& tx) { - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | serializeFlags); - ssTx << tx; + DataStream ssTx; + ssTx << TX_WITH_WITNESS(tx); return HexStr(ssTx); } @@ -168,7 +168,7 @@ void ScriptToUniv(const CScript& script, UniValue& out, bool include_hex, bool i out.pushKV("type", GetTxnOutputType(type)); } -void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex, int serialize_flags, const CTxUndo* txundo, TxVerbosity verbosity) +void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex, const CTxUndo* txundo, TxVerbosity verbosity) { CHECK_NONFATAL(verbosity >= TxVerbosity::SHOW_DETAILS); @@ -177,7 +177,7 @@ void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry // Transaction version is actually unsigned in consensus checks, just signed in memory, // so cast to unsigned before giving it to the user. entry.pushKV("version", static_cast<int64_t>(static_cast<uint32_t>(tx.nVersion))); - entry.pushKV("size", (int)::GetSerializeSize(tx, PROTOCOL_VERSION)); + entry.pushKV("size", tx.GetTotalSize()); entry.pushKV("vsize", (GetTransactionWeight(tx) + WITNESS_SCALE_FACTOR - 1) / WITNESS_SCALE_FACTOR); entry.pushKV("weight", GetTransactionWeight(tx)); entry.pushKV("locktime", (int64_t)tx.nLockTime); @@ -264,6 +264,6 @@ void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry } if (include_hex) { - entry.pushKV("hex", EncodeHexTx(tx, serialize_flags)); // The hex-encoded transaction. Used the name "hex" to be consistent with the verbose output of "getrawtransaction". + entry.pushKV("hex", EncodeHexTx(tx)); // The hex-encoded transaction. Used the name "hex" to be consistent with the verbose output of "getrawtransaction". } } diff --git a/src/external_signer.cpp b/src/external_signer.cpp index 102c58b56a..749bb5f74f 100644 --- a/src/external_signer.cpp +++ b/src/external_signer.cpp @@ -2,12 +2,13 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <external_signer.h> + #include <chainparams.h> #include <common/run_command.h> #include <core_io.h> #include <psbt.h> #include <util/strencodings.h> -#include <external_signer.h> #include <algorithm> #include <stdexcept> @@ -72,7 +73,7 @@ UniValue ExternalSigner::GetDescriptors(const int account) bool ExternalSigner::SignTransaction(PartiallySignedTransaction& psbtx, std::string& error) { // Serialize the PSBT - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssTx{}; ssTx << psbtx; // parse ExternalSigner master fingerprint std::vector<unsigned char> parsed_m_fingerprint = ParseHex(m_fingerprint); diff --git a/src/hash.h b/src/hash.h index d355b703ff..52babf8b1d 100644 --- a/src/hash.h +++ b/src/hash.h @@ -14,7 +14,6 @@ #include <serialize.h> #include <span.h> #include <uint256.h> -#include <version.h> #include <string> #include <vector> @@ -146,23 +145,6 @@ public: } }; -class CHashWriter : public HashWriter -{ -private: - const int nVersion; - -public: - CHashWriter(int nVersionIn) : nVersion{nVersionIn} {} - - int GetVersion() const { return nVersion; } - - template<typename T> - CHashWriter& operator<<(const T& obj) { - ::Serialize(*this, obj); - return (*this); - } -}; - /** Reads data from an underlying stream, while hashing the read data. */ template <typename Source> class HashVerifier : public HashWriter diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 661406e122..c72dbf10bc 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -8,6 +8,7 @@ #include <crypto/hmac_sha256.h> #include <httpserver.h> #include <logging.h> +#include <netaddress.h> #include <rpc/protocol.h> #include <rpc/server.h> #include <util/strencodings.h> diff --git a/src/httpserver.cpp b/src/httpserver.cpp index 647e36adb3..e26ea6a596 100644 --- a/src/httpserver.cpp +++ b/src/httpserver.cpp @@ -15,9 +15,9 @@ #include <netbase.h> #include <node/interface_ui.h> #include <rpc/protocol.h> // For HTTP status codes -#include <shutdown.h> #include <sync.h> #include <util/check.h> +#include <util/signalinterrupt.h> #include <util/strencodings.h> #include <util/threadnames.h> #include <util/translation.h> @@ -224,8 +224,7 @@ static bool InitHTTPAllowList() rpc_allow_subnets.emplace_back(LookupHost("127.0.0.1", false).value(), 8); // always allow IPv4 local subnet rpc_allow_subnets.emplace_back(LookupHost("::1", false).value()); // always allow IPv6 localhost for (const std::string& strAllow : gArgs.GetArgs("-rpcallowip")) { - CSubNet subnet; - LookupSubNet(strAllow, subnet); + const CSubNet subnet{LookupSubNet(strAllow)}; if (!subnet.IsValid()) { uiInterface.ThreadSafeMessageBox( strprintf(Untranslated("Invalid -rpcallowip subnet specification: %s. Valid are a single IP (e.g. 1.2.3.4), a network/netmask (e.g. 1.2.3.4/255.255.255.0) or a network/CIDR (e.g. 1.2.3.4/24)."), strAllow), @@ -285,7 +284,7 @@ static void http_request_cb(struct evhttp_request* req, void* arg) } } } - std::unique_ptr<HTTPRequest> hreq(new HTTPRequest(req)); + auto hreq{std::make_unique<HTTPRequest>(req, *static_cast<const util::SignalInterrupt*>(arg))}; // Early address-based allow check if (!ClientAllowed(hreq->GetPeer())) { @@ -426,7 +425,7 @@ static void libevent_log_cb(int severity, const char *msg) LogPrintLevel(BCLog::LIBEVENT, level, "%s\n", msg); } -bool InitHTTPServer() +bool InitHTTPServer(const util::SignalInterrupt& interrupt) { if (!InitHTTPAllowList()) return false; @@ -455,7 +454,7 @@ bool InitHTTPServer() evhttp_set_timeout(http, gArgs.GetIntArg("-rpcservertimeout", DEFAULT_HTTP_SERVER_TIMEOUT)); evhttp_set_max_headers_size(http, MAX_HEADERS_SIZE); evhttp_set_max_body_size(http, MAX_SIZE); - evhttp_set_gencb(http, http_request_cb, nullptr); + evhttp_set_gencb(http, http_request_cb, (void*)&interrupt); if (!HTTPBindAddresses(http)) { LogPrintf("Unable to bind any endpoint for RPC server\n"); @@ -580,7 +579,8 @@ void HTTPEvent::trigger(struct timeval* tv) else evtimer_add(ev, tv); // trigger after timeval passed } -HTTPRequest::HTTPRequest(struct evhttp_request* _req, bool _replySent) : req(_req), replySent(_replySent) +HTTPRequest::HTTPRequest(struct evhttp_request* _req, const util::SignalInterrupt& interrupt, bool _replySent) + : req(_req), m_interrupt(interrupt), replySent(_replySent) { } @@ -640,7 +640,7 @@ void HTTPRequest::WriteHeader(const std::string& hdr, const std::string& value) void HTTPRequest::WriteReply(int nStatus, const std::string& strReply) { assert(!replySent && req); - if (ShutdownRequested()) { + if (m_interrupt) { WriteHeader("Connection", "close"); } // Send event to main http thread to send reply message diff --git a/src/httpserver.h b/src/httpserver.h index 036a39a023..9a49877f09 100644 --- a/src/httpserver.h +++ b/src/httpserver.h @@ -9,6 +9,10 @@ #include <optional> #include <string> +namespace util { +class SignalInterrupt; +} // namespace util + static const int DEFAULT_HTTP_THREADS=4; static const int DEFAULT_HTTP_WORKQUEUE=16; static const int DEFAULT_HTTP_SERVER_TIMEOUT=30; @@ -21,7 +25,7 @@ class HTTPRequest; /** Initialize HTTP server. * Call this before RegisterHTTPHandler or EventBase(). */ -bool InitHTTPServer(); +bool InitHTTPServer(const util::SignalInterrupt& interrupt); /** Start HTTP server. * This is separate from InitHTTPServer to give users race-condition-free time * to register their handlers between InitHTTPServer and StartHTTPServer. @@ -57,10 +61,11 @@ class HTTPRequest { private: struct evhttp_request* req; + const util::SignalInterrupt& m_interrupt; bool replySent; public: - explicit HTTPRequest(struct evhttp_request* req, bool replySent = false); + explicit HTTPRequest(struct evhttp_request* req, const util::SignalInterrupt& interrupt, bool replySent = false); ~HTTPRequest(); enum RequestMethod { diff --git a/src/i2p.cpp b/src/i2p.cpp index 05a5dde396..685b43ba18 100644 --- a/src/i2p.cpp +++ b/src/i2p.cpp @@ -384,11 +384,26 @@ Binary Session::MyDestination() const static constexpr size_t CERT_LEN_POS = 385; uint16_t cert_len; + + if (m_private_key.size() < CERT_LEN_POS + sizeof(cert_len)) { + throw std::runtime_error(strprintf("The private key is too short (%d < %d)", + m_private_key.size(), + CERT_LEN_POS + sizeof(cert_len))); + } + memcpy(&cert_len, &m_private_key.at(CERT_LEN_POS), sizeof(cert_len)); cert_len = be16toh(cert_len); const size_t dest_len = DEST_LEN_BASE + cert_len; + if (dest_len > m_private_key.size()) { + throw std::runtime_error(strprintf("Certificate length (%d) designates that the private key should " + "be %d bytes, but it is only %d bytes", + cert_len, + dest_len, + m_private_key.size())); + } + return Binary{m_private_key.begin(), m_private_key.begin() + dest_len}; } diff --git a/src/index/base.cpp b/src/index/base.cpp index 8474d01c41..bcfe7215be 100644 --- a/src/index/base.cpp +++ b/src/index/base.cpp @@ -13,7 +13,6 @@ #include <node/context.h> #include <node/database_args.h> #include <node/interface_ui.h> -#include <shutdown.h> #include <tinyformat.h> #include <util/thread.h> #include <util/translation.h> @@ -32,7 +31,7 @@ template <typename... Args> void BaseIndex::FatalErrorf(const char* fmt, const Args&... args) { auto message = tfm::format(fmt, args...); - node::AbortNode(m_chain->context()->exit_status, message); + node::AbortNode(m_chain->context()->shutdown, m_chain->context()->exit_status, message); } CBlockLocator GetLocator(interfaces::Chain& chain, const uint256& block_hash) diff --git a/src/index/blockfilterindex.cpp b/src/index/blockfilterindex.cpp index 5fda798efd..58f777b326 100644 --- a/src/index/blockfilterindex.cpp +++ b/src/index/blockfilterindex.cpp @@ -174,8 +174,8 @@ size_t BlockFilterIndex::WriteFilterToDisk(FlatFilePos& pos, const BlockFilter& assert(filter.GetFilterType() == GetFilterType()); size_t data_size = - GetSerializeSize(filter.GetBlockHash(), CLIENT_VERSION) + - GetSerializeSize(filter.GetEncodedFilter(), CLIENT_VERSION); + GetSerializeSize(filter.GetBlockHash()) + + GetSerializeSize(filter.GetEncodedFilter()); // If writing the filter would overflow the file, flush and move to the next one. if (pos.nPos + data_size > MAX_FLTR_FILE_SIZE) { diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index e16dd0f8bd..4983926e68 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -65,7 +65,7 @@ bool TxIndex::CustomAppend(const interfaces::BlockInfo& block) vPos.reserve(block.data->vtx.size()); for (const auto& tx : block.data->vtx) { vPos.emplace_back(tx->GetHash(), pos); - pos.nTxOffset += ::GetSerializeSize(*tx, CLIENT_VERSION); + pos.nTxOffset += ::GetSerializeSize(TX_WITH_WITNESS(*tx)); } return m_db->WriteTxs(vPos); } @@ -79,7 +79,7 @@ bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRe return false; } - CAutoFile file{m_chainstate->m_blockman.OpenBlockFile(postx, true)}; + AutoFile file{m_chainstate->m_blockman.OpenBlockFile(postx, true)}; if (file.IsNull()) { return error("%s: OpenBlockFile failed", __func__); } @@ -89,7 +89,7 @@ bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRe if (fseek(file.Get(), postx.nTxOffset, SEEK_CUR)) { return error("%s: fseek(...) failed", __func__); } - file >> tx; + file >> TX_WITH_WITNESS(tx); } catch (const std::exception& e) { return error("%s: Deserialize or I/O error - %s", __func__, e.what()); } diff --git a/src/init.cpp b/src/init.cpp index 42331d37e8..39ee9fe60d 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -66,7 +66,6 @@ #include <rpc/util.h> #include <scheduler.h> #include <script/sigcache.h> -#include <shutdown.h> #include <sync.h> #include <timedata.h> #include <torcontrol.h> @@ -155,6 +154,11 @@ static const char* DEFAULT_ASMAP_FILENAME="ip_asn.map"; * The PID file facilities. */ static const char* BITCOIN_PID_FILENAME = "bitcoind.pid"; +/** + * True if this process has created a PID file. + * Used to determine whether we should remove the PID file on shutdown. + */ +static bool g_generated_pid{false}; static fs::path GetPidFile(const ArgsManager& args) { @@ -170,12 +174,34 @@ static fs::path GetPidFile(const ArgsManager& args) #else tfm::format(file, "%d\n", getpid()); #endif + g_generated_pid = true; return true; } else { return InitError(strprintf(_("Unable to create the PID file '%s': %s"), fs::PathToString(GetPidFile(args)), SysErrorString(errno))); } } +static void RemovePidFile(const ArgsManager& args) +{ + if (!g_generated_pid) return; + const auto pid_path{GetPidFile(args)}; + if (std::error_code error; !fs::remove(pid_path, error)) { + std::string msg{error ? error.message() : "File does not exist"}; + LogPrintf("Unable to remove PID file (%s): %s\n", fs::PathToString(pid_path), msg); + } +} + +static std::optional<util::SignalInterrupt> g_shutdown; + +void InitContext(NodeContext& node) +{ + assert(!g_shutdown); + g_shutdown.emplace(); + + node.args = &gArgs; + node.shutdown = &*g_shutdown; +} + ////////////////////////////////////////////////////////////////////////////// // // Shutdown @@ -187,11 +213,9 @@ static fs::path GetPidFile(const ArgsManager& args) // The network-processing threads are all part of a thread group // created by AppInit() or the Qt main() function. // -// A clean exit happens when StartShutdown() or the SIGTERM -// signal handler sets ShutdownRequested(), which makes main thread's -// WaitForShutdown() interrupts the thread group. -// And then, WaitForShutdown() makes all other on-going threads -// in the thread group join the main thread. +// A clean exit happens when the SignalInterrupt object is triggered, which +// makes the main thread's SignalInterrupt::wait() call return, and join all +// other ongoing threads in the thread group to the main thread. // Shutdown() is then called to clean up database connections, and stop other // threads that should only be stopped after the main network-processing // threads have exited. @@ -201,6 +225,11 @@ static fs::path GetPidFile(const ArgsManager& args) // shutdown thing. // +bool ShutdownRequested(node::NodeContext& node) +{ + return bool{*Assert(node.shutdown)}; +} + #if HAVE_SYSTEM static void ShutdownNotify(const ArgsManager& args) { @@ -268,10 +297,9 @@ void Shutdown(NodeContext& node) StopTorControl(); // After everything has been shut down, but before things get flushed, stop the - // CScheduler/checkqueue, scheduler and load block thread. + // scheduler and load block thread. if (node.scheduler) node.scheduler->stop(); if (node.chainman && node.chainman->m_thread_load.joinable()) node.chainman->m_thread_load.join(); - StopScriptCheckWorkerThreads(); // After the threads that potentially access these pointers have been stopped, // destruct and reset all to nullptr. @@ -285,8 +313,12 @@ void Shutdown(NodeContext& node) DumpMempool(*node.mempool, MempoolPath(*node.args)); } - // Drop transactions we were still watching, and record fee estimations. - if (node.fee_estimator) node.fee_estimator->Flush(); + // Drop transactions we were still watching, record fee estimations and unregister + // fee estimator from validation interface. + if (node.fee_estimator) { + node.fee_estimator->Flush(); + UnregisterValidationInterface(node.fee_estimator.get()); + } // FlushStateToDisk generates a ChainStateFlushed callback, which we should avoid missing if (node.chainman) { @@ -343,19 +375,13 @@ void Shutdown(NodeContext& node) node.chain_clients.clear(); UnregisterAllValidationInterfaces(); GetMainSignals().UnregisterBackgroundSignalScheduler(); - node.kernel.reset(); node.mempool.reset(); node.fee_estimator.reset(); node.chainman.reset(); node.scheduler.reset(); + node.kernel.reset(); - try { - if (!fs::remove(GetPidFile(*node.args))) { - LogPrintf("%s: Unable to remove PID file: File does not exist\n", __func__); - } - } catch (const fs::filesystem_error& e) { - LogPrintf("%s: Unable to remove PID file: %s\n", __func__, fsbridge::get_filesystem_error_message(e)); - } + RemovePidFile(*node.args); LogPrintf("%s: done\n", __func__); } @@ -368,7 +394,9 @@ void Shutdown(NodeContext& node) #ifndef WIN32 static void HandleSIGTERM(int) { - StartShutdown(); + // Return value is intentionally ignored because there is not a better way + // of handling this failure in a signal handler. + (void)(*Assert(g_shutdown))(); } static void HandleSIGHUP(int) @@ -378,7 +406,10 @@ static void HandleSIGHUP(int) #else static BOOL WINAPI consoleCtrlHandler(DWORD dwCtrlType) { - StartShutdown(); + if (!(*Assert(g_shutdown))()) { + LogPrintf("Error: failed to send shutdown signal on Ctrl-C\n"); + return false; + } Sleep(INFINITE); return true; } @@ -455,9 +486,14 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-maxorphantx=<n>", strprintf("Keep at most <n> unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-mempoolexpiry=<n>", strprintf("Do not keep transactions in the mempool longer than <n> hours (default: %u)", DEFAULT_MEMPOOL_EXPIRY_HOURS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-minimumchainwork=<hex>", strprintf("Minimum work assumed to exist on a valid chain in hex (default: %s, testnet: %s, signet: %s)", defaultChainParams->GetConsensus().nMinimumChainWork.GetHex(), testnetChainParams->GetConsensus().nMinimumChainWork.GetHex(), signetChainParams->GetConsensus().nMinimumChainWork.GetHex()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); - argsman.AddArg("-par=<n>", strprintf("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)", - -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-par=<n>", strprintf("Set the number of script verification threads (0 = auto, up to %d, <0 = leave that many cores free, default: %d)", + MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-persistmempool", strprintf("Whether to save the mempool on shutdown and load on restart (default: %u)", DEFAULT_PERSIST_MEMPOOL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-persistmempoolv1", + strprintf("Whether a mempool.dat file created by -persistmempool or the savemempool RPC will be written in the legacy format " + "(version 1) or the current format (version 2). This temporary option will be removed in the future. (default: %u)", + DEFAULT_PERSIST_V1_DAT), + ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-pid=<file>", strprintf("Specify pid file. Relative paths will be prefixed by a net-specific datadir location. (default: %s)", BITCOIN_PID_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex. " "Warning: Reverting this setting requires re-downloading the entire blockchain. " @@ -489,7 +525,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-forcednsseed", strprintf("Always query for peer addresses via DNS lookup (default: %u)", DEFAULT_FORCEDNSSEED), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-listen", strprintf("Accept connections from outside (default: %u if no -proxy, -connect or -maxconnections=0)", DEFAULT_LISTEN), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-listenonion", strprintf("Automatically create Tor onion service (default: %d)", DEFAULT_LISTEN_ONION), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-maxconnections=<n>", strprintf("Maintain at most <n> connections to peers (default: %u). This limit does not apply to connections manually added via -addnode or the addnode RPC, which have a separate limit of %u.", DEFAULT_MAX_PEER_CONNECTIONS, MAX_ADDNODE_CONNECTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-maxconnections=<n>", strprintf("Maintain at most <n> automatic connections to peers (default: %u). This limit does not apply to connections manually added via -addnode or the addnode RPC, which have a separate limit of %u.", DEFAULT_MAX_PEER_CONNECTIONS, MAX_ADDNODE_CONNECTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxreceivebuffer=<n>", strprintf("Maximum per-connection receive buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXRECEIVEBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxsendbuffer=<n>", strprintf("Maximum per-connection memory usage for the send buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXSENDBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by outbound peers forward or backward by this amount (default: %u seconds).", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -619,7 +655,6 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-rpccookiefile=<loc>", "Location of the auth cookie. Relative paths will be prefixed by a net-specific datadir location. (default: data dir)", ArgsManager::ALLOW_ANY, OptionsCategory::RPC); argsman.AddArg("-rpcpassword=<pw>", "Password for JSON-RPC connections", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::RPC); argsman.AddArg("-rpcport=<port>", strprintf("Listen for JSON-RPC connections on <port> (default: %u, testnet: %u, signet: %u, regtest: %u)", defaultBaseParams->RPCPort(), testnetBaseParams->RPCPort(), signetBaseParams->RPCPort(), regtestBaseParams->RPCPort()), ArgsManager::ALLOW_ANY | ArgsManager::NETWORK_ONLY, OptionsCategory::RPC); - argsman.AddArg("-rpcserialversion", strprintf("Sets the serialization of raw transaction or block hex returned in non-verbose mode, non-segwit(0) (DEPRECATED) or segwit(1) (default: %d)", DEFAULT_RPC_SERIALIZE_VERSION), ArgsManager::ALLOW_ANY, OptionsCategory::RPC); argsman.AddArg("-rpcservertimeout=<n>", strprintf("Timeout during HTTP requests (default: %d)", DEFAULT_HTTP_SERVER_TIMEOUT), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::RPC); argsman.AddArg("-rpcthreads=<n>", strprintf("Set the number of threads to service RPC calls (default: %d)", DEFAULT_HTTP_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::RPC); argsman.AddArg("-rpcuser=<user>", "Username for JSON-RPC connections", ArgsManager::ALLOW_ANY | ArgsManager::SENSITIVE, OptionsCategory::RPC); @@ -671,8 +706,9 @@ static bool AppInitServers(NodeContext& node) const ArgsManager& args = *Assert(node.args); RPCServer::OnStarted(&OnRPCStarted); RPCServer::OnStopped(&OnRPCStopped); - if (!InitHTTPServer()) + if (!InitHTTPServer(*Assert(node.shutdown))) { return false; + } StartRPC(); node.rpc_interruption_point = RpcInterruptionPoint; if (!StartHTTPRPC(&node)) @@ -988,16 +1024,6 @@ bool AppInitParameterInteraction(const ArgsManager& args) if (args.GetBoolArg("-peerbloomfilters", DEFAULT_PEERBLOOMFILTERS)) nLocalServices = ServiceFlags(nLocalServices | NODE_BLOOM); - if (args.GetIntArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) < 0) - return InitError(Untranslated("rpcserialversion must be non-negative.")); - - if (args.GetIntArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) > 1) - return InitError(Untranslated("Unknown rpcserialversion requested.")); - - if (args.GetIntArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) == 0 && !IsDeprecatedRPCEnabled("serialversion")) { - return InitError(Untranslated("-rpcserialversion=0 is deprecated and will be removed in the future. Specify -deprecatedrpc=serialversion to allow anyway.")); - } - // Also report errors from parsing before daemonization { kernel::Notifications notifications{}; @@ -1028,13 +1054,14 @@ static bool LockDataDirectory(bool probeOnly) { // Make sure only a single Bitcoin process is using the data directory. const fs::path& datadir = gArgs.GetDataDirNet(); - if (!DirIsWritable(datadir)) { + switch (util::LockDirectory(datadir, ".lock", probeOnly)) { + case util::LockResult::ErrorWrite: return InitError(strprintf(_("Cannot write to data directory '%s'; check permissions."), fs::PathToString(datadir))); - } - if (!LockDirectory(datadir, ".lock", probeOnly)) { + case util::LockResult::ErrorLock: return InitError(strprintf(_("Cannot obtain a lock on data directory %s. %s is probably already running."), fs::PathToString(datadir), PACKAGE_NAME)); - } - return true; + case util::LockResult::Success: return true; + } // no default case, so the compiler can warn about missing cases + assert(false); } bool AppInitSanityChecks(const kernel::Context& kernel) @@ -1109,24 +1136,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) return InitError(strprintf(_("Unable to allocate memory for -maxsigcachesize: '%s' MiB"), args.GetIntArg("-maxsigcachesize", DEFAULT_MAX_SIG_CACHE_BYTES >> 20))); } - int script_threads = args.GetIntArg("-par", DEFAULT_SCRIPTCHECK_THREADS); - if (script_threads <= 0) { - // -par=0 means autodetect (number of cores - 1 script threads) - // -par=-n means "leave n cores free" (number of cores - n - 1 script threads) - script_threads += GetNumCores(); - } - - // Subtract 1 because the main thread counts towards the par threads - script_threads = std::max(script_threads - 1, 0); - - // Number of script-checking threads <= MAX_SCRIPTCHECK_THREADS - script_threads = std::min(script_threads, MAX_SCRIPTCHECK_THREADS); - - LogPrintf("Script verification uses %d additional threads\n", script_threads); - if (script_threads >= 1) { - StartScriptCheckWorkerThreads(script_threads); - } - assert(!node.scheduler); node.scheduler = std::make_unique<CScheduler>(); @@ -1139,11 +1148,13 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) }, std::chrono::minutes{1}); // Check disk space every 5 minutes to avoid db corruption. - node.scheduler->scheduleEvery([&args]{ + node.scheduler->scheduleEvery([&args, &node]{ constexpr uint64_t min_disk_space = 50 << 20; // 50 MB if (!CheckDiskSpace(args.GetBlocksDirPath(), min_disk_space)) { LogPrintf("Shutting down due to lack of disk space!\n"); - StartShutdown(); + if (!(*Assert(node.shutdown))()) { + LogPrintf("Error: failed to send shutdown signal after disk space check\n"); + } } }, std::chrono::minutes{5}); @@ -1253,6 +1264,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // Flush estimates to disk periodically CBlockPolicyEstimator* fee_estimator = node.fee_estimator.get(); node.scheduler->scheduleEvery([fee_estimator] { fee_estimator->FlushFeeEstimates(); }, FEE_FLUSH_INTERVAL); + RegisterValidationInterface(fee_estimator); } // Check port numbers @@ -1429,7 +1441,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // ********************************************************* Step 7: load block chain - node.notifications = std::make_unique<KernelNotifications>(node.exit_status); + node.notifications = std::make_unique<KernelNotifications>(*Assert(node.shutdown), node.exit_status); ReadNotificationArgs(args, *node.notifications); fReindex = args.GetBoolArg("-reindex", false); bool fReindexChainState = args.GetBoolArg("-reindex-chainstate", false); @@ -1466,7 +1478,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) assert(!node.chainman); CTxMemPool::Options mempool_opts{ - .estimator = node.fee_estimator.get(), .check_ratio = chainparams.DefaultConsistencyChecks() ? 1 : 0, }; auto result{ApplyArgsManOptions(args, chainparams, mempool_opts)}; @@ -1481,10 +1492,10 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) } LogPrintf("* Using %.1f MiB for in-memory UTXO set (plus up to %.1f MiB of unused mempool space)\n", cache_sizes.coins * (1.0 / 1024 / 1024), mempool_opts.max_size_bytes * (1.0 / 1024 / 1024)); - for (bool fLoaded = false; !fLoaded && !ShutdownRequested();) { + for (bool fLoaded = false; !fLoaded && !ShutdownRequested(node);) { node.mempool = std::make_unique<CTxMemPool>(mempool_opts); - node.chainman = std::make_unique<ChainstateManager>(node.kernel->interrupt, chainman_opts, blockman_opts); + node.chainman = std::make_unique<ChainstateManager>(*Assert(node.shutdown), chainman_opts, blockman_opts); ChainstateManager& chainman = *node.chainman; // This is defined and set here instead of inline in validation.h to avoid a hard @@ -1514,7 +1525,6 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) options.check_blocks = args.GetIntArg("-checkblocks", DEFAULT_CHECKBLOCKS); options.check_level = args.GetIntArg("-checklevel", DEFAULT_CHECKLEVEL); options.require_full_verification = args.IsArgSet("-checkblocks") || args.IsArgSet("-checklevel"); - options.check_interrupt = ShutdownRequested; options.coins_error_cb = [] { uiInterface.ThreadSafeMessageBox( _("Error reading from database, shutting down."), @@ -1549,7 +1559,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) return InitError(error); } - if (!fLoaded && !ShutdownRequested()) { + if (!fLoaded && !ShutdownRequested(node)) { // first suggest a reindex if (!options.reindex) { bool fRet = uiInterface.ThreadSafeQuestion( @@ -1558,7 +1568,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) "", CClientUIInterface::MSG_ERROR | CClientUIInterface::BTN_ABORT); if (fRet) { fReindex = true; - AbortShutdown(); + if (!Assert(node.shutdown)->reset()) { + LogPrintf("Internal error: failed to reset shutdown signal.\n"); + } } else { LogPrintf("Aborted block database rebuild. Exiting.\n"); return false; @@ -1572,7 +1584,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // As LoadBlockIndex can take several minutes, it's possible the user // requested to kill the GUI during the last operation. If so, exit. // As the program has not fully started yet, Shutdown() is possibly overkill. - if (ShutdownRequested()) { + if (ShutdownRequested(node)) { LogPrintf("Shutdown requested. Exiting.\n"); return false; } @@ -1693,7 +1705,9 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) ImportBlocks(chainman, vImportFiles); if (args.GetBoolArg("-stopafterblockimport", DEFAULT_STOPAFTERBLOCKIMPORT)) { LogPrintf("Stopping after block import\n"); - StartShutdown(); + if (!(*Assert(node.shutdown))()) { + LogPrintf("Error: failed to send shutdown signal after finishing block import\n"); + } return; } @@ -1713,16 +1727,16 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) // Wait for genesis block to be processed { WAIT_LOCK(g_genesis_wait_mutex, lock); - // We previously could hang here if StartShutdown() is called prior to + // We previously could hang here if shutdown was requested prior to // ImportBlocks getting started, so instead we just wait on a timer to // check ShutdownRequested() regularly. - while (!fHaveGenesis && !ShutdownRequested()) { + while (!fHaveGenesis && !ShutdownRequested(node)) { g_genesis_wait_cv.wait_for(lock, std::chrono::milliseconds(500)); } block_notify_genesis_wait_connection.disconnect(); } - if (ShutdownRequested()) { + if (ShutdownRequested(node)) { return false; } @@ -1751,11 +1765,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) CConnman::Options connOptions; connOptions.nLocalServices = nLocalServices; - connOptions.nMaxConnections = nMaxConnections; - connOptions.m_max_outbound_full_relay = std::min(MAX_OUTBOUND_FULL_RELAY_CONNECTIONS, connOptions.nMaxConnections); - connOptions.m_max_outbound_block_relay = std::min(MAX_BLOCK_RELAY_ONLY_CONNECTIONS, connOptions.nMaxConnections-connOptions.m_max_outbound_full_relay); - connOptions.nMaxAddnode = MAX_ADDNODE_CONNECTIONS; - connOptions.nMaxFeeler = MAX_FEELER_CONNECTIONS; + connOptions.m_max_automatic_connections = nMaxConnections; connOptions.uiInterface = &uiInterface; connOptions.m_banman = node.banman.get(); connOptions.m_msgproc = node.peerman.get(); diff --git a/src/init.h b/src/init.h index f27d6120ef..ead5f5e0d2 100644 --- a/src/init.h +++ b/src/init.h @@ -26,6 +26,11 @@ namespace node { struct NodeContext; } // namespace node +/** Initialize node context shutdown and args variables. */ +void InitContext(node::NodeContext& node); +/** Return whether node shutdown was requested. */ +bool ShutdownRequested(node::NodeContext& node); + /** Interrupt threads */ void Interrupt(node::NodeContext& node); void Shutdown(node::NodeContext& node); diff --git a/src/init/bitcoin-gui.cpp b/src/init/bitcoin-gui.cpp index ddbdaa6cd0..aceff1e40f 100644 --- a/src/init/bitcoin-gui.cpp +++ b/src/init/bitcoin-gui.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <common/args.h> +#include <init.h> #include <interfaces/chain.h> #include <interfaces/echo.h> #include <interfaces/init.h> @@ -23,7 +23,7 @@ class BitcoinGuiInit : public interfaces::Init public: BitcoinGuiInit(const char* arg0) : m_ipc(interfaces::MakeIpc(EXE_NAME, arg0, *this)) { - m_node.args = &gArgs; + InitContext(m_node); m_node.init = this; } std::unique_ptr<interfaces::Node> makeNode() override { return interfaces::MakeNode(m_node); } diff --git a/src/init/bitcoin-node.cpp b/src/init/bitcoin-node.cpp index b04596f986..97b8dc1161 100644 --- a/src/init/bitcoin-node.cpp +++ b/src/init/bitcoin-node.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <common/args.h> +#include <init.h> #include <interfaces/chain.h> #include <interfaces/echo.h> #include <interfaces/init.h> @@ -25,7 +25,7 @@ public: : m_node(node), m_ipc(interfaces::MakeIpc(EXE_NAME, arg0, *this)) { - m_node.args = &gArgs; + InitContext(m_node); m_node.init = this; } std::unique_ptr<interfaces::Node> makeNode() override { return interfaces::MakeNode(m_node); } diff --git a/src/init/bitcoin-qt.cpp b/src/init/bitcoin-qt.cpp index dd5826d982..3003a8fde1 100644 --- a/src/init/bitcoin-qt.cpp +++ b/src/init/bitcoin-qt.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <common/args.h> +#include <init.h> #include <interfaces/chain.h> #include <interfaces/echo.h> #include <interfaces/init.h> @@ -20,7 +20,7 @@ class BitcoinQtInit : public interfaces::Init public: BitcoinQtInit() { - m_node.args = &gArgs; + InitContext(m_node); m_node.init = this; } std::unique_ptr<interfaces::Node> makeNode() override { return interfaces::MakeNode(m_node); } diff --git a/src/init/bitcoind.cpp b/src/init/bitcoind.cpp index 210608370c..b5df764017 100644 --- a/src/init/bitcoind.cpp +++ b/src/init/bitcoind.cpp @@ -2,7 +2,7 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <common/args.h> +#include <init.h> #include <interfaces/chain.h> #include <interfaces/echo.h> #include <interfaces/init.h> @@ -22,7 +22,7 @@ class BitcoindInit : public interfaces::Init public: BitcoindInit(NodeContext& node) : m_node(node) { - m_node.args = &gArgs; + InitContext(m_node); m_node.init = this; } std::unique_ptr<interfaces::Node> makeNode() override { return interfaces::MakeNode(m_node); } diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index dea868f844..9da5cb9637 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -8,6 +8,7 @@ #include <blockfilter.h> #include <common/settings.h> #include <primitives/transaction.h> // For CTransactionRef +#include <util/result.h> #include <functional> #include <memory> @@ -244,7 +245,7 @@ public: // outputs in the same transaction) or have shared ancestry, the bump fees are calculated // independently, i.e. as if only one of them is spent. This may result in double-fee-bumping. This // caveat can be rectified per use of the sister-function CalculateCombinedBumpFee(…). - virtual std::map<COutPoint, CAmount> CalculateIndividualBumpFees(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) = 0; + virtual std::map<COutPoint, CAmount> calculateIndividualBumpFees(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) = 0; //! Calculate the combined bump fee for an input set per the same strategy // as in CalculateIndividualBumpFees(…). @@ -252,7 +253,7 @@ public: // bump fees per outpoint, but a single bump fee for the shared ancestry. // The combined bump fee may be used to correct overestimation due to // shared ancestry by multiple UTXOs after coin selection. - virtual std::optional<CAmount> CalculateCombinedBumpFee(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) = 0; + virtual std::optional<CAmount> calculateCombinedBumpFee(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) = 0; //! Get the node's package limits. //! Currently only returns the ancestor and descendant count limits, but could be enhanced to @@ -260,7 +261,7 @@ public: virtual void getPackageLimits(unsigned int& limit_ancestor_count, unsigned int& limit_descendant_count) = 0; //! Check if transaction will pass the mempool's chain limits. - virtual bool checkChainLimits(const CTransactionRef& tx) = 0; + virtual util::Result<void> checkChainLimits(const CTransactionRef& tx) = 0; //! Estimate smart fee. virtual CFeeRate estimateSmartFee(int num_blocks, bool conservative, FeeCalculation* calc = nullptr) = 0; @@ -334,9 +335,6 @@ public: //! Run function after given number of seconds. Cancel any previous calls with same name. virtual void rpcRunLater(const std::string& name, std::function<void()> fn, int64_t seconds) = 0; - //! Current RPC serialization flags. - virtual int rpcSerializationFlags() = 0; - //! Get settings value. virtual common::SettingsValue getSetting(const std::string& arg) = 0; @@ -395,6 +393,9 @@ public: //! Set mock time. virtual void setMockTime(int64_t time) = 0; + + //! Mock the scheduler to fast forward in time. + virtual void schedulerMockForward(std::chrono::seconds delta_seconds) = 0; }; //! Return implementation of Chain interface. diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 3f8df57124..aeb2612c07 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -204,8 +204,8 @@ public: //! Unset RPC timer interface. virtual void rpcUnsetTimerInterface(RPCTimerInterface* iface) = 0; - //! Get unspent outputs associated with a transaction. - virtual bool getUnspentOutput(const COutPoint& output, Coin& coin) = 0; + //! Get unspent output associated with a transaction. + virtual std::optional<Coin> getUnspentOutput(const COutPoint& output) = 0; //! Broadcast transaction. virtual TransactionError broadcastTransaction(CTransactionRef tx, CAmount max_tx_fee, std::string& err_string) = 0; diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index 4b896c11a3..6114236623 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -118,7 +118,7 @@ public: wallet::AddressPurpose* purpose) = 0; //! Get wallet address list. - virtual std::vector<WalletAddress> getAddresses() const = 0; + virtual std::vector<WalletAddress> getAddresses() = 0; //! Get receive requests. virtual std::vector<std::string> getAddressReceiveRequests() = 0; diff --git a/src/kernel/chainparams.cpp b/src/kernel/chainparams.cpp index 73ba330ff0..56cb3a63a0 100644 --- a/src/kernel/chainparams.cpp +++ b/src/kernel/chainparams.cpp @@ -136,7 +136,7 @@ public: vSeeds.emplace_back("dnsseed.bitcoin.dashjr.org."); // Luke Dashjr vSeeds.emplace_back("seed.bitcoinstats.com."); // Christian Decker, supports x1 - xf vSeeds.emplace_back("seed.bitcoin.jonasschnelli.ch."); // Jonas Schnelli, only supports x1, x5, x9, and xd - vSeeds.emplace_back("seed.btc.petertodd.org."); // Peter Todd, only supports x1, x5, x9, and xd + vSeeds.emplace_back("seed.btc.petertodd.net."); // Peter Todd, only supports x1, x5, x9, and xd vSeeds.emplace_back("seed.bitcoin.sprovoost.nl."); // Sjors Provoost vSeeds.emplace_back("dnsseed.emzy.de."); // Stephan Oeste vSeeds.emplace_back("seed.bitcoin.wiz.biz."); // Jason Maurice @@ -243,7 +243,7 @@ public: vSeeds.clear(); // nodes with support for servicebits filtering should be at the top vSeeds.emplace_back("testnet-seed.bitcoin.jonasschnelli.ch."); - vSeeds.emplace_back("seed.tbtc.petertodd.org."); + vSeeds.emplace_back("seed.tbtc.petertodd.net."); vSeeds.emplace_back("seed.testnet.bitcoin.sprovoost.nl."); vSeeds.emplace_back("testnet-seed.bluematt.me."); // Just a static list of stable node(s), only supports x9 diff --git a/src/kernel/chainstatemanager_opts.h b/src/kernel/chainstatemanager_opts.h index 917f7d226c..ee20eabd79 100644 --- a/src/kernel/chainstatemanager_opts.h +++ b/src/kernel/chainstatemanager_opts.h @@ -45,6 +45,8 @@ struct ChainstateManagerOpts { DBOptions coins_db{}; CoinsViewOptions coins_view{}; Notifications& notifications; + //! Number of script check worker threads. Zero means no parallel verification. + int worker_threads_num{0}; }; } // namespace kernel diff --git a/src/kernel/coinstats.cpp b/src/kernel/coinstats.cpp index 9bd755ed27..ff8a33e804 100644 --- a/src/kernel/coinstats.cpp +++ b/src/kernel/coinstats.cpp @@ -21,7 +21,6 @@ #include <util/check.h> #include <util/overflow.h> #include <validation.h> -#include <version.h> #include <cassert> #include <iosfwd> @@ -90,7 +89,7 @@ static void ApplyCoinHash(std::nullptr_t, const COutPoint& outpoint, const Coin& //! construction could cause a previously invalid (and potentially malicious) //! UTXO snapshot to be considered valid. template <typename T> -static void ApplyHash(T& hash_obj, const uint256& hash, const std::map<uint32_t, Coin>& outputs) +static void ApplyHash(T& hash_obj, const Txid& hash, const std::map<uint32_t, Coin>& outputs) { for (auto it = outputs.begin(); it != outputs.end(); ++it) { COutPoint outpoint = COutPoint(hash, it->first); @@ -119,7 +118,7 @@ static bool ComputeUTXOStats(CCoinsView* view, CCoinsStats& stats, T hash_obj, c std::unique_ptr<CCoinsViewCursor> pcursor(view->Cursor()); assert(pcursor); - uint256 prevkey; + Txid prevkey; std::map<uint32_t, Coin> outputs; while (pcursor->Valid()) { if (interruption_point) interruption_point(); diff --git a/src/kernel/context.cpp b/src/kernel/context.cpp index 3f4a423531..c60f1638d1 100644 --- a/src/kernel/context.cpp +++ b/src/kernel/context.cpp @@ -14,12 +14,8 @@ namespace kernel { -Context* g_context; - Context::Context() { - assert(!g_context); - g_context = this; std::string sha256_algo = SHA256AutoDetect(); LogPrintf("Using the '%s' SHA256 implementation\n", sha256_algo); RandomInit(); @@ -29,8 +25,6 @@ Context::Context() Context::~Context() { ECC_Stop(); - assert(g_context); - g_context = nullptr; } } // namespace kernel diff --git a/src/kernel/context.h b/src/kernel/context.h index e74e0a6f08..ff4df20473 100644 --- a/src/kernel/context.h +++ b/src/kernel/context.h @@ -18,24 +18,9 @@ namespace kernel { //! State stored directly in this struct should be simple. More complex state //! should be stored to std::unique_ptr members pointing to opaque types. struct Context { - //! Interrupt object that can be used to stop long-running kernel operations. - util::SignalInterrupt interrupt; - - //! Declare default constructor and destructor that are not inline, so code - //! instantiating the kernel::Context struct doesn't need to #include class - //! definitions for all the unique_ptr members. Context(); ~Context(); }; - -//! Global pointer to kernel::Context for legacy code. New code should avoid -//! using this, and require state it needs to be passed to it directly. -//! -//! Having this pointer is useful because it allows state be moved out of global -//! variables into the kernel::Context struct before all global references to -//! that state are removed. This allows the global references to be removed -//! incrementally, instead of all at once. -extern Context* g_context; } // namespace kernel #endif // BITCOIN_KERNEL_CONTEXT_H diff --git a/src/kernel/disconnected_transactions.cpp b/src/kernel/disconnected_transactions.cpp new file mode 100644 index 0000000000..f865fed688 --- /dev/null +++ b/src/kernel/disconnected_transactions.cpp @@ -0,0 +1,90 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <kernel/disconnected_transactions.h> + +#include <assert.h> +#include <core_memusage.h> +#include <memusage.h> +#include <primitives/transaction.h> +#include <util/hasher.h> + +#include <memory> +#include <utility> + +// It's almost certainly a logic bug if we don't clear out queuedTx before +// destruction, as we add to it while disconnecting blocks, and then we +// need to re-process remaining transactions to ensure mempool consistency. +// For now, assert() that we've emptied out this object on destruction. +// This assert() can always be removed if the reorg-processing code were +// to be refactored such that this assumption is no longer true (for +// instance if there was some other way we cleaned up the mempool after a +// reorg, besides draining this object). +DisconnectedBlockTransactions::~DisconnectedBlockTransactions() +{ + assert(queuedTx.empty()); + assert(iters_by_txid.empty()); + assert(cachedInnerUsage == 0); +} + +std::vector<CTransactionRef> DisconnectedBlockTransactions::LimitMemoryUsage() +{ + std::vector<CTransactionRef> evicted; + + while (!queuedTx.empty() && DynamicMemoryUsage() > m_max_mem_usage) { + evicted.emplace_back(queuedTx.front()); + cachedInnerUsage -= RecursiveDynamicUsage(queuedTx.front()); + iters_by_txid.erase(queuedTx.front()->GetHash()); + queuedTx.pop_front(); + } + return evicted; +} + +size_t DisconnectedBlockTransactions::DynamicMemoryUsage() const +{ + return cachedInnerUsage + memusage::DynamicUsage(iters_by_txid) + memusage::DynamicUsage(queuedTx); +} + +[[nodiscard]] std::vector<CTransactionRef> DisconnectedBlockTransactions::AddTransactionsFromBlock(const std::vector<CTransactionRef>& vtx) +{ + iters_by_txid.reserve(iters_by_txid.size() + vtx.size()); + for (auto block_it = vtx.rbegin(); block_it != vtx.rend(); ++block_it) { + auto it = queuedTx.insert(queuedTx.end(), *block_it); + auto [_, inserted] = iters_by_txid.emplace((*block_it)->GetHash(), it); + assert(inserted); // callers may never pass multiple transactions with the same txid + cachedInnerUsage += RecursiveDynamicUsage(*block_it); + } + return LimitMemoryUsage(); +} + +void DisconnectedBlockTransactions::removeForBlock(const std::vector<CTransactionRef>& vtx) +{ + // Short-circuit in the common case of a block being added to the tip + if (queuedTx.empty()) { + return; + } + for (const auto& tx : vtx) { + auto iter = iters_by_txid.find(tx->GetHash()); + if (iter != iters_by_txid.end()) { + auto list_iter = iter->second; + iters_by_txid.erase(iter); + cachedInnerUsage -= RecursiveDynamicUsage(*list_iter); + queuedTx.erase(list_iter); + } + } +} + +void DisconnectedBlockTransactions::clear() +{ + cachedInnerUsage = 0; + iters_by_txid.clear(); + queuedTx.clear(); +} + +std::list<CTransactionRef> DisconnectedBlockTransactions::take() +{ + std::list<CTransactionRef> ret = std::move(queuedTx); + clear(); + return ret; +} diff --git a/src/kernel/disconnected_transactions.h b/src/kernel/disconnected_transactions.h index 7db39ba5ca..401ec435e6 100644 --- a/src/kernel/disconnected_transactions.h +++ b/src/kernel/disconnected_transactions.h @@ -5,8 +5,6 @@ #ifndef BITCOIN_KERNEL_DISCONNECTED_TRANSACTIONS_H #define BITCOIN_KERNEL_DISCONNECTED_TRANSACTIONS_H -#include <core_memusage.h> -#include <memusage.h> #include <primitives/transaction.h> #include <util/hasher.h> @@ -14,8 +12,8 @@ #include <unordered_map> #include <vector> -/** Maximum kilobytes for transactions to store for processing during reorg */ -static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20'000; +/** Maximum bytes for transactions to store for processing during reorg */ +static const unsigned int MAX_DISCONNECTED_TX_POOL_BYTES{20'000'000}; /** * DisconnectedBlockTransactions @@ -38,8 +36,7 @@ static const unsigned int MAX_DISCONNECTED_TX_POOL_SIZE = 20'000; */ class DisconnectedBlockTransactions { private: - /** Cached dynamic memory usage for the CTransactions (memory for the shared pointers is - * included in the container calculations). */ + /** Cached dynamic memory usage for the `CTransactionRef`s */ uint64_t cachedInnerUsage = 0; const size_t m_max_mem_usage; std::list<CTransactionRef> queuedTx; @@ -47,39 +44,15 @@ private: std::unordered_map<uint256, TxList::iterator, SaltedTxidHasher> iters_by_txid; /** Trim the earliest-added entries until we are within memory bounds. */ - std::vector<CTransactionRef> LimitMemoryUsage() - { - std::vector<CTransactionRef> evicted; - - while (!queuedTx.empty() && DynamicMemoryUsage() > m_max_mem_usage) { - evicted.emplace_back(queuedTx.front()); - cachedInnerUsage -= RecursiveDynamicUsage(*queuedTx.front()); - iters_by_txid.erase(queuedTx.front()->GetHash()); - queuedTx.pop_front(); - } - return evicted; - } + std::vector<CTransactionRef> LimitMemoryUsage(); public: - DisconnectedBlockTransactions(size_t max_mem_usage) : m_max_mem_usage{max_mem_usage} {} + DisconnectedBlockTransactions(size_t max_mem_usage) + : m_max_mem_usage{max_mem_usage} {} - // It's almost certainly a logic bug if we don't clear out queuedTx before - // destruction, as we add to it while disconnecting blocks, and then we - // need to re-process remaining transactions to ensure mempool consistency. - // For now, assert() that we've emptied out this object on destruction. - // This assert() can always be removed if the reorg-processing code were - // to be refactored such that this assumption is no longer true (for - // instance if there was some other way we cleaned up the mempool after a - // reorg, besides draining this object). - ~DisconnectedBlockTransactions() { - assert(queuedTx.empty()); - assert(iters_by_txid.empty()); - assert(cachedInnerUsage == 0); - } + ~DisconnectedBlockTransactions(); - size_t DynamicMemoryUsage() const { - return cachedInnerUsage + memusage::DynamicUsage(iters_by_txid) + memusage::DynamicUsage(queuedTx); - } + size_t DynamicMemoryUsage() const; /** Add transactions from the block, iterating through vtx in reverse order. Callers should call * this function for blocks in descending order by block height. @@ -88,50 +61,16 @@ public: * corresponding entry in iters_by_txid. * @returns vector of transactions that were evicted for size-limiting. */ - [[nodiscard]] std::vector<CTransactionRef> AddTransactionsFromBlock(const std::vector<CTransactionRef>& vtx) - { - iters_by_txid.reserve(iters_by_txid.size() + vtx.size()); - for (auto block_it = vtx.rbegin(); block_it != vtx.rend(); ++block_it) { - auto it = queuedTx.insert(queuedTx.end(), *block_it); - iters_by_txid.emplace((*block_it)->GetHash(), it); - cachedInnerUsage += RecursiveDynamicUsage(**block_it); - } - return LimitMemoryUsage(); - } + [[nodiscard]] std::vector<CTransactionRef> AddTransactionsFromBlock(const std::vector<CTransactionRef>& vtx); /** Remove any entries that are in this block. */ - void removeForBlock(const std::vector<CTransactionRef>& vtx) - { - // Short-circuit in the common case of a block being added to the tip - if (queuedTx.empty()) { - return; - } - for (const auto& tx : vtx) { - auto iter = iters_by_txid.find(tx->GetHash()); - if (iter != iters_by_txid.end()) { - auto list_iter = iter->second; - iters_by_txid.erase(iter); - cachedInnerUsage -= RecursiveDynamicUsage(**list_iter); - queuedTx.erase(list_iter); - } - } - } + void removeForBlock(const std::vector<CTransactionRef>& vtx); size_t size() const { return queuedTx.size(); } - void clear() - { - cachedInnerUsage = 0; - iters_by_txid.clear(); - queuedTx.clear(); - } + void clear(); /** Clear all data structures and return the list of transactions. */ - std::list<CTransactionRef> take() - { - std::list<CTransactionRef> ret = std::move(queuedTx); - clear(); - return ret; - } + std::list<CTransactionRef> take(); }; #endif // BITCOIN_KERNEL_DISCONNECTED_TRANSACTIONS_H diff --git a/src/kernel/mempool_entry.h b/src/kernel/mempool_entry.h index 1f175a5ccf..2adeaea652 100644 --- a/src/kernel/mempool_entry.h +++ b/src/kernel/mempool_entry.h @@ -71,6 +71,11 @@ public: typedef std::set<CTxMemPoolEntryRef, CompareIteratorByHash> Children; private: + CTxMemPoolEntry(const CTxMemPoolEntry&) = default; + struct ExplicitCopyTag { + explicit ExplicitCopyTag() = default; + }; + const CTransactionRef tx; mutable Parents m_parents; mutable Children m_children; @@ -83,7 +88,7 @@ private: const bool spendsCoinbase; //!< keep track of transactions that spend a coinbase const int64_t sigOpCost; //!< Total sigop cost CAmount m_modified_fee; //!< Used for determining the priority of the transaction for mining in a block - LockPoints lockPoints; //!< Track the height and time at which tx was final + mutable LockPoints lockPoints; //!< Track the height and time at which tx was final // Information about descendants of this transaction that are in the // mempool; if we remove this transaction we must remove all of these @@ -122,6 +127,13 @@ public: nModFeesWithAncestors{nFee}, nSigOpCostWithAncestors{sigOpCost} {} + CTxMemPoolEntry(ExplicitCopyTag, const CTxMemPoolEntry& entry) : CTxMemPoolEntry(entry) {} + CTxMemPoolEntry& operator=(const CTxMemPoolEntry&) = delete; + CTxMemPoolEntry(CTxMemPoolEntry&&) = delete; + CTxMemPoolEntry& operator=(CTxMemPoolEntry&&) = delete; + + static constexpr ExplicitCopyTag ExplicitCopy{}; + const CTransaction& GetTx() const { return *this->tx; } CTransactionRef GetSharedTx() const { return this->tx; } const CAmount& GetFee() const { return nFee; } @@ -151,7 +163,7 @@ public: } // Update the LockPoints after a reorg - void UpdateLockPoints(const LockPoints& lp) + void UpdateLockPoints(const LockPoints& lp) const { lockPoints = lp; } @@ -172,8 +184,69 @@ public: Parents& GetMemPoolParents() const { return m_parents; } Children& GetMemPoolChildren() const { return m_children; } - mutable size_t vTxHashesIdx; //!< Index in mempool's vTxHashes + mutable size_t idx_randomized; //!< Index in mempool's txns_randomized mutable Epoch::Marker m_epoch_marker; //!< epoch when last touched, useful for graph algorithms }; +using CTxMemPoolEntryRef = CTxMemPoolEntry::CTxMemPoolEntryRef; + +struct TransactionInfo { + const CTransactionRef m_tx; + /* The fee the transaction paid */ + const CAmount m_fee; + /** + * The virtual transaction size. + * + * This is a policy field which considers the sigop cost of the + * transaction as well as its weight, and reinterprets it as bytes. + * + * It is the primary metric by which the mining algorithm selects + * transactions. + */ + const int64_t m_virtual_transaction_size; + /* The block height the transaction entered the mempool */ + const unsigned int txHeight; + + TransactionInfo(const CTransactionRef& tx, const CAmount& fee, const int64_t vsize, const unsigned int height) + : m_tx{tx}, + m_fee{fee}, + m_virtual_transaction_size{vsize}, + txHeight{height} {} +}; + +struct RemovedMempoolTransactionInfo { + TransactionInfo info; + explicit RemovedMempoolTransactionInfo(const CTxMemPoolEntry& entry) + : info{entry.GetSharedTx(), entry.GetFee(), entry.GetTxSize(), entry.GetHeight()} {} +}; + +struct NewMempoolTransactionInfo { + TransactionInfo info; + /* + * This boolean indicates whether the transaction was added + * without enforcing mempool fee limits. + */ + const bool m_mempool_limit_bypassed; + /* This boolean indicates whether the transaction is part of a package. */ + const bool m_submitted_in_package; + /* + * This boolean indicates whether the blockchain is up to date when the + * transaction is added to the mempool. + */ + const bool m_chainstate_is_current; + /* Indicates whether the transaction has unconfirmed parents. */ + const bool m_has_no_mempool_parents; + + explicit NewMempoolTransactionInfo(const CTransactionRef& tx, const CAmount& fee, + const int64_t vsize, const unsigned int height, + const bool mempool_limit_bypassed, const bool submitted_in_package, + const bool chainstate_is_current, + const bool has_no_mempool_parents) + : info{tx, fee, vsize, height}, + m_mempool_limit_bypassed{mempool_limit_bypassed}, + m_submitted_in_package{submitted_in_package}, + m_chainstate_is_current{chainstate_is_current}, + m_has_no_mempool_parents{has_no_mempool_parents} {} +}; + #endif // BITCOIN_KERNEL_MEMPOOL_ENTRY_H diff --git a/src/kernel/mempool_options.h b/src/kernel/mempool_options.h index 757be41b3c..753aebd455 100644 --- a/src/kernel/mempool_options.h +++ b/src/kernel/mempool_options.h @@ -13,8 +13,6 @@ #include <cstdint> #include <optional> -class CBlockPolicyEstimator; - /** Default for -maxmempool, maximum megabytes of mempool memory usage */ static constexpr unsigned int DEFAULT_MAX_MEMPOOL_SIZE_MB{300}; /** Default for -maxmempool when blocksonly is set */ @@ -23,6 +21,8 @@ static constexpr unsigned int DEFAULT_BLOCKSONLY_MAX_MEMPOOL_SIZE_MB{5}; static constexpr unsigned int DEFAULT_MEMPOOL_EXPIRY_HOURS{336}; /** Default for -mempoolfullrbf, if the transaction replaceability signaling is ignored */ static constexpr bool DEFAULT_MEMPOOL_FULL_RBF{false}; +/** Whether to fall back to legacy V1 serialization when writing mempool.dat */ +static constexpr bool DEFAULT_PERSIST_V1_DAT{false}; /** Default for -acceptnonstdtxn */ static constexpr bool DEFAULT_ACCEPT_NON_STD_TXN{false}; @@ -35,8 +35,6 @@ namespace kernel { * Most of the time, this struct should be referenced as CTxMemPool::Options. */ struct MemPoolOptions { - /* Used to estimate appropriate transaction fees. */ - CBlockPolicyEstimator* estimator{nullptr}; /* The ratio used to determine how often sanity checks will run. */ int check_ratio{0}; int64_t max_size_bytes{DEFAULT_MAX_MEMPOOL_SIZE_MB * 1'000'000}; @@ -56,6 +54,7 @@ struct MemPoolOptions { bool permit_bare_multisig{DEFAULT_PERMIT_BAREMULTISIG}; bool require_standard{true}; bool full_rbf{DEFAULT_MEMPOOL_FULL_RBF}; + bool persist_v1_dat{DEFAULT_PERSIST_V1_DAT}; MemPoolLimits limits{}; }; } // namespace kernel diff --git a/src/kernel/mempool_persist.cpp b/src/kernel/mempool_persist.cpp index ff655c5ffa..0808d42452 100644 --- a/src/kernel/mempool_persist.cpp +++ b/src/kernel/mempool_persist.cpp @@ -8,6 +8,7 @@ #include <consensus/amount.h> #include <logging.h> #include <primitives/transaction.h> +#include <random.h> #include <serialize.h> #include <streams.h> #include <sync.h> @@ -34,14 +35,14 @@ using fsbridge::FopenFn; namespace kernel { -static const uint64_t MEMPOOL_DUMP_VERSION = 1; +static const uint64_t MEMPOOL_DUMP_VERSION_NO_XOR_KEY{1}; +static const uint64_t MEMPOOL_DUMP_VERSION{2}; bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active_chainstate, ImportMempoolOptions&& opts) { if (load_path.empty()) return false; - FILE* filestr{opts.mockable_fopen_function(load_path, "rb")}; - CAutoFile file{filestr, CLIENT_VERSION}; + AutoFile file{opts.mockable_fopen_function(load_path, "rb")}; if (file.IsNull()) { LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); return false; @@ -57,9 +58,15 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active try { uint64_t version; file >> version; - if (version != MEMPOOL_DUMP_VERSION) { + std::vector<std::byte> xor_key; + if (version == MEMPOOL_DUMP_VERSION_NO_XOR_KEY) { + // Leave XOR-key empty + } else if (version == MEMPOOL_DUMP_VERSION) { + file >> xor_key; + } else { return false; } + file.SetXor(xor_key); uint64_t num; file >> num; while (num) { @@ -67,7 +74,7 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active CTransactionRef tx; int64_t nTime; int64_t nFeeDelta; - file >> tx; + file >> TX_WITH_WITNESS(tx); file >> nTime; file >> nFeeDelta; @@ -151,20 +158,25 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock auto mid = SteadyClock::now(); - try { - FILE* filestr{mockable_fopen_function(dump_path + ".new", "wb")}; - if (!filestr) { - return false; - } - - CAutoFile file{filestr, CLIENT_VERSION}; + AutoFile file{mockable_fopen_function(dump_path + ".new", "wb")}; + if (file.IsNull()) { + return false; + } - uint64_t version = MEMPOOL_DUMP_VERSION; + try { + const uint64_t version{pool.m_persist_v1_dat ? MEMPOOL_DUMP_VERSION_NO_XOR_KEY : MEMPOOL_DUMP_VERSION}; file << version; + std::vector<std::byte> xor_key(8); + if (!pool.m_persist_v1_dat) { + FastRandomContext{}.fillrand(xor_key); + file << xor_key; + } + file.SetXor(xor_key); + file << (uint64_t)vinfo.size(); for (const auto& i : vinfo) { - file << *(i.tx); + file << TX_WITH_WITNESS(*(i.tx)); file << int64_t{count_seconds(i.m_time)}; file << int64_t{i.nFeeDelta}; mapDeltas.erase(i.tx->GetHash()); diff --git a/src/key.cpp b/src/key.cpp index 0f283ca3e3..512790252a 100644 --- a/src/key.cpp +++ b/src/key.cpp @@ -369,6 +369,13 @@ ECDHSecret CKey::ComputeBIP324ECDHSecret(const EllSwiftPubKey& their_ellswift, c return output; } +CKey GenerateRandomKey(bool compressed) noexcept +{ + CKey key; + key.MakeNewKey(/*fCompressed=*/compressed); + return key; +} + bool CExtKey::Derive(CExtKey &out, unsigned int _nChild) const { if (nDepth == std::numeric_limits<unsigned char>::max()) return false; out.nDepth = nDepth + 1; @@ -420,8 +427,7 @@ void CExtKey::Decode(const unsigned char code[BIP32_EXTKEY_SIZE]) { } bool ECC_InitSanityCheck() { - CKey key; - key.MakeNewKey(true); + CKey key = GenerateRandomKey(); CPubKey pubkey = key.GetPubKey(); return key.VerifyPubKey(pubkey); } @@ -205,6 +205,8 @@ public: bool initiating) const; }; +CKey GenerateRandomKey(bool compressed = true) noexcept; + struct CExtKey { unsigned char nDepth; unsigned char vchFingerprint[4]; diff --git a/src/merkleblock.cpp b/src/merkleblock.cpp index 3ffe1465da..c75f5c5e60 100644 --- a/src/merkleblock.cpp +++ b/src/merkleblock.cpp @@ -27,7 +27,7 @@ std::vector<bool> BytesToBits(const std::vector<unsigned char>& bytes) return ret; } -CMerkleBlock::CMerkleBlock(const CBlock& block, CBloomFilter* filter, const std::set<uint256>* txids) +CMerkleBlock::CMerkleBlock(const CBlock& block, CBloomFilter* filter, const std::set<Txid>* txids) { header = block.GetBlockHeader(); @@ -39,7 +39,7 @@ CMerkleBlock::CMerkleBlock(const CBlock& block, CBloomFilter* filter, const std: for (unsigned int i = 0; i < block.vtx.size(); i++) { - const uint256& hash = block.vtx[i]->GetHash(); + const Txid& hash{block.vtx[i]->GetHash()}; if (txids && txids->count(hash)) { vMatch.push_back(true); } else if (filter && filter->IsRelevantAndUpdate(*block.vtx[i])) { diff --git a/src/merkleblock.h b/src/merkleblock.h index b546b17f52..12b41a581e 100644 --- a/src/merkleblock.h +++ b/src/merkleblock.h @@ -11,6 +11,7 @@ #include <serialize.h> #include <uint256.h> +#include <set> #include <vector> // Helper functions for serialization. @@ -144,7 +145,7 @@ public: CMerkleBlock(const CBlock& block, CBloomFilter& filter) : CMerkleBlock(block, &filter, nullptr) { } // Create from a CBlock, matching the txids in the set - CMerkleBlock(const CBlock& block, const std::set<uint256>& txids) : CMerkleBlock(block, nullptr, &txids) { } + CMerkleBlock(const CBlock& block, const std::set<Txid>& txids) : CMerkleBlock{block, nullptr, &txids} {} CMerkleBlock() {} @@ -152,7 +153,7 @@ public: private: // Combined constructor to consolidate code - CMerkleBlock(const CBlock& block, CBloomFilter* filter, const std::set<uint256>* txids); + CMerkleBlock(const CBlock& block, CBloomFilter* filter, const std::set<Txid>* txids); }; #endif // BITCOIN_MERKLEBLOCK_H diff --git a/src/net.cpp b/src/net.cpp index 09a3d8617a..41f5323d91 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -18,6 +18,7 @@ #include <consensus/consensus.h> #include <crypto/sha256.h> #include <i2p.h> +#include <key.h> #include <logging.h> #include <memusage.h> #include <net_permissions.h> @@ -417,21 +418,25 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo const uint16_t default_port{pszDest != nullptr ? GetDefaultPort(pszDest) : m_params.GetDefaultPort()}; if (pszDest) { - const std::vector<CService> resolved{Lookup(pszDest, default_port, fNameLookup && !HaveNameProxy(), 256)}; + std::vector<CService> resolved{Lookup(pszDest, default_port, fNameLookup && !HaveNameProxy(), 256)}; if (!resolved.empty()) { - const CService& rnd{resolved[GetRand(resolved.size())]}; - addrConnect = CAddress{MaybeFlipIPv6toCJDNS(rnd), NODE_NONE}; - if (!addrConnect.IsValid()) { - LogPrint(BCLog::NET, "Resolver returned invalid address %s for %s\n", addrConnect.ToStringAddrPort(), pszDest); - return nullptr; - } - // It is possible that we already have a connection to the IP/port pszDest resolved to. - // In that case, drop the connection that was just created. - LOCK(m_nodes_mutex); - CNode* pnode = FindNode(static_cast<CService>(addrConnect)); - if (pnode) { - LogPrintf("Failed to open new connection, already connected\n"); - return nullptr; + Shuffle(resolved.begin(), resolved.end(), FastRandomContext()); + // If the connection is made by name, it can be the case that the name resolves to more than one address. + // We don't want to connect any more of them if we are already connected to one + for (const auto& r : resolved) { + addrConnect = CAddress{MaybeFlipIPv6toCJDNS(r), NODE_NONE}; + if (!addrConnect.IsValid()) { + LogPrint(BCLog::NET, "Resolver returned invalid address %s for %s\n", addrConnect.ToStringAddrPort(), pszDest); + return nullptr; + } + // It is possible that we already have a connection to the IP/port pszDest resolved to. + // In that case, drop the connection that was just created. + LOCK(m_nodes_mutex); + CNode* pnode = FindNode(static_cast<CService>(addrConnect)); + if (pnode) { + LogPrintf("Not opening a connection to %s, already connected to %s\n", pszDest, addrConnect.ToStringAddrPort()); + return nullptr; + } } } } @@ -679,11 +684,9 @@ bool CNode::ReceiveMsgBytes(Span<const uint8_t> msg_bytes, bool& complete) return true; } -V1Transport::V1Transport(const NodeId node_id, int nTypeIn, int nVersionIn) noexcept : - m_node_id(node_id), hdrbuf(nTypeIn, nVersionIn), vRecv(nTypeIn, nVersionIn) +V1Transport::V1Transport(const NodeId node_id) noexcept + : m_magic_bytes{Params().MessageStart()}, m_node_id{node_id} { - assert(std::size(Params().MessageStart()) == std::size(m_magic_bytes)); - m_magic_bytes = Params().MessageStart(); LOCK(m_recv_mutex); Reset(); } @@ -816,7 +819,7 @@ bool V1Transport::SetMessageToSend(CSerializedNetMsg& msg) noexcept // serialize header m_header_to_send.clear(); - CVectorWriter{INIT_PROTO_VERSION, m_header_to_send, 0, hdr}; + VectorWriter{m_header_to_send, 0, hdr}; // update state m_message_to_send = std::move(msg); @@ -936,13 +939,6 @@ public: const V2MessageMap V2_MESSAGE_MAP; -CKey GenerateRandomKey() noexcept -{ - CKey key; - key.MakeNewKey(/*fCompressed=*/true); - return key; -} - std::vector<uint8_t> GenerateRandomGarbage() noexcept { std::vector<uint8_t> ret; @@ -966,12 +962,12 @@ void V2Transport::StartSendingHandshake() noexcept // We cannot wipe m_send_garbage as it will still be used as AAD later in the handshake. } -V2Transport::V2Transport(NodeId nodeid, bool initiating, int type_in, int version_in, const CKey& key, Span<const std::byte> ent32, std::vector<uint8_t> garbage) noexcept : - m_cipher{key, ent32}, m_initiating{initiating}, m_nodeid{nodeid}, - m_v1_fallback{nodeid, type_in, version_in}, m_recv_type{type_in}, m_recv_version{version_in}, - m_recv_state{initiating ? RecvState::KEY : RecvState::KEY_MAYBE_V1}, - m_send_garbage{std::move(garbage)}, - m_send_state{initiating ? SendState::AWAITING_KEY : SendState::MAYBE_V1} +V2Transport::V2Transport(NodeId nodeid, bool initiating, const CKey& key, Span<const std::byte> ent32, std::vector<uint8_t> garbage) noexcept + : m_cipher{key, ent32}, m_initiating{initiating}, m_nodeid{nodeid}, + m_v1_fallback{nodeid}, + m_recv_state{initiating ? RecvState::KEY : RecvState::KEY_MAYBE_V1}, + m_send_garbage{std::move(garbage)}, + m_send_state{initiating ? SendState::AWAITING_KEY : SendState::MAYBE_V1} { Assume(m_send_garbage.size() <= MAX_GARBAGE_LEN); // Start sending immediately if we're the initiator of the connection. @@ -981,9 +977,9 @@ V2Transport::V2Transport(NodeId nodeid, bool initiating, int type_in, int versio } } -V2Transport::V2Transport(NodeId nodeid, bool initiating, int type_in, int version_in) noexcept : - V2Transport{nodeid, initiating, type_in, version_in, GenerateRandomKey(), - MakeByteSpan(GetRandHash()), GenerateRandomGarbage()} { } +V2Transport::V2Transport(NodeId nodeid, bool initiating) noexcept + : V2Transport{nodeid, initiating, GenerateRandomKey(), + MakeByteSpan(GetRandHash()), GenerateRandomGarbage()} {} void V2Transport::SetReceiveState(RecvState recv_state) noexcept { @@ -1415,7 +1411,7 @@ std::optional<std::string> V2Transport::GetMessageType(Span<const uint8_t>& cont } // Strip message type bytes of contents. contents = contents.subspan(CMessageHeader::COMMAND_SIZE); - return {std::move(ret)}; + return ret; } CNetMessage V2Transport::GetReceivedMessage(std::chrono::microseconds time, bool& reject_message) noexcept @@ -1427,8 +1423,7 @@ CNetMessage V2Transport::GetReceivedMessage(std::chrono::microseconds time, bool Assume(m_recv_state == RecvState::APP_READY); Span<const uint8_t> contents{m_recv_decode_buffer}; auto msg_type = GetMessageType(contents); - CDataStream ret(m_recv_type, m_recv_version); - CNetMessage msg{std::move(ret)}; + CNetMessage msg{DataStream{}}; // Note that BIP324Cipher::EXPANSION also includes the length descriptor size. msg.m_raw_message_size = m_recv_decode_buffer.size() + BIP324Cipher::EXPANSION; if (msg_type) { @@ -1730,7 +1725,6 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, const CAddress& addr) { int nInbound = 0; - int nMaxInbound = nMaxConnections - m_max_outbound; AddWhitelistPermissionFlags(permission_flags, addr); if (NetPermissions::HasFlag(permission_flags, NetPermissionFlags::Implicit)) { @@ -1776,13 +1770,13 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, // Only accept connections from discouraged peers if our inbound slots aren't (almost) full. bool discouraged = m_banman && m_banman->IsDiscouraged(addr); - if (!NetPermissions::HasFlag(permission_flags, NetPermissionFlags::NoBan) && nInbound + 1 >= nMaxInbound && discouraged) + if (!NetPermissions::HasFlag(permission_flags, NetPermissionFlags::NoBan) && nInbound + 1 >= m_max_inbound && discouraged) { LogPrint(BCLog::NET, "connection from %s dropped (discouraged)\n", addr.ToStringAddrPort()); return; } - if (nInbound >= nMaxInbound) + if (nInbound >= m_max_inbound) { if (!AttemptToEvictConnection()) { // No connection to evict, disconnect the new connection @@ -2714,6 +2708,17 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) continue; } + // Do not make automatic outbound connections to addnode peers, to + // not use our limited outbound slots for them and to ensure + // addnode connections benefit from their intended protections. + if (AddedNodesContain(addr)) { + LogPrintLevel(BCLog::NET, BCLog::Level::Debug, "Not making automatic %s%s connection to %s peer selected for manual (addnode) connection%s\n", + preferred_net.has_value() ? "network-specific " : "", + ConnectionTypeAsString(conn_type), GetNetworkName(addr.GetNetwork()), + fLogIPs ? strprintf(": %s", addr.ToStringAddrPort()) : ""); + continue; + } + addrConnect = addr; break; } @@ -2733,7 +2738,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // different netgroups in ipv4/ipv6 networks + all peers in Tor/I2P/CJDNS networks. // Don't record addrman failure attempts when node is offline. This can be identified since all local // network connections (if any) belong in the same netgroup, and the size of `outbound_ipv46_peer_netgroups` would only be 1. - const bool count_failures{((int)outbound_ipv46_peer_netgroups.size() + outbound_privacy_network_peers) >= std::min(nMaxConnections - 1, 2)}; + const bool count_failures{((int)outbound_ipv46_peer_netgroups.size() + outbound_privacy_network_peers) >= std::min(m_max_automatic_connections - 1, 2)}; // Use BIP324 transport when both us and them have NODE_V2_P2P set. const bool use_v2transport(addrConnect.nServices & GetLocalServices() & NODE_P2P_V2); OpenNetworkConnection(addrConnect, count_failures, std::move(grant), /*strDest=*/nullptr, conn_type, use_v2transport); @@ -2754,7 +2759,7 @@ std::vector<CAddress> CConnman::GetCurrentBlockRelayOnlyConns() const return ret; } -std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const +std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo(bool include_connected) const { std::vector<AddedNodeInfo> ret; @@ -2789,6 +2794,9 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const // strAddNode is an IP:port auto it = mapConnected.find(service); if (it != mapConnected.end()) { + if (!include_connected) { + continue; + } addedNode.resolvedAddress = service; addedNode.fConnected = true; addedNode.fInbound = it->second; @@ -2797,6 +2805,9 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const // strAddNode is a name auto it = mapConnectedByName.find(addr.m_added_node); if (it != mapConnectedByName.end()) { + if (!include_connected) { + continue; + } addedNode.resolvedAddress = it->second.second; addedNode.fConnected = true; addedNode.fInbound = it->second.first; @@ -2815,21 +2826,19 @@ void CConnman::ThreadOpenAddedConnections() while (true) { CSemaphoreGrant grant(*semAddnode); - std::vector<AddedNodeInfo> vInfo = GetAddedNodeInfo(); + std::vector<AddedNodeInfo> vInfo = GetAddedNodeInfo(/*include_connected=*/false); bool tried = false; for (const AddedNodeInfo& info : vInfo) { - if (!info.fConnected) { - if (!grant) { - // If we've used up our semaphore and need a new one, let's not wait here since while we are waiting - // the addednodeinfo state might change. - break; - } - tried = true; - CAddress addr(CService(), NODE_NONE); - OpenNetworkConnection(addr, false, std::move(grant), info.m_params.m_added_node.c_str(), ConnectionType::MANUAL, info.m_params.m_use_v2transport); - if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) return; - grant = CSemaphoreGrant(*semAddnode, /*fTry=*/true); + if (!grant) { + // If we've used up our semaphore and need a new one, let's not wait here since while we are waiting + // the addednodeinfo state might change. + break; } + tried = true; + CAddress addr(CService(), NODE_NONE); + OpenNetworkConnection(addr, false, std::move(grant), info.m_params.m_added_node.c_str(), ConnectionType::MANUAL, info.m_params.m_use_v2transport); + if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) return; + grant = CSemaphoreGrant(*semAddnode, /*fTry=*/true); } // Retry every 60 seconds if a connection was attempted, otherwise two seconds if (!interruptNet.sleep_for(std::chrono::seconds(tried ? 60 : 2))) @@ -3208,18 +3217,17 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) if (semOutbound == nullptr) { // initialize semaphore - semOutbound = std::make_unique<CSemaphore>(std::min(m_max_outbound, nMaxConnections)); + semOutbound = std::make_unique<CSemaphore>(std::min(m_max_automatic_outbound, m_max_automatic_connections)); } if (semAddnode == nullptr) { // initialize semaphore - semAddnode = std::make_unique<CSemaphore>(nMaxAddnode); + semAddnode = std::make_unique<CSemaphore>(m_max_addnode); } // // Start threads // assert(m_msgproc); - InterruptSocks5(false); interruptNet.reset(); flagInterruptMsgProc = false; @@ -3264,6 +3272,12 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) // Dump network addresses scheduler.scheduleEvery([this] { DumpAddresses(); }, DUMP_PEERS_INTERVAL); + // Run the ASMap Health check once and then schedule it to run every 24h. + if (m_netgroupman.UsingASMap()) { + ASMapHealthCheck(); + scheduler.scheduleEvery([this] { ASMapHealthCheck(); }, ASMAP_HEALTH_CHECK_INTERVAL); + } + return true; } @@ -3291,16 +3305,16 @@ void CConnman::Interrupt() condMsgProc.notify_all(); interruptNet(); - InterruptSocks5(true); + g_socks5_interrupt(); if (semOutbound) { - for (int i=0; i<m_max_outbound; i++) { + for (int i=0; i<m_max_automatic_outbound; i++) { semOutbound->post(); } } if (semAddnode) { - for (int i=0; i<nMaxAddnode; i++) { + for (int i=0; i<m_max_addnode; i++) { semAddnode->post(); } } @@ -3369,9 +3383,9 @@ CConnman::~CConnman() Stop(); } -std::vector<CAddress> CConnman::GetAddresses(size_t max_addresses, size_t max_pct, std::optional<Network> network) const +std::vector<CAddress> CConnman::GetAddresses(size_t max_addresses, size_t max_pct, std::optional<Network> network, const bool filtered) const { - std::vector<CAddress> addresses = addrman.GetAddr(max_addresses, max_pct, network); + std::vector<CAddress> addresses = addrman.GetAddr(max_addresses, max_pct, network, filtered); if (m_banman) { addresses.erase(std::remove_if(addresses.begin(), addresses.end(), [this](const CAddress& addr){return m_banman->IsDiscouraged(addr) || m_banman->IsBanned(addr);}), @@ -3426,9 +3440,12 @@ std::vector<CAddress> CConnman::GetAddresses(CNode& requestor, size_t max_addres bool CConnman::AddNode(const AddedNodeParams& add) { + const CService resolved(LookupNumeric(add.m_added_node, GetDefaultPort(add.m_added_node))); + const bool resolved_is_valid{resolved.IsValid()}; + LOCK(m_added_nodes_mutex); for (const auto& it : m_added_node_params) { - if (add.m_added_node == it.m_added_node) return false; + if (add.m_added_node == it.m_added_node || (resolved_is_valid && resolved == LookupNumeric(it.m_added_node, GetDefaultPort(it.m_added_node)))) return false; } m_added_node_params.push_back(add); @@ -3447,6 +3464,17 @@ bool CConnman::RemoveAddedNode(const std::string& strNode) return false; } +bool CConnman::AddedNodesContain(const CAddress& addr) const +{ + AssertLockNotHeld(m_added_nodes_mutex); + const std::string addr_str{addr.ToStringAddr()}; + const std::string addr_port_str{addr.ToStringAddrPort()}; + LOCK(m_added_nodes_mutex); + return (m_added_node_params.size() < 24 // bound the query to a reasonable limit + && std::any_of(m_added_node_params.cbegin(), m_added_node_params.cend(), + [&](const auto& p) { return p.m_added_node == addr_str || p.m_added_node == addr_port_str; })); +} + size_t CConnman::GetNodeCount(ConnectionDirection flags) const { LOCK(m_nodes_mutex); @@ -3631,9 +3659,9 @@ ServiceFlags CConnman::GetLocalServices() const static std::unique_ptr<Transport> MakeTransport(NodeId id, bool use_v2transport, bool inbound) noexcept { if (use_v2transport) { - return std::make_unique<V2Transport>(id, /*initiating=*/!inbound, SER_NETWORK, INIT_PROTO_VERSION); + return std::make_unique<V2Transport>(id, /*initiating=*/!inbound); } else { - return std::make_unique<V1Transport>(id, SER_NETWORK, INIT_PROTO_VERSION); + return std::make_unique<V1Transport>(id); } } @@ -3812,6 +3840,19 @@ void CConnman::PerformReconnections() } } +void CConnman::ASMapHealthCheck() +{ + const std::vector<CAddress> v4_addrs{GetAddresses(/*max_addresses=*/ 0, /*max_pct=*/ 0, Network::NET_IPV4, /*filtered=*/ false)}; + const std::vector<CAddress> v6_addrs{GetAddresses(/*max_addresses=*/ 0, /*max_pct=*/ 0, Network::NET_IPV6, /*filtered=*/ false)}; + std::vector<CNetAddr> clearnet_addrs; + clearnet_addrs.reserve(v4_addrs.size() + v6_addrs.size()); + std::transform(v4_addrs.begin(), v4_addrs.end(), std::back_inserter(clearnet_addrs), + [](const CAddress& addr) { return static_cast<CNetAddr>(addr); }); + std::transform(v6_addrs.begin(), v6_addrs.end(), std::back_inserter(clearnet_addrs), + [](const CAddress& addr) { return static_cast<CNetAddr>(addr); }); + m_netgroupman.ASMapHealthCheck(clearnet_addrs); +} + // Dump binary message to file, with timestamp. static void CaptureMessageToFile(const CAddress& addr, const std::string& msg_type, @@ -20,6 +20,7 @@ #include <netbase.h> #include <netgroup.h> #include <node/connection_types.h> +#include <node/protocol_version.h> #include <policy/feerate.h> #include <protocol.h> #include <random.h> @@ -87,6 +88,8 @@ static const bool DEFAULT_BLOCKSONLY = false; static const int64_t DEFAULT_PEER_CONNECT_TIMEOUT = 60; /** Number of file descriptors required for message capture **/ static const int NUM_FDS_MESSAGE_CAPTURE = 1; +/** Interval for ASMap Health Check **/ +static constexpr std::chrono::hours ASMAP_HEALTH_CHECK_INTERVAL{24}; static constexpr bool DEFAULT_FORCEDNSSEED{false}; static constexpr bool DEFAULT_DNSSEED{true}; @@ -231,15 +234,16 @@ public: * Ideally it should only contain receive time, payload, * type and size. */ -class CNetMessage { +class CNetMessage +{ public: - CDataStream m_recv; //!< received message data + DataStream m_recv; //!< received message data std::chrono::microseconds m_time{0}; //!< time of message receipt uint32_t m_message_size{0}; //!< size of the payload uint32_t m_raw_message_size{0}; //!< used wire size of the message (including header/checksum) std::string m_type; - CNetMessage(CDataStream&& recv_in) : m_recv(std::move(recv_in)) {} + explicit CNetMessage(DataStream&& recv_in) : m_recv(std::move(recv_in)) {} // Only one CNetMessage object will exist for the same message on either // the receive or processing queue. For performance reasons we therefore // delete the copy constructor and assignment operator to avoid the @@ -248,11 +252,6 @@ public: CNetMessage(const CNetMessage&) = delete; CNetMessage& operator=(CNetMessage&&) = default; CNetMessage& operator=(const CNetMessage&) = delete; - - void SetVersion(int nVersionIn) - { - m_recv.SetVersion(nVersionIn); - } }; /** The Transport converts one connection's sent messages to wire bytes, and received bytes back. */ @@ -372,15 +371,15 @@ public: class V1Transport final : public Transport { private: - MessageStartChars m_magic_bytes; + const MessageStartChars m_magic_bytes; const NodeId m_node_id; // Only for logging mutable Mutex m_recv_mutex; //!< Lock for receive state mutable CHash256 hasher GUARDED_BY(m_recv_mutex); mutable uint256 data_hash GUARDED_BY(m_recv_mutex); bool in_data GUARDED_BY(m_recv_mutex); // parsing header (false) or data (true) - CDataStream hdrbuf GUARDED_BY(m_recv_mutex); // partially received header + DataStream hdrbuf GUARDED_BY(m_recv_mutex){}; // partially received header CMessageHeader hdr GUARDED_BY(m_recv_mutex); // complete header - CDataStream vRecv GUARDED_BY(m_recv_mutex); // received message data + DataStream vRecv GUARDED_BY(m_recv_mutex){}; // received message data unsigned int nHdrPos GUARDED_BY(m_recv_mutex); unsigned int nDataPos GUARDED_BY(m_recv_mutex); @@ -419,7 +418,7 @@ private: size_t m_bytes_sent GUARDED_BY(m_send_mutex) {0}; public: - V1Transport(const NodeId node_id, int nTypeIn, int nVersionIn) noexcept; + explicit V1Transport(const NodeId node_id) noexcept; bool ReceivedMessageComplete() const override EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex) { @@ -597,10 +596,6 @@ private: std::vector<uint8_t> m_recv_aad GUARDED_BY(m_recv_mutex); /** Buffer to put decrypted contents in, for converting to CNetMessage. */ std::vector<uint8_t> m_recv_decode_buffer GUARDED_BY(m_recv_mutex); - /** Deserialization type. */ - const int m_recv_type; - /** Deserialization version number. */ - const int m_recv_version; /** Current receiver state. */ RecvState m_recv_state GUARDED_BY(m_recv_mutex); @@ -646,13 +641,11 @@ public: * * @param[in] nodeid the node's NodeId (only for debug log output). * @param[in] initiating whether we are the initiator side. - * @param[in] type_in the serialization type of returned CNetMessages. - * @param[in] version_in the serialization version of returned CNetMessages. */ - V2Transport(NodeId nodeid, bool initiating, int type_in, int version_in) noexcept; + V2Transport(NodeId nodeid, bool initiating) noexcept; /** Construct a V2 transport with specified keys and garbage (test use only). */ - V2Transport(NodeId nodeid, bool initiating, int type_in, int version_in, const CKey& key, Span<const std::byte> ent32, std::vector<uint8_t> garbage) noexcept; + V2Transport(NodeId nodeid, bool initiating, const CKey& key, Span<const std::byte> ent32, std::vector<uint8_t> garbage) noexcept; // Receive side functions. bool ReceivedMessageComplete() const noexcept override EXCLUSIVE_LOCKS_REQUIRED(!m_recv_mutex); @@ -1045,11 +1038,7 @@ public: struct Options { ServiceFlags nLocalServices = NODE_NONE; - int nMaxConnections = 0; - int m_max_outbound_full_relay = 0; - int m_max_outbound_block_relay = 0; - int nMaxAddnode = 0; - int nMaxFeeler = 0; + int m_max_automatic_connections = 0; CClientUIInterface* uiInterface = nullptr; NetEventsInterface* m_msgproc = nullptr; BanMan* m_banman = nullptr; @@ -1076,13 +1065,12 @@ public: AssertLockNotHeld(m_total_bytes_sent_mutex); nLocalServices = connOptions.nLocalServices; - nMaxConnections = connOptions.nMaxConnections; - m_max_outbound_full_relay = std::min(connOptions.m_max_outbound_full_relay, connOptions.nMaxConnections); - m_max_outbound_block_relay = connOptions.m_max_outbound_block_relay; + m_max_automatic_connections = connOptions.m_max_automatic_connections; + m_max_outbound_full_relay = std::min(MAX_OUTBOUND_FULL_RELAY_CONNECTIONS, m_max_automatic_connections); + m_max_outbound_block_relay = std::min(MAX_BLOCK_RELAY_ONLY_CONNECTIONS, m_max_automatic_connections - m_max_outbound_full_relay); + m_max_automatic_outbound = m_max_outbound_full_relay + m_max_outbound_block_relay + m_max_feeler; + m_max_inbound = std::max(0, m_max_automatic_connections - m_max_automatic_outbound); m_use_addrman_outgoing = connOptions.m_use_addrman_outgoing; - nMaxAddnode = connOptions.nMaxAddnode; - nMaxFeeler = connOptions.nMaxFeeler; - m_max_outbound = m_max_outbound_full_relay + m_max_outbound_block_relay + nMaxFeeler; m_client_interface = connOptions.uiInterface; m_banman = connOptions.m_banman; m_msgproc = connOptions.m_msgproc; @@ -1126,6 +1114,7 @@ public: void SetNetworkActive(bool active); void OpenNetworkConnection(const CAddress& addrConnect, bool fCountFailure, CSemaphoreGrant&& grant_outbound, const char* strDest, ConnectionType conn_type, bool use_v2transport) EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex); bool CheckIncomingNonce(uint64_t nonce); + void ASMapHealthCheck(); // alias for thread safety annotations only, not defined RecursiveMutex& GetNodesMutex() const LOCK_RETURNED(m_nodes_mutex); @@ -1160,8 +1149,9 @@ public: * @param[in] max_addresses Maximum number of addresses to return (0 = all). * @param[in] max_pct Maximum percentage of addresses to return (0 = all). * @param[in] network Select only addresses of this network (nullopt = all). + * @param[in] filtered Select only addresses that are considered high quality (false = all). */ - std::vector<CAddress> GetAddresses(size_t max_addresses, size_t max_pct, std::optional<Network> network) const; + std::vector<CAddress> GetAddresses(size_t max_addresses, size_t max_pct, std::optional<Network> network, const bool filtered = true) const; /** * Cache is used to minimize topology leaks, so it should * be used for all non-trusted calls, for example, p2p. @@ -1189,7 +1179,8 @@ public: bool AddNode(const AddedNodeParams& add) EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); bool RemoveAddedNode(const std::string& node) EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); - std::vector<AddedNodeInfo> GetAddedNodeInfo() const EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); + bool AddedNodesContain(const CAddress& addr) const EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); + std::vector<AddedNodeInfo> GetAddedNodeInfo(bool include_connected) const EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); /** * Attempts to open a connection. Currently only used from tests. @@ -1466,7 +1457,18 @@ private: std::unique_ptr<CSemaphore> semOutbound; std::unique_ptr<CSemaphore> semAddnode; - int nMaxConnections; + + /** + * Maximum number of automatic connections permitted, excluding manual + * connections but including inbounds. May be changed by the user and is + * potentially limited by the operating system (number of file descriptors). + */ + int m_max_automatic_connections; + + /* + * Maximum number of peers by connection type. Might vary from defaults + * based on -maxconnections init value. + */ // How many full-relay (tx, block, addr) outbound peers we want int m_max_outbound_full_relay; @@ -1475,9 +1477,11 @@ private: // We do not relay tx or addr messages with these peers int m_max_outbound_block_relay; - int nMaxAddnode; - int nMaxFeeler; - int m_max_outbound; + int m_max_addnode{MAX_ADDNODE_CONNECTIONS}; + int m_max_feeler{MAX_FEELER_CONNECTIONS}; + int m_max_automatic_outbound; + int m_max_inbound; + bool m_use_addrman_outgoing; CClientUIInterface* m_client_interface; NetEventsInterface* m_msgproc; diff --git a/src/net_permissions.cpp b/src/net_permissions.cpp index cf6b58e08d..a134a55264 100644 --- a/src/net_permissions.cpp +++ b/src/net_permissions.cpp @@ -111,8 +111,7 @@ bool NetWhitelistPermissions::TryParse(const std::string& str, NetWhitelistPermi if (!TryParsePermissionFlags(str, flags, offset, error)) return false; const std::string net = str.substr(offset); - CSubNet subnet; - LookupSubNet(net, subnet); + const CSubNet subnet{LookupSubNet(net)}; if (!subnet.IsValid()) { error = strprintf(_("Invalid netmask specified in -whitelist: '%s'"), net); return false; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 3bfb606037..df54a62f28 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -16,9 +16,9 @@ #include <hash.h> #include <headerssync.h> #include <index/blockfilterindex.h> +#include <kernel/chain.h> #include <kernel/mempool_entry.h> #include <logging.h> -#include <kernel/chain.h> #include <merkleblock.h> #include <netbase.h> #include <netmessagemaker.h> @@ -39,7 +39,7 @@ #include <txmempool.h> #include <txorphanage.h> #include <txrequest.h> -#include <util/check.h> // For NDEBUG compile time check +#include <util/check.h> #include <util/strencodings.h> #include <util/trace.h> #include <validation.h> @@ -51,6 +51,7 @@ #include <memory> #include <optional> #include <typeinfo> +#include <utility> /** Headers download timeout. * Timeout = base + per_header * (expected number of headers) */ @@ -514,7 +515,7 @@ public: void RelayTransaction(const uint256& txid, const uint256& wtxid) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex); void SetBestHeight(int height) override { m_best_height = height; }; void UnitTestMisbehaving(NodeId peer_id, int howmuch) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex) { Misbehaving(*Assert(GetPeerRef(peer_id)), howmuch, ""); }; - void ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, + void ProcessMessage(CNode& pfrom, const std::string& msg_type, DataStream& vRecv, const std::chrono::microseconds time_received, const std::atomic<bool>& interruptMsgProc) override EXCLUSIVE_LOCKS_REQUIRED(!m_peer_mutex, !m_recent_confirmed_transactions_mutex, !m_most_recent_block_mutex, !m_headers_presync_mutex, g_msgproc_mutex); void UpdateLastBlockAnnounceTime(NodeId node, int64_t time_in_seconds) override; @@ -669,6 +670,14 @@ private: void AddTxAnnouncement(const CNode& node, const GenTxid& gtxid, std::chrono::microseconds current_time) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + /** Send a message to a peer */ + void PushMessage(CNode& node, CSerializedNetMsg&& msg) const { m_connman.PushMessage(&node, std::move(msg)); } + template <typename... Args> + void MakeAndPushMessage(CNode& node, std::string msg_type, Args&&... args) const + { + m_connman.PushMessage(&node, NetMsg::Make(std::move(msg_type), std::forward<Args>(args)...)); + } + /** Send a version message to a peer */ void PushNodeVersion(CNode& pnode, const Peer& peer); @@ -1024,7 +1033,7 @@ private: * @param[in] peer The peer that we received the request from * @param[in] vRecv The raw message received */ - void ProcessGetCFilters(CNode& node, Peer& peer, CDataStream& vRecv); + void ProcessGetCFilters(CNode& node, Peer& peer, DataStream& vRecv); /** * Handle a cfheaders request. @@ -1035,7 +1044,7 @@ private: * @param[in] peer The peer that we received the request from * @param[in] vRecv The raw message received */ - void ProcessGetCFHeaders(CNode& node, Peer& peer, CDataStream& vRecv); + void ProcessGetCFHeaders(CNode& node, Peer& peer, DataStream& vRecv); /** * Handle a getcfcheckpt request. @@ -1046,7 +1055,7 @@ private: * @param[in] peer The peer that we received the request from * @param[in] vRecv The raw message received */ - void ProcessGetCFCheckPt(CNode& node, Peer& peer, CDataStream& vRecv); + void ProcessGetCFCheckPt(CNode& node, Peer& peer, DataStream& vRecv); /** Checks if address relay is permitted with peer. If needed, initializes * the m_addr_known bloom filter and sets m_addr_relay_enabled to true. @@ -1277,14 +1286,14 @@ void PeerManagerImpl::MaybeSetPeerAsAnnouncingHeaderAndIDs(NodeId nodeid) // As per BIP152, we only get 3 of our peers to announce // blocks using compact encodings. m_connman.ForNode(lNodesAnnouncingHeaderAndIDs.front(), [this](CNode* pnodeStop){ - m_connman.PushMessage(pnodeStop, CNetMsgMaker(pnodeStop->GetCommonVersion()).Make(NetMsgType::SENDCMPCT, /*high_bandwidth=*/false, /*version=*/CMPCTBLOCKS_VERSION)); + MakeAndPushMessage(*pnodeStop, NetMsgType::SENDCMPCT, /*high_bandwidth=*/false, /*version=*/CMPCTBLOCKS_VERSION); // save BIP152 bandwidth state: we select peer to be low-bandwidth pnodeStop->m_bip152_highbandwidth_to = false; return true; }); lNodesAnnouncingHeaderAndIDs.pop_front(); } - m_connman.PushMessage(pfrom, CNetMsgMaker(pfrom->GetCommonVersion()).Make(NetMsgType::SENDCMPCT, /*high_bandwidth=*/true, /*version=*/CMPCTBLOCKS_VERSION)); + MakeAndPushMessage(*pfrom, NetMsgType::SENDCMPCT, /*high_bandwidth=*/true, /*version=*/CMPCTBLOCKS_VERSION); // save BIP152 bandwidth state: we select peer to be high-bandwidth pfrom->m_bip152_highbandwidth_to = true; lNodesAnnouncingHeaderAndIDs.push_back(pfrom->GetId()); @@ -1487,10 +1496,10 @@ void PeerManagerImpl::PushNodeVersion(CNode& pnode, const Peer& peer) uint64_t your_services{addr.nServices}; const bool tx_relay{!RejectIncomingTxs(pnode)}; - m_connman.PushMessage(&pnode, CNetMsgMaker(INIT_PROTO_VERSION).Make(NetMsgType::VERSION, PROTOCOL_VERSION, my_services, nTime, + MakeAndPushMessage(pnode, NetMsgType::VERSION, PROTOCOL_VERSION, my_services, nTime, your_services, CNetAddr::V1(addr_you), // Together the pre-version-31402 serialization of CAddress "addrYou" (without nTime) my_services, CNetAddr::V1(CService{}), // Together the pre-version-31402 serialization of CAddress "addrMe" (without nTime) - nonce, strSubVersion, nNodeStartingHeight, tx_relay)); + nonce, strSubVersion, nNodeStartingHeight, tx_relay); if (fLogIPs) { LogPrint(BCLog::NET, "send version message: version %d, blocks=%d, them=%s, txrelay=%d, peer=%d\n", PROTOCOL_VERSION, nNodeStartingHeight, addr_you.ToStringAddrPort(), tx_relay, nodeid); @@ -1820,6 +1829,8 @@ bool PeerManagerImpl::MaybePunishNodeForTx(NodeId nodeid, const TxValidationStat case TxValidationResult::TX_CONFLICT: case TxValidationResult::TX_MEMPOOL_POLICY: case TxValidationResult::TX_NO_MEMPOOL: + case TxValidationResult::TX_RECONSIDERABLE: + case TxValidationResult::TX_UNKNOWN: break; } return false; @@ -1859,8 +1870,7 @@ std::optional<std::string> PeerManagerImpl::FetchBlock(NodeId peer_id, const CBl // Send block request message to the peer bool success = m_connman.ForNode(peer_id, [this, &invs](CNode* node) { - const CNetMsgMaker msgMaker(node->GetCommonVersion()); - this->m_connman.PushMessage(node, msgMaker.Make(NetMsgType::GETDATA, invs)); + this->MakeAndPushMessage(*node, NetMsgType::GETDATA, invs); return true; }); @@ -1947,9 +1957,9 @@ void PeerManagerImpl::BlockConnected( { LOCK(m_recent_confirmed_transactions_mutex); for (const auto& ptx : pblock->vtx) { - m_recent_confirmed_transactions.insert(ptx->GetHash()); - if (ptx->GetHash() != ptx->GetWitnessHash()) { - m_recent_confirmed_transactions.insert(ptx->GetWitnessHash()); + m_recent_confirmed_transactions.insert(ptx->GetHash().ToUint256()); + if (ptx->HasWitness()) { + m_recent_confirmed_transactions.insert(ptx->GetWitnessHash().ToUint256()); } } } @@ -1983,7 +1993,6 @@ void PeerManagerImpl::BlockDisconnected(const std::shared_ptr<const CBlock> &blo void PeerManagerImpl::NewPoWValidBlock(const CBlockIndex *pindex, const std::shared_ptr<const CBlock>& pblock) { auto pcmpctblock = std::make_shared<const CBlockHeaderAndShortTxIDs>(*pblock); - const CNetMsgMaker msgMaker(PROTOCOL_VERSION); LOCK(cs_main); @@ -1995,7 +2004,7 @@ void PeerManagerImpl::NewPoWValidBlock(const CBlockIndex *pindex, const std::sha uint256 hashBlock(pblock->GetHash()); const std::shared_future<CSerializedNetMsg> lazy_ser{ - std::async(std::launch::deferred, [&] { return msgMaker.Make(NetMsgType::CMPCTBLOCK, *pcmpctblock); })}; + std::async(std::launch::deferred, [&] { return NetMsg::Make(NetMsgType::CMPCTBLOCK, *pcmpctblock); })}; { auto most_recent_block_txs = std::make_unique<std::map<uint256, CTransactionRef>>(); @@ -2026,7 +2035,7 @@ void PeerManagerImpl::NewPoWValidBlock(const CBlockIndex *pindex, const std::sha hashBlock.ToString(), pnode->GetId()); const CSerializedNetMsg& ser_cmpctblock{lazy_ser.get()}; - m_connman.PushMessage(pnode, ser_cmpctblock.Copy()); + PushMessage(*pnode, ser_cmpctblock.Copy()); state.pindexBestHeaderSent = pindex; } }); @@ -2260,7 +2269,6 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& LogPrint(BCLog::NET, "%s: ignoring request from peer=%i for old block that isn't in the main chain\n", __func__, pfrom.GetId()); return; } - const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); // disconnect node in case we have reached the outbound limit for serving historical blocks if (m_connman.OutboundTargetReached(true) && (((m_chainman.m_best_header != nullptr) && (m_chainman.m_best_header->GetBlockTime() - pindex->GetBlockTime() > HISTORICAL_BLOCK_AGE)) || inv.IsMsgFilteredBlk()) && @@ -2294,7 +2302,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& if (!m_chainman.m_blockman.ReadRawBlockFromDisk(block_data, pindex->GetBlockPos())) { assert(!"cannot load block from disk"); } - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, Span{block_data})); + MakeAndPushMessage(pfrom, NetMsgType::BLOCK, Span{block_data}); // Don't set pblock as we've sent the block } else { // Send block from disk @@ -2306,9 +2314,9 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& } if (pblock) { if (inv.IsMsgBlk()) { - m_connman.PushMessage(&pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, *pblock)); + MakeAndPushMessage(pfrom, NetMsgType::BLOCK, TX_NO_WITNESS(*pblock)); } else if (inv.IsMsgWitnessBlk()) { - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); + MakeAndPushMessage(pfrom, NetMsgType::BLOCK, TX_WITH_WITNESS(*pblock)); } else if (inv.IsMsgFilteredBlk()) { bool sendMerkleBlock = false; CMerkleBlock merkleBlock; @@ -2320,7 +2328,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& } } if (sendMerkleBlock) { - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::MERKLEBLOCK, merkleBlock)); + MakeAndPushMessage(pfrom, NetMsgType::MERKLEBLOCK, merkleBlock); // CMerkleBlock just contains hashes, so also push any transactions in the block the client did not see // This avoids hurting performance by pointlessly requiring a round-trip // Note that there is currently no way for a node to request any single transactions we didn't send here - @@ -2329,7 +2337,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& // however we MUST always provide at least what the remote peer needs typedef std::pair<unsigned int, uint256> PairType; for (PairType& pair : merkleBlock.vMatchedTxn) - m_connman.PushMessage(&pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *pblock->vtx[pair.first])); + MakeAndPushMessage(pfrom, NetMsgType::TX, TX_NO_WITNESS(*pblock->vtx[pair.first])); } // else // no response @@ -2340,13 +2348,13 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& // instead we respond with the full, non-compact block. if (CanDirectFetch() && pindex->nHeight >= m_chainman.ActiveChain().Height() - MAX_CMPCTBLOCK_DEPTH) { if (a_recent_compact_block && a_recent_compact_block->header.GetHash() == pindex->GetBlockHash()) { - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::CMPCTBLOCK, *a_recent_compact_block)); + MakeAndPushMessage(pfrom, NetMsgType::CMPCTBLOCK, *a_recent_compact_block); } else { CBlockHeaderAndShortTxIDs cmpctblock{*pblock}; - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); + MakeAndPushMessage(pfrom, NetMsgType::CMPCTBLOCK, cmpctblock); } } else { - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); + MakeAndPushMessage(pfrom, NetMsgType::BLOCK, TX_WITH_WITNESS(*pblock)); } } } @@ -2360,7 +2368,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& // wait for other stuff first. std::vector<CInv> vInv; vInv.emplace_back(MSG_BLOCK, m_chainman.ActiveChain().Tip()->GetBlockHash()); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::INV, vInv)); + MakeAndPushMessage(pfrom, NetMsgType::INV, vInv); peer.m_continuation_block.SetNull(); } } @@ -2394,7 +2402,6 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic std::deque<CInv>::iterator it = peer.m_getdata_requests.begin(); std::vector<CInv> vNotFound; - const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); // Process as many TX items from the front of the getdata queue as // possible, since they're common and it's efficient to batch process @@ -2416,8 +2423,8 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic CTransactionRef tx = FindTxForGetData(*tx_relay, ToGenTxid(inv)); if (tx) { // WTX and WITNESS_TX imply we serialize with witness - int nSendFlags = (inv.IsMsgTx() ? SERIALIZE_TRANSACTION_NO_WITNESS : 0); - m_connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *tx)); + const auto maybe_with_witness = (inv.IsMsgTx() ? TX_NO_WITNESS : TX_WITH_WITNESS); + MakeAndPushMessage(pfrom, NetMsgType::TX, maybe_with_witness(*tx)); m_mempool.RemoveUnbroadcastTx(tx->GetHash()); } else { vNotFound.push_back(inv); @@ -2447,12 +2454,12 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic // of transactions relevant to them, without having to download the // entire memory pool. // Also, other nodes can use these messages to automatically request a - // transaction from some other peer that annnounced it, and stop + // transaction from some other peer that announced it, and stop // waiting for us to respond. // In normal operation, we often send NOTFOUND messages for parents of // transactions that we relay; if a peer is missing a parent, they may // assume we have them and request the parents from us. - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::NOTFOUND, vNotFound)); + MakeAndPushMessage(pfrom, NetMsgType::NOTFOUND, vNotFound); } } @@ -2476,8 +2483,7 @@ void PeerManagerImpl::SendBlockTransactions(CNode& pfrom, Peer& peer, const CBlo resp.txn[i] = block.vtx[req.indexes[i]]; } - const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCKTXN, resp)); + MakeAndPushMessage(pfrom, NetMsgType::BLOCKTXN, resp); } bool PeerManagerImpl::CheckHeadersPoW(const std::vector<CBlockHeader>& headers, const Consensus::Params& consensusParams, Peer& peer) @@ -2705,14 +2711,12 @@ bool PeerManagerImpl::IsAncestorOfBestHeaderOrTip(const CBlockIndex* header) bool PeerManagerImpl::MaybeSendGetHeaders(CNode& pfrom, const CBlockLocator& locator, Peer& peer) { - const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); - const auto current_time = NodeClock::now(); // Only allow a new getheaders message to go out if we don't have a recent // one already in-flight if (current_time - peer.m_last_getheaders_timestamp > HEADERS_RESPONSE_TIME) { - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETHEADERS, locator, uint256())); + MakeAndPushMessage(pfrom, NetMsgType::GETHEADERS, locator, uint256()); peer.m_last_getheaders_timestamp = current_time; return true; } @@ -2726,8 +2730,6 @@ bool PeerManagerImpl::MaybeSendGetHeaders(CNode& pfrom, const CBlockLocator& loc */ void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const Peer& peer, const CBlockIndex& last_header) { - const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); - LOCK(cs_main); CNodeState *nodestate = State(pfrom.GetId()); @@ -2780,7 +2782,7 @@ void PeerManagerImpl::HeadersDirectFetchBlocks(CNode& pfrom, const Peer& peer, c // In any case, we want to download using a compact block, not a regular one vGetData[0] = CInv(MSG_CMPCT_BLOCK, vGetData[0].hash); } - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + MakeAndPushMessage(pfrom, NetMsgType::GETDATA, vGetData); } } } @@ -3003,8 +3005,8 @@ bool PeerManagerImpl::ProcessOrphanTx(Peer& peer) while (CTransactionRef porphanTx = m_orphanage.GetTxToReconsider(peer.m_id)) { const MempoolAcceptResult result = m_chainman.ProcessTransaction(porphanTx); const TxValidationState& state = result.m_state; - const uint256& orphanHash = porphanTx->GetHash(); - const uint256& orphan_wtxid = porphanTx->GetWitnessHash(); + const Txid& orphanHash = porphanTx->GetHash(); + const Wtxid& orphan_wtxid = porphanTx->GetWitnessHash(); if (result.m_result_type == MempoolAcceptResult::ResultType::VALID) { LogPrint(BCLog::TXPACKAGES, " accepted orphan tx %s (wtxid=%s)\n", orphanHash.ToString(), orphan_wtxid.ToString()); @@ -3052,7 +3054,7 @@ bool PeerManagerImpl::ProcessOrphanTx(Peer& peer) // See also comments in https://github.com/bitcoin/bitcoin/pull/18044#discussion_r443419034 // for concerns around weakening security of unupgraded nodes // if we start doing this too early. - m_recent_rejects.insert(porphanTx->GetWitnessHash()); + m_recent_rejects.insert(porphanTx->GetWitnessHash().ToUint256()); // If the transaction failed for TX_INPUTS_NOT_STANDARD, // then we know that the witness was irrelevant to the policy // failure, since this check depends only on the txid @@ -3061,10 +3063,10 @@ bool PeerManagerImpl::ProcessOrphanTx(Peer& peer) // processing of this transaction in the event that child // transactions are later received (resulting in // parent-fetching by txid via the orphan-handling logic). - if (state.GetResult() == TxValidationResult::TX_INPUTS_NOT_STANDARD && porphanTx->GetWitnessHash() != porphanTx->GetHash()) { + if (state.GetResult() == TxValidationResult::TX_INPUTS_NOT_STANDARD && porphanTx->HasWitness()) { // We only add the txid if it differs from the wtxid, to // avoid wasting entries in the rolling bloom filter. - m_recent_rejects.insert(porphanTx->GetHash()); + m_recent_rejects.insert(porphanTx->GetHash().ToUint256()); } } m_orphanage.EraseTx(orphanHash); @@ -3128,7 +3130,7 @@ bool PeerManagerImpl::PrepareBlockFilterRequest(CNode& node, Peer& peer, return true; } -void PeerManagerImpl::ProcessGetCFilters(CNode& node,Peer& peer, CDataStream& vRecv) +void PeerManagerImpl::ProcessGetCFilters(CNode& node, Peer& peer, DataStream& vRecv) { uint8_t filter_type_ser; uint32_t start_height; @@ -3153,13 +3155,11 @@ void PeerManagerImpl::ProcessGetCFilters(CNode& node,Peer& peer, CDataStream& vR } for (const auto& filter : filters) { - CSerializedNetMsg msg = CNetMsgMaker(node.GetCommonVersion()) - .Make(NetMsgType::CFILTER, filter); - m_connman.PushMessage(&node, std::move(msg)); + MakeAndPushMessage(node, NetMsgType::CFILTER, filter); } } -void PeerManagerImpl::ProcessGetCFHeaders(CNode& node, Peer& peer, CDataStream& vRecv) +void PeerManagerImpl::ProcessGetCFHeaders(CNode& node, Peer& peer, DataStream& vRecv) { uint8_t filter_type_ser; uint32_t start_height; @@ -3194,16 +3194,14 @@ void PeerManagerImpl::ProcessGetCFHeaders(CNode& node, Peer& peer, CDataStream& return; } - CSerializedNetMsg msg = CNetMsgMaker(node.GetCommonVersion()) - .Make(NetMsgType::CFHEADERS, + MakeAndPushMessage(node, NetMsgType::CFHEADERS, filter_type_ser, stop_index->GetBlockHash(), prev_header, filter_hashes); - m_connman.PushMessage(&node, std::move(msg)); } -void PeerManagerImpl::ProcessGetCFCheckPt(CNode& node, Peer& peer, CDataStream& vRecv) +void PeerManagerImpl::ProcessGetCFCheckPt(CNode& node, Peer& peer, DataStream& vRecv) { uint8_t filter_type_ser; uint256 stop_hash; @@ -3235,12 +3233,10 @@ void PeerManagerImpl::ProcessGetCFCheckPt(CNode& node, Peer& peer, CDataStream& } } - CSerializedNetMsg msg = CNetMsgMaker(node.GetCommonVersion()) - .Make(NetMsgType::CFCHECKPT, + MakeAndPushMessage(node, NetMsgType::CFCHECKPT, filter_type_ser, stop_index->GetBlockHash(), headers); - m_connman.PushMessage(&node, std::move(msg)); } void PeerManagerImpl::ProcessBlock(CNode& node, const std::shared_ptr<const CBlock>& block, bool force_processing, bool min_pow_checked) @@ -3264,7 +3260,6 @@ void PeerManagerImpl::ProcessCompactBlockTxns(CNode& pfrom, Peer& peer, const Bl { std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); bool fBlockRead{false}; - const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); { LOCK(cs_main); @@ -3300,7 +3295,7 @@ void PeerManagerImpl::ProcessCompactBlockTxns(CNode& pfrom, Peer& peer, const Bl // Might have collided, fall back to getdata now :( std::vector<CInv> invs; invs.emplace_back(MSG_BLOCK | GetFetchFlags(peer), block_transactions.blockhash); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, invs)); + MakeAndPushMessage(pfrom, NetMsgType::GETDATA, invs); } else { RemoveBlockRequest(block_transactions.blockhash, pfrom.GetId()); LogPrint(BCLog::NET, "Peer %d sent us a compact block but it failed to reconstruct, waiting on first download to complete\n", pfrom.GetId()); @@ -3347,7 +3342,7 @@ void PeerManagerImpl::ProcessCompactBlockTxns(CNode& pfrom, Peer& peer, const Bl return; } -void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, +void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, DataStream& vRecv, const std::chrono::microseconds time_received, const std::atomic<bool>& interruptMsgProc) { @@ -3439,10 +3434,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, pfrom.SetCommonVersion(greatest_common_version); pfrom.nVersion = nVersion; - const CNetMsgMaker msg_maker(greatest_common_version); - if (greatest_common_version >= WTXID_RELAY_VERSION) { - m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::WTXIDRELAY)); + MakeAndPushMessage(pfrom, NetMsgType::WTXIDRELAY); } // Signal ADDRv2 support (BIP155). @@ -3451,7 +3444,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // implementations reject messages they don't know. As a courtesy, don't send // it to nodes with a version before 70016, as no software is known to support // BIP155 that doesn't announce at least that protocol version number. - m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::SENDADDRV2)); + MakeAndPushMessage(pfrom, NetMsgType::SENDADDRV2); } pfrom.m_has_all_wanted_services = HasAllDesirableServiceFlags(nServices); @@ -3491,12 +3484,12 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (tx_relay && WITH_LOCK(tx_relay->m_bloom_filter_mutex, return tx_relay->m_relay_txs) && !pfrom.IsAddrFetchConn() && !m_opts.ignore_incoming_txs) { const uint64_t recon_salt = m_txreconciliation->PreRegisterPeer(pfrom.GetId()); - m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::SENDTXRCNCL, - TXRECONCILIATION_VERSION, recon_salt)); + MakeAndPushMessage(pfrom, NetMsgType::SENDTXRCNCL, + TXRECONCILIATION_VERSION, recon_salt); } } - m_connman.PushMessage(&pfrom, msg_maker.Make(NetMsgType::VERACK)); + MakeAndPushMessage(pfrom, NetMsgType::VERACK); // Potentially mark this peer as a preferred download peer. { @@ -3520,7 +3513,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // We skip this for block-relay-only peers. We want to avoid // potentially leaking addr information and we do not want to // indicate to the peer that we will participate in addr relay. - m_connman.PushMessage(&pfrom, CNetMsgMaker(greatest_common_version).Make(NetMsgType::GETADDR)); + MakeAndPushMessage(pfrom, NetMsgType::GETADDR); peer->m_getaddr_sent = true; // When requesting a getaddr, accept an additional MAX_ADDR_TO_SEND addresses in response // (bypassing the MAX_ADDR_PROCESSING_TOKEN_BUCKET limit). @@ -3566,7 +3559,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // If the peer is old enough to have the old alert system, send it the final alert. if (greatest_common_version <= 70012) { const auto finalAlert{ParseHex("60010000000000000000000000ffffff7f00000000ffffff7ffeffff7f01ffffff7f00000000ffffff7f00ffffff7f002f555247454e543a20416c657274206b657920636f6d70726f6d697365642c2075706772616465207265717569726564004630440220653febd6410f470f6bae11cad19c48413becb1ac2c17f908fd0fd53bdc3abd5202206d0e9c96fe88d4a0f01ed9dedae2b6f9e00da94cad0fecaae66ecf689bf71b50")}; - m_connman.PushMessage(&pfrom, CNetMsgMaker(greatest_common_version).Make("alert", Span{finalAlert})); + MakeAndPushMessage(pfrom, "alert", Span{finalAlert}); } // Feeler connections exist only to verify if address is online. @@ -3583,16 +3576,13 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, return; } - // At this point, the outgoing message serialization version can't change. - const CNetMsgMaker msgMaker(pfrom.GetCommonVersion()); - if (msg_type == NetMsgType::VERACK) { if (pfrom.fSuccessfullyConnected) { LogPrint(BCLog::NET, "ignoring redundant verack message from peer=%d\n", pfrom.GetId()); return; } - // Log succesful connections unconditionally for outbound, but not for inbound as those + // Log successful connections unconditionally for outbound, but not for inbound as those // can be triggered by an attacker at high rate. if (!pfrom.IsInboundConn() || LogAcceptCategory(BCLog::NET, BCLog::Level::Debug)) { const auto mapped_as{m_connman.GetMappedAS(pfrom.addr)}; @@ -3610,7 +3600,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // cmpctblock messages. // We send this to non-NODE NETWORK peers as well, because // they may wish to request compact blocks from us - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::SENDCMPCT, /*high_bandwidth=*/false, /*version=*/CMPCTBLOCKS_VERSION)); + MakeAndPushMessage(pfrom, NetMsgType::SENDCMPCT, /*high_bandwidth=*/false, /*version=*/CMPCTBLOCKS_VERSION); } if (m_txreconciliation) { @@ -3770,7 +3760,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, std::vector<CAddress> vAddr; - vRecv >> WithParams(ser_params, vAddr); + vRecv >> ser_params(vAddr); if (!SetupAddressRelay(pfrom, *peer)) { LogPrint(BCLog::NET, "ignoring %s message from %s peer=%d\n", msg_type, pfrom.ConnectionTypeAsString(), pfrom.GetId()); @@ -4117,7 +4107,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because active chain has too little work; sending empty response\n", pfrom.GetId()); // Just respond with an empty headers message, to tell the peer to // go away but not treat us as unresponsive. - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, std::vector<CBlock>())); + MakeAndPushMessage(pfrom, NetMsgType::HEADERS, std::vector<CBlockHeader>()); return; } @@ -4167,7 +4157,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // will re-announce the new block via headers (or compact blocks again) // in the SendMessages logic. nodestate->pindexBestHeaderSent = pindex ? pindex : m_chainman.ActiveChain().Tip(); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + MakeAndPushMessage(pfrom, NetMsgType::HEADERS, TX_WITH_WITNESS(vHeaders)); return; } @@ -4184,7 +4174,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (m_chainman.IsInitialBlockDownload()) return; CTransactionRef ptx; - vRecv >> ptx; + vRecv >> TX_WITH_WITNESS(ptx); const CTransaction& tx = *ptx; const uint256& txid = ptx->GetHash(); @@ -4308,7 +4298,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, m_txrequest.ForgetTxHash(tx.GetWitnessHash()); // DoS prevention: do not allow m_orphanage to grow unbounded (see CVE-2012-3789) - m_orphanage.LimitOrphans(m_opts.max_orphan_txs); + m_orphanage.LimitOrphans(m_opts.max_orphan_txs, m_rng); } else { LogPrint(BCLog::MEMPOOL, "not keeping orphan with rejected parents %s (wtxid=%s)\n", tx.GetHash().ToString(), @@ -4319,8 +4309,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // regardless of what witness is provided, we will not accept // this, so we don't need to allow for redownload of this txid // from any of our non-wtxidrelay peers. - m_recent_rejects.insert(tx.GetHash()); - m_recent_rejects.insert(tx.GetWitnessHash()); + m_recent_rejects.insert(tx.GetHash().ToUint256()); + m_recent_rejects.insert(tx.GetWitnessHash().ToUint256()); m_txrequest.ForgetTxHash(tx.GetHash()); m_txrequest.ForgetTxHash(tx.GetWitnessHash()); } @@ -4339,7 +4329,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // See also comments in https://github.com/bitcoin/bitcoin/pull/18044#discussion_r443419034 // for concerns around weakening security of unupgraded nodes // if we start doing this too early. - m_recent_rejects.insert(tx.GetWitnessHash()); + m_recent_rejects.insert(tx.GetWitnessHash().ToUint256()); m_txrequest.ForgetTxHash(tx.GetWitnessHash()); // If the transaction failed for TX_INPUTS_NOT_STANDARD, // then we know that the witness was irrelevant to the policy @@ -4349,8 +4339,8 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // processing of this transaction in the event that child // transactions are later received (resulting in // parent-fetching by txid via the orphan-handling logic). - if (state.GetResult() == TxValidationResult::TX_INPUTS_NOT_STANDARD && tx.GetWitnessHash() != tx.GetHash()) { - m_recent_rejects.insert(tx.GetHash()); + if (state.GetResult() == TxValidationResult::TX_INPUTS_NOT_STANDARD && tx.HasWitness()) { + m_recent_rejects.insert(tx.GetHash().ToUint256()); m_txrequest.ForgetTxHash(tx.GetHash()); } if (RecursiveDynamicUsage(*ptx) < 100000) { @@ -4469,7 +4459,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // so we just grab the block via normal getdata std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(*peer), blockhash); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + MakeAndPushMessage(pfrom, NetMsgType::GETDATA, vInv); } return; } @@ -4506,7 +4496,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // Duplicate txindexes, the block is now in-flight, so just request it std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(*peer), blockhash); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + MakeAndPushMessage(pfrom, NetMsgType::GETDATA, vInv); } else { // Give up for this peer and wait for other peer(s) RemoveBlockRequest(pindex->GetBlockHash(), pfrom.GetId()); @@ -4525,7 +4515,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // We will try to round-trip any compact blocks we get on failure, // as long as it's first... req.blockhash = pindex->GetBlockHash(); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); + MakeAndPushMessage(pfrom, NetMsgType::GETBLOCKTXN, req); } else if (pfrom.m_bip152_highbandwidth_to && (!pfrom.IsInboundConn() || IsBlockRequestedFromOutbound(blockhash) || @@ -4535,7 +4525,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // - we already have an outbound attempt in flight(so we'll take what we can get), or // - it's not the final parallel download slot (which we may reserve for first outbound) req.blockhash = pindex->GetBlockHash(); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETBLOCKTXN, req)); + MakeAndPushMessage(pfrom, NetMsgType::GETBLOCKTXN, req); } else { // Give up for this peer and wait for other peer(s) RemoveBlockRequest(pindex->GetBlockHash(), pfrom.GetId()); @@ -4564,7 +4554,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // mempool will probably be useless - request the block normally std::vector<CInv> vInv(1); vInv[0] = CInv(MSG_BLOCK | GetFetchFlags(*peer), blockhash); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::GETDATA, vInv)); + MakeAndPushMessage(pfrom, NetMsgType::GETDATA, vInv); return; } else { // If this was an announce-cmpctblock, we want the same treatment as a header message @@ -4685,7 +4675,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); - vRecv >> *pblock; + vRecv >> TX_WITH_WITNESS(*pblock); LogPrint(BCLog::NET, "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom.GetId()); @@ -4794,7 +4784,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // it, if the remote node sends a ping once per second and this node takes 5 // seconds to respond to each, the 5th ping the remote sends would appear to // return very quickly. - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::PONG, nonce)); + MakeAndPushMessage(pfrom, NetMsgType::PONG, nonce); } return; } @@ -5066,8 +5056,6 @@ bool PeerManagerImpl::ProcessMessages(CNode* pfrom, std::atomic<bool>& interrupt CaptureMessage(pfrom->addr, msg.m_type, MakeUCharSpan(msg.m_recv), /*is_incoming=*/true); } - msg.SetVersion(pfrom->GetCommonVersion()); - try { ProcessMessage(*pfrom, msg.m_type, msg.m_recv, msg.m_time, interruptMsgProc); if (interruptMsgProc) return false; @@ -5295,7 +5283,6 @@ void PeerManagerImpl::MaybeSendPing(CNode& node_to, Peer& peer, std::chrono::mic return; } - const CNetMsgMaker msgMaker(node_to.GetCommonVersion()); bool pingSend = false; if (peer.m_ping_queued) { @@ -5317,11 +5304,11 @@ void PeerManagerImpl::MaybeSendPing(CNode& node_to, Peer& peer, std::chrono::mic peer.m_ping_start = now; if (node_to.GetCommonVersion() > BIP0031_VERSION) { peer.m_ping_nonce_sent = nonce; - m_connman.PushMessage(&node_to, msgMaker.Make(NetMsgType::PING, nonce)); + MakeAndPushMessage(node_to, NetMsgType::PING, nonce); } else { // Peer is too old to support ping command with nonce, pong will never arrive. peer.m_ping_nonce_sent = 0; - m_connman.PushMessage(&node_to, msgMaker.Make(NetMsgType::PING)); + MakeAndPushMessage(node_to, NetMsgType::PING); } } } @@ -5375,16 +5362,11 @@ void PeerManagerImpl::MaybeSendAddr(CNode& node, Peer& peer, std::chrono::micros // No addr messages to send if (peer.m_addrs_to_send.empty()) return; - const char* msg_type; - CNetAddr::Encoding ser_enc; if (peer.m_wants_addrv2) { - msg_type = NetMsgType::ADDRV2; - ser_enc = CNetAddr::Encoding::V2; + MakeAndPushMessage(node, NetMsgType::ADDRV2, CAddress::V2_NETWORK(peer.m_addrs_to_send)); } else { - msg_type = NetMsgType::ADDR; - ser_enc = CNetAddr::Encoding::V1; + MakeAndPushMessage(node, NetMsgType::ADDR, CAddress::V1_NETWORK(peer.m_addrs_to_send)); } - m_connman.PushMessage(&node, CNetMsgMaker(node.GetCommonVersion()).Make(msg_type, WithParams(CAddress::SerParams{{ser_enc}, CAddress::Format::Network}, peer.m_addrs_to_send))); peer.m_addrs_to_send.clear(); // we only send the big addr message once @@ -5408,7 +5390,7 @@ void PeerManagerImpl::MaybeSendSendHeaders(CNode& node, Peer& peer) // We send this to non-NODE NETWORK peers as well, because even // non-NODE NETWORK peers can announce blocks (such as pruning // nodes) - m_connman.PushMessage(&node, CNetMsgMaker(node.GetCommonVersion()).Make(NetMsgType::SENDHEADERS)); + MakeAndPushMessage(node, NetMsgType::SENDHEADERS); peer.m_sent_sendheaders = true; } } @@ -5443,7 +5425,7 @@ void PeerManagerImpl::MaybeSendFeefilter(CNode& pto, Peer& peer, std::chrono::mi // We always have a fee filter of at least the min relay fee filterToSend = std::max(filterToSend, m_mempool.m_min_relay_feerate.GetFeePerK()); if (filterToSend != peer.m_fee_filter_sent) { - m_connman.PushMessage(&pto, CNetMsgMaker(pto.GetCommonVersion()).Make(NetMsgType::FEEFILTER, filterToSend)); + MakeAndPushMessage(pto, NetMsgType::FEEFILTER, filterToSend); peer.m_fee_filter_sent = filterToSend; } peer.m_next_send_feefilter = GetExponentialRand(current_time, AVG_FEEFILTER_BROADCAST_INTERVAL); @@ -5520,9 +5502,6 @@ bool PeerManagerImpl::SendMessages(CNode* pto) if (!pto->fSuccessfullyConnected || pto->fDisconnect) return true; - // If we get here, the outgoing message serialization version is set and can't change. - const CNetMsgMaker msgMaker(pto->GetCommonVersion()); - const auto current_time{GetTime<std::chrono::microseconds>()}; if (pto->IsAddrFetchConn() && current_time - pto->m_connected > 10 * AVG_ADDRESS_BROADCAST_INTERVAL) { @@ -5677,17 +5656,17 @@ bool PeerManagerImpl::SendMessages(CNode* pto) { LOCK(m_most_recent_block_mutex); if (m_most_recent_block_hash == pBestIndex->GetBlockHash()) { - cached_cmpctblock_msg = msgMaker.Make(NetMsgType::CMPCTBLOCK, *m_most_recent_compact_block); + cached_cmpctblock_msg = NetMsg::Make(NetMsgType::CMPCTBLOCK, *m_most_recent_compact_block); } } if (cached_cmpctblock_msg.has_value()) { - m_connman.PushMessage(pto, std::move(cached_cmpctblock_msg.value())); + PushMessage(*pto, std::move(cached_cmpctblock_msg.value())); } else { CBlock block; const bool ret{m_chainman.m_blockman.ReadBlockFromDisk(block, *pBestIndex)}; assert(ret); CBlockHeaderAndShortTxIDs cmpctblock{block}; - m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); + MakeAndPushMessage(*pto, NetMsgType::CMPCTBLOCK, cmpctblock); } state.pindexBestHeaderSent = pBestIndex; } else if (peer->m_prefers_headers) { @@ -5700,7 +5679,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) LogPrint(BCLog::NET, "%s: sending header %s to peer=%d\n", __func__, vHeaders.front().GetHash().ToString(), pto->GetId()); } - m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + MakeAndPushMessage(*pto, NetMsgType::HEADERS, TX_WITH_WITNESS(vHeaders)); state.pindexBestHeaderSent = pBestIndex; } else fRevertToInv = true; @@ -5745,7 +5724,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) for (const uint256& hash : peer->m_blocks_for_inv_relay) { vInv.emplace_back(MSG_BLOCK, hash); if (vInv.size() == MAX_INV_SZ) { - m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + MakeAndPushMessage(*pto, NetMsgType::INV, vInv); vInv.clear(); } } @@ -5780,9 +5759,14 @@ bool PeerManagerImpl::SendMessages(CNode* pto) LOCK(tx_relay->m_bloom_filter_mutex); for (const auto& txinfo : vtxinfo) { - const uint256& hash = peer->m_wtxid_relay ? txinfo.tx->GetWitnessHash() : txinfo.tx->GetHash(); - CInv inv(peer->m_wtxid_relay ? MSG_WTX : MSG_TX, hash); - tx_relay->m_tx_inventory_to_send.erase(hash); + CInv inv{ + peer->m_wtxid_relay ? MSG_WTX : MSG_TX, + peer->m_wtxid_relay ? + txinfo.tx->GetWitnessHash().ToUint256() : + txinfo.tx->GetHash().ToUint256(), + }; + tx_relay->m_tx_inventory_to_send.erase(inv.hash); + // Don't send transactions that peers will not put into their mempool if (txinfo.fee < filterrate.GetFee(txinfo.vsize)) { continue; @@ -5790,10 +5774,10 @@ bool PeerManagerImpl::SendMessages(CNode* pto) if (tx_relay->m_bloom_filter) { if (!tx_relay->m_bloom_filter->IsRelevantAndUpdate(*txinfo.tx)) continue; } - tx_relay->m_tx_inventory_known_filter.insert(hash); + tx_relay->m_tx_inventory_known_filter.insert(inv.hash); vInv.push_back(inv); if (vInv.size() == MAX_INV_SZ) { - m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + MakeAndPushMessage(*pto, NetMsgType::INV, vInv); vInv.clear(); } } @@ -5845,7 +5829,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) vInv.push_back(inv); nRelayedTransactions++; if (vInv.size() == MAX_INV_SZ) { - m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + MakeAndPushMessage(*pto, NetMsgType::INV, vInv); vInv.clear(); } tx_relay->m_tx_inventory_known_filter.insert(hash); @@ -5857,7 +5841,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) } } if (!vInv.empty()) - m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::INV, vInv)); + MakeAndPushMessage(*pto, NetMsgType::INV, vInv); // Detect whether we're stalling auto stalling_timeout = m_block_stalling_timeout.load(); @@ -5977,7 +5961,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) gtxid.GetHash().ToString(), pto->GetId()); vGetData.emplace_back(gtxid.IsWtxid() ? MSG_WTX : (MSG_TX | GetFetchFlags(*peer)), gtxid.GetHash()); if (vGetData.size() >= MAX_GETDATA_SZ) { - m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + MakeAndPushMessage(*pto, NetMsgType::GETDATA, vGetData); vGetData.clear(); } m_txrequest.RequestedTx(pto->GetId(), gtxid.GetHash(), current_time + GETDATA_TX_INTERVAL); @@ -5990,7 +5974,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) if (!vGetData.empty()) - m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::GETDATA, vGetData)); + MakeAndPushMessage(*pto, NetMsgType::GETDATA, vGetData); } // release cs_main MaybeSendFeefilter(*pto, *peer, current_time); return true; diff --git a/src/net_processing.h b/src/net_processing.h index 80d07648a4..afdef00067 100644 --- a/src/net_processing.h +++ b/src/net_processing.h @@ -105,7 +105,7 @@ public: virtual void CheckForStaleTipAndEvictPeers() = 0; /** Process a single message from a peer. Public for fuzz testing */ - virtual void ProcessMessage(CNode& pfrom, const std::string& msg_type, CDataStream& vRecv, + virtual void ProcessMessage(CNode& pfrom, const std::string& msg_type, DataStream& vRecv, const std::chrono::microseconds time_received, const std::atomic<bool>& interruptMsgProc) EXCLUSIVE_LOCKS_REQUIRED(g_msgproc_mutex) = 0; /** This function is used for testing the stale tip eviction logic, see denialofservice_tests.cpp */ diff --git a/src/net_types.cpp b/src/net_types.cpp index 2cdc10d8c9..fd6ad80404 100644 --- a/src/net_types.cpp +++ b/src/net_types.cpp @@ -63,9 +63,9 @@ void BanMapFromJson(const UniValue& bans_json, banmap_t& bans) LogPrintf("Dropping entry with unknown version (%s) from ban list\n", version); continue; } - CSubNet subnet; const auto& subnet_str = ban_entry_json[BANMAN_JSON_ADDR_KEY].get_str(); - if (!LookupSubNet(subnet_str, subnet)) { + const CSubNet subnet{LookupSubNet(subnet_str)}; + if (!subnet.IsValid()) { LogPrintf("Dropping entry with unparseable address or subnet (%s) from ban list\n", subnet_str); continue; } diff --git a/src/netbase.cpp b/src/netbase.cpp index 5e1e121bfe..9fbd9f7dea 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -30,7 +30,7 @@ bool fNameLookup = DEFAULT_NAME_LOOKUP; // Need ample time for negotiation for very slow proxies such as Tor std::chrono::milliseconds g_socks5_recv_timeout = 20s; -static std::atomic<bool> interruptSocks5Recv(false); +CThreadInterrupt g_socks5_interrupt; ReachableNets g_reachable_nets; @@ -271,7 +271,7 @@ enum class IntrRecvError { * IntrRecvError::OK only if all of the specified number of bytes were * read. * - * @see This function can be interrupted by calling InterruptSocks5(bool). + * @see This function can be interrupted by calling g_socks5_interrupt(). * Sockets can be made non-blocking with Sock::SetNonBlocking(). */ static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, std::chrono::milliseconds timeout, const Sock& sock) @@ -299,8 +299,9 @@ static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, std::chrono::m return IntrRecvError::NetworkError; } } - if (interruptSocks5Recv) + if (g_socks5_interrupt) { return IntrRecvError::Interrupted; + } curTime = Now<SteadyMilliseconds>(); } return len == 0 ? IntrRecvError::OK : IntrRecvError::Timeout; @@ -333,103 +334,93 @@ static std::string Socks5ErrorString(uint8_t err) bool Socks5(const std::string& strDest, uint16_t port, const ProxyCredentials* auth, const Sock& sock) { - IntrRecvError recvr; - LogPrint(BCLog::NET, "SOCKS5 connecting %s\n", strDest); - if (strDest.size() > 255) { - return error("Hostname too long"); - } - // Construct the version identifier/method selection message - std::vector<uint8_t> vSocks5Init; - vSocks5Init.push_back(SOCKSVersion::SOCKS5); // We want the SOCK5 protocol - if (auth) { - vSocks5Init.push_back(0x02); // 2 method identifiers follow... - vSocks5Init.push_back(SOCKS5Method::NOAUTH); - vSocks5Init.push_back(SOCKS5Method::USER_PASS); - } else { - vSocks5Init.push_back(0x01); // 1 method identifier follows... - vSocks5Init.push_back(SOCKS5Method::NOAUTH); - } - ssize_t ret = sock.Send(vSocks5Init.data(), vSocks5Init.size(), MSG_NOSIGNAL); - if (ret != (ssize_t)vSocks5Init.size()) { - return error("Error sending to proxy"); - } - uint8_t pchRet1[2]; - if (InterruptibleRecv(pchRet1, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { - LogPrintf("Socks5() connect to %s:%d failed: InterruptibleRecv() timeout or other failure\n", strDest, port); - return false; - } - if (pchRet1[0] != SOCKSVersion::SOCKS5) { - return error("Proxy failed to initialize"); - } - if (pchRet1[1] == SOCKS5Method::USER_PASS && auth) { - // Perform username/password authentication (as described in RFC1929) - std::vector<uint8_t> vAuth; - vAuth.push_back(0x01); // Current (and only) version of user/pass subnegotiation - if (auth->username.size() > 255 || auth->password.size() > 255) - return error("Proxy username or password too long"); - vAuth.push_back(auth->username.size()); - vAuth.insert(vAuth.end(), auth->username.begin(), auth->username.end()); - vAuth.push_back(auth->password.size()); - vAuth.insert(vAuth.end(), auth->password.begin(), auth->password.end()); - ret = sock.Send(vAuth.data(), vAuth.size(), MSG_NOSIGNAL); - if (ret != (ssize_t)vAuth.size()) { - return error("Error sending authentication to proxy"); - } - LogPrint(BCLog::PROXY, "SOCKS5 sending proxy authentication %s:%s\n", auth->username, auth->password); - uint8_t pchRetA[2]; - if (InterruptibleRecv(pchRetA, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { - return error("Error reading proxy authentication response"); + try { + IntrRecvError recvr; + LogPrint(BCLog::NET, "SOCKS5 connecting %s\n", strDest); + if (strDest.size() > 255) { + return error("Hostname too long"); } - if (pchRetA[0] != 0x01 || pchRetA[1] != 0x00) { - return error("Proxy authentication unsuccessful"); + // Construct the version identifier/method selection message + std::vector<uint8_t> vSocks5Init; + vSocks5Init.push_back(SOCKSVersion::SOCKS5); // We want the SOCK5 protocol + if (auth) { + vSocks5Init.push_back(0x02); // 2 method identifiers follow... + vSocks5Init.push_back(SOCKS5Method::NOAUTH); + vSocks5Init.push_back(SOCKS5Method::USER_PASS); + } else { + vSocks5Init.push_back(0x01); // 1 method identifier follows... + vSocks5Init.push_back(SOCKS5Method::NOAUTH); } - } else if (pchRet1[1] == SOCKS5Method::NOAUTH) { - // Perform no authentication - } else { - return error("Proxy requested wrong authentication method %02x", pchRet1[1]); - } - std::vector<uint8_t> vSocks5; - vSocks5.push_back(SOCKSVersion::SOCKS5); // VER protocol version - vSocks5.push_back(SOCKS5Command::CONNECT); // CMD CONNECT - vSocks5.push_back(0x00); // RSV Reserved must be 0 - vSocks5.push_back(SOCKS5Atyp::DOMAINNAME); // ATYP DOMAINNAME - vSocks5.push_back(strDest.size()); // Length<=255 is checked at beginning of function - vSocks5.insert(vSocks5.end(), strDest.begin(), strDest.end()); - vSocks5.push_back((port >> 8) & 0xFF); - vSocks5.push_back((port >> 0) & 0xFF); - ret = sock.Send(vSocks5.data(), vSocks5.size(), MSG_NOSIGNAL); - if (ret != (ssize_t)vSocks5.size()) { - return error("Error sending to proxy"); - } - uint8_t pchRet2[4]; - if ((recvr = InterruptibleRecv(pchRet2, 4, g_socks5_recv_timeout, sock)) != IntrRecvError::OK) { - if (recvr == IntrRecvError::Timeout) { - /* If a timeout happens here, this effectively means we timed out while connecting - * to the remote node. This is very common for Tor, so do not print an - * error message. */ + sock.SendComplete(vSocks5Init, g_socks5_recv_timeout, g_socks5_interrupt); + uint8_t pchRet1[2]; + if (InterruptibleRecv(pchRet1, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { + LogPrintf("Socks5() connect to %s:%d failed: InterruptibleRecv() timeout or other failure\n", strDest, port); return false; + } + if (pchRet1[0] != SOCKSVersion::SOCKS5) { + return error("Proxy failed to initialize"); + } + if (pchRet1[1] == SOCKS5Method::USER_PASS && auth) { + // Perform username/password authentication (as described in RFC1929) + std::vector<uint8_t> vAuth; + vAuth.push_back(0x01); // Current (and only) version of user/pass subnegotiation + if (auth->username.size() > 255 || auth->password.size() > 255) + return error("Proxy username or password too long"); + vAuth.push_back(auth->username.size()); + vAuth.insert(vAuth.end(), auth->username.begin(), auth->username.end()); + vAuth.push_back(auth->password.size()); + vAuth.insert(vAuth.end(), auth->password.begin(), auth->password.end()); + sock.SendComplete(vAuth, g_socks5_recv_timeout, g_socks5_interrupt); + LogPrint(BCLog::PROXY, "SOCKS5 sending proxy authentication %s:%s\n", auth->username, auth->password); + uint8_t pchRetA[2]; + if (InterruptibleRecv(pchRetA, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { + return error("Error reading proxy authentication response"); + } + if (pchRetA[0] != 0x01 || pchRetA[1] != 0x00) { + return error("Proxy authentication unsuccessful"); + } + } else if (pchRet1[1] == SOCKS5Method::NOAUTH) { + // Perform no authentication } else { - return error("Error while reading proxy response"); + return error("Proxy requested wrong authentication method %02x", pchRet1[1]); } - } - if (pchRet2[0] != SOCKSVersion::SOCKS5) { - return error("Proxy failed to accept request"); - } - if (pchRet2[1] != SOCKS5Reply::SUCCEEDED) { - // Failures to connect to a peer that are not proxy errors - LogPrintf("Socks5() connect to %s:%d failed: %s\n", strDest, port, Socks5ErrorString(pchRet2[1])); - return false; - } - if (pchRet2[2] != 0x00) { // Reserved field must be 0 - return error("Error: malformed proxy response"); - } - uint8_t pchRet3[256]; - switch (pchRet2[3]) - { + std::vector<uint8_t> vSocks5; + vSocks5.push_back(SOCKSVersion::SOCKS5); // VER protocol version + vSocks5.push_back(SOCKS5Command::CONNECT); // CMD CONNECT + vSocks5.push_back(0x00); // RSV Reserved must be 0 + vSocks5.push_back(SOCKS5Atyp::DOMAINNAME); // ATYP DOMAINNAME + vSocks5.push_back(strDest.size()); // Length<=255 is checked at beginning of function + vSocks5.insert(vSocks5.end(), strDest.begin(), strDest.end()); + vSocks5.push_back((port >> 8) & 0xFF); + vSocks5.push_back((port >> 0) & 0xFF); + sock.SendComplete(vSocks5, g_socks5_recv_timeout, g_socks5_interrupt); + uint8_t pchRet2[4]; + if ((recvr = InterruptibleRecv(pchRet2, 4, g_socks5_recv_timeout, sock)) != IntrRecvError::OK) { + if (recvr == IntrRecvError::Timeout) { + /* If a timeout happens here, this effectively means we timed out while connecting + * to the remote node. This is very common for Tor, so do not print an + * error message. */ + return false; + } else { + return error("Error while reading proxy response"); + } + } + if (pchRet2[0] != SOCKSVersion::SOCKS5) { + return error("Proxy failed to accept request"); + } + if (pchRet2[1] != SOCKS5Reply::SUCCEEDED) { + // Failures to connect to a peer that are not proxy errors + LogPrintf("Socks5() connect to %s:%d failed: %s\n", strDest, port, Socks5ErrorString(pchRet2[1])); + return false; + } + if (pchRet2[2] != 0x00) { // Reserved field must be 0 + return error("Error: malformed proxy response"); + } + uint8_t pchRet3[256]; + switch (pchRet2[3]) { case SOCKS5Atyp::IPV4: recvr = InterruptibleRecv(pchRet3, 4, g_socks5_recv_timeout, sock); break; case SOCKS5Atyp::IPV6: recvr = InterruptibleRecv(pchRet3, 16, g_socks5_recv_timeout, sock); break; - case SOCKS5Atyp::DOMAINNAME: - { + case SOCKS5Atyp::DOMAINNAME: { recvr = InterruptibleRecv(pchRet3, 1, g_socks5_recv_timeout, sock); if (recvr != IntrRecvError::OK) { return error("Error reading from proxy"); @@ -439,15 +430,18 @@ bool Socks5(const std::string& strDest, uint16_t port, const ProxyCredentials* a break; } default: return error("Error: malformed proxy response"); + } + if (recvr != IntrRecvError::OK) { + return error("Error reading from proxy"); + } + if (InterruptibleRecv(pchRet3, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { + return error("Error reading from proxy"); + } + LogPrint(BCLog::NET, "SOCKS5 connected %s\n", strDest); + return true; + } catch (const std::runtime_error& e) { + return error("Error during SOCKS5 proxy handshake: %s", e.what()); } - if (recvr != IntrRecvError::OK) { - return error("Error reading from proxy"); - } - if (InterruptibleRecv(pchRet3, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { - return error("Error reading from proxy"); - } - LogPrint(BCLog::NET, "SOCKS5 connected %s\n", strDest); - return true; } std::unique_ptr<Sock> CreateSockTCP(const CService& address_family) @@ -645,10 +639,12 @@ bool ConnectThroughProxy(const Proxy& proxy, const std::string& strDest, uint16_ return true; } -bool LookupSubNet(const std::string& subnet_str, CSubNet& subnet_out) +CSubNet LookupSubNet(const std::string& subnet_str) { + CSubNet subnet; + assert(!subnet.IsValid()); if (!ContainsNoNUL(subnet_str)) { - return false; + return subnet; } const size_t slash_pos{subnet_str.find_last_of('/')}; @@ -662,28 +658,21 @@ bool LookupSubNet(const std::string& subnet_str, CSubNet& subnet_out) uint8_t netmask; if (ParseUInt8(netmask_str, &netmask)) { // Valid number; assume CIDR variable-length subnet masking. - subnet_out = CSubNet{addr.value(), netmask}; - return subnet_out.IsValid(); + subnet = CSubNet{addr.value(), netmask}; } else { // Invalid number; try full netmask syntax. Never allow lookup for netmask. const std::optional<CNetAddr> full_netmask{LookupHost(netmask_str, /*fAllowLookup=*/false)}; if (full_netmask.has_value()) { - subnet_out = CSubNet{addr.value(), full_netmask.value()}; - return subnet_out.IsValid(); + subnet = CSubNet{addr.value(), full_netmask.value()}; } } } else { // Single IP subnet (<ipv4>/32 or <ipv6>/128). - subnet_out = CSubNet{addr.value()}; - return subnet_out.IsValid(); + subnet = CSubNet{addr.value()}; } } - return false; -} -void InterruptSocks5(bool interrupt) -{ - interruptSocks5Recv = interrupt; + return subnet; } bool IsBadPort(uint16_t port) diff --git a/src/netbase.h b/src/netbase.h index 8b7da4109f..8523f59b4d 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -13,6 +13,7 @@ #include <netaddress.h> #include <serialize.h> #include <util/sock.h> +#include <util/threadinterrupt.h> #include <functional> #include <memory> @@ -227,11 +228,9 @@ CService LookupNumeric(const std::string& name, uint16_t portDefault = 0, DNSLoo * @param[in] subnet_str A string representation of a subnet of the form * `network address [ "/", ( CIDR-style suffix | netmask ) ]` * e.g. "2001:db8::/32", "192.0.2.0/255.255.255.0" or "8.8.8.8". - * @param[out] subnet_out Internal subnet representation, if parsable/resolvable - * from `subnet_str`. - * @returns whether the operation succeeded or not. + * @returns a CSubNet object (that may or may not be valid). */ -bool LookupSubNet(const std::string& subnet_str, CSubNet& subnet_out); +CSubNet LookupSubNet(const std::string& subnet_str); /** * Create a TCP socket in the given address family. @@ -276,7 +275,10 @@ bool ConnectSocketDirectly(const CService &addrConnect, const Sock& sock, int nT */ bool ConnectThroughProxy(const Proxy& proxy, const std::string& strDest, uint16_t port, const Sock& sock, int nTimeout, bool& outProxyConnectionFailed); -void InterruptSocks5(bool interrupt); +/** + * Interrupt SOCKS5 reads or writes. + */ +extern CThreadInterrupt g_socks5_interrupt; /** * Connect to a specified destination service through an already connected diff --git a/src/netgroup.cpp b/src/netgroup.cpp index a03927b152..0ae229b3f3 100644 --- a/src/netgroup.cpp +++ b/src/netgroup.cpp @@ -5,6 +5,7 @@ #include <netgroup.h> #include <hash.h> +#include <logging.h> #include <util/asmap.h> uint256 NetGroupManager::GetAsmapChecksum() const @@ -109,3 +110,23 @@ uint32_t NetGroupManager::GetMappedAS(const CNetAddr& address) const uint32_t mapped_as = Interpret(m_asmap, ip_bits); return mapped_as; } + +void NetGroupManager::ASMapHealthCheck(const std::vector<CNetAddr>& clearnet_addrs) const { + std::set<uint32_t> clearnet_asns{}; + int unmapped_count{0}; + + for (const auto& addr : clearnet_addrs) { + uint32_t asn = GetMappedAS(addr); + if (asn == 0) { + ++unmapped_count; + continue; + } + clearnet_asns.insert(asn); + } + + LogPrintf("ASMap Health Check: %i clearnet peers are mapped to %i ASNs with %i peers being unmapped\n", clearnet_addrs.size(), clearnet_asns.size(), unmapped_count); +} + +bool NetGroupManager::UsingASMap() const { + return m_asmap.size() > 0; +} diff --git a/src/netgroup.h b/src/netgroup.h index 2dd63ec66b..5aa6ef7742 100644 --- a/src/netgroup.h +++ b/src/netgroup.h @@ -41,6 +41,16 @@ public: */ uint32_t GetMappedAS(const CNetAddr& address) const; + /** + * Analyze and log current health of ASMap based buckets. + */ + void ASMapHealthCheck(const std::vector<CNetAddr>& clearnet_addrs) const; + + /** + * Indicates whether ASMap is being used for clearnet bucketing. + */ + bool UsingASMap() const; + private: /** Compressed IP->ASN mapping, loaded from a file when a node starts. * diff --git a/src/netmessagemaker.h b/src/netmessagemaker.h index a121183aab..d04867e45e 100644 --- a/src/netmessagemaker.h +++ b/src/netmessagemaker.h @@ -9,28 +9,15 @@ #include <net.h> #include <serialize.h> -class CNetMsgMaker -{ -public: - explicit CNetMsgMaker(int nVersionIn) : nVersion(nVersionIn){} - +namespace NetMsg { template <typename... Args> - CSerializedNetMsg Make(int nFlags, std::string msg_type, Args&&... args) const + CSerializedNetMsg Make(std::string msg_type, Args&&... args) { CSerializedNetMsg msg; msg.m_type = std::move(msg_type); - CVectorWriter{nFlags | nVersion, msg.data, 0, std::forward<Args>(args)...}; + VectorWriter{msg.data, 0, std::forward<Args>(args)...}; return msg; } - - template <typename... Args> - CSerializedNetMsg Make(std::string msg_type, Args&&... args) const - { - return Make(0, std::move(msg_type), std::forward<Args>(args)...); - } - -private: - const int nVersion; -}; +} // namespace NetMsg #endif // BITCOIN_NETMESSAGEMAKER_H diff --git a/src/node/abort.cpp b/src/node/abort.cpp index a554126b86..1bdc91670d 100644 --- a/src/node/abort.cpp +++ b/src/node/abort.cpp @@ -6,7 +6,7 @@ #include <logging.h> #include <node/interface_ui.h> -#include <shutdown.h> +#include <util/signalinterrupt.h> #include <util/translation.h> #include <warnings.h> @@ -16,12 +16,14 @@ namespace node { -void AbortNode(std::atomic<int>& exit_status, const std::string& debug_message, const bilingual_str& user_message, bool shutdown) +void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const std::string& debug_message, const bilingual_str& user_message) { SetMiscWarning(Untranslated(debug_message)); LogPrintf("*** %s\n", debug_message); InitError(user_message.empty() ? _("A fatal internal error occurred, see debug.log for details") : user_message); exit_status.store(EXIT_FAILURE); - if (shutdown) StartShutdown(); + if (shutdown && !(*shutdown)()) { + LogPrintf("Error: failed to send shutdown signal\n"); + }; } } // namespace node diff --git a/src/node/abort.h b/src/node/abort.h index d6bb0c14d5..28d021cc78 100644 --- a/src/node/abort.h +++ b/src/node/abort.h @@ -10,8 +10,12 @@ #include <atomic> #include <string> +namespace util { +class SignalInterrupt; +} // namespace util + namespace node { -void AbortNode(std::atomic<int>& exit_status, const std::string& debug_message, const bilingual_str& user_message = {}, bool shutdown = true); +void AbortNode(util::SignalInterrupt* shutdown, std::atomic<int>& exit_status, const std::string& debug_message, const bilingual_str& user_message = {}); } // namespace node #endif // BITCOIN_NODE_ABORT_H diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 53f616de23..e6164c2e59 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -4,23 +4,32 @@ #include <node/blockstorage.h> +#include <arith_uint256.h> #include <chain.h> -#include <clientversion.h> +#include <consensus/params.h> #include <consensus/validation.h> #include <dbwrapper.h> #include <flatfile.h> #include <hash.h> -#include <kernel/chain.h> +#include <kernel/blockmanager_opts.h> #include <kernel/chainparams.h> #include <kernel/messagestartchars.h> +#include <kernel/notifications_interface.h> #include <logging.h> #include <pow.h> +#include <primitives/block.h> +#include <primitives/transaction.h> #include <reverse_iterator.h> +#include <serialize.h> #include <signet.h> +#include <span.h> #include <streams.h> #include <sync.h> +#include <tinyformat.h> +#include <uint256.h> #include <undo.h> #include <util/batchpriority.h> +#include <util/check.h> #include <util/fs.h> #include <util/signalinterrupt.h> #include <util/strencodings.h> @@ -387,7 +396,12 @@ bool BlockManager::LoadBlockIndex(const std::optional<uint256>& snapshot_blockha } if (snapshot_blockhash) { - const AssumeutxoData au_data = *Assert(GetParams().AssumeutxoForBlockhash(*snapshot_blockhash)); + const std::optional<AssumeutxoData> maybe_au_data = GetParams().AssumeutxoForBlockhash(*snapshot_blockhash); + if (!maybe_au_data) { + m_opts.notifications.fatalError(strprintf("Assumeutxo data not found for the given blockhash '%s'.", snapshot_blockhash->ToString())); + return false; + } + const AssumeutxoData& au_data = *Assert(maybe_au_data); m_snapshot_height = au_data.height; CBlockIndex* base{LookupBlockIndex(*snapshot_blockhash)}; @@ -568,10 +582,10 @@ const CBlockIndex* BlockManager::GetLastCheckpoint(const CCheckpointData& data) return nullptr; } -bool BlockManager::IsBlockPruned(const CBlockIndex* pblockindex) +bool BlockManager::IsBlockPruned(const CBlockIndex& block) { AssertLockHeld(::cs_main); - return (m_have_pruned && !(pblockindex->nStatus & BLOCK_HAVE_DATA) && pblockindex->nTx > 0); + return m_have_pruned && !(block.nStatus & BLOCK_HAVE_DATA) && (block.nTx > 0); } const CBlockIndex* BlockManager::GetFirstStoredBlock(const CBlockIndex& upper_block, const CBlockIndex* lower_block) @@ -651,13 +665,13 @@ CBlockFileInfo* BlockManager::GetBlockFileInfo(size_t n) bool BlockManager::UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock) const { // Open history file to append - CAutoFile fileout{OpenUndoFile(pos)}; + AutoFile fileout{OpenUndoFile(pos)}; if (fileout.IsNull()) { return error("%s: OpenUndoFile failed", __func__); } // Write index header - unsigned int nSize = GetSerializeSize(blockundo, CLIENT_VERSION); + unsigned int nSize = GetSerializeSize(blockundo); fileout << GetParams().MessageStart() << nSize; // Write undo data @@ -686,7 +700,7 @@ bool BlockManager::UndoReadFromDisk(CBlockUndo& blockundo, const CBlockIndex& in } // Open history file to read - CAutoFile filein{OpenUndoFile(pos, true)}; + AutoFile filein{OpenUndoFile(pos, true)}; if (filein.IsNull()) { return error("%s: OpenUndoFile failed", __func__); } @@ -805,15 +819,15 @@ FlatFileSeq BlockManager::UndoFileSeq() const return FlatFileSeq(m_opts.blocks_dir, "rev", UNDOFILE_CHUNK_SIZE); } -CAutoFile BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const +AutoFile BlockManager::OpenBlockFile(const FlatFilePos& pos, bool fReadOnly) const { - return CAutoFile{BlockFileSeq().Open(pos, fReadOnly), CLIENT_VERSION}; + return AutoFile{BlockFileSeq().Open(pos, fReadOnly)}; } /** Open an undo file (rev?????.dat) */ -CAutoFile BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const +AutoFile BlockManager::OpenUndoFile(const FlatFilePos& pos, bool fReadOnly) const { - return CAutoFile{UndoFileSeq().Open(pos, fReadOnly), CLIENT_VERSION}; + return AutoFile{UndoFileSeq().Open(pos, fReadOnly)}; } fs::path BlockManager::GetBlockPosFilename(const FlatFilePos& pos) const @@ -945,13 +959,13 @@ bool BlockManager::FindUndoPos(BlockValidationState& state, int nFile, FlatFileP bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const { // Open history file to append - CAutoFile fileout{OpenBlockFile(pos)}; + AutoFile fileout{OpenBlockFile(pos)}; if (fileout.IsNull()) { return error("WriteBlockToDisk: OpenBlockFile failed"); } // Write index header - unsigned int nSize = GetSerializeSize(block, fileout.GetVersion()); + unsigned int nSize = GetSerializeSize(TX_WITH_WITNESS(block)); fileout << GetParams().MessageStart() << nSize; // Write block @@ -960,7 +974,7 @@ bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const return error("WriteBlockToDisk: ftell failed"); } pos.nPos = (unsigned int)fileOutPos; - fileout << block; + fileout << TX_WITH_WITNESS(block); return true; } @@ -974,7 +988,7 @@ bool BlockManager::WriteUndoDataForBlock(const CBlockUndo& blockundo, BlockValid // Write undo information to disk if (block.GetUndoPos().IsNull()) { FlatFilePos _pos; - if (!FindUndoPos(state, block.nFile, _pos, ::GetSerializeSize(blockundo, CLIENT_VERSION) + 40)) { + if (!FindUndoPos(state, block.nFile, _pos, ::GetSerializeSize(blockundo) + 40)) { return error("ConnectBlock(): FindUndoPos failed"); } if (!UndoWriteToDisk(blockundo, _pos, block.pprev->GetBlockHash())) { @@ -1011,14 +1025,14 @@ bool BlockManager::ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) cons block.SetNull(); // Open history file to read - CAutoFile filein{OpenBlockFile(pos, true)}; + AutoFile filein{OpenBlockFile(pos, true)}; if (filein.IsNull()) { return error("ReadBlockFromDisk: OpenBlockFile failed for %s", pos.ToString()); } // Read block try { - filein >> block; + filein >> TX_WITH_WITNESS(block); } catch (const std::exception& e) { return error("%s: Deserialize or I/O error - %s at %s", __func__, e.what(), pos.ToString()); } @@ -1054,7 +1068,7 @@ bool BlockManager::ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatF { FlatFilePos hpos = pos; hpos.nPos -= 8; // Seek back 8 bytes for meta header - CAutoFile filein{OpenBlockFile(hpos, true)}; + AutoFile filein{OpenBlockFile(hpos, true)}; if (filein.IsNull()) { return error("%s: OpenBlockFile failed for %s", __func__, pos.ToString()); } @@ -1087,7 +1101,7 @@ bool BlockManager::ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatF FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, const FlatFilePos* dbp) { - unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); + unsigned int nBlockSize = ::GetSerializeSize(TX_WITH_WITNESS(block)); FlatFilePos blockPos; const auto position_known {dbp != nullptr}; if (position_known) { @@ -1146,7 +1160,7 @@ void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFile if (!fs::exists(chainman.m_blockman.GetBlockPosFilename(pos))) { break; // No block files left to reindex } - CAutoFile file{chainman.m_blockman.OpenBlockFile(pos, true)}; + AutoFile file{chainman.m_blockman.OpenBlockFile(pos, true)}; if (file.IsNull()) { break; // This error is logged in OpenBlockFile } @@ -1167,7 +1181,7 @@ void ImportBlocks(ChainstateManager& chainman, std::vector<fs::path> vImportFile // -loadblock= for (const fs::path& path : vImportFiles) { - CAutoFile file{fsbridge::fopen(path, "rb"), CLIENT_VERSION}; + AutoFile file{fsbridge::fopen(path, "rb")}; if (!file.IsNull()) { LogPrintf("Importing blocks file %s...\n", fs::PathToString(path)); chainman.LoadExternalBlockFile(file); diff --git a/src/node/blockstorage.h b/src/node/blockstorage.h index ba44d31581..ce514cc645 100644 --- a/src/node/blockstorage.h +++ b/src/node/blockstorage.h @@ -8,21 +8,26 @@ #include <attributes.h> #include <chain.h> #include <dbwrapper.h> +#include <flatfile.h> #include <kernel/blockmanager_opts.h> -#include <kernel/chain.h> #include <kernel/chainparams.h> #include <kernel/cs_main.h> #include <kernel/messagestartchars.h> +#include <primitives/block.h> +#include <streams.h> #include <sync.h> +#include <uint256.h> #include <util/fs.h> #include <util/hasher.h> +#include <array> #include <atomic> #include <cstdint> #include <functional> #include <limits> #include <map> #include <memory> +#include <optional> #include <set> #include <string> #include <unordered_map> @@ -30,14 +35,9 @@ #include <vector> class BlockValidationState; -class CAutoFile; -class CBlock; class CBlockUndo; -class CChainParams; class Chainstate; class ChainstateManager; -struct CCheckpointData; -struct FlatFilePos; namespace Consensus { struct Params; } @@ -162,7 +162,7 @@ private: FlatFileSeq BlockFileSeq() const; FlatFileSeq UndoFileSeq() const; - CAutoFile OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const; + AutoFile OpenUndoFile(const FlatFilePos& pos, bool fReadOnly = false) const; bool WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const; bool UndoWriteToDisk(const CBlockUndo& blockundo, FlatFilePos& pos, const uint256& hashBlock) const; @@ -344,13 +344,13 @@ public: bool m_have_pruned = false; //! Check whether the block associated with this index entry is pruned or not. - bool IsBlockPruned(const CBlockIndex* pblockindex) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + bool IsBlockPruned(const CBlockIndex& block) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); //! Create or update a prune lock identified by its name void UpdatePruneLock(const std::string& name, const PruneLockInfo& lock_info) EXCLUSIVE_LOCKS_REQUIRED(::cs_main); /** Open a block file (blk?????.dat) */ - CAutoFile OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false) const; + AutoFile OpenBlockFile(const FlatFilePos& pos, bool fReadOnly = false) const; /** Translation to a filesystem path */ fs::path GetBlockPosFilename(const FlatFilePos& pos) const; diff --git a/src/node/chainstate.cpp b/src/node/chainstate.cpp index eb1994177a..bf1fc06b0b 100644 --- a/src/node/chainstate.cpp +++ b/src/node/chainstate.cpp @@ -17,6 +17,7 @@ #include <txdb.h> #include <uint256.h> #include <util/fs.h> +#include <util/signalinterrupt.h> #include <util/time.h> #include <util/translation.h> #include <validation.h> @@ -55,14 +56,14 @@ static ChainstateLoadResult CompleteChainstateInitialization( } } - if (options.check_interrupt && options.check_interrupt()) return {ChainstateLoadStatus::INTERRUPTED, {}}; + if (chainman.m_interrupt) return {ChainstateLoadStatus::INTERRUPTED, {}}; // LoadBlockIndex will load m_have_pruned if we've ever removed a // block file from disk. // Note that it also sets fReindex global based on the disk flag! // From here on, fReindex and options.reindex values may be different! if (!chainman.LoadBlockIndex()) { - if (options.check_interrupt && options.check_interrupt()) return {ChainstateLoadStatus::INTERRUPTED, {}}; + if (chainman.m_interrupt) return {ChainstateLoadStatus::INTERRUPTED, {}}; return {ChainstateLoadStatus::FAILURE, _("Error loading block database")}; } diff --git a/src/node/chainstate.h b/src/node/chainstate.h index 2e35035c28..a6e9a0331b 100644 --- a/src/node/chainstate.h +++ b/src/node/chainstate.h @@ -32,7 +32,6 @@ struct ChainstateLoadOptions { bool require_full_verification{true}; int64_t check_blocks{DEFAULT_CHECKBLOCKS}; int64_t check_level{DEFAULT_CHECKLEVEL}; - std::function<bool()> check_interrupt; std::function<void()> coins_error_cb; }; diff --git a/src/node/chainstatemanager_args.cpp b/src/node/chainstatemanager_args.cpp index 87d9238c18..1cc126cb05 100644 --- a/src/node/chainstatemanager_args.cpp +++ b/src/node/chainstatemanager_args.cpp @@ -6,7 +6,8 @@ #include <arith_uint256.h> #include <common/args.h> -#include <kernel/chainstatemanager_opts.h> +#include <common/system.h> +#include <logging.h> #include <node/coins_view_args.h> #include <node/database_args.h> #include <tinyformat.h> @@ -16,6 +17,7 @@ #include <util/translation.h> #include <validation.h> +#include <algorithm> #include <chrono> #include <string> @@ -41,6 +43,16 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& args, ChainstateManage ReadDatabaseArgs(args, opts.coins_db); ReadCoinsViewArgs(args, opts.coins_view); + int script_threads = args.GetIntArg("-par", DEFAULT_SCRIPTCHECK_THREADS); + if (script_threads <= 0) { + // -par=0 means autodetect (number of cores - 1 script threads) + // -par=-n means "leave n cores free" (number of cores - n - 1 script threads) + script_threads += GetNumCores(); + } + // Subtract 1 because the main thread counts towards the par threads. + opts.worker_threads_num = std::clamp(script_threads - 1, 0, MAX_SCRIPTCHECK_THREADS); + LogPrintf("Script verification uses %d additional threads\n", opts.worker_threads_num); + return {}; } } // namespace node diff --git a/src/node/chainstatemanager_args.h b/src/node/chainstatemanager_args.h index 701515953e..b2cdba68b8 100644 --- a/src/node/chainstatemanager_args.h +++ b/src/node/chainstatemanager_args.h @@ -10,6 +10,11 @@ class ArgsManager; +/** Maximum number of dedicated script-checking threads allowed */ +static constexpr int MAX_SCRIPTCHECK_THREADS{15}; +/** -par default (number of script-checking threads, 0 = auto) */ +static constexpr int DEFAULT_SCRIPTCHECK_THREADS{0}; + namespace node { [[nodiscard]] util::Result<void> ApplyArgsManOptions(const ArgsManager& args, ChainstateManager::Options& opts); } // namespace node diff --git a/src/node/context.h b/src/node/context.h index dae900ff7f..4f3b640b2d 100644 --- a/src/node/context.h +++ b/src/node/context.h @@ -50,6 +50,8 @@ struct NodeContext { std::unique_ptr<kernel::Context> kernel; //! Init interface for initializing current process and connecting to other processes. interfaces::Init* init{nullptr}; + //! Interrupt object used to track whether node shutdown was requested. + util::SignalInterrupt* shutdown{nullptr}; std::unique_ptr<AddrMan> addrman; std::unique_ptr<CConnman> connman; std::unique_ptr<CTxMemPool> mempool; diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index f6dbe4f008..5a8b7fc105 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -39,13 +39,14 @@ #include <primitives/transaction.h> #include <rpc/protocol.h> #include <rpc/server.h> -#include <shutdown.h> #include <support/allocators/secure.h> #include <sync.h> #include <txmempool.h> #include <uint256.h> #include <univalue.h> #include <util/check.h> +#include <util/result.h> +#include <util/signalinterrupt.h> #include <util/translation.h> #include <validation.h> #include <validationinterface.h> @@ -120,14 +121,16 @@ public: } void startShutdown() override { - StartShutdown(); + if (!(*Assert(Assert(m_context)->shutdown))()) { + LogPrintf("Error: failed to send shutdown signal\n"); + } // Stop RPC for clean shutdown if any of waitfor* commands is executed. if (args().GetBoolArg("-server", false)) { InterruptRPC(); StopRPC(); } } - bool shutdownRequested() override { return ShutdownRequested(); } + bool shutdownRequested() override { return ShutdownRequested(*Assert(m_context)); }; bool isSettingIgnored(const std::string& name) override { bool ignored = false; @@ -328,10 +331,12 @@ public: std::vector<std::string> listRpcCommands() override { return ::tableRPC.listCommands(); } void rpcSetTimerInterfaceIfUnset(RPCTimerInterface* iface) override { RPCSetTimerInterfaceIfUnset(iface); } void rpcUnsetTimerInterface(RPCTimerInterface* iface) override { RPCUnsetTimerInterface(iface); } - bool getUnspentOutput(const COutPoint& output, Coin& coin) override + std::optional<Coin> getUnspentOutput(const COutPoint& output) override { LOCK(::cs_main); - return chainman().ActiveChainstate().CoinsTip().GetCoin(output, coin); + Coin coin; + if (chainman().ActiveChainstate().CoinsTip().GetCoin(output, coin)) return coin; + return {}; } TransactionError broadcastTransaction(CTransactionRef tx, CAmount max_tx_fee, std::string& err_string) override { @@ -426,9 +431,9 @@ public: explicit NotificationsProxy(std::shared_ptr<Chain::Notifications> notifications) : m_notifications(std::move(notifications)) {} virtual ~NotificationsProxy() = default; - void TransactionAddedToMempool(const CTransactionRef& tx, uint64_t mempool_sequence) override + void TransactionAddedToMempool(const NewMempoolTransactionInfo& tx, uint64_t mempool_sequence) override { - m_notifications->transactionAddedToMempool(tx); + m_notifications->transactionAddedToMempool(tx.info.m_tx); } void TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason, uint64_t mempool_sequence) override { @@ -648,8 +653,9 @@ public: { if (!m_node.mempool) return false; LOCK(m_node.mempool->cs); - auto it = m_node.mempool->GetIter(txid); - return it && (*it)->GetCountWithDescendants() > 1; + const auto entry{m_node.mempool->GetEntry(Txid::FromUint256(txid))}; + if (entry == nullptr) return false; + return entry->GetCountWithDescendants() > 1; } bool broadcastTransaction(const CTransactionRef& tx, const CAmount& max_tx_fee, @@ -669,7 +675,7 @@ public: m_node.mempool->GetTransactionAncestry(txid, ancestors, descendants, ancestorsize, ancestorfees); } - std::map<COutPoint, CAmount> CalculateIndividualBumpFees(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) override + std::map<COutPoint, CAmount> calculateIndividualBumpFees(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) override { if (!m_node.mempool) { std::map<COutPoint, CAmount> bump_fees; @@ -681,7 +687,7 @@ public: return MiniMiner(*m_node.mempool, outpoints).CalculateBumpFees(target_feerate); } - std::optional<CAmount> CalculateCombinedBumpFee(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) override + std::optional<CAmount> calculateCombinedBumpFee(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) override { if (!m_node.mempool) { return 0; @@ -697,14 +703,13 @@ public: limit_ancestor_count = limits.ancestor_count; limit_descendant_count = limits.descendant_count; } - bool checkChainLimits(const CTransactionRef& tx) override + util::Result<void> checkChainLimits(const CTransactionRef& tx) override { - if (!m_node.mempool) return true; + if (!m_node.mempool) return {}; LockPoints lp; CTxMemPoolEntry entry(tx, 0, 0, 0, 0, false, 0, lp); - const CTxMemPool::Limits& limits{m_node.mempool->m_limits}; LOCK(m_node.mempool->cs); - return m_node.mempool->CalculateMemPoolAncestors(entry, limits).has_value(); + return m_node.mempool->CheckPackageLimits({tx}, entry.GetTxSize()); } CFeeRate estimateSmartFee(int num_blocks, bool conservative, FeeCalculation* calc) override { @@ -746,7 +751,7 @@ public: { return chainman().IsInitialBlockDownload(); } - bool shutdownRequested() override { return ShutdownRequested(); } + bool shutdownRequested() override { return ShutdownRequested(m_node); } void initMessage(const std::string& message) override { ::uiInterface.InitMessage(message); } void initWarning(const bilingual_str& message) override { InitWarning(message); } void initError(const bilingual_str& message) override { InitError(message); } @@ -772,7 +777,6 @@ public: { RPCRunLater(name, std::move(fn), seconds); } - int rpcSerializationFlags() override { return RPCSerializationFlags(); } common::SettingsValue getSetting(const std::string& name) override { return args().GetSetting(name); @@ -806,7 +810,7 @@ public: { if (!m_node.mempool) return; LOCK2(::cs_main, m_node.mempool->cs); - for (const CTxMemPoolEntry& entry : m_node.mempool->mapTx) { + for (const CTxMemPoolEntry& entry : m_node.mempool->entryAll()) { notifications.transactionAddedToMempool(entry.GetSharedTx()); } } diff --git a/src/node/kernel_notifications.cpp b/src/node/kernel_notifications.cpp index 7224127c72..1fd3bad296 100644 --- a/src/node/kernel_notifications.cpp +++ b/src/node/kernel_notifications.cpp @@ -15,7 +15,6 @@ #include <logging.h> #include <node/abort.h> #include <node/interface_ui.h> -#include <shutdown.h> #include <util/check.h> #include <util/strencodings.h> #include <util/string.h> @@ -62,7 +61,9 @@ kernel::InterruptResult KernelNotifications::blockTip(SynchronizationState state { uiInterface.NotifyBlockTip(state, &index); if (m_stop_at_height && index.nHeight >= m_stop_at_height) { - StartShutdown(); + if (!m_shutdown()) { + LogPrintf("Error: failed to send shutdown signal after reaching stop height\n"); + } return kernel::Interrupted{}; } return {}; @@ -85,12 +86,13 @@ void KernelNotifications::warning(const bilingual_str& warning) void KernelNotifications::flushError(const std::string& debug_message) { - AbortNode(m_exit_status, debug_message); + AbortNode(&m_shutdown, m_exit_status, debug_message); } void KernelNotifications::fatalError(const std::string& debug_message, const bilingual_str& user_message) { - node::AbortNode(m_exit_status, debug_message, user_message, m_shutdown_on_fatal_error); + node::AbortNode(m_shutdown_on_fatal_error ? &m_shutdown : nullptr, + m_exit_status, debug_message, user_message); } void ReadNotificationArgs(const ArgsManager& args, KernelNotifications& notifications) diff --git a/src/node/kernel_notifications.h b/src/node/kernel_notifications.h index b2dfc03398..38d8600ac6 100644 --- a/src/node/kernel_notifications.h +++ b/src/node/kernel_notifications.h @@ -16,6 +16,10 @@ class CBlockIndex; enum class SynchronizationState; struct bilingual_str; +namespace util { +class SignalInterrupt; +} // namespace util + namespace node { static constexpr int DEFAULT_STOPATHEIGHT{0}; @@ -23,7 +27,7 @@ static constexpr int DEFAULT_STOPATHEIGHT{0}; class KernelNotifications : public kernel::Notifications { public: - KernelNotifications(std::atomic<int>& exit_status) : m_exit_status{exit_status} {} + KernelNotifications(util::SignalInterrupt& shutdown, std::atomic<int>& exit_status) : m_shutdown(shutdown), m_exit_status{exit_status} {} [[nodiscard]] kernel::InterruptResult blockTip(SynchronizationState state, CBlockIndex& index) override; @@ -42,6 +46,7 @@ public: //! Useful for tests, can be set to false to avoid shutdown on fatal error. bool m_shutdown_on_fatal_error{true}; private: + util::SignalInterrupt& m_shutdown; std::atomic<int>& m_exit_status; }; diff --git a/src/node/mempool_args.cpp b/src/node/mempool_args.cpp index f63d9875fc..ac26600919 100644 --- a/src/node/mempool_args.cpp +++ b/src/node/mempool_args.cpp @@ -93,6 +93,8 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& argsman, const CChainP mempool_opts.full_rbf = argsman.GetBoolArg("-mempoolfullrbf", mempool_opts.full_rbf); + mempool_opts.persist_v1_dat = argsman.GetBoolArg("-persistmempoolv1", mempool_opts.persist_v1_dat); + ApplyArgsManOptions(argsman, mempool_opts.limits); return {}; diff --git a/src/node/miner.cpp b/src/node/miner.cpp index caa2991819..ce5452d1f9 100644 --- a/src/node/miner.cpp +++ b/src/node/miner.cpp @@ -188,7 +188,7 @@ void BlockAssembler::onlyUnconfirmed(CTxMemPool::setEntries& testSet) { for (CTxMemPool::setEntries::iterator iit = testSet.begin(); iit != testSet.end(); ) { // Only test txs not already in the block - if (inBlock.count(*iit)) { + if (inBlock.count((*iit)->GetSharedTx()->GetHash())) { testSet.erase(iit++); } else { iit++; @@ -229,7 +229,7 @@ void BlockAssembler::AddToBlock(CTxMemPool::txiter iter) ++nBlockTx; nBlockSigOpsCost += iter->GetSigOpCost(); nFees += iter->GetFee(); - inBlock.insert(iter); + inBlock.insert(iter->GetSharedTx()->GetHash()); bool fPrintPriority = gArgs.GetBoolArg("-printpriority", DEFAULT_PRINTPRIORITY); if (fPrintPriority) { @@ -298,7 +298,7 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele // because some of their txs are already in the block indexed_modified_transaction_set mapModifiedTx; // Keep track of entries that failed inclusion, to avoid duplicate work - CTxMemPool::setEntries failedTx; + std::set<Txid> failedTx; CTxMemPool::indexed_transaction_set::index<ancestor_score>::type::iterator mi = mempool.mapTx.get<ancestor_score>().begin(); CTxMemPool::txiter iter; @@ -326,7 +326,7 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele if (mi != mempool.mapTx.get<ancestor_score>().end()) { auto it = mempool.mapTx.project<0>(mi); assert(it != mempool.mapTx.end()); - if (mapModifiedTx.count(it) || inBlock.count(it) || failedTx.count(it)) { + if (mapModifiedTx.count(it) || inBlock.count(it->GetSharedTx()->GetHash()) || failedTx.count(it->GetSharedTx()->GetHash())) { ++mi; continue; } @@ -360,7 +360,7 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele // We skip mapTx entries that are inBlock, and mapModifiedTx shouldn't // contain anything that is inBlock. - assert(!inBlock.count(iter)); + assert(!inBlock.count(iter->GetSharedTx()->GetHash())); uint64_t packageSize = iter->GetSizeWithAncestors(); CAmount packageFees = iter->GetModFeesWithAncestors(); @@ -382,7 +382,7 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele // we must erase failed entries so that we can consider the // next best entry on the next loop iteration mapModifiedTx.get<ancestor_score>().erase(modit); - failedTx.insert(iter); + failedTx.insert(iter->GetSharedTx()->GetHash()); } ++nConsecutiveFailed; @@ -404,7 +404,7 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele if (!TestPackageTransactions(ancestors)) { if (fUsingModified) { mapModifiedTx.get<ancestor_score>().erase(modit); - failedTx.insert(iter); + failedTx.insert(iter->GetSharedTx()->GetHash()); } continue; } diff --git a/src/node/miner.h b/src/node/miner.h index 4173521585..06a917228d 100644 --- a/src/node/miner.h +++ b/src/node/miner.h @@ -142,7 +142,7 @@ private: uint64_t nBlockTx; uint64_t nBlockSigOpsCost; CAmount nFees; - CTxMemPool::setEntries inBlock; + std::unordered_set<Txid, SaltedTxidHasher> inBlock; // Chain context for the block int nHeight; diff --git a/src/node/mini_miner.cpp b/src/node/mini_miner.cpp index 2827242f96..58422c4439 100644 --- a/src/node/mini_miner.cpp +++ b/src/node/mini_miner.cpp @@ -4,9 +4,14 @@ #include <node/mini_miner.h> +#include <boost/multi_index/detail/hash_index_iterator.hpp> +#include <boost/operators.hpp> #include <consensus/amount.h> #include <policy/feerate.h> #include <primitives/transaction.h> +#include <sync.h> +#include <txmempool.h> +#include <uint256.h> #include <util/check.h> #include <algorithm> @@ -72,7 +77,12 @@ MiniMiner::MiniMiner(const CTxMemPool& mempool, const std::vector<COutPoint>& ou // Add every entry to m_entries_by_txid and m_entries, except the ones that will be replaced. for (const auto& txiter : cluster) { if (!m_to_be_replaced.count(txiter->GetTx().GetHash())) { - auto [mapiter, success] = m_entries_by_txid.emplace(txiter->GetTx().GetHash(), MiniMinerMempoolEntry(txiter)); + auto [mapiter, success] = m_entries_by_txid.emplace(txiter->GetTx().GetHash(), + MiniMinerMempoolEntry{/*tx_in=*/txiter->GetSharedTx(), + /*vsize_self=*/txiter->GetTxSize(), + /*vsize_ancestor=*/txiter->GetSizeWithAncestors(), + /*fee_self=*/txiter->GetModifiedFee(), + /*fee_ancestor=*/txiter->GetModFeesWithAncestors()}); m_entries.push_back(mapiter); } else { auto outpoints_it = m_requested_outpoints_by_txid.find(txiter->GetTx().GetHash()); @@ -122,6 +132,48 @@ MiniMiner::MiniMiner(const CTxMemPool& mempool, const std::vector<COutPoint>& ou SanityCheck(); } +MiniMiner::MiniMiner(const std::vector<MiniMinerMempoolEntry>& manual_entries, + const std::map<Txid, std::set<Txid>>& descendant_caches) +{ + for (const auto& entry : manual_entries) { + const auto& txid = entry.GetTx().GetHash(); + // We need to know the descendant set of every transaction. + if (!Assume(descendant_caches.count(txid) > 0)) { + m_ready_to_calculate = false; + return; + } + // Just forward these args onto MiniMinerMempoolEntry + auto [mapiter, success] = m_entries_by_txid.emplace(txid, entry); + // Txids must be unique; this txid shouldn't already be an entry in m_entries_by_txid + if (Assume(success)) m_entries.push_back(mapiter); + } + // Descendant cache is already built, but we need to translate them to m_entries_by_txid iters. + for (const auto& [txid, desc_txids] : descendant_caches) { + // Descendant cache should include at least the tx itself. + if (!Assume(!desc_txids.empty())) { + m_ready_to_calculate = false; + return; + } + std::vector<MockEntryMap::iterator> descendants; + for (const auto& desc_txid : desc_txids) { + auto desc_it{m_entries_by_txid.find(desc_txid)}; + // Descendants should only include transactions with corresponding entries. + if (!Assume(desc_it != m_entries_by_txid.end())) { + m_ready_to_calculate = false; + return; + } else { + descendants.emplace_back(desc_it); + } + } + m_descendant_set_by_txid.emplace(txid, descendants); + } + Assume(m_to_be_replaced.empty()); + Assume(m_requested_outpoints_by_txid.empty()); + Assume(m_bump_fees.empty()); + Assume(m_inclusion_order.empty()); + SanityCheck(); +} + // Compare by min(ancestor feerate, individual feerate), then iterator // // Under the ancestor-based mining approach, high-feerate children can pay for parents, but high-feerate @@ -201,8 +253,10 @@ void MiniMiner::SanityCheck() const [&](const auto& txid){return m_entries_by_txid.find(txid) == m_entries_by_txid.end();})); } -void MiniMiner::BuildMockTemplate(const CFeeRate& target_feerate) +void MiniMiner::BuildMockTemplate(std::optional<CFeeRate> target_feerate) { + const auto num_txns{m_entries_by_txid.size()}; + uint32_t sequence_num{0}; while (!m_entries_by_txid.empty()) { // Sort again, since transaction removal may change some m_entries' ancestor feerates. std::sort(m_entries.begin(), m_entries.end(), AncestorFeerateComparator()); @@ -213,7 +267,8 @@ void MiniMiner::BuildMockTemplate(const CFeeRate& target_feerate) const auto ancestor_package_size = (*best_iter)->second.GetSizeWithAncestors(); const auto ancestor_package_fee = (*best_iter)->second.GetModFeesWithAncestors(); // Stop here. Everything that didn't "make it into the block" has bumpfee. - if (ancestor_package_fee < target_feerate.GetFee(ancestor_package_size)) { + if (target_feerate.has_value() && + ancestor_package_fee < target_feerate->GetFee(ancestor_package_size)) { break; } @@ -237,14 +292,32 @@ void MiniMiner::BuildMockTemplate(const CFeeRate& target_feerate) to_process.erase(iter); } } + // Track the order in which transactions were selected. + for (const auto& ancestor : ancestors) { + m_inclusion_order.emplace(Txid::FromUint256(ancestor->first), sequence_num); + } DeleteAncestorPackage(ancestors); SanityCheck(); + ++sequence_num; + } + if (!target_feerate.has_value()) { + Assume(m_in_block.size() == num_txns); + } else { + Assume(m_in_block.empty() || m_total_fees >= target_feerate->GetFee(m_total_vsize)); } - Assume(m_in_block.empty() || m_total_fees >= target_feerate.GetFee(m_total_vsize)); + Assume(m_in_block.empty() || sequence_num > 0); + Assume(m_in_block.size() == m_inclusion_order.size()); // Do not try to continue building the block template with a different feerate. m_ready_to_calculate = false; } + +std::map<Txid, uint32_t> MiniMiner::Linearize() +{ + BuildMockTemplate(std::nullopt); + return m_inclusion_order; +} + std::map<COutPoint, CAmount> MiniMiner::CalculateBumpFees(const CFeeRate& target_feerate) { if (!m_ready_to_calculate) return {}; diff --git a/src/node/mini_miner.h b/src/node/mini_miner.h index 9d9d66bf0b..de62c0af75 100644 --- a/src/node/mini_miner.h +++ b/src/node/mini_miner.h @@ -5,33 +5,45 @@ #ifndef BITCOIN_NODE_MINI_MINER_H #define BITCOIN_NODE_MINI_MINER_H -#include <txmempool.h> +#include <consensus/amount.h> +#include <primitives/transaction.h> +#include <uint256.h> +#include <map> #include <memory> #include <optional> +#include <set> #include <stdint.h> +#include <vector> + +class CFeeRate; +class CTxMemPool; namespace node { // Container for tracking updates to ancestor feerate as we include ancestors in the "block" class MiniMinerMempoolEntry { - const CAmount fee_individual; const CTransactionRef tx; const int64_t vsize_individual; - CAmount fee_with_ancestors; int64_t vsize_with_ancestors; + const CAmount fee_individual; + CAmount fee_with_ancestors; // This class must be constructed while holding mempool.cs. After construction, the object's // methods can be called without holding that lock. public: - explicit MiniMinerMempoolEntry(CTxMemPool::txiter entry) : - fee_individual{entry->GetModifiedFee()}, - tx{entry->GetSharedTx()}, - vsize_individual(entry->GetTxSize()), - fee_with_ancestors{entry->GetModFeesWithAncestors()}, - vsize_with_ancestors(entry->GetSizeWithAncestors()) + explicit MiniMinerMempoolEntry(const CTransactionRef& tx_in, + int64_t vsize_self, + int64_t vsize_ancestor, + CAmount fee_self, + CAmount fee_ancestor): + tx{tx_in}, + vsize_individual{vsize_self}, + vsize_with_ancestors{vsize_ancestor}, + fee_individual{fee_self}, + fee_with_ancestors{fee_ancestor} { } CAmount GetModifiedFee() const { return fee_individual; } @@ -55,8 +67,14 @@ struct IteratorComparator } }; -/** A minimal version of BlockAssembler. Allows us to run the mining algorithm on a subset of - * mempool transactions, ignoring consensus rules, to calculate mining scores. */ +/** A minimal version of BlockAssembler, using the same ancestor set scoring algorithm. Allows us to + * run this algorithm on a limited set of transactions (e.g. subset of mempool or transactions that + * are not yet in mempool) instead of the entire mempool, ignoring consensus rules. + * Callers may use this to: + * - Calculate the "bump fee" needed to spend an unconfirmed UTXO at a given feerate + * - "Linearize" a list of transactions to see the order in which they would be selected for + * inclusion in a block + */ class MiniMiner { // When true, a caller may use CalculateBumpFees(). Becomes false if we failed to retrieve @@ -72,7 +90,11 @@ class MiniMiner // the same tx will have the same bumpfee. Excludes non-mempool transactions. std::map<uint256, std::vector<COutPoint>> m_requested_outpoints_by_txid; - // What we're trying to calculate. + // Txid to a number representing the order in which this transaction was included (smaller + // number = included earlier). Transactions included in an ancestor set together have the same + // sequence number. + std::map<Txid, uint32_t> m_inclusion_order; + // What we're trying to calculate. Outpoint to the fee needed to bring the transaction to the target feerate. std::map<COutPoint, CAmount> m_bump_fees; // The constructed block template @@ -102,14 +124,32 @@ public: /** Returns true if CalculateBumpFees may be called, false if not. */ bool IsReadyToCalculate() const { return m_ready_to_calculate; } - /** Build a block template until the target feerate is hit. */ - void BuildMockTemplate(const CFeeRate& target_feerate); + /** Build a block template until the target feerate is hit. If target_feerate is not given, + * builds a block template until all transactions have been selected. */ + void BuildMockTemplate(std::optional<CFeeRate> target_feerate); /** Returns set of txids in the block template if one has been constructed. */ std::set<uint256> GetMockTemplateTxids() const { return m_in_block; } + /** Constructor that takes a list of outpoints that may or may not belong to transactions in the + * mempool. Copies out information about the relevant transactions in the mempool into + * MiniMinerMempoolEntrys. + */ MiniMiner(const CTxMemPool& mempool, const std::vector<COutPoint>& outpoints); + /** Constructor in which the MiniMinerMempoolEntry entries have been constructed manually. + * It is assumed that all entries are unique and their values are correct, otherwise results + * computed by MiniMiner may be incorrect. Callers should check IsReadyToCalculate() after + * construction. + * @param[in] descendant_caches A map from each transaction to the set of txids of this + * transaction's descendant set, including itself. Each tx in + * manual_entries must have a corresponding entry in this map, and + * all of the txids in a descendant set must correspond to a tx in + * manual_entries. + */ + MiniMiner(const std::vector<MiniMinerMempoolEntry>& manual_entries, + const std::map<Txid, std::set<Txid>>& descendant_caches); + /** Construct a new block template and, for each outpoint corresponding to a transaction that * did not make it into the block, calculate the cost of bumping those transactions (and their * ancestors) to the minimum feerate. Returns a map from outpoint to bump fee, or an empty map @@ -120,6 +160,12 @@ public: * not make it into the block to the target feerate. Returns the total bump fee, or std::nullopt * if it cannot be calculated. */ std::optional<CAmount> CalculateTotalBumpFees(const CFeeRate& target_feerate); + + /** Construct a new block template with all of the transactions and calculate the order in which + * they are selected. Returns the sequence number (lower = selected earlier) with which each + * transaction was selected, indexed by txid, or an empty map if it cannot be calculated. + */ + std::map<Txid, uint32_t> Linearize(); }; } // namespace node diff --git a/src/version.h b/src/node/protocol_version.h index 204df3758b..43c1bf6c5b 100644 --- a/src/version.h +++ b/src/node/protocol_version.h @@ -1,9 +1,9 @@ -// Copyright (c) 2012-2020 The Bitcoin Core developers +// Copyright (c) 2012-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#ifndef BITCOIN_VERSION_H -#define BITCOIN_VERSION_H +#ifndef BITCOIN_NODE_PROTOCOL_VERSION_H +#define BITCOIN_NODE_PROTOCOL_VERSION_H /** * network protocol versioning @@ -35,7 +35,4 @@ static const int INVALID_CB_NO_BAN_VERSION = 70015; //! "wtxidrelay" command for wtxid-based relay starts with this version static const int WTXID_RELAY_VERSION = 70016; -// Make sure that none of the values above collide with -// `SERIALIZE_TRANSACTION_NO_WITNESS`. - -#endif // BITCOIN_VERSION_H +#endif // BITCOIN_NODE_PROTOCOL_VERSION_H diff --git a/src/node/transaction.cpp b/src/node/transaction.cpp index 026c8084dd..e8ab2326c1 100644 --- a/src/node/transaction.cpp +++ b/src/node/transaction.cpp @@ -40,7 +40,7 @@ TransactionError BroadcastTransaction(NodeContext& node, const CTransactionRef t assert(node.peerman); std::promise<void> promise; - uint256 txid = tx->GetHash(); + Txid txid = tx->GetHash(); uint256 wtxid = tx->GetWitnessHash(); bool callback_set = false; diff --git a/src/policy/feerate.h b/src/policy/feerate.h index 41f4a4d06b..2e50172914 100644 --- a/src/policy/feerate.h +++ b/src/policy/feerate.h @@ -71,6 +71,8 @@ public: friend bool operator!=(const CFeeRate& a, const CFeeRate& b) { return a.nSatoshisPerK != b.nSatoshisPerK; } CFeeRate& operator+=(const CFeeRate& a) { nSatoshisPerK += a.nSatoshisPerK; return *this; } std::string ToString(const FeeEstimateMode& fee_estimate_mode = FeeEstimateMode::BTC_KVB) const; + friend CFeeRate operator*(const CFeeRate& f, int a) { return CFeeRate(a * f.nSatoshisPerK); } + friend CFeeRate operator*(int a, const CFeeRate& f) { return CFeeRate(a * f.nSatoshisPerK); } SERIALIZE_METHODS(CFeeRate, obj) { READWRITE(obj.nSatoshisPerK); } }; diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 87bfa4cfc3..5440548636 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -260,6 +260,11 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, unsigned int curFarBucket = maxbucketindex; unsigned int bestFarBucket = maxbucketindex; + // We'll always group buckets into sets that meet sufficientTxVal -- + // this ensures that we're using consistent groups between different + // confirmation targets. + double partialNum = 0; + bool foundAnswer = false; unsigned int bins = unconfTxs.size(); bool newBucketRange = true; @@ -275,6 +280,7 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, } curFarBucket = bucket; nConf += confAvg[periodTarget - 1][bucket]; + partialNum += txCtAvg[bucket]; totalNum += txCtAvg[bucket]; failNum += failAvg[periodTarget - 1][bucket]; for (unsigned int confct = confTarget; confct < GetMaxConfirms(); confct++) @@ -284,7 +290,14 @@ double TxConfirmStats::EstimateMedianVal(int confTarget, double sufficientTxVal, // we can test for success // (Only count the confirmed data points, so that each confirmation count // will be looking at the same amount of data and same bucket breaks) - if (totalNum >= sufficientTxVal / (1 - decay)) { + + if (partialNum < sufficientTxVal / (1 - decay)) { + // the buckets we've added in this round aren't sufficient + // so keep adding + continue; + } else { + partialNum = 0; // reset for the next range we'll add + double curPct = nConf / (totalNum + failNum + extraNum); // Check to see if we are no longer getting confirmed at the success rate @@ -502,15 +515,10 @@ void TxConfirmStats::removeTx(unsigned int entryHeight, unsigned int nBestSeenHe } } -// This function is called from CTxMemPool::removeUnchecked to ensure -// txs removed from the mempool for any reason are no longer -// tracked. Txs that were part of a block have already been removed in -// processBlockTx to ensure they are never double tracked, but it is -// of no harm to try to remove them again. -bool CBlockPolicyEstimator::removeTx(uint256 hash, bool inBlock) +bool CBlockPolicyEstimator::removeTx(uint256 hash) { LOCK(m_cs_fee_estimator); - return _removeTx(hash, inBlock); + return _removeTx(hash, /*inBlock=*/false); } bool CBlockPolicyEstimator::_removeTx(const uint256& hash, bool inBlock) @@ -566,11 +574,26 @@ CBlockPolicyEstimator::CBlockPolicyEstimator(const fs::path& estimation_filepath CBlockPolicyEstimator::~CBlockPolicyEstimator() = default; -void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate) +void CBlockPolicyEstimator::TransactionAddedToMempool(const NewMempoolTransactionInfo& tx, uint64_t /*unused*/) +{ + processTransaction(tx); +} + +void CBlockPolicyEstimator::TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason /*unused*/, uint64_t /*unused*/) +{ + removeTx(tx->GetHash()); +} + +void CBlockPolicyEstimator::MempoolTransactionsRemovedForBlock(const std::vector<RemovedMempoolTransactionInfo>& txs_removed_for_block, unsigned int nBlockHeight) +{ + processBlock(txs_removed_for_block, nBlockHeight); +} + +void CBlockPolicyEstimator::processTransaction(const NewMempoolTransactionInfo& tx) { LOCK(m_cs_fee_estimator); - unsigned int txHeight = entry.GetHeight(); - uint256 hash = entry.GetTx().GetHash(); + const unsigned int txHeight = tx.info.txHeight; + const auto& hash = tx.info.m_tx->GetHash(); if (mapMemPoolTxs.count(hash)) { LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy error mempool tx %s already being tracked\n", hash.ToString()); @@ -584,31 +607,37 @@ void CBlockPolicyEstimator::processTransaction(const CTxMemPoolEntry& entry, boo // It will be synced next time a block is processed. return; } + // This transaction should only count for fee estimation if: + // - it's not being re-added during a reorg which bypasses typical mempool fee limits + // - the node is not behind + // - the transaction is not dependent on any other transactions in the mempool + // - it's not part of a package. + const bool validForFeeEstimation = !tx.m_mempool_limit_bypassed && !tx.m_submitted_in_package && tx.m_chainstate_is_current && tx.m_has_no_mempool_parents; // Only want to be updating estimates when our blockchain is synced, // otherwise we'll miscalculate how many blocks its taking to get included. - if (!validFeeEstimate) { + if (!validForFeeEstimation) { untrackedTxs++; return; } trackedTxs++; // Feerates are stored and reported as BTC-per-kb: - CFeeRate feeRate(entry.GetFee(), entry.GetTxSize()); + const CFeeRate feeRate(tx.info.m_fee, tx.info.m_virtual_transaction_size); mapMemPoolTxs[hash].blockHeight = txHeight; - unsigned int bucketIndex = feeStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + unsigned int bucketIndex = feeStats->NewTx(txHeight, static_cast<double>(feeRate.GetFeePerK())); mapMemPoolTxs[hash].bucketIndex = bucketIndex; - unsigned int bucketIndex2 = shortStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + unsigned int bucketIndex2 = shortStats->NewTx(txHeight, static_cast<double>(feeRate.GetFeePerK())); assert(bucketIndex == bucketIndex2); - unsigned int bucketIndex3 = longStats->NewTx(txHeight, (double)feeRate.GetFeePerK()); + unsigned int bucketIndex3 = longStats->NewTx(txHeight, static_cast<double>(feeRate.GetFeePerK())); assert(bucketIndex == bucketIndex3); } -bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry) +bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const RemovedMempoolTransactionInfo& tx) { AssertLockHeld(m_cs_fee_estimator); - if (!_removeTx(entry->GetTx().GetHash(), true)) { + if (!_removeTx(tx.info.m_tx->GetHash(), true)) { // This transaction wasn't being tracked for fee estimation return false; } @@ -616,7 +645,7 @@ bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxM // How many blocks did it take for miners to include this transaction? // blocksToConfirm is 1-based, so a transaction included in the earliest // possible block has confirmation count of 1 - int blocksToConfirm = nBlockHeight - entry->GetHeight(); + int blocksToConfirm = nBlockHeight - tx.info.txHeight; if (blocksToConfirm <= 0) { // This can't happen because we don't process transactions from a block with a height // lower than our greatest seen height @@ -625,16 +654,16 @@ bool CBlockPolicyEstimator::processBlockTx(unsigned int nBlockHeight, const CTxM } // Feerates are stored and reported as BTC-per-kb: - CFeeRate feeRate(entry->GetFee(), entry->GetTxSize()); + CFeeRate feeRate(tx.info.m_fee, tx.info.m_virtual_transaction_size); - feeStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); - shortStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); - longStats->Record(blocksToConfirm, (double)feeRate.GetFeePerK()); + feeStats->Record(blocksToConfirm, static_cast<double>(feeRate.GetFeePerK())); + shortStats->Record(blocksToConfirm, static_cast<double>(feeRate.GetFeePerK())); + longStats->Record(blocksToConfirm, static_cast<double>(feeRate.GetFeePerK())); return true; } -void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, - std::vector<const CTxMemPoolEntry*>& entries) +void CBlockPolicyEstimator::processBlock(const std::vector<RemovedMempoolTransactionInfo>& txs_removed_for_block, + unsigned int nBlockHeight) { LOCK(m_cs_fee_estimator); if (nBlockHeight <= nBestSeenHeight) { @@ -663,8 +692,8 @@ void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, unsigned int countedTxs = 0; // Update averages with data points from current block - for (const auto& entry : entries) { - if (processBlockTx(nBlockHeight, entry)) + for (const auto& tx : txs_removed_for_block) { + if (processBlockTx(nBlockHeight, tx)) countedTxs++; } @@ -675,7 +704,7 @@ void CBlockPolicyEstimator::processBlock(unsigned int nBlockHeight, LogPrint(BCLog::ESTIMATEFEE, "Blockpolicy estimates updated by %u of %u block txs, since last block %u of %u tracked, mempool map size %u, max target %u from %s\n", - countedTxs, entries.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size(), + countedTxs, txs_removed_for_block.size(), trackedTxs, trackedTxs + untrackedTxs, mapMemPoolTxs.size(), MaxUsableEstimate(), HistoricalBlockSpan() > BlockSpan() ? "historical" : "current"); trackedTxs = 0; @@ -1031,8 +1060,8 @@ void CBlockPolicyEstimator::FlushUnconfirmed() std::chrono::hours CBlockPolicyEstimator::GetFeeEstimatorFileAge() { - auto file_time = std::filesystem::last_write_time(m_estimation_filepath); - auto now = std::filesystem::file_time_type::clock::now(); + auto file_time{fs::last_write_time(m_estimation_filepath)}; + auto now{fs::file_time_type::clock::now()}; return std::chrono::duration_cast<std::chrono::hours>(now - file_time); } diff --git a/src/policy/fees.h b/src/policy/fees.h index 69bda195be..f34f66d3f0 100644 --- a/src/policy/fees.h +++ b/src/policy/fees.h @@ -12,6 +12,7 @@ #include <threadsafety.h> #include <uint256.h> #include <util/fs.h> +#include <validationinterface.h> #include <array> #include <chrono> @@ -35,8 +36,9 @@ static constexpr std::chrono::hours MAX_FILE_AGE{60}; static constexpr bool DEFAULT_ACCEPT_STALE_FEE_ESTIMATES{false}; class AutoFile; -class CTxMemPoolEntry; class TxConfirmStats; +struct RemovedMempoolTransactionInfo; +struct NewMempoolTransactionInfo; /* Identifier for each of the 3 different TxConfirmStats which will track * history over different time horizons. */ @@ -143,7 +145,7 @@ struct FeeCalculation * a certain number of blocks. Every time a block is added to the best chain, this class records * stats on the transactions included in that block */ -class CBlockPolicyEstimator +class CBlockPolicyEstimator : public CValidationInterface { private: /** Track confirm delays up to 12 blocks for short horizon */ @@ -198,19 +200,19 @@ private: public: /** Create new BlockPolicyEstimator and initialize stats tracking classes with default values */ CBlockPolicyEstimator(const fs::path& estimation_filepath, const bool read_stale_estimates); - ~CBlockPolicyEstimator(); + virtual ~CBlockPolicyEstimator(); /** Process all the transactions that have been included in a block */ - void processBlock(unsigned int nBlockHeight, - std::vector<const CTxMemPoolEntry*>& entries) + void processBlock(const std::vector<RemovedMempoolTransactionInfo>& txs_removed_for_block, + unsigned int nBlockHeight) EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** Process a transaction accepted to the mempool*/ - void processTransaction(const CTxMemPoolEntry& entry, bool validFeeEstimate) + void processTransaction(const NewMempoolTransactionInfo& tx) EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); - /** Remove a transaction from the mempool tracking stats*/ - bool removeTx(uint256 hash, bool inBlock) + /** Remove a transaction from the mempool tracking stats for non BLOCK removal reasons*/ + bool removeTx(uint256 hash) EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); /** DEPRECATED. Return a feerate estimate */ @@ -260,6 +262,15 @@ public: /** Calculates the age of the file, since last modified */ std::chrono::hours GetFeeEstimatorFileAge(); +protected: + /** Overridden from CValidationInterface. */ + void TransactionAddedToMempool(const NewMempoolTransactionInfo& tx, uint64_t /*unused*/) override + EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); + void TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason /*unused*/, uint64_t /*unused*/) override + EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); + void MempoolTransactionsRemovedForBlock(const std::vector<RemovedMempoolTransactionInfo>& txs_removed_for_block, unsigned int nBlockHeight) override + EXCLUSIVE_LOCKS_REQUIRED(!m_cs_fee_estimator); + private: mutable Mutex m_cs_fee_estimator; @@ -290,7 +301,7 @@ private: std::map<double, unsigned int> bucketMap GUARDED_BY(m_cs_fee_estimator); // Map of bucket upper-bound to index into all vectors by bucket /** Process a transaction confirmed in a block*/ - bool processBlockTx(unsigned int nBlockHeight, const CTxMemPoolEntry* entry) EXCLUSIVE_LOCKS_REQUIRED(m_cs_fee_estimator); + bool processBlockTx(unsigned int nBlockHeight, const RemovedMempoolTransactionInfo& tx) EXCLUSIVE_LOCKS_REQUIRED(m_cs_fee_estimator); /** Helper for estimateSmartFee */ double estimateCombinedFee(unsigned int confTarget, double successThreshold, bool checkShorterHorizon, EstimationResult *result) const EXCLUSIVE_LOCKS_REQUIRED(m_cs_fee_estimator); diff --git a/src/policy/packages.cpp b/src/policy/packages.cpp index 47a9274a31..3a63a9fe46 100644 --- a/src/policy/packages.cpp +++ b/src/policy/packages.cpp @@ -6,16 +6,77 @@ #include <policy/policy.h> #include <primitives/transaction.h> #include <uint256.h> -#include <util/hasher.h> +#include <util/check.h> #include <algorithm> #include <cassert> #include <iterator> #include <memory> #include <numeric> -#include <unordered_set> -bool CheckPackage(const Package& txns, PackageValidationState& state) +/** IsTopoSortedPackage where a set of txids has been pre-populated. The set is assumed to be correct and + * is mutated within this function (even if return value is false). */ +bool IsTopoSortedPackage(const Package& txns, std::unordered_set<uint256, SaltedTxidHasher>& later_txids) +{ + // Avoid misusing this function: later_txids should contain the txids of txns. + Assume(txns.size() == later_txids.size()); + + // later_txids always contains the txids of this transaction and the ones that come later in + // txns. If any transaction's input spends a tx in that set, we've found a parent placed later + // than its child. + for (const auto& tx : txns) { + for (const auto& input : tx->vin) { + if (later_txids.find(input.prevout.hash) != later_txids.end()) { + // The parent is a subsequent transaction in the package. + return false; + } + } + // Avoid misusing this function: later_txids must contain every tx. + Assume(later_txids.erase(tx->GetHash()) == 1); + } + + // Avoid misusing this function: later_txids should have contained the txids of txns. + Assume(later_txids.empty()); + return true; +} + +bool IsTopoSortedPackage(const Package& txns) +{ + std::unordered_set<uint256, SaltedTxidHasher> later_txids; + std::transform(txns.cbegin(), txns.cend(), std::inserter(later_txids, later_txids.end()), + [](const auto& tx) { return tx->GetHash(); }); + + return IsTopoSortedPackage(txns, later_txids); +} + +bool IsConsistentPackage(const Package& txns) +{ + // Don't allow any conflicting transactions, i.e. spending the same inputs, in a package. + std::unordered_set<COutPoint, SaltedOutpointHasher> inputs_seen; + for (const auto& tx : txns) { + if (tx->vin.empty()) { + // This function checks consistency based on inputs, and we can't do that if there are + // no inputs. Duplicate empty transactions are also not consistent with one another. + // This doesn't create false negatives, as unconfirmed transactions are not allowed to + // have no inputs. + return false; + } + for (const auto& input : tx->vin) { + if (inputs_seen.find(input.prevout) != inputs_seen.end()) { + // This input is also present in another tx in the package. + return false; + } + } + // Batch-add all the inputs for a tx at a time. If we added them 1 at a time, we could + // catch duplicate inputs within a single tx. This is a more severe, consensus error, + // and we want to report that from CheckTransaction instead. + std::transform(tx->vin.cbegin(), tx->vin.cend(), std::inserter(inputs_seen, inputs_seen.end()), + [](const auto& input) { return input.prevout; }); + } + return true; +} + +bool IsWellFormedPackage(const Package& txns, PackageValidationState& state, bool require_sorted) { const unsigned int package_count = txns.size(); @@ -30,10 +91,6 @@ bool CheckPackage(const Package& txns, PackageValidationState& state) return state.Invalid(PackageValidationResult::PCKG_POLICY, "package-too-large"); } - // Require the package to be sorted in order of dependency, i.e. parents appear before children. - // An unsorted package will fail anyway on missing-inputs, but it's better to quit earlier and - // fail on something less ambiguous (missing-inputs could also be an orphan or trying to - // spend nonexistent coins). std::unordered_set<uint256, SaltedTxidHasher> later_txids; std::transform(txns.cbegin(), txns.cend(), std::inserter(later_txids, later_txids.end()), [](const auto& tx) { return tx->GetHash(); }); @@ -44,30 +101,17 @@ bool CheckPackage(const Package& txns, PackageValidationState& state) return state.Invalid(PackageValidationResult::PCKG_POLICY, "package-contains-duplicates"); } - for (const auto& tx : txns) { - for (const auto& input : tx->vin) { - if (later_txids.find(input.prevout.hash) != later_txids.end()) { - // The parent is a subsequent transaction in the package. - return state.Invalid(PackageValidationResult::PCKG_POLICY, "package-not-sorted"); - } - } - later_txids.erase(tx->GetHash()); + // Require the package to be sorted in order of dependency, i.e. parents appear before children. + // An unsorted package will fail anyway on missing-inputs, but it's better to quit earlier and + // fail on something less ambiguous (missing-inputs could also be an orphan or trying to + // spend nonexistent coins). + if (require_sorted && !IsTopoSortedPackage(txns, later_txids)) { + return state.Invalid(PackageValidationResult::PCKG_POLICY, "package-not-sorted"); } // Don't allow any conflicting transactions, i.e. spending the same inputs, in a package. - std::unordered_set<COutPoint, SaltedOutpointHasher> inputs_seen; - for (const auto& tx : txns) { - for (const auto& input : tx->vin) { - if (inputs_seen.find(input.prevout) != inputs_seen.end()) { - // This input is also present in another tx in the package. - return state.Invalid(PackageValidationResult::PCKG_POLICY, "conflict-in-package"); - } - } - // Batch-add all the inputs for a tx at a time. If we added them 1 at a time, we could - // catch duplicate inputs within a single tx. This is a more severe, consensus error, - // and we want to report that from CheckTransaction instead. - std::transform(tx->vin.cbegin(), tx->vin.cend(), std::inserter(inputs_seen, inputs_seen.end()), - [](const auto& input) { return input.prevout; }); + if (!IsConsistentPackage(txns)) { + return state.Invalid(PackageValidationResult::PCKG_POLICY, "conflict-in-package"); } return true; } diff --git a/src/policy/packages.h b/src/policy/packages.h index cf37857e4b..537d8476e2 100644 --- a/src/policy/packages.h +++ b/src/policy/packages.h @@ -9,8 +9,10 @@ #include <consensus/validation.h> #include <policy/policy.h> #include <primitives/transaction.h> +#include <util/hasher.h> #include <cstdint> +#include <unordered_set> #include <vector> /** Default maximum number of transactions in a package. */ @@ -49,13 +51,32 @@ using Package = std::vector<CTransactionRef>; class PackageValidationState : public ValidationState<PackageValidationResult> {}; +/** If any direct dependencies exist between transactions (i.e. a child spending the output of a + * parent), checks that all parents appear somewhere in the list before their respective children. + * No other ordering is enforced. This function cannot detect indirect dependencies (e.g. a + * transaction's grandparent if its parent is not present). + * @returns true if sorted. False if any tx spends the output of a tx that appears later in txns. + */ +bool IsTopoSortedPackage(const Package& txns); + +/** Checks that these transactions don't conflict, i.e., spend the same prevout. This includes + * checking that there are no duplicate transactions. Since these checks require looking at the inputs + * of a transaction, returns false immediately if any transactions have empty vin. + * + * Does not check consistency of a transaction with oneself; does not check if a transaction spends + * the same prevout multiple times (see bad-txns-inputs-duplicate in CheckTransaction()). + * + * @returns true if there are no conflicts. False if any two transactions spend the same prevout. + * */ +bool IsConsistentPackage(const Package& txns); + /** Context-free package policy checks: * 1. The number of transactions cannot exceed MAX_PACKAGE_COUNT. * 2. The total weight cannot exceed MAX_PACKAGE_WEIGHT. * 3. If any dependencies exist between transactions, parents must appear before children. * 4. Transactions cannot conflict, i.e., spend the same inputs. */ -bool CheckPackage(const Package& txns, PackageValidationState& state); +bool IsWellFormedPackage(const Package& txns, PackageValidationState& state, bool require_sorted); /** Context-free check that a package is exactly one child and its parents; not all parents need to * be present, but the package must not contain any transactions that are not the child's parents. diff --git a/src/policy/policy.h b/src/policy/policy.h index d1c8148800..6a7980c312 100644 --- a/src/policy/policy.h +++ b/src/policy/policy.h @@ -98,7 +98,7 @@ static constexpr unsigned int MANDATORY_SCRIPT_VERIFY_FLAGS{SCRIPT_VERIFY_P2SH | * Standard script verification flags that standard transactions will comply * with. However we do not ban/disconnect nodes that forward txs violating * the additional (non-mandatory) rules here, to improve forwards and - * backwards compatability. + * backwards compatibility. */ static constexpr unsigned int STANDARD_SCRIPT_VERIFY_FLAGS{MANDATORY_SCRIPT_VERIFY_FLAGS | SCRIPT_VERIFY_STRICTENC | diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index d032b74008..76ab2b1a96 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -12,6 +12,7 @@ #include <tinyformat.h> #include <txmempool.h> #include <uint256.h> +#include <util/check.h> #include <util/moneystr.h> #include <util/rbf.h> @@ -35,7 +36,7 @@ RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) // If all the inputs have nSequence >= maxint-1, it still might be // signaled for RBF if any unconfirmed parents have signaled. - const CTxMemPoolEntry entry{*pool.mapTx.find(tx.GetHash())}; + const auto& entry{*Assert(pool.GetEntry(tx.GetHash()))}; auto ancestors{pool.AssumeCalculateMemPoolAncestors(__func__, entry, CTxMemPool::Limits::NoLimits(), /*fSearchForParents=*/false)}; diff --git a/src/primitives/block.cpp b/src/primitives/block.cpp index 3d21708820..27da161e1e 100644 --- a/src/primitives/block.cpp +++ b/src/primitives/block.cpp @@ -10,7 +10,7 @@ uint256 CBlockHeader::GetHash() const { - return (CHashWriter{PROTOCOL_VERSION} << *this).GetHash(); + return (HashWriter{} << *this).GetHash(); } std::string CBlock::ToString() const diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index 2c913bf432..b4a860dd9e 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -12,8 +12,9 @@ #include <tinyformat.h> #include <uint256.h> #include <util/strencodings.h> -#include <version.h> +#include <util/transaction_identifier.h> +#include <algorithm> #include <cassert> #include <stdexcept> @@ -29,7 +30,7 @@ CTxIn::CTxIn(COutPoint prevoutIn, CScript scriptSigIn, uint32_t nSequenceIn) nSequence = nSequenceIn; } -CTxIn::CTxIn(uint256 hashPrevTx, uint32_t nOut, CScript scriptSigIn, uint32_t nSequenceIn) +CTxIn::CTxIn(Txid hashPrevTx, uint32_t nOut, CScript scriptSigIn, uint32_t nSequenceIn) { prevout = COutPoint(hashPrevTx, nOut); scriptSig = scriptSigIn; @@ -65,26 +66,34 @@ std::string CTxOut::ToString() const CMutableTransaction::CMutableTransaction() : nVersion(CTransaction::CURRENT_VERSION), nLockTime(0) {} CMutableTransaction::CMutableTransaction(const CTransaction& tx) : vin(tx.vin), vout(tx.vout), nVersion(tx.nVersion), nLockTime(tx.nLockTime) {} -uint256 CMutableTransaction::GetHash() const +Txid CMutableTransaction::GetHash() const { - return (CHashWriter{SERIALIZE_TRANSACTION_NO_WITNESS} << *this).GetHash(); + return Txid::FromUint256((HashWriter{} << TX_NO_WITNESS(*this)).GetHash()); } -uint256 CTransaction::ComputeHash() const +bool CTransaction::ComputeHasWitness() const { - return (CHashWriter{SERIALIZE_TRANSACTION_NO_WITNESS} << *this).GetHash(); + return std::any_of(vin.begin(), vin.end(), [](const auto& input) { + return !input.scriptWitness.IsNull(); + }); } -uint256 CTransaction::ComputeWitnessHash() const +Txid CTransaction::ComputeHash() const +{ + return Txid::FromUint256((HashWriter{} << TX_NO_WITNESS(*this)).GetHash()); +} + +Wtxid CTransaction::ComputeWitnessHash() const { if (!HasWitness()) { - return hash; + return Wtxid::FromUint256(hash.ToUint256()); } - return (CHashWriter{0} << *this).GetHash(); + + return Wtxid::FromUint256((HashWriter{} << TX_WITH_WITNESS(*this)).GetHash()); } -CTransaction::CTransaction(const CMutableTransaction& tx) : vin(tx.vin), vout(tx.vout), nVersion(tx.nVersion), nLockTime(tx.nLockTime), hash{ComputeHash()}, m_witness_hash{ComputeWitnessHash()} {} -CTransaction::CTransaction(CMutableTransaction&& tx) : vin(std::move(tx.vin)), vout(std::move(tx.vout)), nVersion(tx.nVersion), nLockTime(tx.nLockTime), hash{ComputeHash()}, m_witness_hash{ComputeWitnessHash()} {} +CTransaction::CTransaction(const CMutableTransaction& tx) : vin(tx.vin), vout(tx.vout), nVersion(tx.nVersion), nLockTime(tx.nLockTime), m_has_witness{ComputeHasWitness()}, hash{ComputeHash()}, m_witness_hash{ComputeWitnessHash()} {} +CTransaction::CTransaction(CMutableTransaction&& tx) : vin(std::move(tx.vin)), vout(std::move(tx.vout)), nVersion(tx.nVersion), nLockTime(tx.nLockTime), m_has_witness{ComputeHasWitness()}, hash{ComputeHash()}, m_witness_hash{ComputeWitnessHash()} {} CAmount CTransaction::GetValueOut() const { @@ -100,7 +109,7 @@ CAmount CTransaction::GetValueOut() const unsigned int CTransaction::GetTotalSize() const { - return ::GetSerializeSize(*this, PROTOCOL_VERSION); + return ::GetSerializeSize(TX_WITH_WITNESS(*this)); } std::string CTransaction::ToString() const diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index bd7eb16bec..ccbeb3ec49 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -6,11 +6,12 @@ #ifndef BITCOIN_PRIMITIVES_TRANSACTION_H #define BITCOIN_PRIMITIVES_TRANSACTION_H +#include <attributes.h> #include <consensus/amount.h> -#include <prevector.h> #include <script/script.h> #include <serialize.h> #include <uint256.h> +#include <util/transaction_identifier.h> // IWYU pragma: export #include <cstddef> #include <cstdint> @@ -23,25 +24,17 @@ #include <utility> #include <vector> -/** - * A flag that is ORed into the protocol version to designate that a transaction - * should be (un)serialized without witness data. - * Make sure that this does not collide with any of the values in `version.h` - * or with `ADDRV2_FORMAT`. - */ -static const int SERIALIZE_TRANSACTION_NO_WITNESS = 0x40000000; - /** An outpoint - a combination of a transaction hash and an index n into its vout */ class COutPoint { public: - uint256 hash; + Txid hash; uint32_t n; static constexpr uint32_t NULL_INDEX = std::numeric_limits<uint32_t>::max(); COutPoint(): n(NULL_INDEX) { } - COutPoint(const uint256& hashIn, uint32_t nIn): hash(hashIn), n(nIn) { } + COutPoint(const Txid& hashIn, uint32_t nIn): hash(hashIn), n(nIn) { } SERIALIZE_METHODS(COutPoint, obj) { READWRITE(obj.hash, obj.n); } @@ -50,8 +43,7 @@ public: friend bool operator<(const COutPoint& a, const COutPoint& b) { - int cmp = a.hash.Compare(b.hash); - return cmp < 0 || (cmp == 0 && a.n < b.n); + return std::tie(a.hash, a.n) < std::tie(b.hash, b.n); } friend bool operator==(const COutPoint& a, const COutPoint& b) @@ -132,7 +124,7 @@ public: } explicit CTxIn(COutPoint prevoutIn, CScript scriptSigIn=CScript(), uint32_t nSequenceIn=SEQUENCE_FINAL); - CTxIn(uint256 hashPrevTx, uint32_t nOut, CScript scriptSigIn=CScript(), uint32_t nSequenceIn=SEQUENCE_FINAL); + CTxIn(Txid hashPrevTx, uint32_t nOut, CScript scriptSigIn=CScript(), uint32_t nSequenceIn=SEQUENCE_FINAL); SERIALIZE_METHODS(CTxIn, obj) { READWRITE(obj.prevout, obj.scriptSig, obj.nSequence); } @@ -196,6 +188,13 @@ public: struct CMutableTransaction; +struct TransactionSerParams { + const bool allow_witness; + SER_PARAMS_OPFUNC +}; +static constexpr TransactionSerParams TX_WITH_WITNESS{.allow_witness = true}; +static constexpr TransactionSerParams TX_NO_WITNESS{.allow_witness = false}; + /** * Basic transaction serialization format: * - int32_t nVersion @@ -214,8 +213,9 @@ struct CMutableTransaction; * - uint32_t nLockTime */ template<typename Stream, typename TxType> -inline void UnserializeTransaction(TxType& tx, Stream& s) { - const bool fAllowWitness = !(s.GetVersion() & SERIALIZE_TRANSACTION_NO_WITNESS); +void UnserializeTransaction(TxType& tx, Stream& s, const TransactionSerParams& params) +{ + const bool fAllowWitness = params.allow_witness; s >> tx.nVersion; unsigned char flags = 0; @@ -253,8 +253,9 @@ inline void UnserializeTransaction(TxType& tx, Stream& s) { } template<typename Stream, typename TxType> -inline void SerializeTransaction(const TxType& tx, Stream& s) { - const bool fAllowWitness = !(s.GetVersion() & SERIALIZE_TRANSACTION_NO_WITNESS); +void SerializeTransaction(const TxType& tx, Stream& s, const TransactionSerParams& params) +{ + const bool fAllowWitness = params.allow_witness; s << tx.nVersion; unsigned char flags = 0; @@ -309,11 +310,14 @@ public: private: /** Memory only. */ - const uint256 hash; - const uint256 m_witness_hash; + const bool m_has_witness; + const Txid hash; + const Wtxid m_witness_hash; + + Txid ComputeHash() const; + Wtxid ComputeWitnessHash() const; - uint256 ComputeHash() const; - uint256 ComputeWitnessHash() const; + bool ComputeHasWitness() const; public: /** Convert a CMutableTransaction into a CTransaction. */ @@ -322,20 +326,22 @@ public: template <typename Stream> inline void Serialize(Stream& s) const { - SerializeTransaction(*this, s); + SerializeTransaction(*this, s, s.GetParams()); } /** This deserializing constructor is provided instead of an Unserialize method. * Unserialize is not possible, since it would require overwriting const fields. */ template <typename Stream> - CTransaction(deserialize_type, Stream& s) : CTransaction(CMutableTransaction(deserialize, s)) {} + CTransaction(deserialize_type, const TransactionSerParams& params, Stream& s) : CTransaction(CMutableTransaction(deserialize, params, s)) {} + template <typename Stream> + CTransaction(deserialize_type, ParamsStream<TransactionSerParams,Stream>& s) : CTransaction(CMutableTransaction(deserialize, s)) {} bool IsNull() const { return vin.empty() && vout.empty(); } - const uint256& GetHash() const { return hash; } - const uint256& GetWitnessHash() const { return m_witness_hash; }; + const Txid& GetHash() const LIFETIMEBOUND { return hash; } + const Wtxid& GetWitnessHash() const LIFETIMEBOUND { return m_witness_hash; }; // Return sum of txouts. CAmount GetValueOut() const; @@ -364,15 +370,7 @@ public: std::string ToString() const; - bool HasWitness() const - { - for (size_t i = 0; i < vin.size(); i++) { - if (!vin[i].scriptWitness.IsNull()) { - return true; - } - } - return false; - } + bool HasWitness() const { return m_has_witness; } }; /** A mutable version of CTransaction. */ @@ -388,24 +386,28 @@ struct CMutableTransaction template <typename Stream> inline void Serialize(Stream& s) const { - SerializeTransaction(*this, s); + SerializeTransaction(*this, s, s.GetParams()); } - template <typename Stream> inline void Unserialize(Stream& s) { - UnserializeTransaction(*this, s); + UnserializeTransaction(*this, s, s.GetParams()); + } + + template <typename Stream> + CMutableTransaction(deserialize_type, const TransactionSerParams& params, Stream& s) { + UnserializeTransaction(*this, s, params); } template <typename Stream> - CMutableTransaction(deserialize_type, Stream& s) { + CMutableTransaction(deserialize_type, ParamsStream<TransactionSerParams,Stream>& s) { Unserialize(s); } /** Compute the hash of this CMutableTransaction. This is computed on the * fly, as opposed to GetHash() in CTransaction, which uses a cached result. */ - uint256 GetHash() const; + Txid GetHash() const; bool HasWitness() const { @@ -432,7 +434,7 @@ public: static GenTxid Txid(const uint256& hash) { return GenTxid{false, hash}; } static GenTxid Wtxid(const uint256& hash) { return GenTxid{true, hash}; } bool IsWtxid() const { return m_is_wtxid; } - const uint256& GetHash() const { return m_hash; } + const uint256& GetHash() const LIFETIMEBOUND { return m_hash; } friend bool operator==(const GenTxid& a, const GenTxid& b) { return a.m_is_wtxid == b.m_is_wtxid && a.m_hash == b.m_hash; } friend bool operator<(const GenTxid& a, const GenTxid& b) { return std::tie(a.m_is_wtxid, a.m_hash) < std::tie(b.m_is_wtxid, b.m_hash); } }; diff --git a/src/protocol.cpp b/src/protocol.cpp index f956728af2..27a0a2ffc1 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -91,9 +91,8 @@ const static std::vector<std::string> g_all_net_message_types{ }; CMessageHeader::CMessageHeader(const MessageStartChars& pchMessageStartIn, const char* pszCommand, unsigned int nMessageSizeIn) + : pchMessageStart{pchMessageStartIn} { - pchMessageStart = pchMessageStartIn; - // Copy the command name size_t i = 0; for (; i < COMMAND_SIZE && pszCommand[i] != 0; ++i) pchCommand[i] = pszCommand[i]; diff --git a/src/protocol.h b/src/protocol.h index a58d671a70..e405253632 100644 --- a/src/protocol.h +++ b/src/protocol.h @@ -445,7 +445,7 @@ public: } // Invoke V1/V2 serializer for CService parent object. const auto ser_params{use_v2 ? CNetAddr::V2 : CNetAddr::V1}; - READWRITE(WithParams(ser_params, AsBase<CService>(obj))); + READWRITE(ser_params(AsBase<CService>(obj))); } //! Always included in serialization. The behavior is unspecified if the value is not representable as uint32_t. diff --git a/src/psbt.cpp b/src/psbt.cpp index 76a2fd8241..b2ee3ce7a5 100644 --- a/src/psbt.cpp +++ b/src/psbt.cpp @@ -545,7 +545,7 @@ bool DecodeBase64PSBT(PartiallySignedTransaction& psbt, const std::string& base6 bool DecodeRawPSBT(PartiallySignedTransaction& psbt, Span<const std::byte> tx_data, std::string& error) { - CDataStream ss_data(tx_data, SER_NETWORK, PROTOCOL_VERSION); + DataStream ss_data{tx_data}; try { ss_data >> psbt; if (!ss_data.empty()) { diff --git a/src/psbt.h b/src/psbt.h index 5b4daafed5..3f74083717 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -89,13 +89,15 @@ struct PSBTProprietary template<typename Stream, typename... X> void SerializeToVector(Stream& s, const X&... args) { - WriteCompactSize(s, GetSerializeSizeMany(s.GetVersion(), args...)); + SizeComputer sizecomp; + SerializeMany(sizecomp, args...); + WriteCompactSize(s, sizecomp.size()); SerializeMany(s, args...); } // Takes a stream and multiple arguments and unserializes them first as a vector then each object individually in the order provided in the arguments template<typename Stream, typename... X> -void UnserializeFromVector(Stream& s, X&... args) +void UnserializeFromVector(Stream& s, X&&... args) { size_t expected_size = ReadCompactSize(s); size_t remaining_before = s.size(); @@ -226,8 +228,7 @@ struct PSBTInput // Write the utxo if (non_witness_utxo) { SerializeToVector(s, CompactSizeWriter(PSBT_IN_NON_WITNESS_UTXO)); - OverrideStream<Stream> os{&s, s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS}; - SerializeToVector(os, non_witness_utxo); + SerializeToVector(s, TX_NO_WITNESS(non_witness_utxo)); } if (!witness_utxo.IsNull()) { SerializeToVector(s, CompactSizeWriter(PSBT_IN_WITNESS_UTXO)); @@ -315,7 +316,7 @@ struct PSBTInput const auto& [leaf_hashes, origin] = leaf_origin; SerializeToVector(s, PSBT_IN_TAP_BIP32_DERIVATION, xonly); std::vector<unsigned char> value; - CVectorWriter s_value{s.GetVersion(), value, 0}; + VectorWriter s_value{value, 0}; s_value << leaf_hashes; SerializeKeyOrigin(s_value, origin); s << value; @@ -381,7 +382,7 @@ struct PSBTInput } // Type is compact size uint at beginning of key - SpanReader skey{s.GetVersion(), key}; + SpanReader skey{key}; uint64_t type = ReadCompactSize(skey); // Do stuff based on type @@ -394,8 +395,7 @@ struct PSBTInput throw std::ios_base::failure("Non-witness utxo key is more than one byte type"); } // Set the stream to unserialize with witness since this is always a valid network transaction - OverrideStream<Stream> os{&s, s.GetVersion() & ~SERIALIZE_TRANSACTION_NO_WITNESS}; - UnserializeFromVector(os, non_witness_utxo); + UnserializeFromVector(s, TX_WITH_WITNESS(non_witness_utxo)); break; } case PSBT_IN_WITNESS_UTXO: @@ -590,7 +590,7 @@ struct PSBTInput } else if (key.size() != 65) { throw std::ios_base::failure("Input Taproot script signature key is not 65 bytes"); } - SpanReader s_key{s.GetVersion(), Span{key}.subspan(1)}; + SpanReader s_key{Span{key}.subspan(1)}; XOnlyPubKey xonly; uint256 hash; s_key >> xonly; @@ -632,7 +632,7 @@ struct PSBTInput } else if (key.size() != 33) { throw std::ios_base::failure("Input Taproot BIP32 keypath key is not at 33 bytes"); } - SpanReader s_key{s.GetVersion(), Span{key}.subspan(1)}; + SpanReader s_key{Span{key}.subspan(1)}; XOnlyPubKey xonly; s_key >> xonly; std::set<uint256> leaf_hashes; @@ -757,7 +757,7 @@ struct PSBTOutput if (!m_tap_tree.empty()) { SerializeToVector(s, PSBT_OUT_TAP_TREE); std::vector<unsigned char> value; - CVectorWriter s_value{s.GetVersion(), value, 0}; + VectorWriter s_value{value, 0}; for (const auto& [depth, leaf_ver, script] : m_tap_tree) { s_value << depth; s_value << leaf_ver; @@ -771,7 +771,7 @@ struct PSBTOutput const auto& [leaf_hashes, origin] = leaf; SerializeToVector(s, PSBT_OUT_TAP_BIP32_DERIVATION, xonly); std::vector<unsigned char> value; - CVectorWriter s_value{s.GetVersion(), value, 0}; + VectorWriter s_value{value, 0}; s_value << leaf_hashes; SerializeKeyOrigin(s_value, origin); s << value; @@ -807,7 +807,7 @@ struct PSBTOutput } // Type is compact size uint at beginning of key - SpanReader skey{s.GetVersion(), key}; + SpanReader skey{key}; uint64_t type = ReadCompactSize(skey); // Do stuff based on type @@ -856,7 +856,7 @@ struct PSBTOutput } std::vector<unsigned char> tree_v; s >> tree_v; - SpanReader s_tree{s.GetVersion(), tree_v}; + SpanReader s_tree{tree_v}; if (s_tree.empty()) { throw std::ios_base::failure("Output Taproot tree must not be empty"); } @@ -984,8 +984,7 @@ struct PartiallySignedTransaction SerializeToVector(s, CompactSizeWriter(PSBT_GLOBAL_UNSIGNED_TX)); // Write serialized tx to a stream - OverrideStream<Stream> os{&s, s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS}; - SerializeToVector(os, *tx); + SerializeToVector(s, TX_NO_WITNESS(*tx)); // Write xpubs for (const auto& xpub_pair : m_xpubs) { @@ -1061,7 +1060,7 @@ struct PartiallySignedTransaction } // Type is compact size uint at beginning of key - SpanReader skey{s.GetVersion(), key}; + SpanReader skey{key}; uint64_t type = ReadCompactSize(skey); // Do stuff based on type @@ -1075,8 +1074,7 @@ struct PartiallySignedTransaction } CMutableTransaction mtx; // Set the stream to serialize with non-witness since this should always be non-witness - OverrideStream<Stream> os{&s, s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS}; - UnserializeFromVector(os, mtx); + UnserializeFromVector(s, TX_NO_WITNESS(mtx)); tx = std::move(mtx); // Make sure that all scriptSigs and scriptWitnesses are empty for (const CTxIn& txin : tx->vin) { diff --git a/src/qt/bitcoin.cpp b/src/qt/bitcoin.cpp index bad4e72794..33c305f0d4 100644 --- a/src/qt/bitcoin.cpp +++ b/src/qt/bitcoin.cpp @@ -547,6 +547,34 @@ int GuiMain(int argc, char* argv[]) return EXIT_FAILURE; } + // Error out when loose non-argument tokens are encountered on command line + // However, allow BIP-21 URIs only if no options follow + bool payment_server_token_seen = false; + for (int i = 1; i < argc; i++) { + QString arg(argv[i]); + bool invalid_token = !arg.startsWith("-"); +#ifdef ENABLE_WALLET + if (arg.startsWith(BITCOIN_IPC_PREFIX, Qt::CaseInsensitive)) { + invalid_token &= false; + payment_server_token_seen = true; + } +#endif + if (payment_server_token_seen && arg.startsWith("-")) { + InitError(Untranslated(strprintf("Options ('%s') cannot follow a BIP-21 payment URI", argv[i]))); + QMessageBox::critical(nullptr, PACKAGE_NAME, + // message cannot be translated because translations have not been initialized + QString::fromStdString("Options ('%1') cannot follow a BIP-21 payment URI").arg(QString::fromStdString(argv[i]))); + return EXIT_FAILURE; + } + if (invalid_token) { + InitError(Untranslated(strprintf("Command line contains unexpected token '%s', see bitcoin-qt -h for a list of options.", argv[i]))); + QMessageBox::critical(nullptr, PACKAGE_NAME, + // message cannot be translated because translations have not been initialized + QString::fromStdString("Command line contains unexpected token '%1', see bitcoin-qt -h for a list of options.").arg(QString::fromStdString(argv[i]))); + return EXIT_FAILURE; + } + } + // Now that the QApplication is setup and we have parsed our parameters, we can set the platform style app.setupPlatformStyle(); @@ -633,7 +661,10 @@ int GuiMain(int argc, char* argv[]) app.installEventFilter(new GUIUtil::LabelOutOfFocusEventFilter(&app)); #if defined(Q_OS_WIN) // Install global event filter for processing Windows session related Windows messages (WM_QUERYENDSESSION and WM_ENDSESSION) - qApp->installNativeEventFilter(new WinShutdownMonitor()); + // Note: it is safe to call app.node() in the lambda below despite the fact + // that app.createNode() hasn't been called yet, because native events will + // not be processed until the Qt event loop is executed. + qApp->installNativeEventFilter(new WinShutdownMonitor([&app] { app.node().startShutdown(); })); #endif // Install qDebug() message handler to route to debug.log qInstallMessageHandler(DebugMessageHandler); diff --git a/src/qt/bitcoingui.h b/src/qt/bitcoingui.h index 6fdc4c60d8..ba91eeb1ff 100644 --- a/src/qt/bitcoingui.h +++ b/src/qt/bitcoingui.h @@ -315,7 +315,7 @@ public Q_SLOTS: /** Simply calls showNormalIfMinimized(true) */ void toggleHidden(); - /** called by a timer to check if ShutdownRequested() has been set **/ + /** called by a timer to check if shutdown has been requested */ void detectShutdown(); /** Show progress dialog e.g. for verifychain */ diff --git a/src/qt/coincontroldialog.cpp b/src/qt/coincontroldialog.cpp index 70aa9bc8da..9e49fd87fb 100644 --- a/src/qt/coincontroldialog.cpp +++ b/src/qt/coincontroldialog.cpp @@ -181,7 +181,7 @@ void CoinControlDialog::showMenu(const QPoint &point) if (item->data(COLUMN_ADDRESS, TxHashRole).toString().length() == 64) // transaction hash is 64 characters (this means it is a child node, so it is not a parent node in tree mode) { m_copy_transaction_outpoint_action->setEnabled(true); - if (model->wallet().isLockedCoin(COutPoint(uint256S(item->data(COLUMN_ADDRESS, TxHashRole).toString().toStdString()), item->data(COLUMN_ADDRESS, VOutRole).toUInt()))) + if (model->wallet().isLockedCoin(COutPoint(TxidFromString(item->data(COLUMN_ADDRESS, TxHashRole).toString().toStdString()), item->data(COLUMN_ADDRESS, VOutRole).toUInt()))) { lockAction->setEnabled(false); unlockAction->setEnabled(true); @@ -244,7 +244,7 @@ void CoinControlDialog::lockCoin() if (contextMenuItem->checkState(COLUMN_CHECKBOX) == Qt::Checked) contextMenuItem->setCheckState(COLUMN_CHECKBOX, Qt::Unchecked); - COutPoint outpt(uint256S(contextMenuItem->data(COLUMN_ADDRESS, TxHashRole).toString().toStdString()), contextMenuItem->data(COLUMN_ADDRESS, VOutRole).toUInt()); + COutPoint outpt(TxidFromString(contextMenuItem->data(COLUMN_ADDRESS, TxHashRole).toString().toStdString()), contextMenuItem->data(COLUMN_ADDRESS, VOutRole).toUInt()); model->wallet().lockCoin(outpt, /* write_to_db = */ true); contextMenuItem->setDisabled(true); contextMenuItem->setIcon(COLUMN_CHECKBOX, platformStyle->SingleColorIcon(":/icons/lock_closed")); @@ -254,7 +254,7 @@ void CoinControlDialog::lockCoin() // context menu action: unlock coin void CoinControlDialog::unlockCoin() { - COutPoint outpt(uint256S(contextMenuItem->data(COLUMN_ADDRESS, TxHashRole).toString().toStdString()), contextMenuItem->data(COLUMN_ADDRESS, VOutRole).toUInt()); + COutPoint outpt(TxidFromString(contextMenuItem->data(COLUMN_ADDRESS, TxHashRole).toString().toStdString()), contextMenuItem->data(COLUMN_ADDRESS, VOutRole).toUInt()); model->wallet().unlockCoin(outpt); contextMenuItem->setDisabled(false); contextMenuItem->setIcon(COLUMN_CHECKBOX, QIcon()); @@ -346,7 +346,7 @@ void CoinControlDialog::viewItemChanged(QTreeWidgetItem* item, int column) { if (column == COLUMN_CHECKBOX && item->data(COLUMN_ADDRESS, TxHashRole).toString().length() == 64) // transaction hash is 64 characters (this means it is a child node, so it is not a parent node in tree mode) { - COutPoint outpt(uint256S(item->data(COLUMN_ADDRESS, TxHashRole).toString().toStdString()), item->data(COLUMN_ADDRESS, VOutRole).toUInt()); + COutPoint outpt(TxidFromString(item->data(COLUMN_ADDRESS, TxHashRole).toString().toStdString()), item->data(COLUMN_ADDRESS, VOutRole).toUInt()); if (item->checkState(COLUMN_CHECKBOX) == Qt::Unchecked) m_coin_control.UnSelect(outpt); diff --git a/src/qt/guiutil.cpp b/src/qt/guiutil.cpp index eb9c65caf2..ebfd14cc25 100644 --- a/src/qt/guiutil.cpp +++ b/src/qt/guiutil.cpp @@ -669,7 +669,7 @@ fs::path QStringToPath(const QString &path) QString PathToQString(const fs::path &path) { - return QString::fromStdString(path.u8string()); + return QString::fromStdString(path.utf8string()); } QString NetworkToQString(Network net) @@ -723,8 +723,7 @@ QString ConnectionTypeToQString(ConnectionType conn_type, bool prepend_direction QString formatDurationStr(std::chrono::seconds dur) { - using days = std::chrono::duration<int, std::ratio<86400>>; // can remove this line after C++20 - const auto d{std::chrono::duration_cast<days>(dur)}; + const auto d{std::chrono::duration_cast<std::chrono::days>(dur)}; const auto h{std::chrono::duration_cast<std::chrono::hours>(dur - d)}; const auto m{std::chrono::duration_cast<std::chrono::minutes>(dur - d - h)}; const auto s{std::chrono::duration_cast<std::chrono::seconds>(dur - d - h - m)}; diff --git a/src/qt/optionsdialog.cpp b/src/qt/optionsdialog.cpp index 512fce473d..6e1d36effb 100644 --- a/src/qt/optionsdialog.cpp +++ b/src/qt/optionsdialog.cpp @@ -17,9 +17,9 @@ #include <common/system.h> #include <interfaces/node.h> +#include <node/chainstatemanager_args.h> #include <netbase.h> #include <txdb.h> -#include <validation.h> #include <chrono> diff --git a/src/qt/optionsmodel.cpp b/src/qt/optionsmodel.cpp index c1563fe1e2..43564dad16 100644 --- a/src/qt/optionsmodel.cpp +++ b/src/qt/optionsmodel.cpp @@ -17,6 +17,7 @@ #include <mapport.h> #include <net.h> #include <netbase.h> +#include <node/chainstatemanager_args.h> #include <txdb.h> // for -dbcache defaults #include <util/string.h> #include <validation.h> // For DEFAULT_SCRIPTCHECK_THREADS diff --git a/src/qt/paymentserver.h b/src/qt/paymentserver.h index 63f4faa772..f73aa1e61e 100644 --- a/src/qt/paymentserver.h +++ b/src/qt/paymentserver.h @@ -54,6 +54,8 @@ class QLocalServer; class QUrl; QT_END_NAMESPACE +extern const QString BITCOIN_IPC_PREFIX; + class PaymentServer : public QObject { Q_OBJECT diff --git a/src/qt/psbtoperationsdialog.cpp b/src/qt/psbtoperationsdialog.cpp index 26f1696de1..353709c7f5 100644 --- a/src/qt/psbtoperationsdialog.cpp +++ b/src/qt/psbtoperationsdialog.cpp @@ -128,14 +128,14 @@ void PSBTOperationsDialog::broadcastTransaction() } void PSBTOperationsDialog::copyToClipboard() { - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssTx{}; ssTx << m_transaction_data; GUIUtil::setClipboard(EncodeBase64(ssTx.str()).c_str()); showStatus(tr("PSBT copied to clipboard."), StatusLevel::INFO); } void PSBTOperationsDialog::saveTransaction() { - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssTx{}; ssTx << m_transaction_data; QString selected_filter; @@ -167,19 +167,20 @@ void PSBTOperationsDialog::saveTransaction() { } void PSBTOperationsDialog::updateTransactionDisplay() { - m_ui->transactionDescription->setText(QString::fromStdString(renderTransaction(m_transaction_data))); + m_ui->transactionDescription->setText(renderTransaction(m_transaction_data)); showTransactionStatus(m_transaction_data); } -std::string PSBTOperationsDialog::renderTransaction(const PartiallySignedTransaction &psbtx) +QString PSBTOperationsDialog::renderTransaction(const PartiallySignedTransaction &psbtx) { - QString tx_description = ""; + QString tx_description; + QLatin1String bullet_point(" * "); CAmount totalAmount = 0; for (const CTxOut& out : psbtx.tx->vout) { CTxDestination address; ExtractDestination(out.scriptPubKey, address); totalAmount += out.nValue; - tx_description.append(tr(" * Sends %1 to %2") + tx_description.append(bullet_point).append(tr("Sends %1 to %2") .arg(BitcoinUnits::formatWithUnit(BitcoinUnit::BTC, out.nValue)) .arg(QString::fromStdString(EncodeDestination(address)))); // Check if the address is one of ours @@ -188,7 +189,7 @@ std::string PSBTOperationsDialog::renderTransaction(const PartiallySignedTransac } PSBTAnalysis analysis = AnalyzePSBT(psbtx); - tx_description.append(" * "); + tx_description.append(bullet_point); if (!*analysis.fee) { // This happens if the transaction is missing input UTXO information. tx_description.append(tr("Unable to calculate transaction fee or total transaction amount.")); @@ -217,7 +218,7 @@ std::string PSBTOperationsDialog::renderTransaction(const PartiallySignedTransac tx_description.append(tr("Transaction has %1 unsigned inputs.").arg(QString::number(num_unsigned))); } - return tx_description.toStdString(); + return tx_description; } void PSBTOperationsDialog::showStatus(const QString &msg, StatusLevel level) { diff --git a/src/qt/psbtoperationsdialog.h b/src/qt/psbtoperationsdialog.h index f37bdbe39a..23f7dbf227 100644 --- a/src/qt/psbtoperationsdialog.h +++ b/src/qt/psbtoperationsdialog.h @@ -6,6 +6,7 @@ #define BITCOIN_QT_PSBTOPERATIONSDIALOG_H #include <QDialog> +#include <QString> #include <psbt.h> #include <qt/clientmodel.h> @@ -46,7 +47,7 @@ private: size_t couldSignInputs(const PartiallySignedTransaction &psbtx); void updateTransactionDisplay(); - std::string renderTransaction(const PartiallySignedTransaction &psbtx); + QString renderTransaction(const PartiallySignedTransaction &psbtx); void showStatus(const QString &msg, StatusLevel level); void showTransactionStatus(const PartiallySignedTransaction &psbtx); }; diff --git a/src/qt/sendcoinsdialog.cpp b/src/qt/sendcoinsdialog.cpp index dea9b92395..3c331c70ef 100644 --- a/src/qt/sendcoinsdialog.cpp +++ b/src/qt/sendcoinsdialog.cpp @@ -302,7 +302,7 @@ bool SendCoinsDialog::PrepareSendText(QString& question_string, QString& informa // generate amount string with wallet name in case of multiwallet QString amount = BitcoinUnits::formatWithUnit(model->getOptionsModel()->getDisplayUnit(), rcp.amount); if (model->isMultiwallet()) { - amount.append(tr(" from wallet '%1'").arg(GUIUtil::HtmlEscape(model->getWalletName()))); + amount = tr("%1 from wallet '%2'").arg(amount, GUIUtil::HtmlEscape(model->getWalletName())); } // generate address string @@ -397,7 +397,7 @@ bool SendCoinsDialog::PrepareSendText(QString& question_string, QString& informa void SendCoinsDialog::presentPSBT(PartiallySignedTransaction& psbtx) { // Serialize the PSBT - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssTx{}; ssTx << psbtx; GUIUtil::setClipboard(EncodeBase64(ssTx.str()).c_str()); QMessageBox msgBox(this); diff --git a/src/qt/test/addressbooktests.cpp b/src/qt/test/addressbooktests.cpp index f17a6b7f74..f7d66f316e 100644 --- a/src/qt/test/addressbooktests.cpp +++ b/src/qt/test/addressbooktests.cpp @@ -85,8 +85,7 @@ void TestAddAddressesToSendBook(interfaces::Node& node) } auto build_address = [&wallet]() { - CKey key; - key.MakeNewKey(true); + CKey key = GenerateRandomKey(); CTxDestination dest(GetDestinationForKey( key.GetPubKey(), wallet->m_default_address_type)); diff --git a/src/qt/test/apptests.cpp b/src/qt/test/apptests.cpp index b05009965f..9007b183aa 100644 --- a/src/qt/test/apptests.cpp +++ b/src/qt/test/apptests.cpp @@ -11,7 +11,6 @@ #include <qt/bitcoingui.h> #include <qt/networkstyle.h> #include <qt/rpcconsole.h> -#include <shutdown.h> #include <test/util/setup_common.h> #include <validation.h> diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index dae6a2dea9..a916e4ead6 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -323,7 +323,7 @@ QString TransactionDesc::toHTML(interfaces::Node& node, interfaces::Wallet& wall if (!GetPaymentRequestMerchant(r.second, merchant)) { merchant.clear(); } else { - merchant += tr(" (Certificate was not verified)"); + merchant = tr("%1 (Certificate was not verified)").arg(merchant); } if (!merchant.isNull()) { strHTML += "<b>" + tr("Merchant") + ":</b> " + GUIUtil::HtmlEscape(merchant) + "<br>"; @@ -360,12 +360,10 @@ QString TransactionDesc::toHTML(interfaces::Node& node, interfaces::Wallet& wall { COutPoint prevout = txin.prevout; - Coin prev; - if(node.getUnspentOutput(prevout, prev)) - { + if (auto prev{node.getUnspentOutput(prevout)}) { { strHTML += "<li>"; - const CTxOut &vout = prev.out; + const CTxOut& vout = prev->out; CTxDestination address; if (ExtractDestination(vout.scriptPubKey, address)) { diff --git a/src/qt/transactiontablemodel.cpp b/src/qt/transactiontablemodel.cpp index 486e51c777..d4267fcf61 100644 --- a/src/qt/transactiontablemodel.cpp +++ b/src/qt/transactiontablemodel.cpp @@ -219,7 +219,7 @@ public: // If a status update is needed (blocks came in since last check), // try to update the status of this transaction from the wallet. - // Otherwise, simply re-use the cached status. + // Otherwise, simply reuse the cached status. interfaces::WalletTxStatus wtx; int numBlocks; int64_t block_time; diff --git a/src/qt/transactionview.cpp b/src/qt/transactionview.cpp index 67af62285d..7e24dbd3ec 100644 --- a/src/qt/transactionview.cpp +++ b/src/qt/transactionview.cpp @@ -531,6 +531,9 @@ void TransactionView::showDetails() TransactionDescDialog *dlg = new TransactionDescDialog(selection.at(0)); dlg->setAttribute(Qt::WA_DeleteOnClose); m_opened_dialogs.append(dlg); + connect(dlg, &QObject::destroyed, [this, dlg] { + m_opened_dialogs.removeOne(dlg); + }); dlg->show(); } } diff --git a/src/qt/walletcontroller.cpp b/src/qt/walletcontroller.cpp index b1ef489cc3..c7fe62f4e9 100644 --- a/src/qt/walletcontroller.cpp +++ b/src/qt/walletcontroller.cpp @@ -472,10 +472,10 @@ void MigrateWalletActivity::migrate(WalletModel* wallet_model) if (res) { m_success_message = tr("The wallet '%1' was migrated successfully.").arg(GUIUtil::HtmlEscape(res->wallet->getWalletName())); if (res->watchonly_wallet_name) { - m_success_message += tr(" Watchonly scripts have been migrated to a new wallet named '%1'.").arg(GUIUtil::HtmlEscape(res->watchonly_wallet_name.value())); + m_success_message += QChar(' ') + tr("Watchonly scripts have been migrated to a new wallet named '%1'.").arg(GUIUtil::HtmlEscape(res->watchonly_wallet_name.value())); } if (res->solvables_wallet_name) { - m_success_message += tr(" Solvable but not watched scripts have been migrated to a new wallet named '%1'.").arg(GUIUtil::HtmlEscape(res->solvables_wallet_name.value())); + m_success_message += QChar(' ') + tr("Solvable but not watched scripts have been migrated to a new wallet named '%1'.").arg(GUIUtil::HtmlEscape(res->solvables_wallet_name.value())); } m_wallet_model = m_wallet_controller->getOrCreateWallet(std::move(res->wallet)); } else { diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index ee3327530c..9821f8504a 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -187,8 +187,7 @@ WalletModel::SendCoinsReturn WalletModel::prepareTransaction(WalletModelTransact setAddress.insert(rcp.address); ++nAddresses; - CScript scriptPubKey = GetScriptForDestination(DecodeDestination(rcp.address.toStdString())); - CRecipient recipient = {scriptPubKey, rcp.amount, rcp.fSubtractFeeFromAmount}; + CRecipient recipient{DecodeDestination(rcp.address.toStdString()), rcp.amount, rcp.fSubtractFeeFromAmount}; vecSend.push_back(recipient); total += rcp.amount; @@ -261,8 +260,8 @@ void WalletModel::sendCoins(WalletModelTransaction& transaction) auto& newTx = transaction.getWtx(); wallet().commitTransaction(newTx, /*value_map=*/{}, std::move(vOrderForm)); - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); - ssTx << *newTx; + DataStream ssTx; + ssTx << TX_WITH_WITNESS(*newTx); transaction_array.append((const char*)ssTx.data(), ssTx.size()); } @@ -544,7 +543,7 @@ bool WalletModel::bumpFee(uint256 hash, uint256& new_hash) return false; } // Serialize the PSBT - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssTx{}; ssTx << psbtx; GUIUtil::setClipboard(EncodeBase64(ssTx.str()).c_str()); Q_EMIT message(tr("PSBT copied"), tr("Copied to clipboard", "Fee-bump PSBT saved"), CClientUIInterface::MSG_INFORMATION); diff --git a/src/qt/winshutdownmonitor.cpp b/src/qt/winshutdownmonitor.cpp index 97a9ec318c..0b5278c192 100644 --- a/src/qt/winshutdownmonitor.cpp +++ b/src/qt/winshutdownmonitor.cpp @@ -5,7 +5,6 @@ #include <qt/winshutdownmonitor.h> #if defined(Q_OS_WIN) -#include <shutdown.h> #include <windows.h> @@ -25,7 +24,7 @@ bool WinShutdownMonitor::nativeEventFilter(const QByteArray &eventType, void *pM { // Initiate a client shutdown after receiving a WM_QUERYENDSESSION and block // Windows session end until we have finished client shutdown. - StartShutdown(); + m_shutdown_fn(); *pnResult = FALSE; return true; } diff --git a/src/qt/winshutdownmonitor.h b/src/qt/winshutdownmonitor.h index 72655da3da..78f287637f 100644 --- a/src/qt/winshutdownmonitor.h +++ b/src/qt/winshutdownmonitor.h @@ -8,6 +8,7 @@ #ifdef WIN32 #include <QByteArray> #include <QString> +#include <functional> #include <windef.h> // for HWND @@ -16,11 +17,16 @@ class WinShutdownMonitor : public QAbstractNativeEventFilter { public: + WinShutdownMonitor(std::function<void()> shutdown_fn) : m_shutdown_fn{std::move(shutdown_fn)} {} + /** Implements QAbstractNativeEventFilter interface for processing Windows messages */ bool nativeEventFilter(const QByteArray &eventType, void *pMessage, long *pnResult) override; /** Register the reason for blocking shutdown on Windows to allow clean client exit */ static void registerShutdownBlockReason(const QString& strReason, const HWND& mainWinId); + +private: + std::function<void()> m_shutdown_fn; }; #endif diff --git a/src/random.cpp b/src/random.cpp index 9bd8ff9f1a..ce4266a567 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -38,6 +38,9 @@ #ifdef HAVE_SYSCTL_ARND #include <sys/sysctl.h> #endif +#if defined(HAVE_STRONG_GETAUXVAL) && defined(__aarch64__) +#include <sys/auxv.h> +#endif [[noreturn]] static void RandFailure() { @@ -173,6 +176,62 @@ static uint64_t GetRdSeed() noexcept #endif } +#elif defined(__aarch64__) && defined(HWCAP2_RNG) + +static bool g_rndr_supported = false; + +static void InitHardwareRand() +{ + if (getauxval(AT_HWCAP2) & HWCAP2_RNG) { + g_rndr_supported = true; + } +} + +static void ReportHardwareRand() +{ + // This must be done in a separate function, as InitHardwareRand() may be indirectly called + // from global constructors, before logging is initialized. + if (g_rndr_supported) { + LogPrintf("Using RNDR and RNDRRS as additional entropy sources\n"); + } +} + +/** Read 64 bits of entropy using rndr. + * + * Must only be called when RNDR is supported. + */ +static uint64_t GetRNDR() noexcept +{ + uint8_t ok; + uint64_t r1; + do { + // https://developer.arm.com/documentation/ddi0601/2022-12/AArch64-Registers/RNDR--Random-Number + __asm__ volatile("mrs %0, s3_3_c2_c4_0; cset %w1, ne;" + : "=r"(r1), "=r"(ok)::"cc"); + if (ok) break; + __asm__ volatile("yield"); + } while (true); + return r1; +} + +/** Read 64 bits of entropy using rndrrs. + * + * Must only be called when RNDRRS is supported. + */ +static uint64_t GetRNDRRS() noexcept +{ + uint8_t ok; + uint64_t r1; + do { + // https://developer.arm.com/documentation/ddi0601/2022-12/AArch64-Registers/RNDRRS--Reseeded-Random-Number + __asm__ volatile("mrs %0, s3_3_c2_c4_1; cset %w1, ne;" + : "=r"(r1), "=r"(ok)::"cc"); + if (ok) break; + __asm__ volatile("yield"); + } while (true); + return r1; +} + #else /* Access to other hardware random number generators could be added here later, * assuming it is sufficiently fast (in the order of a few hundred CPU cycles). @@ -191,6 +250,12 @@ static void SeedHardwareFast(CSHA512& hasher) noexcept { hasher.Write((const unsigned char*)&out, sizeof(out)); return; } +#elif defined(__aarch64__) && defined(HWCAP2_RNG) + if (g_rndr_supported) { + uint64_t out = GetRNDR(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + return; + } #endif } @@ -216,6 +281,14 @@ static void SeedHardwareSlow(CSHA512& hasher) noexcept { } return; } +#elif defined(__aarch64__) && defined(HWCAP2_RNG) + if (g_rndr_supported) { + for (int i = 0; i < 4; ++i) { + uint64_t out = GetRNDRRS(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + } + return; + } #endif } diff --git a/src/rest.cpp b/src/rest.cpp index e094039366..fbbf6cfa84 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -26,8 +26,8 @@ #include <txmempool.h> #include <util/any.h> #include <util/check.h> +#include <util/strencodings.h> #include <validation.h> -#include <version.h> #include <any> #include <string> @@ -264,7 +264,7 @@ static bool rest_headers(const std::any& context, case RESTResponseFormat::JSON: { UniValue jsonHeaders(UniValue::VARR); for (const CBlockIndex *pindex : headers) { - jsonHeaders.push_back(blockheaderToJSON(tip, pindex)); + jsonHeaders.push_back(blockheaderToJSON(*tip, *pindex)); } std::string strJSON = jsonHeaders.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); @@ -304,10 +304,9 @@ static bool rest_block(const std::any& context, if (!pblockindex) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not found"); } - - if (chainman.m_blockman.IsBlockPruned(pblockindex)) + if (chainman.m_blockman.IsBlockPruned(*pblockindex)) { return RESTERR(req, HTTP_NOT_FOUND, hashStr + " not available (pruned data)"); - + } } if (!chainman.m_blockman.ReadBlockFromDisk(block, *pblockindex)) { @@ -316,8 +315,8 @@ static bool rest_block(const std::any& context, switch (rf) { case RESTResponseFormat::BINARY: { - CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); - ssBlock << block; + DataStream ssBlock; + ssBlock << TX_WITH_WITNESS(block); std::string binaryBlock = ssBlock.str(); req->WriteHeader("Content-Type", "application/octet-stream"); req->WriteReply(HTTP_OK, binaryBlock); @@ -325,8 +324,8 @@ static bool rest_block(const std::any& context, } case RESTResponseFormat::HEX: { - CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); - ssBlock << block; + DataStream ssBlock; + ssBlock << TX_WITH_WITNESS(block); std::string strHex = HexStr(ssBlock) + "\n"; req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, strHex); @@ -334,7 +333,7 @@ static bool rest_block(const std::any& context, } case RESTResponseFormat::JSON: { - UniValue objBlock = blockToJSON(chainman.m_blockman, block, tip, pblockindex, tx_verbosity); + UniValue objBlock = blockToJSON(chainman.m_blockman, block, *tip, *pblockindex, tx_verbosity); std::string strJSON = objBlock.write() + "\n"; req->WriteHeader("Content-Type", "application/json"); req->WriteReply(HTTP_OK, strJSON); @@ -722,8 +721,8 @@ static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string switch (rf) { case RESTResponseFormat::BINARY: { - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); - ssTx << tx; + DataStream ssTx; + ssTx << TX_WITH_WITNESS(tx); std::string binaryTx = ssTx.str(); req->WriteHeader("Content-Type", "application/octet-stream"); @@ -732,8 +731,8 @@ static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string } case RESTResponseFormat::HEX: { - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); - ssTx << tx; + DataStream ssTx; + ssTx << TX_WITH_WITNESS(tx); std::string strHex = HexStr(ssTx) + "\n"; req->WriteHeader("Content-Type", "text/plain"); @@ -789,7 +788,6 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: for (size_t i = (fCheckMemPool) ? 1 : 0; i < uriParts.size(); i++) { - uint256 txid; int32_t nOutput; std::string strTxid = uriParts[i].substr(0, uriParts[i].find('-')); std::string strOutput = uriParts[i].substr(uriParts[i].find('-')+1); @@ -797,8 +795,7 @@ static bool rest_getutxos(const std::any& context, HTTPRequest* req, const std:: if (!ParseInt32(strOutput, &nOutput) || !IsHex(strTxid)) return RESTERR(req, HTTP_BAD_REQUEST, "Parse error"); - txid.SetHex(strTxid); - vOutPoints.emplace_back(txid, (uint32_t)nOutput); + vOutPoints.emplace_back(TxidFromString(strTxid), (uint32_t)nOutput); } if (vOutPoints.size() > 0) diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index d51439f466..7d04fb06d5 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -73,13 +73,11 @@ static CUpdatedBlock latestblock GUARDED_BY(cs_blockchange); /* Calculate the difficulty for a given block index. */ -double GetDifficulty(const CBlockIndex* blockindex) +double GetDifficulty(const CBlockIndex& blockindex) { - CHECK_NONFATAL(blockindex); - - int nShift = (blockindex->nBits >> 24) & 0xff; + int nShift = (blockindex.nBits >> 24) & 0xff; double dDiff = - (double)0x0000ffff / (double)(blockindex->nBits & 0x00ffffff); + (double)0x0000ffff / (double)(blockindex.nBits & 0x00ffffff); while (nShift < 29) { @@ -95,14 +93,14 @@ double GetDifficulty(const CBlockIndex* blockindex) return dDiff; } -static int ComputeNextBlockAndDepth(const CBlockIndex* tip, const CBlockIndex* blockindex, const CBlockIndex*& next) +static int ComputeNextBlockAndDepth(const CBlockIndex& tip, const CBlockIndex& blockindex, const CBlockIndex*& next) { - next = tip->GetAncestor(blockindex->nHeight + 1); - if (next && next->pprev == blockindex) { - return tip->nHeight - blockindex->nHeight + 1; + next = tip.GetAncestor(blockindex.nHeight + 1); + if (next && next->pprev == &blockindex) { + return tip.nHeight - blockindex.nHeight + 1; } next = nullptr; - return blockindex == tip ? 1 : -1; + return &blockindex == &tip ? 1 : -1; } static const CBlockIndex* ParseHashOrHeight(const UniValue& param, ChainstateManager& chainman) @@ -133,41 +131,41 @@ static const CBlockIndex* ParseHashOrHeight(const UniValue& param, ChainstateMan } } -UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex) +UniValue blockheaderToJSON(const CBlockIndex& tip, const CBlockIndex& blockindex) { // Serialize passed information without accessing chain state of the active chain! AssertLockNotHeld(cs_main); // For performance reasons UniValue result(UniValue::VOBJ); - result.pushKV("hash", blockindex->GetBlockHash().GetHex()); + result.pushKV("hash", blockindex.GetBlockHash().GetHex()); const CBlockIndex* pnext; int confirmations = ComputeNextBlockAndDepth(tip, blockindex, pnext); result.pushKV("confirmations", confirmations); - result.pushKV("height", blockindex->nHeight); - result.pushKV("version", blockindex->nVersion); - result.pushKV("versionHex", strprintf("%08x", blockindex->nVersion)); - result.pushKV("merkleroot", blockindex->hashMerkleRoot.GetHex()); - result.pushKV("time", (int64_t)blockindex->nTime); - result.pushKV("mediantime", (int64_t)blockindex->GetMedianTimePast()); - result.pushKV("nonce", (uint64_t)blockindex->nNonce); - result.pushKV("bits", strprintf("%08x", blockindex->nBits)); + result.pushKV("height", blockindex.nHeight); + result.pushKV("version", blockindex.nVersion); + result.pushKV("versionHex", strprintf("%08x", blockindex.nVersion)); + result.pushKV("merkleroot", blockindex.hashMerkleRoot.GetHex()); + result.pushKV("time", blockindex.nTime); + result.pushKV("mediantime", blockindex.GetMedianTimePast()); + result.pushKV("nonce", blockindex.nNonce); + result.pushKV("bits", strprintf("%08x", blockindex.nBits)); result.pushKV("difficulty", GetDifficulty(blockindex)); - result.pushKV("chainwork", blockindex->nChainWork.GetHex()); - result.pushKV("nTx", (uint64_t)blockindex->nTx); + result.pushKV("chainwork", blockindex.nChainWork.GetHex()); + result.pushKV("nTx", blockindex.nTx); - if (blockindex->pprev) - result.pushKV("previousblockhash", blockindex->pprev->GetBlockHash().GetHex()); + if (blockindex.pprev) + result.pushKV("previousblockhash", blockindex.pprev->GetBlockHash().GetHex()); if (pnext) result.pushKV("nextblockhash", pnext->GetBlockHash().GetHex()); return result; } -UniValue blockToJSON(BlockManager& blockman, const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, TxVerbosity verbosity) +UniValue blockToJSON(BlockManager& blockman, const CBlock& block, const CBlockIndex& tip, const CBlockIndex& blockindex, TxVerbosity verbosity) { UniValue result = blockheaderToJSON(tip, blockindex); - result.pushKV("strippedsize", (int)::GetSerializeSize(block, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS)); - result.pushKV("size", (int)::GetSerializeSize(block, PROTOCOL_VERSION)); + result.pushKV("strippedsize", (int)::GetSerializeSize(TX_NO_WITNESS(block))); + result.pushKV("size", (int)::GetSerializeSize(TX_WITH_WITNESS(block))); result.pushKV("weight", (int)::GetBlockWeight(block)); UniValue txs(UniValue::VARR); @@ -182,14 +180,14 @@ UniValue blockToJSON(BlockManager& blockman, const CBlock& block, const CBlockIn case TxVerbosity::SHOW_DETAILS_AND_PREVOUT: CBlockUndo blockUndo; const bool is_not_pruned{WITH_LOCK(::cs_main, return !blockman.IsBlockPruned(blockindex))}; - const bool have_undo{is_not_pruned && blockman.UndoReadFromDisk(blockUndo, *blockindex)}; + const bool have_undo{is_not_pruned && blockman.UndoReadFromDisk(blockUndo, blockindex)}; for (size_t i = 0; i < block.vtx.size(); ++i) { const CTransactionRef& tx = block.vtx.at(i); // coinbase transaction (i.e. i == 0) doesn't have undo data const CTxUndo* txundo = (have_undo && i > 0) ? &blockUndo.vtxundo.at(i - 1) : nullptr; UniValue objTx(UniValue::VOBJ); - TxToUniv(*tx, /*block_hash=*/uint256(), /*entry=*/objTx, /*include_hex=*/true, RPCSerializationFlags(), txundo, verbosity); + TxToUniv(*tx, /*block_hash=*/uint256(), /*entry=*/objTx, /*include_hex=*/true, txundo, verbosity); txs.push_back(objTx); } break; @@ -418,7 +416,7 @@ static RPCHelpMan getdifficulty() { ChainstateManager& chainman = EnsureAnyChainman(request.context); LOCK(cs_main); - return GetDifficulty(chainman.ActiveChain().Tip()); + return GetDifficulty(*CHECK_NONFATAL(chainman.ActiveChain().Tip())); }, }; } @@ -571,22 +569,22 @@ static RPCHelpMan getblockheader() return strHex; } - return blockheaderToJSON(tip, pblockindex); + return blockheaderToJSON(*tip, *pblockindex); }, }; } -static CBlock GetBlockChecked(BlockManager& blockman, const CBlockIndex* pblockindex) +static CBlock GetBlockChecked(BlockManager& blockman, const CBlockIndex& blockindex) { CBlock block; { LOCK(cs_main); - if (blockman.IsBlockPruned(pblockindex)) { + if (blockman.IsBlockPruned(blockindex)) { throw JSONRPCError(RPC_MISC_ERROR, "Block not available (pruned data)"); } } - if (!blockman.ReadBlockFromDisk(block, *pblockindex)) { + if (!blockman.ReadBlockFromDisk(block, blockindex)) { // Block not found on disk. This could be because we have the block // header in our index but not yet have the block or did not accept the // block. Or if the block was pruned right after we released the lock above. @@ -596,21 +594,21 @@ static CBlock GetBlockChecked(BlockManager& blockman, const CBlockIndex* pblocki return block; } -static CBlockUndo GetUndoChecked(BlockManager& blockman, const CBlockIndex* pblockindex) +static CBlockUndo GetUndoChecked(BlockManager& blockman, const CBlockIndex& blockindex) { CBlockUndo blockUndo; // The Genesis block does not have undo data - if (pblockindex->nHeight == 0) return blockUndo; + if (blockindex.nHeight == 0) return blockUndo; { LOCK(cs_main); - if (blockman.IsBlockPruned(pblockindex)) { + if (blockman.IsBlockPruned(blockindex)) { throw JSONRPCError(RPC_MISC_ERROR, "Undo data not available (pruned data)"); } } - if (!blockman.UndoReadFromDisk(blockUndo, *pblockindex)) { + if (!blockman.UndoReadFromDisk(blockUndo, blockindex)) { throw JSONRPCError(RPC_MISC_ERROR, "Can't read undo data from disk"); } @@ -736,12 +734,11 @@ static RPCHelpMan getblock() } } - const CBlock block{GetBlockChecked(chainman.m_blockman, pblockindex)}; + const CBlock block{GetBlockChecked(chainman.m_blockman, *pblockindex)}; - if (verbosity <= 0) - { - CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); - ssBlock << block; + if (verbosity <= 0) { + DataStream ssBlock; + ssBlock << TX_WITH_WITNESS(block); std::string strHex = HexStr(ssBlock); return strHex; } @@ -755,7 +752,7 @@ static RPCHelpMan getblock() tx_verbosity = TxVerbosity::SHOW_DETAILS_AND_PREVOUT; } - return blockToJSON(chainman.m_blockman, block, tip, pblockindex, tx_verbosity); + return blockToJSON(chainman.m_blockman, block, *tip, *pblockindex, tx_verbosity); }, }; } @@ -1057,7 +1054,7 @@ static RPCHelpMan gettxout() UniValue ret(UniValue::VOBJ); - uint256 hash(ParseHashV(request.params[0], "txid")); + auto hash{Txid::FromUint256(ParseHashV(request.params[0], "txid"))}; COutPoint out{hash, request.params[1].getInt<uint32_t>()}; bool fMempool = true; if (!request.params[2].isNull()) @@ -1257,7 +1254,7 @@ RPCHelpMan getblockchaininfo() obj.pushKV("blocks", height); obj.pushKV("headers", chainman.m_best_header ? chainman.m_best_header->nHeight : -1); obj.pushKV("bestblockhash", tip.GetBlockHash().GetHex()); - obj.pushKV("difficulty", GetDifficulty(&tip)); + obj.pushKV("difficulty", GetDifficulty(tip)); obj.pushKV("time", tip.GetBlockTime()); obj.pushKV("mediantime", tip.GetMedianTimePast()); obj.pushKV("verificationprogress", GuessVerificationProgress(chainman.GetParams().TxData(), &tip)); @@ -1815,8 +1812,8 @@ static RPCHelpMan getblockstats() } } - const CBlock& block = GetBlockChecked(chainman.m_blockman, &pindex); - const CBlockUndo& blockUndo = GetUndoChecked(chainman.m_blockman, &pindex); + const CBlock& block = GetBlockChecked(chainman.m_blockman, pindex); + const CBlockUndo& blockUndo = GetUndoChecked(chainman.m_blockman, pindex); const bool do_all = stats.size() == 0; // Calculate everything if nothing selected (default) const bool do_mediantxsize = do_all || stats.count("mediantxsize") != 0; @@ -1861,7 +1858,7 @@ static RPCHelpMan getblockstats() for (const CTxOut& out : tx->vout) { tx_total_out += out.nValue; - size_t out_size = GetSerializeSize(out, PROTOCOL_VERSION) + PER_UTXO_OVERHEAD; + size_t out_size = GetSerializeSize(out) + PER_UTXO_OVERHEAD; utxo_size_inc += out_size; // The Genesis block and the repeated BIP30 block coinbases don't change the UTXO @@ -1913,7 +1910,7 @@ static RPCHelpMan getblockstats() const CTxOut& prevoutput = coin.out; tx_total_in += prevoutput.nValue; - size_t prevout_size = GetSerializeSize(prevoutput, PROTOCOL_VERSION) + PER_UTXO_OVERHEAD; + size_t prevout_size = GetSerializeSize(prevoutput) + PER_UTXO_OVERHEAD; utxo_size_inc -= prevout_size; utxo_size_inc_actual -= prevout_size; } @@ -2014,7 +2011,7 @@ bool FindScriptPubKey(std::atomic<int>& scan_progress, const std::atomic<bool>& } if (count % 256 == 0) { // update progress reference every 256 item - uint32_t high = 0x100 * *key.hash.begin() + *(key.hash.begin() + 1); + uint32_t high = 0x100 * *UCharCast(key.hash.begin()) + *(UCharCast(key.hash.begin()) + 1); scan_progress = (int)(high * 100.0 / 65536.0 + 0.5); } if (needles.count(coin.out.scriptPubKey)) { @@ -2275,8 +2272,8 @@ public: static bool CheckBlockFilterMatches(BlockManager& blockman, const CBlockIndex& blockindex, const GCSFilter::ElementSet& needles) { - const CBlock block{GetBlockChecked(blockman, &blockindex)}; - const CBlockUndo block_undo{GetUndoChecked(blockman, &blockindex)}; + const CBlock block{GetBlockChecked(blockman, blockindex)}; + const CBlockUndo block_undo{GetUndoChecked(blockman, blockindex)}; // Check if any of the outputs match the scriptPubKey for (const auto& tx : block.vtx) { @@ -2603,7 +2600,7 @@ static RPCHelpMan dumptxoutset() if (fs::exists(path)) { throw JSONRPCError( RPC_INVALID_PARAMETER, - path.u8string() + " already exists. If you are sure this is what you want, " + path.utf8string() + " already exists. If you are sure this is what you want, " "move it out of the way first"); } @@ -2612,7 +2609,7 @@ static RPCHelpMan dumptxoutset() if (afile.IsNull()) { throw JSONRPCError( RPC_INVALID_PARAMETER, - "Couldn't open file " + temppath.u8string() + " for writing."); + "Couldn't open file " + temppath.utf8string() + " for writing."); } NodeContext& node = EnsureAnyNodeContext(request.context); @@ -2620,7 +2617,7 @@ static RPCHelpMan dumptxoutset() node, node.chainman->ActiveChainstate(), afile, path, temppath); fs::rename(temppath, path); - result.pushKV("path", path.u8string()); + result.pushKV("path", path.utf8string()); return result; }, }; @@ -2692,7 +2689,7 @@ UniValue CreateUTXOSnapshot( result.pushKV("coins_written", maybe_stats->coins_count); result.pushKV("base_hash", tip->GetBlockHash().ToString()); result.pushKV("base_height", tip->nHeight); - result.pushKV("path", path.u8string()); + result.pushKV("path", path.utf8string()); result.pushKV("txoutset_hash", maybe_stats->hashSerialized.ToString()); result.pushKV("nchaintx", tip->nChainTx); return result; @@ -2745,7 +2742,7 @@ static RPCHelpMan loadtxoutset() if (afile.IsNull()) { throw JSONRPCError( RPC_INVALID_PARAMETER, - "Couldn't open file " + path.u8string() + " for reading."); + "Couldn't open file " + path.utf8string() + " for reading."); } SnapshotMetadata metadata; @@ -2845,7 +2842,7 @@ return RPCHelpMan{ data.pushKV("blocks", (int)chain.Height()); data.pushKV("bestblockhash", tip->GetBlockHash().GetHex()); - data.pushKV("difficulty", (double)GetDifficulty(tip)); + data.pushKV("difficulty", GetDifficulty(*tip)); data.pushKV("verificationprogress", GuessVerificationProgress(Params().TxData(), tip)); data.pushKV("coins_db_cache_bytes", cs.m_coinsdb_cache_size_bytes); data.pushKV("coins_tip_cache_bytes", cs.m_coinstip_cache_size_bytes); diff --git a/src/rpc/blockchain.h b/src/rpc/blockchain.h index 0a86085db0..c2021c3608 100644 --- a/src/rpc/blockchain.h +++ b/src/rpc/blockchain.h @@ -32,16 +32,16 @@ static constexpr int NUM_GETBLOCKSTATS_PERCENTILES = 5; * @return A floating point number that is a multiple of the main net minimum * difficulty (4295032833 hashes). */ -double GetDifficulty(const CBlockIndex* blockindex); +double GetDifficulty(const CBlockIndex& blockindex); /** Callback for when block tip changed. */ void RPCNotifyBlockChange(const CBlockIndex*); /** Block description to JSON */ -UniValue blockToJSON(node::BlockManager& blockman, const CBlock& block, const CBlockIndex* tip, const CBlockIndex* blockindex, TxVerbosity verbosity) LOCKS_EXCLUDED(cs_main); +UniValue blockToJSON(node::BlockManager& blockman, const CBlock& block, const CBlockIndex& tip, const CBlockIndex& blockindex, TxVerbosity verbosity) LOCKS_EXCLUDED(cs_main); /** Block header to JSON */ -UniValue blockheaderToJSON(const CBlockIndex* tip, const CBlockIndex* blockindex) LOCKS_EXCLUDED(cs_main); +UniValue blockheaderToJSON(const CBlockIndex& tip, const CBlockIndex& blockindex) LOCKS_EXCLUDED(cs_main); /** Used by getblockstats to get feerates at different percentiles by weight */ void CalculatePercentilesByWeight(CAmount result[NUM_GETBLOCKSTATS_PERCENTILES], std::vector<std::pair<CAmount, int64_t>>& scores, int64_t total_weight); diff --git a/src/rpc/fees.cpp b/src/rpc/fees.cpp index 62396d4c58..57ba486ed9 100644 --- a/src/rpc/fees.cpp +++ b/src/rpc/fees.cpp @@ -14,6 +14,7 @@ #include <txmempool.h> #include <univalue.h> #include <util/fees.h> +#include <validationinterface.h> #include <algorithm> #include <array> @@ -67,6 +68,7 @@ static RPCHelpMan estimatesmartfee() const NodeContext& node = EnsureAnyNodeContext(request.context); const CTxMemPool& mempool = EnsureMemPool(node); + SyncWithValidationInterfaceQueue(); unsigned int max_target = fee_estimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); unsigned int conf_target = ParseConfirmTarget(request.params[0], max_target); bool conservative = true; @@ -155,6 +157,7 @@ static RPCHelpMan estimaterawfee() { CBlockPolicyEstimator& fee_estimator = EnsureAnyFeeEstimator(request.context); + SyncWithValidationInterfaceQueue(); unsigned int max_target = fee_estimator.HighestTargetTracked(FeeEstimateHorizon::LONG_HALFLIFE); unsigned int conf_target = ParseConfirmTarget(request.params[0], max_target); double threshold = 0.95; diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 136969eb87..f6d9d42f0f 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -21,6 +21,7 @@ #include <univalue.h> #include <util/fs.h> #include <util/moneystr.h> +#include <util/strencodings.h> #include <util/time.h> #include <utility> @@ -289,7 +290,7 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool info.pushKV("descendantsize", e.GetSizeWithDescendants()); info.pushKV("ancestorcount", e.GetCountWithAncestors()); info.pushKV("ancestorsize", e.GetSizeWithAncestors()); - info.pushKV("wtxid", pool.vTxHashes[e.vTxHashesIdx].first.ToString()); + info.pushKV("wtxid", e.GetTx().GetWitnessHash().ToString()); UniValue fees(UniValue::VOBJ); fees.pushKV("base", ValueFromAmount(e.GetFee())); @@ -315,9 +316,7 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool info.pushKV("depends", depends); UniValue spent(UniValue::VARR); - const CTxMemPool::txiter& it = pool.mapTx.find(tx.GetHash()); - const CTxMemPoolEntry::Children& children = it->GetMemPoolChildrenConst(); - for (const CTxMemPoolEntry& child : children) { + for (const CTxMemPoolEntry& child : e.GetMemPoolChildrenConst()) { spent.push_back(child.GetTx().GetHash().ToString()); } @@ -344,14 +343,13 @@ UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose, bool include_mempoo } LOCK(pool.cs); UniValue o(UniValue::VOBJ); - for (const CTxMemPoolEntry& e : pool.mapTx) { - const uint256& hash = e.GetTx().GetHash(); + for (const CTxMemPoolEntry& e : pool.entryAll()) { UniValue info(UniValue::VOBJ); entryToJSON(pool, info, e); // Mempool has unique entries so there is no advantage in using // UniValue::pushKV, which checks if the key already exists in O(N). // UniValue::pushKVEnd is used instead which currently is O(1). - o.pushKVEnd(hash.ToString(), info); + o.pushKVEnd(e.GetTx().GetHash().ToString(), info); } return o; } else { @@ -460,12 +458,12 @@ static RPCHelpMan getmempoolancestors() const CTxMemPool& mempool = EnsureAnyMemPool(request.context); LOCK(mempool.cs); - CTxMemPool::txiter it = mempool.mapTx.find(hash); - if (it == mempool.mapTx.end()) { + const auto entry{mempool.GetEntry(Txid::FromUint256(hash))}; + if (entry == nullptr) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); } - auto ancestors{mempool.AssumeCalculateMemPoolAncestors(__func__, *it, CTxMemPool::Limits::NoLimits(), /*fSearchForParents=*/false)}; + auto ancestors{mempool.AssumeCalculateMemPoolAncestors(self.m_name, *entry, CTxMemPool::Limits::NoLimits(), /*fSearchForParents=*/false)}; if (!fVerbose) { UniValue o(UniValue::VARR); @@ -521,15 +519,15 @@ static RPCHelpMan getmempooldescendants() const CTxMemPool& mempool = EnsureAnyMemPool(request.context); LOCK(mempool.cs); - CTxMemPool::txiter it = mempool.mapTx.find(hash); - if (it == mempool.mapTx.end()) { + const auto it{mempool.GetIter(hash)}; + if (!it) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); } CTxMemPool::setEntries setDescendants; - mempool.CalculateDescendants(it, setDescendants); + mempool.CalculateDescendants(*it, setDescendants); // CTxMemPool::CalculateDescendants will include the given tx - setDescendants.erase(it); + setDescendants.erase(*it); if (!fVerbose) { UniValue o(UniValue::VARR); @@ -573,14 +571,13 @@ static RPCHelpMan getmempoolentry() const CTxMemPool& mempool = EnsureAnyMemPool(request.context); LOCK(mempool.cs); - CTxMemPool::txiter it = mempool.mapTx.find(hash); - if (it == mempool.mapTx.end()) { + const auto entry{mempool.GetEntry(Txid::FromUint256(hash))}; + if (entry == nullptr) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); } - const CTxMemPoolEntry &e = *it; UniValue info(UniValue::VOBJ); - entryToJSON(mempool, info, e); + entryToJSON(mempool, info, *entry); return info; }, }; @@ -636,7 +633,7 @@ static RPCHelpMan gettxspendingprevout() {"vout", UniValueType(UniValue::VNUM)}, }, /*fAllowNull=*/false, /*fStrict=*/true); - const uint256 txid(ParseHashO(o, "txid")); + const Txid txid = Txid::FromUint256(ParseHashO(o, "txid")); const int nOutput{o.find_value("vout").getInt<int>()}; if (nOutput < 0) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, vout cannot be negative"); @@ -809,7 +806,7 @@ static RPCHelpMan savemempool() } UniValue ret(UniValue::VOBJ); - ret.pushKV("filename", dump_path.u8string()); + ret.pushKV("filename", dump_path.utf8string()); return ret; }, @@ -821,7 +818,7 @@ static RPCHelpMan submitpackage() return RPCHelpMan{"submitpackage", "Submit a package of raw transactions (serialized, hex-encoded) to local node.\n" "The package must consist of a child with its parents, and none of the parents may depend on one another.\n" - "The package will be validated according to consensus and mempool policy rules. If all transactions pass, they will be accepted to mempool.\n" + "The package will be validated according to consensus and mempool policy rules. If any transaction passes, it will be accepted to mempool.\n" "This RPC is experimental and the interface may be unstable. Refer to doc/policy/packages.md for documentation on package policies.\n" "Warning: successful submission does not mean the transactions will propagate throughout the network.\n" , @@ -835,19 +832,21 @@ static RPCHelpMan submitpackage() RPCResult{ RPCResult::Type::OBJ, "", "", { + {RPCResult::Type::STR, "package_msg", "The transaction package result message. \"success\" indicates all transactions were accepted into or are already in the mempool."}, {RPCResult::Type::OBJ_DYN, "tx-results", "transaction results keyed by wtxid", { {RPCResult::Type::OBJ, "wtxid", "transaction wtxid", { {RPCResult::Type::STR_HEX, "txid", "The transaction hash in hex"}, {RPCResult::Type::STR_HEX, "other-wtxid", /*optional=*/true, "The wtxid of a different transaction with the same txid but different witness found in the mempool. This means the submitted transaction was ignored."}, - {RPCResult::Type::NUM, "vsize", "Virtual transaction size as defined in BIP 141."}, - {RPCResult::Type::OBJ, "fees", "Transaction fees", { + {RPCResult::Type::NUM, "vsize", /*optional=*/true, "Sigops-adjusted virtual transaction size."}, + {RPCResult::Type::OBJ, "fees", /*optional=*/true, "Transaction fees", { {RPCResult::Type::STR_AMOUNT, "base", "transaction fee in " + CURRENCY_UNIT}, {RPCResult::Type::STR_AMOUNT, "effective-feerate", /*optional=*/true, "if the transaction was not already in the mempool, the effective feerate in " + CURRENCY_UNIT + " per KvB. For example, the package feerate and/or feerate with modified fees from prioritisetransaction."}, {RPCResult::Type::ARR, "effective-includes", /*optional=*/true, "if effective-feerate is provided, the wtxids of the transactions whose fees and vsizes are included in effective-feerate.", {{RPCResult::Type::STR_HEX, "", "transaction wtxid in hex"}, }}, }}, + {RPCResult::Type::STR, "error", /*optional=*/true, "The transaction error string, if it was rejected by the mempool"}, }} }}, {RPCResult::Type::ARR, "replaced-transactions", /*optional=*/true, "List of txids of replaced transactions", @@ -887,57 +886,77 @@ static RPCHelpMan submitpackage() Chainstate& chainstate = EnsureChainman(node).ActiveChainstate(); const auto package_result = WITH_LOCK(::cs_main, return ProcessNewPackage(chainstate, mempool, txns, /*test_accept=*/ false)); - // First catch any errors. + std::string package_msg = "success"; + + // First catch package-wide errors, continue if we can switch(package_result.m_state.GetResult()) { - case PackageValidationResult::PCKG_RESULT_UNSET: break; - case PackageValidationResult::PCKG_POLICY: + case PackageValidationResult::PCKG_RESULT_UNSET: { - throw JSONRPCTransactionError(TransactionError::INVALID_PACKAGE, - package_result.m_state.GetRejectReason()); + // Belt-and-suspenders check; everything should be successful here + CHECK_NONFATAL(package_result.m_tx_results.size() == txns.size()); + for (const auto& tx : txns) { + CHECK_NONFATAL(mempool.exists(GenTxid::Txid(tx->GetHash()))); + } + break; } case PackageValidationResult::PCKG_MEMPOOL_ERROR: { + // This only happens with internal bug; user should stop and report throw JSONRPCTransactionError(TransactionError::MEMPOOL_ERROR, package_result.m_state.GetRejectReason()); } + case PackageValidationResult::PCKG_POLICY: case PackageValidationResult::PCKG_TX: { - for (const auto& tx : txns) { - auto it = package_result.m_tx_results.find(tx->GetWitnessHash()); - if (it != package_result.m_tx_results.end() && it->second.m_state.IsInvalid()) { - throw JSONRPCTransactionError(TransactionError::MEMPOOL_REJECTED, - strprintf("%s failed: %s", tx->GetHash().ToString(), it->second.m_state.GetRejectReason())); - } - } - // If a PCKG_TX error was returned, there must have been an invalid transaction. - NONFATAL_UNREACHABLE(); + // Package-wide error we want to return, but we also want to return individual responses + package_msg = package_result.m_state.GetRejectReason(); + CHECK_NONFATAL(package_result.m_tx_results.size() == txns.size() || + package_result.m_tx_results.empty()); + break; } } + size_t num_broadcast{0}; for (const auto& tx : txns) { + // We don't want to re-submit the txn for validation in BroadcastTransaction + if (!mempool.exists(GenTxid::Txid(tx->GetHash()))) { + continue; + } + + // We do not expect an error here; we are only broadcasting things already/still in mempool std::string err_string; const auto err = BroadcastTransaction(node, tx, err_string, /*max_tx_fee=*/0, /*relay=*/true, /*wait_callback=*/true); if (err != TransactionError::OK) { throw JSONRPCTransactionError(err, - strprintf("transaction broadcast failed: %s (all transactions were submitted, %d transactions were broadcast successfully)", + strprintf("transaction broadcast failed: %s (%d transactions were broadcast successfully)", err_string, num_broadcast)); } num_broadcast++; } + UniValue rpc_result{UniValue::VOBJ}; + rpc_result.pushKV("package_msg", package_msg); UniValue tx_result_map{UniValue::VOBJ}; std::set<uint256> replaced_txids; for (const auto& tx : txns) { - auto it = package_result.m_tx_results.find(tx->GetWitnessHash()); - CHECK_NONFATAL(it != package_result.m_tx_results.end()); UniValue result_inner{UniValue::VOBJ}; result_inner.pushKV("txid", tx->GetHash().GetHex()); + auto it = package_result.m_tx_results.find(tx->GetWitnessHash()); + if (it == package_result.m_tx_results.end()) { + // No results, report error and continue + result_inner.pushKV("error", "unevaluated"); + continue; + } const auto& tx_result = it->second; - if (it->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS) { + switch(it->second.m_result_type) { + case MempoolAcceptResult::ResultType::DIFFERENT_WITNESS: result_inner.pushKV("other-wtxid", it->second.m_other_wtxid.value().GetHex()); - } - if (it->second.m_result_type == MempoolAcceptResult::ResultType::VALID || - it->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY) { + break; + case MempoolAcceptResult::ResultType::INVALID: + result_inner.pushKV("error", it->second.m_state.ToString()); + break; + case MempoolAcceptResult::ResultType::VALID: + case MempoolAcceptResult::ResultType::MEMPOOL_ENTRY: result_inner.pushKV("vsize", int64_t{it->second.m_vsize.value()}); UniValue fees(UniValue::VOBJ); fees.pushKV("base", ValueFromAmount(it->second.m_base_fees.value())); @@ -958,6 +977,7 @@ static RPCHelpMan submitpackage() replaced_txids.insert(ptx->GetHash()); } } + break; } tx_result_map.pushKV(tx->GetWitnessHash().GetHex(), result_inner); } diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 76170c3201..31551dfe98 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -27,7 +27,6 @@ #include <script/descriptor.h> #include <script/script.h> #include <script/signingprovider.h> -#include <shutdown.h> #include <timedata.h> #include <txmempool.h> #include <univalue.h> @@ -49,13 +48,22 @@ using node::UpdateTime; /** * Return average network hashes per second based on the last 'lookup' blocks, - * or from the last difficulty change if 'lookup' is nonpositive. - * If 'height' is nonnegative, compute the estimate at the time when a given block was found. + * or from the last difficulty change if 'lookup' is -1. + * If 'height' is -1, compute the estimate from current chain tip. + * If 'height' is a valid block height, compute the estimate at the time when a given block was found. */ static UniValue GetNetworkHashPS(int lookup, int height, const CChain& active_chain) { + if (lookup < -1 || lookup == 0) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid nblocks. Must be a positive number or -1."); + } + + if (height < -1 || height > active_chain.Height()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "Block does not exist at specified height"); + } + const CBlockIndex* pb = active_chain.Tip(); - if (height >= 0 && height < active_chain.Height()) { + if (height >= 0) { pb = active_chain[height]; } @@ -63,7 +71,7 @@ static UniValue GetNetworkHashPS(int lookup, int height, const CChain& active_ch return 0; // If lookup is -1, then use blocks since last difficulty change. - if (lookup <= 0) + if (lookup == -1) lookup = pb->nHeight % Params().GetConsensus().DifficultyAdjustmentInterval() + 1; // If lookup is larger than chain, then set it to chain length. @@ -97,7 +105,7 @@ static RPCHelpMan getnetworkhashps() "Pass in [blocks] to override # of blocks, -1 specifies since last difficulty change.\n" "Pass in [height] to estimate the network speed at the time when a certain block was found.\n", { - {"nblocks", RPCArg::Type::NUM, RPCArg::Default{120}, "The number of blocks, or -1 for blocks since last difficulty change."}, + {"nblocks", RPCArg::Type::NUM, RPCArg::Default{120}, "The number of previous blocks to calculate estimate from, or -1 for blocks since last difficulty change."}, {"height", RPCArg::Type::NUM, RPCArg::Default{-1}, "To estimate at the time of the given height."}, }, RPCResult{ @@ -120,11 +128,11 @@ static bool GenerateBlock(ChainstateManager& chainman, CBlock& block, uint64_t& block_out.reset(); block.hashMerkleRoot = BlockMerkleRoot(block); - while (max_tries > 0 && block.nNonce < std::numeric_limits<uint32_t>::max() && !CheckProofOfWork(block.GetHash(), block.nBits, chainman.GetConsensus()) && !ShutdownRequested()) { + while (max_tries > 0 && block.nNonce < std::numeric_limits<uint32_t>::max() && !CheckProofOfWork(block.GetHash(), block.nBits, chainman.GetConsensus()) && !chainman.m_interrupt) { ++block.nNonce; --max_tries; } - if (max_tries == 0 || ShutdownRequested()) { + if (max_tries == 0 || chainman.m_interrupt) { return false; } if (block.nNonce == std::numeric_limits<uint32_t>::max()) { @@ -145,7 +153,7 @@ static bool GenerateBlock(ChainstateManager& chainman, CBlock& block, uint64_t& static UniValue generateBlocks(ChainstateManager& chainman, const CTxMemPool& mempool, const CScript& coinbase_script, int nGenerate, uint64_t nMaxTries) { UniValue blockHashes(UniValue::VARR); - while (nGenerate > 0 && !ShutdownRequested()) { + while (nGenerate > 0 && !chainman.m_interrupt) { std::unique_ptr<CBlockTemplate> pblocktemplate(BlockAssembler{chainman.ActiveChainstate(), &mempool}.CreateNewBlock(coinbase_script)); if (!pblocktemplate.get()) throw JSONRPCError(RPC_INTERNAL_ERROR, "Couldn't create new block"); @@ -390,8 +398,8 @@ static RPCHelpMan generateblock() UniValue obj(UniValue::VOBJ); obj.pushKV("hash", block_out->GetHash().GetHex()); if (!process_new_block) { - CDataStream block_ser{SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()}; - block_ser << *block_out; + DataStream block_ser; + block_ser << TX_WITH_WITNESS(*block_out); obj.pushKV("hex", HexStr(block_ser)); } return obj; @@ -432,7 +440,7 @@ static RPCHelpMan getmininginfo() obj.pushKV("blocks", active_chain.Height()); if (BlockAssembler::m_last_block_weight) obj.pushKV("currentblockweight", *BlockAssembler::m_last_block_weight); if (BlockAssembler::m_last_block_num_txs) obj.pushKV("currentblocktx", *BlockAssembler::m_last_block_num_txs); - obj.pushKV("difficulty", (double)GetDifficulty(active_chain.Tip())); + obj.pushKV("difficulty", GetDifficulty(*CHECK_NONFATAL(active_chain.Tip()))); obj.pushKV("networkhashps", getnetworkhashps().HandleRequest(request)); obj.pushKV("pooledtx", (uint64_t)mempool.size()); obj.pushKV("chain", chainman.GetParams().GetChainTypeString()); diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 63788c3a03..5fea8c22c9 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -15,7 +15,9 @@ #include <net_types.h> // For banmap_t #include <netbase.h> #include <node/context.h> +#include <node/protocol_version.h> #include <policy/settings.h> +#include <protocol.h> #include <rpc/blockchain.h> #include <rpc/protocol.h> #include <rpc/server_util.h> @@ -28,7 +30,6 @@ #include <util/time.h> #include <util/translation.h> #include <validation.h> -#include <version.h> #include <warnings.h> #include <optional> @@ -98,6 +99,18 @@ static RPCHelpMan ping() }; } +/** Returns, given services flags, a list of humanly readable (known) network services */ +static UniValue GetServicesNames(ServiceFlags services) +{ + UniValue servicesNames(UniValue::VARR); + + for (const auto& flag : serviceFlagsToStr(services)) { + servicesNames.push_back(flag); + } + + return servicesNames; +} + static RPCHelpMan getpeerinfo() { return RPCHelpMan{ @@ -487,7 +500,7 @@ static RPCHelpMan getaddednodeinfo() NodeContext& node = EnsureAnyNodeContext(request.context); const CConnman& connman = EnsureConnman(node); - std::vector<AddedNodeInfo> vInfo = connman.GetAddedNodeInfo(); + std::vector<AddedNodeInfo> vInfo = connman.GetAddedNodeInfo(/*include_connected=*/true); if (!request.params[0].isNull()) { bool found = false; @@ -734,7 +747,7 @@ static RPCHelpMan setban() } } else - LookupSubNet(request.params[0].get_str(), subNet); + subNet = LookupSubNet(request.params[0].get_str()); if (! (isSubnet ? subNet.IsValid() : netAddr.IsValid()) ) throw JSONRPCError(RPC_CLIENT_INVALID_IP_OR_SUBNET, "Error: Invalid IP/Subnet"); diff --git a/src/rpc/node.cpp b/src/rpc/node.cpp index 6b3662996c..b085828215 100644 --- a/src/rpc/node.cpp +++ b/src/rpc/node.cpp @@ -91,6 +91,9 @@ static RPCHelpMan mockscheduler() const NodeContext& node_context{EnsureAnyNodeContext(request.context)}; CHECK_NONFATAL(node_context.scheduler)->MockForward(std::chrono::seconds{delta_seconds}); SyncWithValidationInterfaceQueue(); + for (const auto& chain_client : node_context.chain_clients) { + chain_client->schedulerMockForward(std::chrono::seconds(delta_seconds)); + } return UniValue::VNULL; }, diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 16705b3ce2..634be2f7fb 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -62,7 +62,7 @@ static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& // Blockchain contextual information (confirmations and blocktime) is not // available to code in bitcoin-common, so we query them here and push the // data into the returned UniValue. - TxToUniv(tx, /*block_hash=*/uint256(), entry, /*include_hex=*/true, RPCSerializationFlags(), txundo, verbosity); + TxToUniv(tx, /*block_hash=*/uint256(), entry, /*include_hex=*/true, txundo, verbosity); if (!hashBlock.IsNull()) { LOCK(cs_main); @@ -383,7 +383,7 @@ static RPCHelpMan getrawtransaction() } if (verbosity <= 0) { - return EncodeHexTx(*tx, RPCSerializationFlags()); + return EncodeHexTx(*tx); } UniValue result(UniValue::VOBJ); @@ -394,7 +394,7 @@ static RPCHelpMan getrawtransaction() // If request is verbosity >= 1 but no blockhash was given, then look up the blockindex if (request.params[2].isNull()) { LOCK(cs_main); - blockindex = chainman.m_blockman.LookupBlockIndex(hash_block); + blockindex = chainman.m_blockman.LookupBlockIndex(hash_block); // May be nullptr for mempool transactions } if (verbosity == 1) { TxToJSON(*tx, hash_block, result, chainman.ActiveChainstate()); @@ -403,10 +403,8 @@ static RPCHelpMan getrawtransaction() CBlockUndo blockUndo; CBlock block; - const bool is_block_pruned{WITH_LOCK(cs_main, return chainman.m_blockman.IsBlockPruned(blockindex))}; - if (tx->IsCoinBase() || - !blockindex || is_block_pruned || + if (tx->IsCoinBase() || !blockindex || WITH_LOCK(::cs_main, return chainman.m_blockman.IsBlockPruned(*blockindex)) || !(chainman.m_blockman.UndoReadFromDisk(blockUndo, *blockindex) && chainman.m_blockman.ReadBlockFromDisk(block, *blockindex))) { TxToJSON(*tx, hash_block, result, chainman.ActiveChainstate()); return result; @@ -1492,7 +1490,7 @@ static RPCHelpMan combinepsbt() throw JSONRPCTransactionError(error); } - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssTx{}; ssTx << merged_psbt; return EncodeBase64(ssTx); }, @@ -1537,11 +1535,11 @@ static RPCHelpMan finalizepsbt() bool complete = FinalizeAndExtractPSBT(psbtx, mtx); UniValue result(UniValue::VOBJ); - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssTx{}; std::string result_str; if (complete && extract) { - ssTx << mtx; + ssTx << TX_WITH_WITNESS(mtx); result_str = HexStr(ssTx); result.pushKV("hex", result_str); } else { @@ -1588,7 +1586,7 @@ static RPCHelpMan createpsbt() } // Serialize the PSBT - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssTx{}; ssTx << psbtx; return EncodeBase64(ssTx); @@ -1655,7 +1653,7 @@ static RPCHelpMan converttopsbt() } // Serialize the PSBT - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssTx{}; ssTx << psbtx; return EncodeBase64(ssTx); @@ -1702,7 +1700,7 @@ static RPCHelpMan utxoupdatepsbt() /*sighash_type=*/SIGHASH_ALL, /*finalize=*/false); - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssTx{}; ssTx << psbtx; return EncodeBase64(ssTx); }, @@ -1803,7 +1801,7 @@ static RPCHelpMan joinpsbts() } shuffled_psbt.unknown.insert(merged_psbt.unknown.begin(), merged_psbt.unknown.end()); - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssTx{}; ssTx << shuffled_psbt; return EncodeBase64(ssTx); }, @@ -1983,7 +1981,7 @@ RPCHelpMan descriptorprocesspsbt() complete &= PSBTInputSigned(input); } - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssTx{}; ssTx << psbtx; UniValue result(UniValue::VOBJ); @@ -1994,8 +1992,8 @@ RPCHelpMan descriptorprocesspsbt() CMutableTransaction mtx; PartiallySignedTransaction psbtx_copy = psbtx; CHECK_NONFATAL(FinalizeAndExtractPSBT(psbtx_copy, mtx)); - CDataStream ssTx_final(SER_NETWORK, PROTOCOL_VERSION); - ssTx_final << mtx; + DataStream ssTx_final; + ssTx_final << TX_WITH_WITNESS(mtx); result.pushKV("hex", HexStr(ssTx_final)); } return result; diff --git a/src/rpc/rawtransaction_util.cpp b/src/rpc/rawtransaction_util.cpp index 3a6fa39e4d..c471986a44 100644 --- a/src/rpc/rawtransaction_util.cpp +++ b/src/rpc/rawtransaction_util.cpp @@ -34,7 +34,7 @@ void AddInputs(CMutableTransaction& rawTx, const UniValue& inputs_in, std::optio const UniValue& input = inputs[idx]; const UniValue& o = input.get_obj(); - uint256 txid = ParseHashO(o, "txid"); + Txid txid = Txid::FromUint256(ParseHashO(o, "txid")); const UniValue& vout_v = o.find_value("vout"); if (!vout_v.isNum()) @@ -185,7 +185,7 @@ void ParsePrevouts(const UniValue& prevTxsUnival, FillableSigningProvider* keyst {"scriptPubKey", UniValueType(UniValue::VSTR)}, }); - uint256 txid = ParseHashO(prevOut, "txid"); + Txid txid = Txid::FromUint256(ParseHashO(prevOut, "txid")); int nOut = prevOut.find_value("vout").getInt<int>(); if (nOut < 0) { diff --git a/src/rpc/request.cpp b/src/rpc/request.cpp index 4c67da8b70..b7acd62ee3 100644 --- a/src/rpc/request.cpp +++ b/src/rpc/request.cpp @@ -80,6 +80,8 @@ static fs::path GetAuthCookieFile(bool temp=false) return AbsPathForConfigVal(gArgs, arg); } +static bool g_generated_cookie = false; + bool GenerateAuthCookie(std::string *cookie_out) { const size_t COOKIE_SIZE = 32; @@ -105,6 +107,7 @@ bool GenerateAuthCookie(std::string *cookie_out) LogPrintf("Unable to rename cookie authentication file %s to %s\n", fs::PathToString(filepath_tmp), fs::PathToString(filepath)); return false; } + g_generated_cookie = true; LogPrintf("Generated RPC authentication cookie %s\n", fs::PathToString(filepath)); if (cookie_out) @@ -131,7 +134,10 @@ bool GetAuthCookie(std::string *cookie_out) void DeleteAuthCookie() { try { - fs::remove(GetAuthCookieFile()); + if (g_generated_cookie) { + // Delete the cookie file if it was generated by this process + fs::remove(GetAuthCookieFile()); + } } catch (const fs::filesystem_error& e) { LogPrintf("%s: Unable to remove random auth cookie file: %s\n", __func__, fsbridge::get_filesystem_error_message(e)); } diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index d3c5a19326..e1ac510cba 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -8,9 +8,11 @@ #include <common/args.h> #include <common/system.h> #include <logging.h> +#include <node/context.h> +#include <rpc/server_util.h> #include <rpc/util.h> -#include <shutdown.h> #include <sync.h> +#include <util/signalinterrupt.h> #include <util/strencodings.h> #include <util/string.h> #include <util/time.h> @@ -179,7 +181,7 @@ static RPCHelpMan stop() { // Event loop will exit after current HTTP requests have been handled, so // this reply will get back to the client. - StartShutdown(); + CHECK_NONFATAL((*CHECK_NONFATAL(EnsureAnyNodeContext(jsonRequest.context).shutdown))()); if (jsonRequest.params[0].isNum()) { UninterruptibleSleep(std::chrono::milliseconds{jsonRequest.params[0].getInt<int>()}); } @@ -243,7 +245,7 @@ static RPCHelpMan getrpcinfo() UniValue result(UniValue::VOBJ); result.pushKV("active_commands", active_commands); - const std::string path = LogInstance().m_file_path.u8string(); + const std::string path = LogInstance().m_file_path.utf8string(); UniValue log_path(UniValue::VSTR, path); result.pushKV("logpath", log_path); @@ -409,7 +411,7 @@ static inline JSONRPCRequest transformNamedArguments(const JSONRPCRequest& in, c } // Process expected parameters. If any parameters were left unspecified in // the request before a parameter that was specified, null values need to be - // inserted at the unspecifed parameter positions, and the "hole" variable + // inserted at the unspecified parameter positions, and the "hole" variable // below tracks the number of null values that need to be inserted. // The "initial_hole_size" variable stores the size of the initial hole, // i.e. how many initial positional arguments were left unspecified. This is @@ -595,12 +597,4 @@ void RPCRunLater(const std::string& name, std::function<void()> func, int64_t nS deadlineTimers.emplace(name, std::unique_ptr<RPCTimerBase>(timerInterface->NewTimer(func, nSeconds*1000))); } -int RPCSerializationFlags() -{ - int flag = 0; - if (gArgs.GetIntArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) == 0) - flag |= SERIALIZE_TRANSACTION_NO_WITNESS; - return flag; -} - CRPCTable tableRPC; diff --git a/src/rpc/server.h b/src/rpc/server.h index 24658ddb8b..b8348e4aa6 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -16,8 +16,6 @@ #include <univalue.h> -static const unsigned int DEFAULT_RPC_SERIALIZE_VERSION = 1; - class CRPCCommand; namespace RPCServer @@ -183,7 +181,4 @@ void InterruptRPC(); void StopRPC(); std::string JSONRPCExecBatch(const JSONRPCRequest& jreq, const UniValue& vReq); -// Retrieves any serialization flags requested in command line argument -int RPCSerializationFlags(); - #endif // BITCOIN_RPC_SERVER_H diff --git a/src/rpc/txoutproof.cpp b/src/rpc/txoutproof.cpp index d74959cecc..7958deb677 100644 --- a/src/rpc/txoutproof.cpp +++ b/src/rpc/txoutproof.cpp @@ -41,13 +41,13 @@ static RPCHelpMan gettxoutproof() RPCExamples{""}, [&](const RPCHelpMan& self, const JSONRPCRequest& request) -> UniValue { - std::set<uint256> setTxids; + std::set<Txid> setTxids; UniValue txids = request.params[0].get_array(); if (txids.empty()) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Parameter 'txids' cannot be empty"); } for (unsigned int idx = 0; idx < txids.size(); idx++) { - auto ret = setTxids.insert(ParseHashV(txids[idx], "txid")); + auto ret{setTxids.insert(Txid::FromUint256(ParseHashV(txids[idx], "txid")))}; if (!ret.second) { throw JSONRPCError(RPC_INVALID_PARAMETER, std::string("Invalid parameter, duplicated txid: ") + txids[idx].get_str()); } @@ -69,7 +69,7 @@ static RPCHelpMan gettxoutproof() // Loop through txids and try to find which block they're in. Exit loop once a block is found. for (const auto& tx : setTxids) { - const Coin& coin = AccessByTxid(active_chainstate.CoinsTip(), tx); + const Coin& coin{AccessByTxid(active_chainstate.CoinsTip(), tx)}; if (!coin.IsSpent()) { pblockindex = active_chainstate.m_chain[coin.nHeight]; break; diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index a11366bd47..cf48ee11e7 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -20,6 +20,7 @@ #include <util/string.h> #include <util/translation.h> +#include <string_view> #include <tuple> const std::string UNIX_EPOCH_TIME = "UNIX epoch time"; @@ -74,29 +75,29 @@ CAmount AmountFromValue(const UniValue& value, int decimals) return amount; } -uint256 ParseHashV(const UniValue& v, std::string strName) +uint256 ParseHashV(const UniValue& v, std::string_view name) { const std::string& strHex(v.get_str()); if (64 != strHex.length()) - throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%s must be of length %d (not %d, for '%s')", strName, 64, strHex.length(), strHex)); + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%s must be of length %d (not %d, for '%s')", name, 64, strHex.length(), strHex)); if (!IsHex(strHex)) // Note: IsHex("") is false - throw JSONRPCError(RPC_INVALID_PARAMETER, strName+" must be hexadecimal string (not '"+strHex+"')"); + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%s must be hexadecimal string (not '%s')", name, strHex)); return uint256S(strHex); } -uint256 ParseHashO(const UniValue& o, std::string strKey) +uint256 ParseHashO(const UniValue& o, std::string_view strKey) { return ParseHashV(o.find_value(strKey), strKey); } -std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName) +std::vector<unsigned char> ParseHexV(const UniValue& v, std::string_view name) { std::string strHex; if (v.isStr()) strHex = v.get_str(); if (!IsHex(strHex)) - throw JSONRPCError(RPC_INVALID_PARAMETER, strName+" must be hexadecimal string (not '"+strHex+"')"); + throw JSONRPCError(RPC_INVALID_PARAMETER, strprintf("%s must be hexadecimal string (not '%s')", name, strHex)); return ParseHex(strHex); } -std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey) +std::vector<unsigned char> ParseHexO(const UniValue& o, std::string_view strKey) { return ParseHexV(o.find_value(strKey), strKey); } @@ -1088,7 +1089,7 @@ UniValue RPCResult::MatchesType(const UniValue& result) const if (UniValue::VARR == result.getType()) { UniValue errors(UniValue::VOBJ); for (size_t i{0}; i < result.get_array().size(); ++i) { - // If there are more results than documented, re-use the last doc_inner. + // If there are more results than documented, reuse the last doc_inner. const RPCResult& doc_inner{m_inner.at(std::min(m_inner.size() - 1, i))}; UniValue match{doc_inner.MatchesType(result.get_array()[i])}; if (!match.isTrue()) errors.pushKV(strprintf("%d", i), match); @@ -1303,17 +1304,6 @@ std::vector<CScript> EvalDescriptorStringOrObject(const UniValue& scanobject, Fl return ret; } -UniValue GetServicesNames(ServiceFlags services) -{ - UniValue servicesNames(UniValue::VARR); - - for (const auto& flag : serviceFlagsToStr(services)) { - servicesNames.push_back(flag); - } - - return servicesNames; -} - /** Convert a vector of bilingual strings to a UniValue::VARR containing their original untranslated values. */ [[nodiscard]] static UniValue BilingualStringsToUniValue(const std::vector<bilingual_str>& bilingual_strings) { diff --git a/src/rpc/util.h b/src/rpc/util.h index 392540ffad..e2d5ed333c 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -9,7 +9,6 @@ #include <consensus/amount.h> #include <node/transaction.h> #include <outputtype.h> -#include <protocol.h> #include <pubkey.h> #include <rpc/protocol.h> #include <rpc/request.h> @@ -26,6 +25,7 @@ #include <map> #include <optional> #include <string> +#include <string_view> #include <type_traits> #include <utility> #include <variant> @@ -59,7 +59,6 @@ extern const std::string UNIX_EPOCH_TIME; extern const std::string EXAMPLE_ADDRESS[2]; class FillableSigningProvider; -class CPubKey; class CScript; struct Sections; @@ -91,10 +90,10 @@ void RPCTypeCheckObj(const UniValue& o, * Utilities: convert hex-encoded Values * (throws error if not hex). */ -uint256 ParseHashV(const UniValue& v, std::string strName); -uint256 ParseHashO(const UniValue& o, std::string strKey); -std::vector<unsigned char> ParseHexV(const UniValue& v, std::string strName); -std::vector<unsigned char> ParseHexO(const UniValue& o, std::string strKey); +uint256 ParseHashV(const UniValue& v, std::string_view name); +uint256 ParseHashO(const UniValue& o, std::string_view strKey); +std::vector<unsigned char> ParseHexV(const UniValue& v, std::string_view name); +std::vector<unsigned char> ParseHexO(const UniValue& o, std::string_view strKey); /** * Validate and return a CAmount from a UniValue number or string. @@ -132,9 +131,6 @@ std::pair<int64_t, int64_t> ParseDescriptorRange(const UniValue& value); /** Evaluate a descriptor given as a string, or as a {"desc":...,"range":...} object, with default range of 1000. */ std::vector<CScript> EvalDescriptorStringOrObject(const UniValue& scanobject, FlatSigningProvider& provider, const bool expand_priv = false); -/** Returns, given services flags, a list of humanly readable (known) network services */ -UniValue GetServicesNames(ServiceFlags services); - /** * Serializing JSON objects depends on the outer type. Only arrays and * dictionaries can be nested in json. The top-level outer type is "NONE". @@ -406,7 +402,7 @@ public: * RPC method implementations. The helper internally checks whether the * user-passed argument isNull() and parses (from JSON) and returns the * user-passed argument, or the default value derived from the RPCArg - * documention, or a falsy value if no default was given. + * documentation, or a falsy value if no default was given. * * Use Arg<Type>(i) to get the argument or its default value. Otherwise, * use MaybeArg<Type>(i) to get the optional argument or a falsy value. diff --git a/src/script/bitcoinconsensus.cpp b/src/script/bitcoinconsensus.cpp index 71005cfb6e..c4eccacf41 100644 --- a/src/script/bitcoinconsensus.cpp +++ b/src/script/bitcoinconsensus.cpp @@ -8,7 +8,6 @@ #include <primitives/transaction.h> #include <pubkey.h> #include <script/interpreter.h> -#include <version.h> namespace { @@ -16,8 +15,7 @@ namespace { class TxInputStream { public: - TxInputStream(int nVersionIn, const unsigned char *txTo, size_t txToLen) : - m_version(nVersionIn), + TxInputStream(const unsigned char *txTo, size_t txToLen) : m_data(txTo), m_remaining(txToLen) {} @@ -48,9 +46,7 @@ public: return *this; } - int GetVersion() const { return m_version; } private: - const int m_version; const unsigned char* m_data; size_t m_remaining; }; @@ -84,8 +80,8 @@ static int verify_script(const unsigned char *scriptPubKey, unsigned int scriptP } try { - TxInputStream stream(PROTOCOL_VERSION, txTo, txToLen); - CTransaction tx(deserialize, stream); + TxInputStream stream(txTo, txToLen); + CTransaction tx(deserialize, TX_WITH_WITNESS, stream); std::vector<CTxOut> spent_outputs; if (spentOutputs != nullptr) { @@ -102,7 +98,7 @@ static int verify_script(const unsigned char *scriptPubKey, unsigned int scriptP if (nIn >= tx.vin.size()) return set_error(err, bitcoinconsensus_ERR_TX_INDEX); - if (GetSerializeSize(tx, PROTOCOL_VERSION) != txToLen) + if (GetSerializeSize(TX_WITH_WITNESS(tx)) != txToLen) return set_error(err, bitcoinconsensus_ERR_TX_SIZE_MISMATCH); // Regardless of the verification result, the tx did not error. diff --git a/src/script/descriptor.cpp b/src/script/descriptor.cpp index 7e62d75583..c6bc5f8f1d 100644 --- a/src/script/descriptor.cpp +++ b/src/script/descriptor.cpp @@ -1777,15 +1777,15 @@ std::unique_ptr<DescriptorImpl> ParseScript(uint32_t& key_exp_index, Span<const const auto script_ctx{ctx == ParseScriptContext::P2WSH ? miniscript::MiniscriptContext::P2WSH : miniscript::MiniscriptContext::TAPSCRIPT}; KeyParser parser(/*out = */&out, /* in = */nullptr, /* ctx = */script_ctx, key_exp_index); auto node = miniscript::FromString(std::string(expr.begin(), expr.end()), parser); + if (parser.m_key_parsing_error != "") { + error = std::move(parser.m_key_parsing_error); + return nullptr; + } if (node) { if (ctx != ParseScriptContext::P2WSH && ctx != ParseScriptContext::P2TR) { error = "Miniscript expressions can only be used in wsh or tr."; return nullptr; } - if (parser.m_key_parsing_error != "") { - error = std::move(parser.m_key_parsing_error); - return nullptr; - } if (!node->IsSane() || node->IsNotSatisfiable()) { // Try to find the first insane sub for better error reporting. auto insane_node = node.get(); diff --git a/src/script/interpreter.cpp b/src/script/interpreter.cpp index 5f4a1aceb2..c969ce45f1 100644 --- a/src/script/interpreter.cpp +++ b/src/script/interpreter.cpp @@ -1934,7 +1934,7 @@ static bool VerifyWitnessProgram(const CScriptWitness& witness, int witversion, if ((control[0] & TAPROOT_LEAF_MASK) == TAPROOT_LEAF_TAPSCRIPT) { // Tapscript (leaf version 0xc0) exec_script = CScript(script.begin(), script.end()); - execdata.m_validation_weight_left = ::GetSerializeSize(witness.stack, PROTOCOL_VERSION) + VALIDATION_WEIGHT_OFFSET; + execdata.m_validation_weight_left = ::GetSerializeSize(witness.stack) + VALIDATION_WEIGHT_OFFSET; execdata.m_validation_weight_left_init = true; return ExecuteWitnessScript(stack, exec_script, flags, SigVersion::TAPSCRIPT, checker, execdata, serror); } diff --git a/src/script/interpreter.h b/src/script/interpreter.h index ac1013302d..836c2e7982 100644 --- a/src/script/interpreter.h +++ b/src/script/interpreter.h @@ -6,21 +6,23 @@ #ifndef BITCOIN_SCRIPT_INTERPRETER_H #define BITCOIN_SCRIPT_INTERPRETER_H +#include <consensus/amount.h> #include <hash.h> -#include <script/script_error.h> -#include <span.h> #include <primitives/transaction.h> +#include <script/script_error.h> // IWYU pragma: export +#include <span.h> +#include <uint256.h> +#include <cstddef> +#include <cstdint> #include <optional> #include <vector> -#include <stdint.h> class CPubKey; -class XOnlyPubKey; class CScript; -class CTransaction; -class CTxOut; -class uint256; +class CScriptNum; +class XOnlyPubKey; +struct CScriptWitness; /** Signature hash types/flags */ enum diff --git a/src/script/script.cpp b/src/script/script.cpp index 1594d3cc79..80e8d26bcf 100644 --- a/src/script/script.cpp +++ b/src/script/script.cpp @@ -1,11 +1,14 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2021 The Bitcoin Core developers +// Copyright (c) 2009-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <script/script.h> +#include <crypto/common.h> #include <hash.h> +#include <uint256.h> +#include <util/hash_type.h> #include <util/strencodings.h> #include <string> diff --git a/src/script/script.h b/src/script/script.h index c329a2afd6..66d63fae89 100644 --- a/src/script/script.h +++ b/src/script/script.h @@ -1,5 +1,5 @@ // Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2022 The Bitcoin Core developers +// Copyright (c) 2009-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,18 +8,19 @@ #include <attributes.h> #include <crypto/common.h> -#include <prevector.h> +#include <prevector.h> // IWYU pragma: export #include <serialize.h> #include <uint256.h> #include <util/hash_type.h> -#include <assert.h> -#include <climits> +#include <cassert> +#include <cstdint> +#include <cstring> #include <limits> #include <stdexcept> -#include <stdint.h> -#include <string.h> #include <string> +#include <type_traits> +#include <utility> #include <vector> // Maximum number of bytes pushable to the stack diff --git a/src/script/sign.cpp b/src/script/sign.cpp index 251a8420f7..be4b357568 100644 --- a/src/script/sign.cpp +++ b/src/script/sign.cpp @@ -379,7 +379,7 @@ static bool SignTaproot(const SigningProvider& provider, const BaseSignatureCrea result_stack.emplace_back(std::begin(script), std::end(script)); // Push the script result_stack.push_back(*control_blocks.begin()); // Push the smallest control block if (smallest_result_stack.size() == 0 || - GetSerializeSize(result_stack, PROTOCOL_VERSION) < GetSerializeSize(smallest_result_stack, PROTOCOL_VERSION)) { + GetSerializeSize(result_stack) < GetSerializeSize(smallest_result_stack)) { smallest_result_stack = std::move(result_stack); } } diff --git a/src/secp256k1/.cirrus.yml b/src/secp256k1/.cirrus.yml new file mode 100644 index 0000000000..04aa8f2409 --- /dev/null +++ b/src/secp256k1/.cirrus.yml @@ -0,0 +1,95 @@ +env: + ### cirrus config + CIRRUS_CLONE_DEPTH: 1 + ### compiler options + HOST: + WRAPPER_CMD: + # Specific warnings can be disabled with -Wno-error=foo. + # -pedantic-errors is not equivalent to -Werror=pedantic and thus not implied by -Werror according to the GCC manual. + WERROR_CFLAGS: -Werror -pedantic-errors + MAKEFLAGS: -j4 + BUILD: check + ### secp256k1 config + ECMULTWINDOW: auto + ECMULTGENPRECISION: auto + ASM: no + WIDEMUL: auto + WITH_VALGRIND: yes + EXTRAFLAGS: + ### secp256k1 modules + EXPERIMENTAL: no + ECDH: no + RECOVERY: no + SCHNORRSIG: no + ELLSWIFT: no + ### test options + SECP256K1_TEST_ITERS: + BENCH: yes + SECP256K1_BENCH_ITERS: 2 + CTIMETESTS: yes + # Compile and run the tests + EXAMPLES: yes + +cat_logs_snippet: &CAT_LOGS + always: + cat_tests_log_script: + - cat tests.log || true + cat_noverify_tests_log_script: + - cat noverify_tests.log || true + cat_exhaustive_tests_log_script: + - cat exhaustive_tests.log || true + cat_ctime_tests_log_script: + - cat ctime_tests.log || true + cat_bench_log_script: + - cat bench.log || true + cat_config_log_script: + - cat config.log || true + cat_test_env_script: + - cat test_env.log || true + cat_ci_env_script: + - env + +linux_arm64_container_snippet: &LINUX_ARM64_CONTAINER + env_script: + - env | tee /tmp/env + build_script: + - DOCKER_BUILDKIT=1 docker build --file "ci/linux-debian.Dockerfile" --tag="ci_secp256k1_arm" + - docker image prune --force # Cleanup stale layers + test_script: + - docker run --rm --mount "type=bind,src=./,dst=/ci_secp256k1" --env-file /tmp/env --replace --name "ci_secp256k1_arm" "ci_secp256k1_arm" bash -c "cd /ci_secp256k1/ && ./ci/ci.sh" + +task: + name: "ARM64: Linux (Debian stable)" + persistent_worker: + labels: + type: arm64 + env: + ECDH: yes + RECOVERY: yes + SCHNORRSIG: yes + ELLSWIFT: yes + matrix: + # Currently only gcc-snapshot, the other compilers are tested on GHA with QEMU + - env: { CC: 'gcc-snapshot' } + << : *LINUX_ARM64_CONTAINER + << : *CAT_LOGS + +task: + name: "ARM64: Linux (Debian stable), Valgrind" + persistent_worker: + labels: + type: arm64 + env: + ECDH: yes + RECOVERY: yes + SCHNORRSIG: yes + ELLSWIFT: yes + WRAPPER_CMD: 'valgrind --error-exitcode=42' + SECP256K1_TEST_ITERS: 2 + matrix: + - env: { CC: 'gcc' } + - env: { CC: 'clang' } + - env: { CC: 'gcc-snapshot' } + - env: { CC: 'clang-snapshot' } + << : *LINUX_ARM64_CONTAINER + << : *CAT_LOGS diff --git a/src/secp256k1/.github/actions/run-in-docker-action/action.yml b/src/secp256k1/.github/actions/run-in-docker-action/action.yml index d357c3cf75..dbfaa4fece 100644 --- a/src/secp256k1/.github/actions/run-in-docker-action/action.yml +++ b/src/secp256k1/.github/actions/run-in-docker-action/action.yml @@ -14,9 +14,9 @@ inputs: runs: using: "composite" steps: - - uses: docker/setup-buildx-action@v2 + - uses: docker/setup-buildx-action@v3 - - uses: docker/build-push-action@v4 + - uses: docker/build-push-action@v5 id: main_builder continue-on-error: true with: @@ -26,7 +26,7 @@ runs: load: true cache-from: type=gha - - uses: docker/build-push-action@v4 + - uses: docker/build-push-action@v5 id: retry_builder if: steps.main_builder.outcome == 'failure' with: diff --git a/src/secp256k1/.github/workflows/ci.yml b/src/secp256k1/.github/workflows/ci.yml index b9a9eaa82e..4ad905af52 100644 --- a/src/secp256k1/.github/workflows/ci.yml +++ b/src/secp256k1/.github/workflows/ci.yml @@ -47,14 +47,14 @@ jobs: runs-on: ubuntu-latest steps: - name: Set up Docker Buildx - uses: docker/setup-buildx-action@v2 + uses: docker/setup-buildx-action@v3 with: # See: https://github.com/moby/buildkit/issues/3969. driver-opts: | network=host - name: Build container - uses: docker/build-push-action@v4 + uses: docker/build-push-action@v5 with: file: ./ci/linux-debian.Dockerfile tags: linux-debian-image @@ -792,7 +792,7 @@ jobs: - name: Check installation with Autotools env: - CI_INSTALL: ${{ runner.temp }}/${{ github.run_id }}${{ github.action }} + CI_INSTALL: ${{ runner.temp }}/${{ github.run_id }}${{ github.action }}/install run: | ./autogen.sh && ./configure --prefix=${{ env.CI_INSTALL }} && make clean && make install && ls -RlAh ${{ env.CI_INSTALL }} gcc -o ecdsa examples/ecdsa.c $(PKG_CONFIG_PATH=${{ env.CI_INSTALL }}/lib/pkgconfig pkg-config --cflags --libs libsecp256k1) -Wl,-rpath,"${{ env.CI_INSTALL }}/lib" && ./ecdsa diff --git a/src/secp256k1/CHANGELOG.md b/src/secp256k1/CHANGELOG.md index e8d8db5a1e..04ac9b7e5a 100644 --- a/src/secp256k1/CHANGELOG.md +++ b/src/secp256k1/CHANGELOG.md @@ -5,6 +5,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +## [0.4.1] - 2023-12-21 + +#### Changed + - The point multiplication algorithm used for ECDH operations (module `ecdh`) was replaced with a slightly faster one. + - Optional handwritten x86_64 assembly for field operations was removed because modern C compilers are able to output more efficient assembly. This change results in a significant speedup of some library functions when handwritten x86_64 assembly is enabled (`--with-asm=x86_64` in GNU Autotools, `-DSECP256K1_ASM=x86_64` in CMake), which is the default on x86_64. Benchmarks with GCC 10.5.0 show a 10% speedup for `secp256k1_ecdsa_verify` and `secp256k1_schnorrsig_verify`. + +#### ABI Compatibility +The ABI is backward compatible with versions 0.4.0 and 0.3.x. + ## [0.4.0] - 2023-09-04 #### Added @@ -104,7 +115,8 @@ This version was in fact never released. The number was given by the build system since the introduction of autotools in Jan 2014 (ea0fe5a5bf0c04f9cc955b2966b614f5f378c6f6). Therefore, this version number does not uniquely identify a set of source files. -[unreleased]: https://github.com/bitcoin-core/secp256k1/compare/v0.4.0...HEAD +[unreleased]: https://github.com/bitcoin-core/secp256k1/compare/v0.4.1...HEAD +[0.4.1]: https://github.com/bitcoin-core/secp256k1/compare/v0.4.0...v0.4.1 [0.4.0]: https://github.com/bitcoin-core/secp256k1/compare/v0.3.2...v0.4.0 [0.3.2]: https://github.com/bitcoin-core/secp256k1/compare/v0.3.1...v0.3.2 [0.3.1]: https://github.com/bitcoin-core/secp256k1/compare/v0.3.0...v0.3.1 diff --git a/src/secp256k1/CMakeLists.txt b/src/secp256k1/CMakeLists.txt index cdac47ba9d..cf0dc3ba93 100644 --- a/src/secp256k1/CMakeLists.txt +++ b/src/secp256k1/CMakeLists.txt @@ -11,7 +11,7 @@ project(libsecp256k1 # The package (a.k.a. release) version is based on semantic versioning 2.0.0 of # the API. All changes in experimental modules are treated as # backwards-compatible and therefore at most increase the minor version. - VERSION 0.4.0 + VERSION 0.4.2 DESCRIPTION "Optimized C library for ECDSA signatures and secret/public key operations on curve secp256k1." HOMEPAGE_URL "https://github.com/bitcoin-core/secp256k1" LANGUAGES C @@ -35,7 +35,7 @@ endif() # All changes in experimental modules are treated as if they don't affect the # interface and therefore only increase the revision. set(${PROJECT_NAME}_LIB_VERSION_CURRENT 3) -set(${PROJECT_NAME}_LIB_VERSION_REVISION 0) +set(${PROJECT_NAME}_LIB_VERSION_REVISION 2) set(${PROJECT_NAME}_LIB_VERSION_AGE 1) set(CMAKE_C_STANDARD 90) @@ -107,7 +107,7 @@ if(SECP256K1_TEST_OVERRIDE_WIDE_MULTIPLY) endif() mark_as_advanced(FORCE SECP256K1_TEST_OVERRIDE_WIDE_MULTIPLY) -set(SECP256K1_ASM "AUTO" CACHE STRING "Assembly optimizations to use: \"AUTO\", \"OFF\", \"x86_64\" or \"arm32\" (experimental). [default=AUTO]") +set(SECP256K1_ASM "AUTO" CACHE STRING "Assembly to use: \"AUTO\", \"OFF\", \"x86_64\" or \"arm32\" (experimental). [default=AUTO]") set_property(CACHE SECP256K1_ASM PROPERTY STRINGS "AUTO" "OFF" "x86_64" "arm32") check_string_option_value(SECP256K1_ASM) if(SECP256K1_ASM STREQUAL "arm32") @@ -117,7 +117,7 @@ if(SECP256K1_ASM STREQUAL "arm32") if(HAVE_ARM32_ASM) add_compile_definitions(USE_EXTERNAL_ASM=1) else() - message(FATAL_ERROR "ARM32 assembly optimization requested but not available.") + message(FATAL_ERROR "ARM32 assembly requested but not available.") endif() elseif(SECP256K1_ASM) include(CheckX86_64Assembly) @@ -128,14 +128,14 @@ elseif(SECP256K1_ASM) elseif(SECP256K1_ASM STREQUAL "AUTO") set(SECP256K1_ASM "OFF") else() - message(FATAL_ERROR "x86_64 assembly optimization requested but not available.") + message(FATAL_ERROR "x86_64 assembly requested but not available.") endif() endif() option(SECP256K1_EXPERIMENTAL "Allow experimental configuration options." OFF) if(NOT SECP256K1_EXPERIMENTAL) if(SECP256K1_ASM STREQUAL "arm32") - message(FATAL_ERROR "ARM32 assembly optimization is experimental. Use -DSECP256K1_EXPERIMENTAL=ON to allow.") + message(FATAL_ERROR "ARM32 assembly is experimental. Use -DSECP256K1_EXPERIMENTAL=ON to allow.") endif() endif() @@ -280,7 +280,7 @@ message("Parameters:") message(" ecmult window size .................. ${SECP256K1_ECMULT_WINDOW_SIZE}") message(" ecmult gen precision bits ........... ${SECP256K1_ECMULT_GEN_PREC_BITS}") message("Optional features:") -message(" assembly optimization ............... ${SECP256K1_ASM}") +message(" assembly ............................ ${SECP256K1_ASM}") message(" external callbacks .................. ${SECP256K1_USE_EXTERNAL_DEFAULT_CALLBACKS}") if(SECP256K1_TEST_OVERRIDE_WIDE_MULTIPLY) message(" wide multiplication (test-only) ..... ${SECP256K1_TEST_OVERRIDE_WIDE_MULTIPLY}") diff --git a/src/secp256k1/CONTRIBUTING.md b/src/secp256k1/CONTRIBUTING.md new file mode 100644 index 0000000000..a5e457913a --- /dev/null +++ b/src/secp256k1/CONTRIBUTING.md @@ -0,0 +1,107 @@ +# Contributing to libsecp256k1 + +## Scope + +libsecp256k1 is a library for elliptic curve cryptography on the curve secp256k1, not a general-purpose cryptography library. +The library primarily serves the needs of the Bitcoin Core project but provides additional functionality for the benefit of the wider Bitcoin ecosystem. + +## Adding new functionality or modules + +The libsecp256k1 project welcomes contributions in the form of new functionality or modules, provided they are within the project's scope. + +It is the responsibility of the contributors to convince the maintainers that the proposed functionality is within the project's scope, high-quality and maintainable. +Contributors are recommended to provide the following in addition to the new code: + +* **Specification:** + A specification can help significantly in reviewing the new code as it provides documentation and context. + It may justify various design decisions, give a motivation and outline security goals. + If the specification contains pseudocode, a reference implementation or test vectors, these can be used to compare with the proposed libsecp256k1 code. +* **Security Arguments:** + In addition to a defining the security goals, it should be argued that the new functionality meets these goals. + Depending on the nature of the new functionality, a wide range of security arguments are acceptable, ranging from being "obviously secure" to rigorous proofs of security. +* **Relevance Arguments:** + The relevance of the new functionality for the Bitcoin ecosystem should be argued by outlining clear use cases. + +These are not the only factors taken into account when considering to add new functionality. +The proposed new libsecp256k1 code must be of high quality, including API documentation and tests, as well as featuring a misuse-resistant API design. + +We recommend reaching out to other contributors (see [Communication Channels](#communication-channels)) and get feedback before implementing new functionality. + +## Communication channels + +Most communication about libsecp256k1 occurs on the GitHub repository: in issues, pull request or on the discussion board. + +Additionally, there is an IRC channel dedicated to libsecp256k1, with biweekly meetings (see channel topic). +The channel is `#secp256k1` on Libera Chat. +The easiest way to participate on IRC is with the web client, [web.libera.chat](https://web.libera.chat/#secp256k1). +Chat history logs can be found at https://gnusha.org/secp256k1/. + +## Contributor workflow & peer review + +The Contributor Workflow & Peer Review in libsecp256k1 are similar to Bitcoin Core's workflow and review processes described in its [CONTRIBUTING.md](https://github.com/bitcoin/bitcoin/blob/master/CONTRIBUTING.md). + +### Coding conventions + +In addition, libsecp256k1 tries to maintain the following coding conventions: + +* No runtime heap allocation (e.g., no `malloc`) unless explicitly requested by the caller (via `secp256k1_context_create` or `secp256k1_scratch_space_create`, for example). Morever, it should be possible to use the library without any heap allocations. +* The tests should cover all lines and branches of the library (see [Test coverage](#coverage)). +* Operations involving secret data should be tested for being constant time with respect to the secrets (see [src/ctime_tests.c](src/ctime_tests.c)). +* Local variables containing secret data should be cleared explicitly to try to delete secrets from memory. +* Use `secp256k1_memcmp_var` instead of `memcmp` (see [#823](https://github.com/bitcoin-core/secp256k1/issues/823)). + +#### Style conventions + +* Commits should be atomic and diffs should be easy to read. For this reason, do not mix any formatting fixes or code moves with actual code changes. Make sure each individual commit is hygienic: that it builds successfully on its own without warnings, errors, regressions, or test failures. +* New code should adhere to the style of existing, in particular surrounding, code. Other than that, we do not enforce strict rules for code formatting. +* The code conforms to C89. Most notably, that means that only `/* ... */` comments are allowed (no `//` line comments). Moreover, any declarations in a `{ ... }` block (e.g., a function) must appear at the beginning of the block before any statements. When you would like to declare a variable in the middle of a block, you can open a new block: + ```C + void secp256k_foo(void) { + unsigned int x; /* declaration */ + int y = 2*x; /* declaration */ + x = 17; /* statement */ + { + int a, b; /* declaration */ + a = x + y; /* statement */ + secp256k_bar(x, &b); /* statement */ + } + } + ``` +* Use `unsigned int` instead of just `unsigned`. +* Use `void *ptr` instead of `void* ptr`. +* Arguments of the publicly-facing API must have a specific order defined in [include/secp256k1.h](include/secp256k1.h). +* User-facing comment lines in headers should be limited to 80 chars if possible. +* All identifiers in file scope should start with `secp256k1_`. +* Avoid trailing whitespace. + +### Tests + +#### Coverage + +This library aims to have full coverage of reachable lines and branches. + +To create a test coverage report, configure with `--enable-coverage` (use of GCC is necessary): + + $ ./configure --enable-coverage + +Run the tests: + + $ make check + +To create a report, `gcovr` is recommended, as it includes branch coverage reporting: + + $ gcovr --exclude 'src/bench*' --print-summary + +To create a HTML report with coloured and annotated source code: + + $ mkdir -p coverage + $ gcovr --exclude 'src/bench*' --html --html-details -o coverage/coverage.html + +#### Exhaustive tests + +There are tests of several functions in which a small group replaces secp256k1. +These tests are *exhaustive* since they provide all elements and scalars of the small group as input arguments (see [src/tests_exhaustive.c](src/tests_exhaustive.c)). + +### Benchmarks + +See `src/bench*.c` for examples of benchmarks. diff --git a/src/secp256k1/Makefile.am b/src/secp256k1/Makefile.am index 32bc729a41..5498617915 100644 --- a/src/secp256k1/Makefile.am +++ b/src/secp256k1/Makefile.am @@ -37,7 +37,6 @@ noinst_HEADERS += src/field_10x26_impl.h noinst_HEADERS += src/field_5x52.h noinst_HEADERS += src/field_5x52_impl.h noinst_HEADERS += src/field_5x52_int128_impl.h -noinst_HEADERS += src/field_5x52_asm_impl.h noinst_HEADERS += src/modinv32.h noinst_HEADERS += src/modinv32_impl.h noinst_HEADERS += src/modinv64.h @@ -46,6 +45,7 @@ noinst_HEADERS += src/precomputed_ecmult.h noinst_HEADERS += src/precomputed_ecmult_gen.h noinst_HEADERS += src/assumptions.h noinst_HEADERS += src/checkmem.h +noinst_HEADERS += src/testutil.h noinst_HEADERS += src/util.h noinst_HEADERS += src/int128.h noinst_HEADERS += src/int128_impl.h diff --git a/src/secp256k1/README.md b/src/secp256k1/README.md index 19dabe8505..4013e6a93b 100644 --- a/src/secp256k1/README.md +++ b/src/secp256k1/README.md @@ -1,11 +1,10 @@ libsecp256k1 ============ -[](https://cirrus-ci.com/github/bitcoin-core/secp256k1)  [](https://web.libera.chat/#secp256k1) -Optimized C library for ECDSA signatures and secret/public key operations on curve secp256k1. +High-performance high-assurance C library for digital signatures and other cryptographic primitives on the secp256k1 elliptic curve. This library is intended to be the highest quality publicly available library for cryptography on the secp256k1 curve. However, the primary focus of its development has been for usage in the Bitcoin system and usage unlike Bitcoin's may be less well tested, verified, or suffer from a less well thought out interface. Correct usage requires some care and consideration that the library is fit for your application's purpose. @@ -34,7 +33,7 @@ Implementation details * Expose only higher level interfaces to minimize the API surface and improve application security. ("Be difficult to use insecurely.") * Field operations * Optimized implementation of arithmetic modulo the curve's field size (2^256 - 0x1000003D1). - * Using 5 52-bit limbs (including hand-optimized assembly for x86_64, by Diederik Huys). + * Using 5 52-bit limbs * Using 10 26-bit limbs (including hand-optimized assembly for 32-bit ARM, by Wladimir J. van der Laan). * This is an experimental feature that has not received enough scrutiny to satisfy the standard of quality of this library but is made available for testing and review by the community. * Scalar operations @@ -117,28 +116,6 @@ Usage examples can be found in the [examples](examples) directory. To compile th To compile the Schnorr signature and ECDH examples, you also need to configure with `--enable-module-schnorrsig` and `--enable-module-ecdh`. -Test coverage ------------ - -This library aims to have full coverage of the reachable lines and branches. - -To create a test coverage report, configure with `--enable-coverage` (use of GCC is necessary): - - $ ./configure --enable-coverage - -Run the tests: - - $ make check - -To create a report, `gcovr` is recommended, as it includes branch coverage reporting: - - $ gcovr --exclude 'src/bench*' --print-summary - -To create a HTML report with coloured and annotated source code: - - $ mkdir -p coverage - $ gcovr --exclude 'src/bench*' --html --html-details -o coverage/coverage.html - Benchmark ------------ If configured with `--enable-benchmark` (which is the default), binaries for benchmarking the libsecp256k1 functions will be present in the root directory after the build. @@ -155,3 +132,8 @@ Reporting a vulnerability ------------ See [SECURITY.md](SECURITY.md) + +Contributing to libsecp256k1 +------------ + +See [CONTRIBUTING.md](CONTRIBUTING.md) diff --git a/src/secp256k1/ci/ci.sh b/src/secp256k1/ci/ci.sh index 719e7851ef..9cc715955e 100755 --- a/src/secp256k1/ci/ci.sh +++ b/src/secp256k1/ci/ci.sh @@ -83,7 +83,21 @@ esac --host="$HOST" $EXTRAFLAGS # We have set "-j<n>" in MAKEFLAGS. -make +build_exit_code=0 +make > make.log 2>&1 || build_exit_code=$? +cat make.log +if [ $build_exit_code -ne 0 ]; then + case "${CC:-undefined}" in + *snapshot*) + # Ignore internal compiler errors in gcc-snapshot and clang-snapshot + grep -e "internal compiler error:" -e "PLEASE submit a bug report" make.log + return $?; + ;; + *) + return 1; + ;; + esac +fi # Print information about binaries so that we can see that the architecture is correct file *tests* || true diff --git a/src/secp256k1/ci/linux-debian.Dockerfile b/src/secp256k1/ci/linux-debian.Dockerfile index e719907e89..5ce715b41b 100644 --- a/src/secp256k1/ci/linux-debian.Dockerfile +++ b/src/secp256k1/ci/linux-debian.Dockerfile @@ -29,11 +29,15 @@ RUN apt-get update && apt-get install --no-install-recommends -y \ gcc-i686-linux-gnu libc6-dev-i386-cross libc6-dbg:i386 libubsan1:i386 libasan8:i386 \ gcc-s390x-linux-gnu libc6-dev-s390x-cross libc6-dbg:s390x \ gcc-arm-linux-gnueabihf libc6-dev-armhf-cross libc6-dbg:armhf \ - gcc-aarch64-linux-gnu libc6-dev-arm64-cross libc6-dbg:arm64 \ gcc-powerpc64le-linux-gnu libc6-dev-ppc64el-cross libc6-dbg:ppc64el \ gcc-mingw-w64-x86-64-win32 wine64 wine \ gcc-mingw-w64-i686-win32 wine32 \ - python3 + python3 && \ + if ! ( dpkg --print-architecture | grep --quiet "arm64" ) ; then \ + apt-get install --no-install-recommends -y \ + gcc-aarch64-linux-gnu libc6-dev-arm64-cross libc6-dbg:arm64 ;\ + fi && \ + apt-get clean && rm -rf /var/lib/apt/lists/* # Build and install gcc snapshot ARG GCC_SNAPSHOT_MAJOR=14 @@ -44,7 +48,7 @@ RUN apt-get update && apt-get install --no-install-recommends -y wget libgmp-dev sha512sum --check --ignore-missing sha512.sum && \ # We should have downloaded exactly one tar.xz file ls && \ - [[ $(ls *.tar.xz | wc -l) -eq "1" ]] && \ + [ $(ls *.tar.xz | wc -l) -eq "1" ] && \ tar xf *.tar.xz && \ mkdir gcc-build && cd gcc-build && \ ../*/configure --prefix=/opt/gcc-snapshot --enable-languages=c --disable-bootstrap --disable-multilib --without-isl && \ diff --git a/src/secp256k1/cmake/GeneratePkgConfigFile.cmake b/src/secp256k1/cmake/GeneratePkgConfigFile.cmake new file mode 100644 index 0000000000..9c1d7f1ddd --- /dev/null +++ b/src/secp256k1/cmake/GeneratePkgConfigFile.cmake @@ -0,0 +1,8 @@ +function(generate_pkg_config_file in_file) + set(prefix ${CMAKE_INSTALL_PREFIX}) + set(exec_prefix \${prefix}) + set(libdir \${exec_prefix}/${CMAKE_INSTALL_LIBDIR}) + set(includedir \${prefix}/${CMAKE_INSTALL_INCLUDEDIR}) + set(PACKAGE_VERSION ${PROJECT_VERSION}) + configure_file(${in_file} ${PROJECT_NAME}.pc @ONLY) +endfunction() diff --git a/src/secp256k1/configure.ac b/src/secp256k1/configure.ac index e3877850d3..2c1596775e 100644 --- a/src/secp256k1/configure.ac +++ b/src/secp256k1/configure.ac @@ -5,8 +5,8 @@ AC_PREREQ([2.60]) # backwards-compatible and therefore at most increase the minor version. define(_PKG_VERSION_MAJOR, 0) define(_PKG_VERSION_MINOR, 4) -define(_PKG_VERSION_PATCH, 0) -define(_PKG_VERSION_IS_RELEASE, true) +define(_PKG_VERSION_PATCH, 2) +define(_PKG_VERSION_IS_RELEASE, false) # The library version is based on libtool versioning of the ABI. The set of # rules for updating the version can be found here: @@ -14,7 +14,7 @@ define(_PKG_VERSION_IS_RELEASE, true) # All changes in experimental modules are treated as if they don't affect the # interface and therefore only increase the revision. define(_LIB_VERSION_CURRENT, 3) -define(_LIB_VERSION_REVISION, 0) +define(_LIB_VERSION_REVISION, 2) define(_LIB_VERSION_AGE, 1) AC_INIT([libsecp256k1],m4_join([.], _PKG_VERSION_MAJOR, _PKG_VERSION_MINOR, _PKG_VERSION_PATCH)m4_if(_PKG_VERSION_IS_RELEASE, [true], [], [-dev]),[https://github.com/bitcoin-core/secp256k1/issues],[libsecp256k1],[https://github.com/bitcoin-core/secp256k1]) @@ -201,7 +201,7 @@ AC_ARG_ENABLE(external_default_callbacks, AC_ARG_WITH([test-override-wide-multiply], [] ,[set_widemul=$withval], [set_widemul=auto]) AC_ARG_WITH([asm], [AS_HELP_STRING([--with-asm=x86_64|arm32|no|auto], -[assembly optimizations to use (experimental: arm32) [default=auto]])],[req_asm=$withval], [req_asm=auto]) +[assembly to use (experimental: arm32) [default=auto]])],[req_asm=$withval], [req_asm=auto]) AC_ARG_WITH([ecmult-window], [AS_HELP_STRING([--with-ecmult-window=SIZE|auto], [window size for ecmult precomputation for verification, specified as integer in range [2..24].] @@ -279,24 +279,24 @@ else x86_64) SECP_X86_64_ASM_CHECK if test x"$has_x86_64_asm" != x"yes"; then - AC_MSG_ERROR([x86_64 assembly optimization requested but not available]) + AC_MSG_ERROR([x86_64 assembly requested but not available]) fi ;; arm32) SECP_ARM32_ASM_CHECK if test x"$has_arm32_asm" != x"yes"; then - AC_MSG_ERROR([ARM32 assembly optimization requested but not available]) + AC_MSG_ERROR([ARM32 assembly requested but not available]) fi ;; no) ;; *) - AC_MSG_ERROR([invalid assembly optimization selection]) + AC_MSG_ERROR([invalid assembly selection]) ;; esac fi -# Select assembly optimization +# Select assembly enable_external_asm=no case $set_asm in @@ -309,7 +309,7 @@ arm32) no) ;; *) - AC_MSG_ERROR([invalid assembly optimizations]) + AC_MSG_ERROR([invalid assembly selection]) ;; esac @@ -425,7 +425,7 @@ if test x"$enable_experimental" = x"yes"; then AC_MSG_NOTICE([******]) else if test x"$set_asm" = x"arm32"; then - AC_MSG_ERROR([ARM32 assembly optimization is experimental. Use --enable-experimental to allow.]) + AC_MSG_ERROR([ARM32 assembly is experimental. Use --enable-experimental to allow.]) fi fi diff --git a/src/secp256k1/doc/release-process.md b/src/secp256k1/doc/release-process.md index ea6087c9ff..51e337a5ab 100644 --- a/src/secp256k1/doc/release-process.md +++ b/src/secp256k1/doc/release-process.md @@ -24,16 +24,21 @@ Perform these checks before creating a release: 2. Check installation with autotools: ```shell dir=$(mktemp -d) -./autogen.sh && ./configure --prefix=$dir && make clean && make install && ls -l $dir/include $dir/lib +./autogen.sh && ./configure --prefix=$dir && make clean && make install && ls -RlAh $dir gcc -o ecdsa examples/ecdsa.c $(PKG_CONFIG_PATH=$dir/lib/pkgconfig pkg-config --cflags --libs libsecp256k1) -Wl,-rpath,"$dir/lib" && ./ecdsa ``` 3. Check installation with CMake: ```shell dir=$(mktemp -d) build=$(mktemp -d) -cmake -B $build -DCMAKE_INSTALL_PREFIX=$dir && cmake --build $build --target install && ls -l $dir/include $dir/lib* +cmake -B $build -DCMAKE_INSTALL_PREFIX=$dir && cmake --build $build --target install && ls -RlAh $dir gcc -o ecdsa examples/ecdsa.c -I $dir/include -L $dir/lib*/ -l secp256k1 -Wl,-rpath,"$dir/lib",-rpath,"$dir/lib64" && ./ecdsa ``` +4. Use the [`check-abi.sh`](/tools/check-abi.sh) tool to ensure there are no unexpected ABI incompatibilities and that the version number and release notes accurately reflect all potential ABI changes. To run this tool, the `abi-dumper` and `abi-compliance-checker` packages are required. + +```shell +tools/check-abi.sh +``` ## Regular release @@ -41,7 +46,7 @@ gcc -o ecdsa examples/ecdsa.c -I $dir/include -L $dir/lib*/ -l secp256k1 -Wl,-rp * finalizes the release notes in [CHANGELOG.md](../CHANGELOG.md) by * adding a section for the release (make sure that the version number is a link to a diff between the previous and new version), * removing the `[Unreleased]` section header, and - * including an entry for `### ABI Compatibility` if it doesn't exist that mentions the library soname of the release, + * including an entry for `### ABI Compatibility` if it doesn't exist, * sets `_PKG_VERSION_IS_RELEASE` to `true` in `configure.ac`, and * if this is not a patch release * updates `_PKG_VERSION_*` and `_LIB_VERSION_*` in `configure.ac` and diff --git a/src/secp256k1/src/CMakeLists.txt b/src/secp256k1/src/CMakeLists.txt index b305751b08..4cbaeb914d 100644 --- a/src/secp256k1/src/CMakeLists.txt +++ b/src/secp256k1/src/CMakeLists.txt @@ -161,5 +161,13 @@ if(SECP256K1_INSTALL) ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config.cmake ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}-config-version.cmake DESTINATION ${CMAKE_INSTALL_LIBDIR}/cmake/${PROJECT_NAME} -) + ) + + include(GeneratePkgConfigFile) + generate_pkg_config_file(${PROJECT_SOURCE_DIR}/libsecp256k1.pc.in) + install( + FILES + ${CMAKE_CURRENT_BINARY_DIR}/${PROJECT_NAME}.pc + DESTINATION ${CMAKE_INSTALL_LIBDIR}/pkgconfig + ) endif() diff --git a/src/secp256k1/src/asm/field_10x26_arm.s b/src/secp256k1/src/asm/field_10x26_arm.s index 42cbf879ef..664b921400 100644 --- a/src/secp256k1/src/asm/field_10x26_arm.s +++ b/src/secp256k1/src/asm/field_10x26_arm.s @@ -913,3 +913,4 @@ secp256k1_fe_sqr_inner: ldmfd sp!, {r4, r5, r6, r7, r8, r9, r10, r11, pc} .size secp256k1_fe_sqr_inner, .-secp256k1_fe_sqr_inner + .section .note.GNU-stack,"",%progbits diff --git a/src/secp256k1/src/bench_internal.c b/src/secp256k1/src/bench_internal.c index f3686dd289..a700684922 100644 --- a/src/secp256k1/src/bench_internal.c +++ b/src/secp256k1/src/bench_internal.c @@ -14,10 +14,28 @@ #include "field_impl.h" #include "group_impl.h" #include "scalar_impl.h" -#include "ecmult_const_impl.h" #include "ecmult_impl.h" #include "bench.h" +static void help(int default_iters) { + printf("Benchmarks various internal routines.\n"); + printf("\n"); + printf("The default number of iterations for each benchmark is %d. This can be\n", default_iters); + printf("customized using the SECP256K1_BENCH_ITERS environment variable.\n"); + printf("\n"); + printf("Usage: ./bench_internal [args]\n"); + printf("By default, all benchmarks will be run.\n"); + printf("args:\n"); + printf(" help : display this help and exit\n"); + printf(" scalar : all scalar operations (add, half, inverse, mul, negate, split)\n"); + printf(" field : all field operations (half, inverse, issquare, mul, normalize, sqr, sqrt)\n"); + printf(" group : all group operations (add, double, to_affine)\n"); + printf(" ecmult : all point multiplication operations (ecmult_wnaf) \n"); + printf(" hash : all hash algorithms (hmac, rng6979, sha256)\n"); + printf(" context : all context object operations (context_create)\n"); + printf("\n"); +} + typedef struct { secp256k1_scalar scalar[2]; secp256k1_fe fe[4]; @@ -98,6 +116,18 @@ static void bench_scalar_negate(void* arg, int iters) { } } +static void bench_scalar_half(void* arg, int iters) { + int i; + bench_inv *data = (bench_inv*)arg; + secp256k1_scalar s = data->scalar[0]; + + for (i = 0; i < iters; i++) { + secp256k1_scalar_half(&s, &s); + } + + data->scalar[0] = s; +} + static void bench_scalar_mul(void* arg, int iters) { int i; bench_inv *data = (bench_inv*)arg; @@ -309,18 +339,6 @@ static void bench_ecmult_wnaf(void* arg, int iters) { CHECK(bits <= 256*iters); } -static void bench_wnaf_const(void* arg, int iters) { - int i, bits = 0, overflow = 0; - bench_inv *data = (bench_inv*)arg; - - for (i = 0; i < iters; i++) { - bits += secp256k1_wnaf_const(data->wnaf, &data->scalar[0], WINDOW_A, 256); - overflow += secp256k1_scalar_add(&data->scalar[0], &data->scalar[0], &data->scalar[1]); - } - CHECK(overflow >= 0); - CHECK(bits <= 256*iters); -} - static void bench_sha256(void* arg, int iters) { int i; bench_inv *data = (bench_inv*)arg; @@ -366,10 +384,22 @@ static void bench_context(void* arg, int iters) { int main(int argc, char **argv) { bench_inv data; - int iters = get_iters(20000); + int default_iters = 20000; + int iters = get_iters(default_iters); int d = argc == 1; /* default */ + + if (argc > 1) { + if (have_flag(argc, argv, "-h") + || have_flag(argc, argv, "--help") + || have_flag(argc, argv, "help")) { + help(default_iters); + return 0; + } + } + print_output_table_header_row(); + if (d || have_flag(argc, argv, "scalar") || have_flag(argc, argv, "half")) run_benchmark("scalar_half", bench_scalar_half, bench_setup, NULL, &data, 10, iters*100); if (d || have_flag(argc, argv, "scalar") || have_flag(argc, argv, "add")) run_benchmark("scalar_add", bench_scalar_add, bench_setup, NULL, &data, 10, iters*100); if (d || have_flag(argc, argv, "scalar") || have_flag(argc, argv, "negate")) run_benchmark("scalar_negate", bench_scalar_negate, bench_setup, NULL, &data, 10, iters*100); if (d || have_flag(argc, argv, "scalar") || have_flag(argc, argv, "mul")) run_benchmark("scalar_mul", bench_scalar_mul, bench_setup, NULL, &data, 10, iters*10); @@ -394,7 +424,6 @@ int main(int argc, char **argv) { if (d || have_flag(argc, argv, "group") || have_flag(argc, argv, "add")) run_benchmark("group_add_zinv_var", bench_group_add_zinv_var, bench_setup, NULL, &data, 10, iters*10); if (d || have_flag(argc, argv, "group") || have_flag(argc, argv, "to_affine")) run_benchmark("group_to_affine_var", bench_group_to_affine_var, bench_setup, NULL, &data, 10, iters); - if (d || have_flag(argc, argv, "ecmult") || have_flag(argc, argv, "wnaf")) run_benchmark("wnaf_const", bench_wnaf_const, bench_setup, NULL, &data, 10, iters); if (d || have_flag(argc, argv, "ecmult") || have_flag(argc, argv, "wnaf")) run_benchmark("ecmult_wnaf", bench_ecmult_wnaf, bench_setup, NULL, &data, 10, iters); if (d || have_flag(argc, argv, "hash") || have_flag(argc, argv, "sha256")) run_benchmark("hash_sha256", bench_sha256, bench_setup, NULL, &data, 10, iters); diff --git a/src/secp256k1/src/ecdsa_impl.h b/src/secp256k1/src/ecdsa_impl.h index e71254d9f9..ce36e85e6a 100644 --- a/src/secp256k1/src/ecdsa_impl.h +++ b/src/secp256k1/src/ecdsa_impl.h @@ -66,8 +66,7 @@ static int secp256k1_der_read_len(size_t *len, const unsigned char **sigp, const } if (lenleft > sizeof(size_t)) { /* The resulting length would exceed the range of a size_t, so - * certainly longer than the passed array size. - */ + * it is certainly longer than the passed array size. */ return 0; } while (lenleft > 0) { @@ -76,7 +75,9 @@ static int secp256k1_der_read_len(size_t *len, const unsigned char **sigp, const lenleft--; } if (*len > (size_t)(sigend - *sigp)) { - /* Result exceeds the length of the passed array. */ + /* Result exceeds the length of the passed array. + (Checking this is the responsibility of the caller but it + can't hurt do it here, too.) */ return 0; } if (*len < 128) { diff --git a/src/secp256k1/src/ecmult_const_impl.h b/src/secp256k1/src/ecmult_const_impl.h index 06f9e53ffd..7dc4aac25d 100644 --- a/src/secp256k1/src/ecmult_const_impl.h +++ b/src/secp256k1/src/ecmult_const_impl.h @@ -1,5 +1,5 @@ /*********************************************************************** - * Copyright (c) 2015 Pieter Wuille, Andrew Poelstra * + * Copyright (c) 2015, 2022 Pieter Wuille, Andrew Poelstra * * Distributed under the MIT software license, see the accompanying * * file COPYING or https://www.opensource.org/licenses/mit-license.php.* ***********************************************************************/ @@ -12,208 +12,259 @@ #include "ecmult_const.h" #include "ecmult_impl.h" +#if defined(EXHAUSTIVE_TEST_ORDER) +/* We need 2^ECMULT_CONST_GROUP_SIZE - 1 to be less than EXHAUSTIVE_TEST_ORDER, because + * the tables cannot have infinities in them (this breaks the effective-affine technique's + * z-ratio tracking) */ +# if EXHAUSTIVE_TEST_ORDER == 199 +# define ECMULT_CONST_GROUP_SIZE 4 +# elif EXHAUSTIVE_TEST_ORDER == 13 +# define ECMULT_CONST_GROUP_SIZE 3 +# elif EXHAUSTIVE_TEST_ORDER == 7 +# define ECMULT_CONST_GROUP_SIZE 2 +# else +# error "Unknown EXHAUSTIVE_TEST_ORDER" +# endif +#else +/* Group size 4 or 5 appears optimal. */ +# define ECMULT_CONST_GROUP_SIZE 5 +#endif + +#define ECMULT_CONST_TABLE_SIZE (1L << (ECMULT_CONST_GROUP_SIZE - 1)) +#define ECMULT_CONST_GROUPS ((129 + ECMULT_CONST_GROUP_SIZE - 1) / ECMULT_CONST_GROUP_SIZE) +#define ECMULT_CONST_BITS (ECMULT_CONST_GROUPS * ECMULT_CONST_GROUP_SIZE) + /** Fill a table 'pre' with precomputed odd multiples of a. * * The resulting point set is brought to a single constant Z denominator, stores the X and Y - * coordinates as ge_storage points in pre, and stores the global Z in globalz. - * It only operates on tables sized for WINDOW_A wnaf multiples. + * coordinates as ge points in pre, and stores the global Z in globalz. + * + * 'pre' must be an array of size ECMULT_CONST_TABLE_SIZE. */ -static void secp256k1_ecmult_odd_multiples_table_globalz_windowa(secp256k1_ge *pre, secp256k1_fe *globalz, const secp256k1_gej *a) { - secp256k1_fe zr[ECMULT_TABLE_SIZE(WINDOW_A)]; +static void secp256k1_ecmult_const_odd_multiples_table_globalz(secp256k1_ge *pre, secp256k1_fe *globalz, const secp256k1_gej *a) { + secp256k1_fe zr[ECMULT_CONST_TABLE_SIZE]; - secp256k1_ecmult_odd_multiples_table(ECMULT_TABLE_SIZE(WINDOW_A), pre, zr, globalz, a); - secp256k1_ge_table_set_globalz(ECMULT_TABLE_SIZE(WINDOW_A), pre, zr); + secp256k1_ecmult_odd_multiples_table(ECMULT_CONST_TABLE_SIZE, pre, zr, globalz, a); + secp256k1_ge_table_set_globalz(ECMULT_CONST_TABLE_SIZE, pre, zr); } -/* This is like `ECMULT_TABLE_GET_GE` but is constant time */ -#define ECMULT_CONST_TABLE_GET_GE(r,pre,n,w) do { \ - int m = 0; \ - /* Extract the sign-bit for a constant time absolute-value. */ \ - int volatile mask = (n) >> (sizeof(n) * CHAR_BIT - 1); \ - int abs_n = ((n) + mask) ^ mask; \ - int idx_n = abs_n >> 1; \ +/* Given a table 'pre' with odd multiples of a point, put in r the signed-bit multiplication of n with that point. + * + * For example, if ECMULT_CONST_GROUP_SIZE is 4, then pre is expected to contain 8 entries: + * [1*P, 3*P, 5*P, 7*P, 9*P, 11*P, 13*P, 15*P]. n is then expected to be a 4-bit integer (range 0-15), and its + * bits are interpreted as signs of powers of two to look up. + * + * For example, if n=4, which is 0100 in binary, which is interpreted as [- + - -], so the looked up value is + * [ -(2^3) + (2^2) - (2^1) - (2^0) ]*P = -7*P. Every valid n translates to an odd number in range [-15,15], + * which means we just need to look up one of the precomputed values, and optionally negate it. + */ +#define ECMULT_CONST_TABLE_GET_GE(r,pre,n) do { \ + unsigned int m = 0; \ + /* If the top bit of n is 0, we want the negation. */ \ + volatile unsigned int negative = ((n) >> (ECMULT_CONST_GROUP_SIZE - 1)) ^ 1; \ + /* Let n[i] be the i-th bit of n, then the index is + * sum(cnot(n[i]) * 2^i, i=0..l-2) + * where cnot(b) = b if n[l-1] = 1 and 1 - b otherwise. + * For example, if n = 4, in binary 0100, the index is 3, in binary 011. + * + * Proof: + * Let + * x = sum((2*n[i] - 1)*2^i, i=0..l-1) + * = 2*sum(n[i] * 2^i, i=0..l-1) - 2^l + 1 + * be the value represented by n. + * The index is (x - 1)/2 if x > 0 and -(x + 1)/2 otherwise. + * Case x > 0: + * n[l-1] = 1 + * index = sum(n[i] * 2^i, i=0..l-1) - 2^(l-1) + * = sum(n[i] * 2^i, i=0..l-2) + * Case x <= 0: + * n[l-1] = 0 + * index = -(2*sum(n[i] * 2^i, i=0..l-1) - 2^l + 2)/2 + * = 2^(l-1) - 1 - sum(n[i] * 2^i, i=0..l-1) + * = sum((1 - n[i]) * 2^i, i=0..l-2) + */ \ + unsigned int index = ((unsigned int)(-negative) ^ n) & ((1U << (ECMULT_CONST_GROUP_SIZE - 1)) - 1U); \ secp256k1_fe neg_y; \ - VERIFY_CHECK(((n) & 1) == 1); \ - VERIFY_CHECK((n) >= -((1 << ((w)-1)) - 1)); \ - VERIFY_CHECK((n) <= ((1 << ((w)-1)) - 1)); \ - VERIFY_SETUP(secp256k1_fe_clear(&(r)->x)); \ - VERIFY_SETUP(secp256k1_fe_clear(&(r)->y)); \ - /* Unconditionally set r->x = (pre)[m].x. r->y = (pre)[m].y. because it's either the correct one \ + VERIFY_CHECK((n) < (1U << ECMULT_CONST_GROUP_SIZE)); \ + VERIFY_CHECK(index < (1U << (ECMULT_CONST_GROUP_SIZE - 1))); \ + /* Unconditionally set r->x = (pre)[m].x. r->y = (pre)[m].y. because it's either the correct one * or will get replaced in the later iterations, this is needed to make sure `r` is initialized. */ \ (r)->x = (pre)[m].x; \ (r)->y = (pre)[m].y; \ - for (m = 1; m < ECMULT_TABLE_SIZE(w); m++) { \ + for (m = 1; m < ECMULT_CONST_TABLE_SIZE; m++) { \ /* This loop is used to avoid secret data in array indices. See * the comment in ecmult_gen_impl.h for rationale. */ \ - secp256k1_fe_cmov(&(r)->x, &(pre)[m].x, m == idx_n); \ - secp256k1_fe_cmov(&(r)->y, &(pre)[m].y, m == idx_n); \ + secp256k1_fe_cmov(&(r)->x, &(pre)[m].x, m == index); \ + secp256k1_fe_cmov(&(r)->y, &(pre)[m].y, m == index); \ } \ (r)->infinity = 0; \ secp256k1_fe_negate(&neg_y, &(r)->y, 1); \ - secp256k1_fe_cmov(&(r)->y, &neg_y, (n) != abs_n); \ + secp256k1_fe_cmov(&(r)->y, &neg_y, negative); \ } while(0) -/** Convert a number to WNAF notation. - * The number becomes represented by sum(2^{wi} * wnaf[i], i=0..WNAF_SIZE(w)+1) - return_val. - * It has the following guarantees: - * - each wnaf[i] an odd integer between -(1 << w) and (1 << w) - * - each wnaf[i] is nonzero - * - the number of words set is always WNAF_SIZE(w) + 1 - * - * Adapted from `The Width-w NAF Method Provides Small Memory and Fast Elliptic Scalar - * Multiplications Secure against Side Channel Attacks`, Okeya and Tagaki. M. Joye (Ed.) - * CT-RSA 2003, LNCS 2612, pp. 328-443, 2003. Springer-Verlag Berlin Heidelberg 2003 - * - * Numbers reference steps of `Algorithm SPA-resistant Width-w NAF with Odd Scalar` on pp. 335 - */ -static int secp256k1_wnaf_const(int *wnaf, const secp256k1_scalar *scalar, int w, int size) { - int global_sign; - int skew; - int word = 0; - - /* 1 2 3 */ - int u_last; - int u; - - int flip; - secp256k1_scalar s = *scalar; - - VERIFY_CHECK(w > 0); - VERIFY_CHECK(size > 0); +/* For K as defined in the comment of secp256k1_ecmult_const, we have several precomputed + * formulas/constants. + * - in exhaustive test mode, we give an explicit expression to compute it at compile time: */ +#ifdef EXHAUSTIVE_TEST_ORDER +static const secp256k1_scalar secp256k1_ecmult_const_K = ((SECP256K1_SCALAR_CONST(0, 0, 0, (1U << (ECMULT_CONST_BITS - 128)) - 2U, 0, 0, 0, 0) + EXHAUSTIVE_TEST_ORDER - 1U) * (1U + EXHAUSTIVE_TEST_LAMBDA)) % EXHAUSTIVE_TEST_ORDER; +/* - for the real secp256k1 group we have constants for various ECMULT_CONST_BITS values. */ +#elif ECMULT_CONST_BITS == 129 +/* For GROUP_SIZE = 1,3. */ +static const secp256k1_scalar secp256k1_ecmult_const_K = SECP256K1_SCALAR_CONST(0xac9c52b3ul, 0x3fa3cf1ful, 0x5ad9e3fdul, 0x77ed9ba4ul, 0xa880b9fcul, 0x8ec739c2ul, 0xe0cfc810ul, 0xb51283ceul); +#elif ECMULT_CONST_BITS == 130 +/* For GROUP_SIZE = 2,5. */ +static const secp256k1_scalar secp256k1_ecmult_const_K = SECP256K1_SCALAR_CONST(0xa4e88a7dul, 0xcb13034eul, 0xc2bdd6bful, 0x7c118d6bul, 0x589ae848ul, 0x26ba29e4ul, 0xb5c2c1dcul, 0xde9798d9ul); +#elif ECMULT_CONST_BITS == 132 +/* For GROUP_SIZE = 4,6 */ +static const secp256k1_scalar secp256k1_ecmult_const_K = SECP256K1_SCALAR_CONST(0x76b1d93dul, 0x0fae3c6bul, 0x3215874bul, 0x94e93813ul, 0x7937fe0dul, 0xb66bcaaful, 0xb3749ca5ul, 0xd7b6171bul); +#else +# error "Unknown ECMULT_CONST_BITS" +#endif - /* Note that we cannot handle even numbers by negating them to be odd, as is - * done in other implementations, since if our scalars were specified to have - * width < 256 for performance reasons, their negations would have width 256 - * and we'd lose any performance benefit. Instead, we use a variation of a - * technique from Section 4.2 of the Okeya/Tagaki paper, which is to add 1 to the - * number we are encoding when it is even, returning a skew value indicating - * this, and having the caller compensate after doing the multiplication. +static void secp256k1_ecmult_const(secp256k1_gej *r, const secp256k1_ge *a, const secp256k1_scalar *q) { + /* The approach below combines the signed-digit logic from Mike Hamburg's + * "Fast and compact elliptic-curve cryptography" (https://eprint.iacr.org/2012/309) + * Section 3.3, with the GLV endomorphism. * - * In fact, we _do_ want to negate numbers to minimize their bit-lengths (and in - * particular, to ensure that the outputs from the endomorphism-split fit into - * 128 bits). If we negate, the parity of our number flips, affecting whether - * we want to add to the scalar to ensure that it's odd. */ - flip = secp256k1_scalar_is_high(&s); - skew = flip ^ secp256k1_scalar_is_even(&s); - secp256k1_scalar_cadd_bit(&s, 0, skew); - global_sign = secp256k1_scalar_cond_negate(&s, flip); - - /* 4 */ - u_last = secp256k1_scalar_shr_int(&s, w); - do { - int even; - - /* 4.1 4.4 */ - u = secp256k1_scalar_shr_int(&s, w); - /* 4.2 */ - even = ((u & 1) == 0); - /* In contrast to the original algorithm, u_last is always > 0 and - * therefore we do not need to check its sign. In particular, it's easy - * to see that u_last is never < 0 because u is never < 0. Moreover, - * u_last is never = 0 because u is never even after a loop - * iteration. The same holds analogously for the initial value of - * u_last (in the first loop iteration). */ - VERIFY_CHECK(u_last > 0); - VERIFY_CHECK((u_last & 1) == 1); - u += even; - u_last -= even * (1 << w); - - /* 4.3, adapted for global sign change */ - wnaf[word++] = u_last * global_sign; - - u_last = u; - } while (word * w < size); - wnaf[word] = u * global_sign; - - VERIFY_CHECK(secp256k1_scalar_is_zero(&s)); - VERIFY_CHECK(word == WNAF_SIZE_BITS(size, w)); - return skew; -} - -static void secp256k1_ecmult_const(secp256k1_gej *r, const secp256k1_ge *a, const secp256k1_scalar *scalar) { - secp256k1_ge pre_a[ECMULT_TABLE_SIZE(WINDOW_A)]; - secp256k1_ge tmpa; - secp256k1_fe Z; - - int skew_1; - secp256k1_ge pre_a_lam[ECMULT_TABLE_SIZE(WINDOW_A)]; - int wnaf_lam[1 + WNAF_SIZE(WINDOW_A - 1)]; - int skew_lam; - secp256k1_scalar q_1, q_lam; - int wnaf_1[1 + WNAF_SIZE(WINDOW_A - 1)]; - - int i; + * The idea there is to interpret the bits of a scalar as signs (1 = +, 0 = -), and compute a + * point multiplication in that fashion. Let v be an n-bit non-negative integer (0 <= v < 2^n), + * and v[i] its i'th bit (so v = sum(v[i] * 2^i, i=0..n-1)). Then define: + * + * C_l(v, A) = sum((2*v[i] - 1) * 2^i*A, i=0..l-1) + * + * Then it holds that C_l(v, A) = sum((2*v[i] - 1) * 2^i*A, i=0..l-1) + * = (2*sum(v[i] * 2^i, i=0..l-1) + 1 - 2^l) * A + * = (2*v + 1 - 2^l) * A + * + * Thus, one can compute q*A as C_256((q + 2^256 - 1) / 2, A). This is the basis for the + * paper's signed-digit multi-comb algorithm for multiplication using a precomputed table. + * + * It is appealing to try to combine this with the GLV optimization: the idea that a scalar + * s can be written as s1 + lambda*s2, where lambda is a curve-specific constant such that + * lambda*A is easy to compute, and where s1 and s2 are small. In particular we have the + * secp256k1_scalar_split_lambda function which performs such a split with the resulting s1 + * and s2 in range (-2^128, 2^128) mod n. This does work, but is uninteresting: + * + * To compute q*A: + * - Let s1, s2 = split_lambda(q) + * - Let R1 = C_256((s1 + 2^256 - 1) / 2, A) + * - Let R2 = C_256((s2 + 2^256 - 1) / 2, lambda*A) + * - Return R1 + R2 + * + * The issue is that while s1 and s2 are small-range numbers, (s1 + 2^256 - 1) / 2 (mod n) + * and (s2 + 2^256 - 1) / 2 (mod n) are not, undoing the benefit of the splitting. + * + * To make it work, we want to modify the input scalar q first, before splitting, and then only + * add a 2^128 offset of the split results (so that they end up in the single 129-bit range + * [0,2^129]). A slightly smaller offset would work due to the bounds on the split, but we pick + * 2^128 for simplicity. Let s be the scalar fed to split_lambda, and f(q) the function to + * compute it from q: + * + * To compute q*A: + * - Compute s = f(q) + * - Let s1, s2 = split_lambda(s) + * - Let v1 = s1 + 2^128 (mod n) + * - Let v2 = s2 + 2^128 (mod n) + * - Let R1 = C_l(v1, A) + * - Let R2 = C_l(v2, lambda*A) + * - Return R1 + R2 + * + * l will thus need to be at least 129, but we may overshoot by a few bits (see + * further), so keep it as a variable. + * + * To solve for s, we reason: + * q*A = R1 + R2 + * <=> q*A = C_l(s1 + 2^128, A) + C_l(s2 + 2^128, lambda*A) + * <=> q*A = (2*(s1 + 2^128) + 1 - 2^l) * A + (2*(s2 + 2^128) + 1 - 2^l) * lambda*A + * <=> q*A = (2*(s1 + s2*lambda) + (2^129 + 1 - 2^l) * (1 + lambda)) * A + * <=> q = 2*(s1 + s2*lambda) + (2^129 + 1 - 2^l) * (1 + lambda) (mod n) + * <=> q = 2*s + (2^129 + 1 - 2^l) * (1 + lambda) (mod n) + * <=> s = (q + (2^l - 2^129 - 1) * (1 + lambda)) / 2 (mod n) + * <=> f(q) = (q + K) / 2 (mod n) + * where K = (2^l - 2^129 - 1)*(1 + lambda) (mod n) + * + * We will process the computation of C_l(v1, A) and C_l(v2, lambda*A) in groups of + * ECMULT_CONST_GROUP_SIZE, so we set l to the smallest multiple of ECMULT_CONST_GROUP_SIZE + * that is not less than 129; this equals ECMULT_CONST_BITS. + */ + /* The offset to add to s1 and s2 to make them non-negative. Equal to 2^128. */ + static const secp256k1_scalar S_OFFSET = SECP256K1_SCALAR_CONST(0, 0, 0, 1, 0, 0, 0, 0); + secp256k1_scalar s, v1, v2; + secp256k1_ge pre_a[ECMULT_CONST_TABLE_SIZE]; + secp256k1_ge pre_a_lam[ECMULT_CONST_TABLE_SIZE]; + secp256k1_fe global_z; + int group, i; + + /* We're allowed to be non-constant time in the point, and the code below (in particular, + * secp256k1_ecmult_const_odd_multiples_table_globalz) cannot deal with infinity in a + * constant-time manner anyway. */ if (secp256k1_ge_is_infinity(a)) { secp256k1_gej_set_infinity(r); return; } - /* build wnaf representation for q. */ - /* split q into q_1 and q_lam (where q = q_1 + q_lam*lambda, and q_1 and q_lam are ~128 bit) */ - secp256k1_scalar_split_lambda(&q_1, &q_lam, scalar); - skew_1 = secp256k1_wnaf_const(wnaf_1, &q_1, WINDOW_A - 1, 128); - skew_lam = secp256k1_wnaf_const(wnaf_lam, &q_lam, WINDOW_A - 1, 128); + /* Compute v1 and v2. */ + secp256k1_scalar_add(&s, q, &secp256k1_ecmult_const_K); + secp256k1_scalar_half(&s, &s); + secp256k1_scalar_split_lambda(&v1, &v2, &s); + secp256k1_scalar_add(&v1, &v1, &S_OFFSET); + secp256k1_scalar_add(&v2, &v2, &S_OFFSET); - /* Calculate odd multiples of a. +#ifdef VERIFY + /* Verify that v1 and v2 are in range [0, 2^129-1]. */ + for (i = 129; i < 256; ++i) { + VERIFY_CHECK(secp256k1_scalar_get_bits(&v1, i, 1) == 0); + VERIFY_CHECK(secp256k1_scalar_get_bits(&v2, i, 1) == 0); + } +#endif + + /* Calculate odd multiples of A and A*lambda. * All multiples are brought to the same Z 'denominator', which is stored - * in Z. Due to secp256k1' isomorphism we can do all operations pretending + * in global_z. Due to secp256k1' isomorphism we can do all operations pretending * that the Z coordinate was 1, use affine addition formulae, and correct * the Z coordinate of the result once at the end. */ - VERIFY_CHECK(!a->infinity); secp256k1_gej_set_ge(r, a); - secp256k1_ecmult_odd_multiples_table_globalz_windowa(pre_a, &Z, r); - for (i = 0; i < ECMULT_TABLE_SIZE(WINDOW_A); i++) { - secp256k1_fe_normalize_weak(&pre_a[i].y); - } - for (i = 0; i < ECMULT_TABLE_SIZE(WINDOW_A); i++) { + secp256k1_ecmult_const_odd_multiples_table_globalz(pre_a, &global_z, r); + for (i = 0; i < ECMULT_CONST_TABLE_SIZE; i++) { secp256k1_ge_mul_lambda(&pre_a_lam[i], &pre_a[i]); } - /* first loop iteration (separated out so we can directly set r, rather - * than having it start at infinity, get doubled several times, then have - * its new value added to it) */ - i = wnaf_1[WNAF_SIZE_BITS(128, WINDOW_A - 1)]; - VERIFY_CHECK(i != 0); - ECMULT_CONST_TABLE_GET_GE(&tmpa, pre_a, i, WINDOW_A); - secp256k1_gej_set_ge(r, &tmpa); - i = wnaf_lam[WNAF_SIZE_BITS(128, WINDOW_A - 1)]; - VERIFY_CHECK(i != 0); - ECMULT_CONST_TABLE_GET_GE(&tmpa, pre_a_lam, i, WINDOW_A); - secp256k1_gej_add_ge(r, r, &tmpa); - /* remaining loop iterations */ - for (i = WNAF_SIZE_BITS(128, WINDOW_A - 1) - 1; i >= 0; i--) { - int n; + /* Next, we compute r = C_l(v1, A) + C_l(v2, lambda*A). + * + * We proceed in groups of ECMULT_CONST_GROUP_SIZE bits, operating on that many bits + * at a time, from high in v1, v2 to low. Call these bits1 (from v1) and bits2 (from v2). + * + * Now note that ECMULT_CONST_TABLE_GET_GE(&t, pre_a, bits1) loads into t a point equal + * to C_{ECMULT_CONST_GROUP_SIZE}(bits1, A), and analogously for pre_lam_a / bits2. + * This means that all we need to do is add these looked up values together, multiplied + * by 2^(ECMULT_GROUP_SIZE * group). + */ + for (group = ECMULT_CONST_GROUPS - 1; group >= 0; --group) { + /* Using the _var get_bits function is ok here, since it's only variable in offset and count, not in the scalar. */ + unsigned int bits1 = secp256k1_scalar_get_bits_var(&v1, group * ECMULT_CONST_GROUP_SIZE, ECMULT_CONST_GROUP_SIZE); + unsigned int bits2 = secp256k1_scalar_get_bits_var(&v2, group * ECMULT_CONST_GROUP_SIZE, ECMULT_CONST_GROUP_SIZE); + secp256k1_ge t; int j; - for (j = 0; j < WINDOW_A - 1; ++j) { - secp256k1_gej_double(r, r); - } - - n = wnaf_1[i]; - ECMULT_CONST_TABLE_GET_GE(&tmpa, pre_a, n, WINDOW_A); - VERIFY_CHECK(n != 0); - secp256k1_gej_add_ge(r, r, &tmpa); - n = wnaf_lam[i]; - ECMULT_CONST_TABLE_GET_GE(&tmpa, pre_a_lam, n, WINDOW_A); - VERIFY_CHECK(n != 0); - secp256k1_gej_add_ge(r, r, &tmpa); - } - - { - /* Correct for wNAF skew */ - secp256k1_gej tmpj; - - secp256k1_ge_neg(&tmpa, &pre_a[0]); - secp256k1_gej_add_ge(&tmpj, r, &tmpa); - secp256k1_gej_cmov(r, &tmpj, skew_1); - secp256k1_ge_neg(&tmpa, &pre_a_lam[0]); - secp256k1_gej_add_ge(&tmpj, r, &tmpa); - secp256k1_gej_cmov(r, &tmpj, skew_lam); + ECMULT_CONST_TABLE_GET_GE(&t, pre_a, bits1); + if (group == ECMULT_CONST_GROUPS - 1) { + /* Directly set r in the first iteration. */ + secp256k1_gej_set_ge(r, &t); + } else { + /* Shift the result so far up. */ + for (j = 0; j < ECMULT_CONST_GROUP_SIZE; ++j) { + secp256k1_gej_double(r, r); + } + secp256k1_gej_add_ge(r, r, &t); + } + ECMULT_CONST_TABLE_GET_GE(&t, pre_a_lam, bits2); + secp256k1_gej_add_ge(r, r, &t); } - secp256k1_fe_mul(&r->z, &r->z, &Z); + /* Map the result back to the secp256k1 curve from the isomorphic curve. */ + secp256k1_fe_mul(&r->z, &r->z, &global_z); } static int secp256k1_ecmult_const_xonly(secp256k1_fe* r, const secp256k1_fe *n, const secp256k1_fe *d, const secp256k1_scalar *q, int known_on_curve) { @@ -296,9 +347,7 @@ static int secp256k1_ecmult_const_xonly(secp256k1_fe* r, const secp256k1_fe *n, secp256k1_fe_mul(&g, &g, n); if (d) { secp256k1_fe b; -#ifdef VERIFY VERIFY_CHECK(!secp256k1_fe_normalizes_to_zero(d)); -#endif secp256k1_fe_sqr(&b, d); VERIFY_CHECK(SECP256K1_B <= 8); /* magnitude of b will be <= 8 after the next call */ secp256k1_fe_mul_int(&b, SECP256K1_B); @@ -331,13 +380,9 @@ static int secp256k1_ecmult_const_xonly(secp256k1_fe* r, const secp256k1_fe *n, p.infinity = 0; /* Perform x-only EC multiplication of P with q. */ -#ifdef VERIFY VERIFY_CHECK(!secp256k1_scalar_is_zero(q)); -#endif secp256k1_ecmult_const(&rj, &p, q); -#ifdef VERIFY VERIFY_CHECK(!secp256k1_gej_is_infinity(&rj)); -#endif /* The resulting (X, Y, Z) point on the effective-affine isomorphic curve corresponds to * (X, Y, Z*v) on the secp256k1 curve. The affine version of that has X coordinate diff --git a/src/secp256k1/src/field.h b/src/secp256k1/src/field.h index ccd228e1ae..bd589bf8a8 100644 --- a/src/secp256k1/src/field.h +++ b/src/secp256k1/src/field.h @@ -184,7 +184,8 @@ static int secp256k1_fe_equal(const secp256k1_fe *a, const secp256k1_fe *b); */ static int secp256k1_fe_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b); -/** Set a field element equal to a provided 32-byte big endian value, reducing it. +/** Set a field element equal to the element represented by a provided 32-byte big endian value + * interpreted modulo p. * * On input, r does not need to be initialized. a must be a pointer to an initialized 32-byte array. * On output, r = a (mod p). It will have magnitude 1, and not be normalized. @@ -345,8 +346,10 @@ static int secp256k1_fe_is_square_var(const secp256k1_fe *a); /** Check invariants on a field element (no-op unless VERIFY is enabled). */ static void secp256k1_fe_verify(const secp256k1_fe *a); +#define SECP256K1_FE_VERIFY(a) secp256k1_fe_verify(a) /** Check that magnitude of a is at most m (no-op unless VERIFY is enabled). */ static void secp256k1_fe_verify_magnitude(const secp256k1_fe *a, int m); +#define SECP256K1_FE_VERIFY_MAGNITUDE(a, m) secp256k1_fe_verify_magnitude(a, m) #endif /* SECP256K1_FIELD_H */ diff --git a/src/secp256k1/src/field_10x26_impl.h b/src/secp256k1/src/field_10x26_impl.h index 8445db1639..666068c712 100644 --- a/src/secp256k1/src/field_10x26_impl.h +++ b/src/secp256k1/src/field_10x26_impl.h @@ -403,11 +403,7 @@ void secp256k1_fe_sqr_inner(uint32_t *r, const uint32_t *a); #else -#ifdef VERIFY #define VERIFY_BITS(x, n) VERIFY_CHECK(((x) >> (n)) == 0) -#else -#define VERIFY_BITS(x, n) do { } while(0) -#endif SECP256K1_INLINE static void secp256k1_fe_mul_inner(uint32_t *r, const uint32_t *a, const uint32_t * SECP256K1_RESTRICT b) { uint64_t c, d; diff --git a/src/secp256k1/src/field_5x52_asm_impl.h b/src/secp256k1/src/field_5x52_asm_impl.h deleted file mode 100644 index 04a9af2105..0000000000 --- a/src/secp256k1/src/field_5x52_asm_impl.h +++ /dev/null @@ -1,504 +0,0 @@ -/*********************************************************************** - * Copyright (c) 2013-2014 Diederik Huys, Pieter Wuille * - * Distributed under the MIT software license, see the accompanying * - * file COPYING or https://www.opensource.org/licenses/mit-license.php.* - ***********************************************************************/ - -/** - * Changelog: - * - March 2013, Diederik Huys: original version - * - November 2014, Pieter Wuille: updated to use Peter Dettman's parallel multiplication algorithm - * - December 2014, Pieter Wuille: converted from YASM to GCC inline assembly - */ - -#ifndef SECP256K1_FIELD_INNER5X52_IMPL_H -#define SECP256K1_FIELD_INNER5X52_IMPL_H - -#include "util.h" - -SECP256K1_INLINE static void secp256k1_fe_mul_inner(uint64_t *r, const uint64_t *a, const uint64_t * SECP256K1_RESTRICT b) { -/** - * Registers: rdx:rax = multiplication accumulator - * r9:r8 = c - * r15:rcx = d - * r10-r14 = a0-a4 - * rbx = b - * rdi = r - * rsi = a / t? - */ - uint64_t tmp1, tmp2, tmp3; -__asm__ __volatile__( - "movq 0(%%rsi),%%r10\n" - "movq 8(%%rsi),%%r11\n" - "movq 16(%%rsi),%%r12\n" - "movq 24(%%rsi),%%r13\n" - "movq 32(%%rsi),%%r14\n" - - /* d += a3 * b0 */ - "movq 0(%%rbx),%%rax\n" - "mulq %%r13\n" - "movq %%rax,%%rcx\n" - "movq %%rdx,%%r15\n" - /* d += a2 * b1 */ - "movq 8(%%rbx),%%rax\n" - "mulq %%r12\n" - "addq %%rax,%%rcx\n" - "adcq %%rdx,%%r15\n" - /* d += a1 * b2 */ - "movq 16(%%rbx),%%rax\n" - "mulq %%r11\n" - "addq %%rax,%%rcx\n" - "adcq %%rdx,%%r15\n" - /* d = a0 * b3 */ - "movq 24(%%rbx),%%rax\n" - "mulq %%r10\n" - "addq %%rax,%%rcx\n" - "adcq %%rdx,%%r15\n" - /* c = a4 * b4 */ - "movq 32(%%rbx),%%rax\n" - "mulq %%r14\n" - "movq %%rax,%%r8\n" - "movq %%rdx,%%r9\n" - /* d += (c & M) * R */ - "movq $0xfffffffffffff,%%rdx\n" - "andq %%rdx,%%rax\n" - "movq $0x1000003d10,%%rdx\n" - "mulq %%rdx\n" - "addq %%rax,%%rcx\n" - "adcq %%rdx,%%r15\n" - /* c >>= 52 (%%r8 only) */ - "shrdq $52,%%r9,%%r8\n" - /* t3 (tmp1) = d & M */ - "movq %%rcx,%%rsi\n" - "movq $0xfffffffffffff,%%rdx\n" - "andq %%rdx,%%rsi\n" - "movq %%rsi,%q1\n" - /* d >>= 52 */ - "shrdq $52,%%r15,%%rcx\n" - "xorq %%r15,%%r15\n" - /* d += a4 * b0 */ - "movq 0(%%rbx),%%rax\n" - "mulq %%r14\n" - "addq %%rax,%%rcx\n" - "adcq %%rdx,%%r15\n" - /* d += a3 * b1 */ - "movq 8(%%rbx),%%rax\n" - "mulq %%r13\n" - "addq %%rax,%%rcx\n" - "adcq %%rdx,%%r15\n" - /* d += a2 * b2 */ - "movq 16(%%rbx),%%rax\n" - "mulq %%r12\n" - "addq %%rax,%%rcx\n" - "adcq %%rdx,%%r15\n" - /* d += a1 * b3 */ - "movq 24(%%rbx),%%rax\n" - "mulq %%r11\n" - "addq %%rax,%%rcx\n" - "adcq %%rdx,%%r15\n" - /* d += a0 * b4 */ - "movq 32(%%rbx),%%rax\n" - "mulq %%r10\n" - "addq %%rax,%%rcx\n" - "adcq %%rdx,%%r15\n" - /* d += c * R */ - "movq %%r8,%%rax\n" - "movq $0x1000003d10,%%rdx\n" - "mulq %%rdx\n" - "addq %%rax,%%rcx\n" - "adcq %%rdx,%%r15\n" - /* t4 = d & M (%%rsi) */ - "movq %%rcx,%%rsi\n" - "movq $0xfffffffffffff,%%rdx\n" - "andq %%rdx,%%rsi\n" - /* d >>= 52 */ - "shrdq $52,%%r15,%%rcx\n" - "xorq %%r15,%%r15\n" - /* tx = t4 >> 48 (tmp3) */ - "movq %%rsi,%%rax\n" - "shrq $48,%%rax\n" - "movq %%rax,%q3\n" - /* t4 &= (M >> 4) (tmp2) */ - "movq $0xffffffffffff,%%rax\n" - "andq %%rax,%%rsi\n" - "movq %%rsi,%q2\n" - /* c = a0 * b0 */ - "movq 0(%%rbx),%%rax\n" - "mulq %%r10\n" - "movq %%rax,%%r8\n" - "movq %%rdx,%%r9\n" - /* d += a4 * b1 */ - "movq 8(%%rbx),%%rax\n" - "mulq %%r14\n" - "addq %%rax,%%rcx\n" - "adcq %%rdx,%%r15\n" - /* d += a3 * b2 */ - "movq 16(%%rbx),%%rax\n" - "mulq %%r13\n" - "addq %%rax,%%rcx\n" - "adcq %%rdx,%%r15\n" - /* d += a2 * b3 */ - "movq 24(%%rbx),%%rax\n" - "mulq %%r12\n" - "addq %%rax,%%rcx\n" - "adcq %%rdx,%%r15\n" - /* d += a1 * b4 */ - "movq 32(%%rbx),%%rax\n" - "mulq %%r11\n" - "addq %%rax,%%rcx\n" - "adcq %%rdx,%%r15\n" - /* u0 = d & M (%%rsi) */ - "movq %%rcx,%%rsi\n" - "movq $0xfffffffffffff,%%rdx\n" - "andq %%rdx,%%rsi\n" - /* d >>= 52 */ - "shrdq $52,%%r15,%%rcx\n" - "xorq %%r15,%%r15\n" - /* u0 = (u0 << 4) | tx (%%rsi) */ - "shlq $4,%%rsi\n" - "movq %q3,%%rax\n" - "orq %%rax,%%rsi\n" - /* c += u0 * (R >> 4) */ - "movq $0x1000003d1,%%rax\n" - "mulq %%rsi\n" - "addq %%rax,%%r8\n" - "adcq %%rdx,%%r9\n" - /* r[0] = c & M */ - "movq %%r8,%%rax\n" - "movq $0xfffffffffffff,%%rdx\n" - "andq %%rdx,%%rax\n" - "movq %%rax,0(%%rdi)\n" - /* c >>= 52 */ - "shrdq $52,%%r9,%%r8\n" - "xorq %%r9,%%r9\n" - /* c += a1 * b0 */ - "movq 0(%%rbx),%%rax\n" - "mulq %%r11\n" - "addq %%rax,%%r8\n" - "adcq %%rdx,%%r9\n" - /* c += a0 * b1 */ - "movq 8(%%rbx),%%rax\n" - "mulq %%r10\n" - "addq %%rax,%%r8\n" - "adcq %%rdx,%%r9\n" - /* d += a4 * b2 */ - "movq 16(%%rbx),%%rax\n" - "mulq %%r14\n" - "addq %%rax,%%rcx\n" - "adcq %%rdx,%%r15\n" - /* d += a3 * b3 */ - "movq 24(%%rbx),%%rax\n" - "mulq %%r13\n" - "addq %%rax,%%rcx\n" - "adcq %%rdx,%%r15\n" - /* d += a2 * b4 */ - "movq 32(%%rbx),%%rax\n" - "mulq %%r12\n" - "addq %%rax,%%rcx\n" - "adcq %%rdx,%%r15\n" - /* c += (d & M) * R */ - "movq %%rcx,%%rax\n" - "movq $0xfffffffffffff,%%rdx\n" - "andq %%rdx,%%rax\n" - "movq $0x1000003d10,%%rdx\n" - "mulq %%rdx\n" - "addq %%rax,%%r8\n" - "adcq %%rdx,%%r9\n" - /* d >>= 52 */ - "shrdq $52,%%r15,%%rcx\n" - "xorq %%r15,%%r15\n" - /* r[1] = c & M */ - "movq %%r8,%%rax\n" - "movq $0xfffffffffffff,%%rdx\n" - "andq %%rdx,%%rax\n" - "movq %%rax,8(%%rdi)\n" - /* c >>= 52 */ - "shrdq $52,%%r9,%%r8\n" - "xorq %%r9,%%r9\n" - /* c += a2 * b0 */ - "movq 0(%%rbx),%%rax\n" - "mulq %%r12\n" - "addq %%rax,%%r8\n" - "adcq %%rdx,%%r9\n" - /* c += a1 * b1 */ - "movq 8(%%rbx),%%rax\n" - "mulq %%r11\n" - "addq %%rax,%%r8\n" - "adcq %%rdx,%%r9\n" - /* c += a0 * b2 (last use of %%r10 = a0) */ - "movq 16(%%rbx),%%rax\n" - "mulq %%r10\n" - "addq %%rax,%%r8\n" - "adcq %%rdx,%%r9\n" - /* fetch t3 (%%r10, overwrites a0), t4 (%%rsi) */ - "movq %q2,%%rsi\n" - "movq %q1,%%r10\n" - /* d += a4 * b3 */ - "movq 24(%%rbx),%%rax\n" - "mulq %%r14\n" - "addq %%rax,%%rcx\n" - "adcq %%rdx,%%r15\n" - /* d += a3 * b4 */ - "movq 32(%%rbx),%%rax\n" - "mulq %%r13\n" - "addq %%rax,%%rcx\n" - "adcq %%rdx,%%r15\n" - /* c += (d & M) * R */ - "movq %%rcx,%%rax\n" - "movq $0xfffffffffffff,%%rdx\n" - "andq %%rdx,%%rax\n" - "movq $0x1000003d10,%%rdx\n" - "mulq %%rdx\n" - "addq %%rax,%%r8\n" - "adcq %%rdx,%%r9\n" - /* d >>= 52 (%%rcx only) */ - "shrdq $52,%%r15,%%rcx\n" - /* r[2] = c & M */ - "movq %%r8,%%rax\n" - "movq $0xfffffffffffff,%%rdx\n" - "andq %%rdx,%%rax\n" - "movq %%rax,16(%%rdi)\n" - /* c >>= 52 */ - "shrdq $52,%%r9,%%r8\n" - "xorq %%r9,%%r9\n" - /* c += t3 */ - "addq %%r10,%%r8\n" - /* c += d * R */ - "movq %%rcx,%%rax\n" - "movq $0x1000003d10,%%rdx\n" - "mulq %%rdx\n" - "addq %%rax,%%r8\n" - "adcq %%rdx,%%r9\n" - /* r[3] = c & M */ - "movq %%r8,%%rax\n" - "movq $0xfffffffffffff,%%rdx\n" - "andq %%rdx,%%rax\n" - "movq %%rax,24(%%rdi)\n" - /* c >>= 52 (%%r8 only) */ - "shrdq $52,%%r9,%%r8\n" - /* c += t4 (%%r8 only) */ - "addq %%rsi,%%r8\n" - /* r[4] = c */ - "movq %%r8,32(%%rdi)\n" -: "+S"(a), "=&m"(tmp1), "=&m"(tmp2), "=&m"(tmp3) -: "b"(b), "D"(r) -: "%rax", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory" -); -} - -SECP256K1_INLINE static void secp256k1_fe_sqr_inner(uint64_t *r, const uint64_t *a) { -/** - * Registers: rdx:rax = multiplication accumulator - * r9:r8 = c - * rcx:rbx = d - * r10-r14 = a0-a4 - * r15 = M (0xfffffffffffff) - * rdi = r - * rsi = a / t? - */ - uint64_t tmp1, tmp2, tmp3; -__asm__ __volatile__( - "movq 0(%%rsi),%%r10\n" - "movq 8(%%rsi),%%r11\n" - "movq 16(%%rsi),%%r12\n" - "movq 24(%%rsi),%%r13\n" - "movq 32(%%rsi),%%r14\n" - "movq $0xfffffffffffff,%%r15\n" - - /* d = (a0*2) * a3 */ - "leaq (%%r10,%%r10,1),%%rax\n" - "mulq %%r13\n" - "movq %%rax,%%rbx\n" - "movq %%rdx,%%rcx\n" - /* d += (a1*2) * a2 */ - "leaq (%%r11,%%r11,1),%%rax\n" - "mulq %%r12\n" - "addq %%rax,%%rbx\n" - "adcq %%rdx,%%rcx\n" - /* c = a4 * a4 */ - "movq %%r14,%%rax\n" - "mulq %%r14\n" - "movq %%rax,%%r8\n" - "movq %%rdx,%%r9\n" - /* d += (c & M) * R */ - "andq %%r15,%%rax\n" - "movq $0x1000003d10,%%rdx\n" - "mulq %%rdx\n" - "addq %%rax,%%rbx\n" - "adcq %%rdx,%%rcx\n" - /* c >>= 52 (%%r8 only) */ - "shrdq $52,%%r9,%%r8\n" - /* t3 (tmp1) = d & M */ - "movq %%rbx,%%rsi\n" - "andq %%r15,%%rsi\n" - "movq %%rsi,%q1\n" - /* d >>= 52 */ - "shrdq $52,%%rcx,%%rbx\n" - "xorq %%rcx,%%rcx\n" - /* a4 *= 2 */ - "addq %%r14,%%r14\n" - /* d += a0 * a4 */ - "movq %%r10,%%rax\n" - "mulq %%r14\n" - "addq %%rax,%%rbx\n" - "adcq %%rdx,%%rcx\n" - /* d+= (a1*2) * a3 */ - "leaq (%%r11,%%r11,1),%%rax\n" - "mulq %%r13\n" - "addq %%rax,%%rbx\n" - "adcq %%rdx,%%rcx\n" - /* d += a2 * a2 */ - "movq %%r12,%%rax\n" - "mulq %%r12\n" - "addq %%rax,%%rbx\n" - "adcq %%rdx,%%rcx\n" - /* d += c * R */ - "movq %%r8,%%rax\n" - "movq $0x1000003d10,%%rdx\n" - "mulq %%rdx\n" - "addq %%rax,%%rbx\n" - "adcq %%rdx,%%rcx\n" - /* t4 = d & M (%%rsi) */ - "movq %%rbx,%%rsi\n" - "andq %%r15,%%rsi\n" - /* d >>= 52 */ - "shrdq $52,%%rcx,%%rbx\n" - "xorq %%rcx,%%rcx\n" - /* tx = t4 >> 48 (tmp3) */ - "movq %%rsi,%%rax\n" - "shrq $48,%%rax\n" - "movq %%rax,%q3\n" - /* t4 &= (M >> 4) (tmp2) */ - "movq $0xffffffffffff,%%rax\n" - "andq %%rax,%%rsi\n" - "movq %%rsi,%q2\n" - /* c = a0 * a0 */ - "movq %%r10,%%rax\n" - "mulq %%r10\n" - "movq %%rax,%%r8\n" - "movq %%rdx,%%r9\n" - /* d += a1 * a4 */ - "movq %%r11,%%rax\n" - "mulq %%r14\n" - "addq %%rax,%%rbx\n" - "adcq %%rdx,%%rcx\n" - /* d += (a2*2) * a3 */ - "leaq (%%r12,%%r12,1),%%rax\n" - "mulq %%r13\n" - "addq %%rax,%%rbx\n" - "adcq %%rdx,%%rcx\n" - /* u0 = d & M (%%rsi) */ - "movq %%rbx,%%rsi\n" - "andq %%r15,%%rsi\n" - /* d >>= 52 */ - "shrdq $52,%%rcx,%%rbx\n" - "xorq %%rcx,%%rcx\n" - /* u0 = (u0 << 4) | tx (%%rsi) */ - "shlq $4,%%rsi\n" - "movq %q3,%%rax\n" - "orq %%rax,%%rsi\n" - /* c += u0 * (R >> 4) */ - "movq $0x1000003d1,%%rax\n" - "mulq %%rsi\n" - "addq %%rax,%%r8\n" - "adcq %%rdx,%%r9\n" - /* r[0] = c & M */ - "movq %%r8,%%rax\n" - "andq %%r15,%%rax\n" - "movq %%rax,0(%%rdi)\n" - /* c >>= 52 */ - "shrdq $52,%%r9,%%r8\n" - "xorq %%r9,%%r9\n" - /* a0 *= 2 */ - "addq %%r10,%%r10\n" - /* c += a0 * a1 */ - "movq %%r10,%%rax\n" - "mulq %%r11\n" - "addq %%rax,%%r8\n" - "adcq %%rdx,%%r9\n" - /* d += a2 * a4 */ - "movq %%r12,%%rax\n" - "mulq %%r14\n" - "addq %%rax,%%rbx\n" - "adcq %%rdx,%%rcx\n" - /* d += a3 * a3 */ - "movq %%r13,%%rax\n" - "mulq %%r13\n" - "addq %%rax,%%rbx\n" - "adcq %%rdx,%%rcx\n" - /* c += (d & M) * R */ - "movq %%rbx,%%rax\n" - "andq %%r15,%%rax\n" - "movq $0x1000003d10,%%rdx\n" - "mulq %%rdx\n" - "addq %%rax,%%r8\n" - "adcq %%rdx,%%r9\n" - /* d >>= 52 */ - "shrdq $52,%%rcx,%%rbx\n" - "xorq %%rcx,%%rcx\n" - /* r[1] = c & M */ - "movq %%r8,%%rax\n" - "andq %%r15,%%rax\n" - "movq %%rax,8(%%rdi)\n" - /* c >>= 52 */ - "shrdq $52,%%r9,%%r8\n" - "xorq %%r9,%%r9\n" - /* c += a0 * a2 (last use of %%r10) */ - "movq %%r10,%%rax\n" - "mulq %%r12\n" - "addq %%rax,%%r8\n" - "adcq %%rdx,%%r9\n" - /* fetch t3 (%%r10, overwrites a0),t4 (%%rsi) */ - "movq %q2,%%rsi\n" - "movq %q1,%%r10\n" - /* c += a1 * a1 */ - "movq %%r11,%%rax\n" - "mulq %%r11\n" - "addq %%rax,%%r8\n" - "adcq %%rdx,%%r9\n" - /* d += a3 * a4 */ - "movq %%r13,%%rax\n" - "mulq %%r14\n" - "addq %%rax,%%rbx\n" - "adcq %%rdx,%%rcx\n" - /* c += (d & M) * R */ - "movq %%rbx,%%rax\n" - "andq %%r15,%%rax\n" - "movq $0x1000003d10,%%rdx\n" - "mulq %%rdx\n" - "addq %%rax,%%r8\n" - "adcq %%rdx,%%r9\n" - /* d >>= 52 (%%rbx only) */ - "shrdq $52,%%rcx,%%rbx\n" - /* r[2] = c & M */ - "movq %%r8,%%rax\n" - "andq %%r15,%%rax\n" - "movq %%rax,16(%%rdi)\n" - /* c >>= 52 */ - "shrdq $52,%%r9,%%r8\n" - "xorq %%r9,%%r9\n" - /* c += t3 */ - "addq %%r10,%%r8\n" - /* c += d * R */ - "movq %%rbx,%%rax\n" - "movq $0x1000003d10,%%rdx\n" - "mulq %%rdx\n" - "addq %%rax,%%r8\n" - "adcq %%rdx,%%r9\n" - /* r[3] = c & M */ - "movq %%r8,%%rax\n" - "andq %%r15,%%rax\n" - "movq %%rax,24(%%rdi)\n" - /* c >>= 52 (%%r8 only) */ - "shrdq $52,%%r9,%%r8\n" - /* c += t4 (%%r8 only) */ - "addq %%rsi,%%r8\n" - /* r[4] = c */ - "movq %%r8,32(%%rdi)\n" -: "+S"(a), "=&m"(tmp1), "=&m"(tmp2), "=&m"(tmp3) -: "D"(r) -: "%rax", "%rbx", "%rcx", "%rdx", "%r8", "%r9", "%r10", "%r11", "%r12", "%r13", "%r14", "%r15", "cc", "memory" -); -} - -#endif /* SECP256K1_FIELD_INNER5X52_IMPL_H */ diff --git a/src/secp256k1/src/field_5x52_impl.h b/src/secp256k1/src/field_5x52_impl.h index ecb70502c2..76031f755e 100644 --- a/src/secp256k1/src/field_5x52_impl.h +++ b/src/secp256k1/src/field_5x52_impl.h @@ -12,11 +12,7 @@ #include "field.h" #include "modinv64_impl.h" -#if defined(USE_ASM_X86_64) -#include "field_5x52_asm_impl.h" -#else #include "field_5x52_int128_impl.h" -#endif #ifdef VERIFY static void secp256k1_fe_impl_verify(const secp256k1_fe *a) { diff --git a/src/secp256k1/src/field_5x52_int128_impl.h b/src/secp256k1/src/field_5x52_int128_impl.h index b2a391dec9..f23f8ee1c4 100644 --- a/src/secp256k1/src/field_5x52_int128_impl.h +++ b/src/secp256k1/src/field_5x52_int128_impl.h @@ -12,13 +12,8 @@ #include "int128.h" #include "util.h" -#ifdef VERIFY #define VERIFY_BITS(x, n) VERIFY_CHECK(((x) >> (n)) == 0) #define VERIFY_BITS_128(x, n) VERIFY_CHECK(secp256k1_u128_check_bits((x), (n))) -#else -#define VERIFY_BITS(x, n) do { } while(0) -#define VERIFY_BITS_128(x, n) do { } while(0) -#endif SECP256K1_INLINE static void secp256k1_fe_mul_inner(uint64_t *r, const uint64_t *a, const uint64_t * SECP256K1_RESTRICT b) { secp256k1_uint128 c, d; @@ -89,18 +84,18 @@ SECP256K1_INLINE static void secp256k1_fe_mul_inner(uint64_t *r, const uint64_t secp256k1_u128_accum_mul(&d, a2, b[3]); secp256k1_u128_accum_mul(&d, a3, b[2]); secp256k1_u128_accum_mul(&d, a4, b[1]); - VERIFY_BITS_128(&d, 115); + VERIFY_BITS_128(&d, 114); /* [d t4+(tx<<48) t3 0 0 c] = [p8 0 0 p5 p4 p3 0 0 p0] */ u0 = secp256k1_u128_to_u64(&d) & M; secp256k1_u128_rshift(&d, 52); VERIFY_BITS(u0, 52); - VERIFY_BITS_128(&d, 63); + VERIFY_BITS_128(&d, 62); /* [d u0 t4+(tx<<48) t3 0 0 c] = [p8 0 0 p5 p4 p3 0 0 p0] */ /* [d 0 t4+(tx<<48)+(u0<<52) t3 0 0 c] = [p8 0 0 p5 p4 p3 0 0 p0] */ u0 = (u0 << 4) | tx; VERIFY_BITS(u0, 56); /* [d 0 t4+(u0<<48) t3 0 0 c] = [p8 0 0 p5 p4 p3 0 0 p0] */ secp256k1_u128_accum_mul(&c, u0, R >> 4); - VERIFY_BITS_128(&c, 115); + VERIFY_BITS_128(&c, 113); /* [d 0 t4 t3 0 0 c] = [p8 0 0 p5 p4 p3 0 0 p0] */ r[0] = secp256k1_u128_to_u64(&c) & M; secp256k1_u128_rshift(&c, 52); VERIFY_BITS(r[0], 52); @@ -159,7 +154,7 @@ SECP256K1_INLINE static void secp256k1_fe_mul_inner(uint64_t *r, const uint64_t SECP256K1_INLINE static void secp256k1_fe_sqr_inner(uint64_t *r, const uint64_t *a) { secp256k1_uint128 c, d; uint64_t a0 = a[0], a1 = a[1], a2 = a[2], a3 = a[3], a4 = a[4]; - int64_t t3, t4, tx, u0; + uint64_t t3, t4, tx, u0; const uint64_t M = 0xFFFFFFFFFFFFFULL, R = 0x1000003D10ULL; VERIFY_BITS(a[0], 56); diff --git a/src/secp256k1/src/field_impl.h b/src/secp256k1/src/field_impl.h index 80d34b9ef2..989e9cdb2f 100644 --- a/src/secp256k1/src/field_impl.h +++ b/src/secp256k1/src/field_impl.h @@ -20,12 +20,11 @@ SECP256K1_INLINE static int secp256k1_fe_equal(const secp256k1_fe *a, const secp256k1_fe *b) { secp256k1_fe na; -#ifdef VERIFY - secp256k1_fe_verify(a); - secp256k1_fe_verify(b); - secp256k1_fe_verify_magnitude(a, 1); - secp256k1_fe_verify_magnitude(b, 31); -#endif + SECP256K1_FE_VERIFY(a); + SECP256K1_FE_VERIFY(b); + SECP256K1_FE_VERIFY_MAGNITUDE(a, 1); + SECP256K1_FE_VERIFY_MAGNITUDE(b, 31); + secp256k1_fe_negate(&na, a, 1); secp256k1_fe_add(&na, b); return secp256k1_fe_normalizes_to_zero(&na); @@ -44,11 +43,9 @@ static int secp256k1_fe_sqrt(secp256k1_fe * SECP256K1_RESTRICT r, const secp256k secp256k1_fe x2, x3, x6, x9, x11, x22, x44, x88, x176, x220, x223, t1; int j, ret; -#ifdef VERIFY VERIFY_CHECK(r != a); - secp256k1_fe_verify(a); - secp256k1_fe_verify_magnitude(a, 8); -#endif + SECP256K1_FE_VERIFY(a); + SECP256K1_FE_VERIFY_MAGNITUDE(a, 8); /** The binary representation of (p + 1)/4 has 3 blocks of 1s, with lengths in * { 2, 22, 223 }. Use an addition chain to calculate 2^n - 1 for each block: @@ -151,11 +148,11 @@ static void secp256k1_fe_verify_magnitude(const secp256k1_fe *a, int m) { (void) static void secp256k1_fe_impl_verify(const secp256k1_fe *a); static void secp256k1_fe_verify(const secp256k1_fe *a) { /* Magnitude between 0 and 32. */ - secp256k1_fe_verify_magnitude(a, 32); + SECP256K1_FE_VERIFY_MAGNITUDE(a, 32); /* Normalized is 0 or 1. */ VERIFY_CHECK((a->normalized == 0) || (a->normalized == 1)); /* If normalized, magnitude must be 0 or 1. */ - if (a->normalized) secp256k1_fe_verify_magnitude(a, 1); + if (a->normalized) SECP256K1_FE_VERIFY_MAGNITUDE(a, 1); /* Invoke implementation-specific checks. */ secp256k1_fe_impl_verify(a); } @@ -168,59 +165,71 @@ static void secp256k1_fe_verify_magnitude(const secp256k1_fe *a, int m) { static void secp256k1_fe_impl_normalize(secp256k1_fe *r); SECP256K1_INLINE static void secp256k1_fe_normalize(secp256k1_fe *r) { - secp256k1_fe_verify(r); + SECP256K1_FE_VERIFY(r); + secp256k1_fe_impl_normalize(r); r->magnitude = 1; r->normalized = 1; - secp256k1_fe_verify(r); + + SECP256K1_FE_VERIFY(r); } static void secp256k1_fe_impl_normalize_weak(secp256k1_fe *r); SECP256K1_INLINE static void secp256k1_fe_normalize_weak(secp256k1_fe *r) { - secp256k1_fe_verify(r); + SECP256K1_FE_VERIFY(r); + secp256k1_fe_impl_normalize_weak(r); r->magnitude = 1; - secp256k1_fe_verify(r); + + SECP256K1_FE_VERIFY(r); } static void secp256k1_fe_impl_normalize_var(secp256k1_fe *r); SECP256K1_INLINE static void secp256k1_fe_normalize_var(secp256k1_fe *r) { - secp256k1_fe_verify(r); + SECP256K1_FE_VERIFY(r); + secp256k1_fe_impl_normalize_var(r); r->magnitude = 1; r->normalized = 1; - secp256k1_fe_verify(r); + + SECP256K1_FE_VERIFY(r); } static int secp256k1_fe_impl_normalizes_to_zero(const secp256k1_fe *r); SECP256K1_INLINE static int secp256k1_fe_normalizes_to_zero(const secp256k1_fe *r) { - secp256k1_fe_verify(r); + SECP256K1_FE_VERIFY(r); + return secp256k1_fe_impl_normalizes_to_zero(r); } static int secp256k1_fe_impl_normalizes_to_zero_var(const secp256k1_fe *r); SECP256K1_INLINE static int secp256k1_fe_normalizes_to_zero_var(const secp256k1_fe *r) { - secp256k1_fe_verify(r); + SECP256K1_FE_VERIFY(r); + return secp256k1_fe_impl_normalizes_to_zero_var(r); } static void secp256k1_fe_impl_set_int(secp256k1_fe *r, int a); SECP256K1_INLINE static void secp256k1_fe_set_int(secp256k1_fe *r, int a) { VERIFY_CHECK(0 <= a && a <= 0x7FFF); + secp256k1_fe_impl_set_int(r, a); r->magnitude = (a != 0); r->normalized = 1; - secp256k1_fe_verify(r); + + SECP256K1_FE_VERIFY(r); } static void secp256k1_fe_impl_add_int(secp256k1_fe *r, int a); SECP256K1_INLINE static void secp256k1_fe_add_int(secp256k1_fe *r, int a) { VERIFY_CHECK(0 <= a && a <= 0x7FFF); - secp256k1_fe_verify(r); + SECP256K1_FE_VERIFY(r); + secp256k1_fe_impl_add_int(r, a); r->magnitude += 1; r->normalized = 0; - secp256k1_fe_verify(r); + + SECP256K1_FE_VERIFY(r); } static void secp256k1_fe_impl_clear(secp256k1_fe *a); @@ -228,29 +237,33 @@ SECP256K1_INLINE static void secp256k1_fe_clear(secp256k1_fe *a) { a->magnitude = 0; a->normalized = 1; secp256k1_fe_impl_clear(a); - secp256k1_fe_verify(a); + + SECP256K1_FE_VERIFY(a); } static int secp256k1_fe_impl_is_zero(const secp256k1_fe *a); SECP256K1_INLINE static int secp256k1_fe_is_zero(const secp256k1_fe *a) { - secp256k1_fe_verify(a); + SECP256K1_FE_VERIFY(a); VERIFY_CHECK(a->normalized); + return secp256k1_fe_impl_is_zero(a); } static int secp256k1_fe_impl_is_odd(const secp256k1_fe *a); SECP256K1_INLINE static int secp256k1_fe_is_odd(const secp256k1_fe *a) { - secp256k1_fe_verify(a); + SECP256K1_FE_VERIFY(a); VERIFY_CHECK(a->normalized); + return secp256k1_fe_impl_is_odd(a); } static int secp256k1_fe_impl_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b); SECP256K1_INLINE static int secp256k1_fe_cmp_var(const secp256k1_fe *a, const secp256k1_fe *b) { - secp256k1_fe_verify(a); - secp256k1_fe_verify(b); + SECP256K1_FE_VERIFY(a); + SECP256K1_FE_VERIFY(b); VERIFY_CHECK(a->normalized); VERIFY_CHECK(b->normalized); + return secp256k1_fe_impl_cmp_var(a, b); } @@ -259,7 +272,8 @@ SECP256K1_INLINE static void secp256k1_fe_set_b32_mod(secp256k1_fe *r, const uns secp256k1_fe_impl_set_b32_mod(r, a); r->magnitude = 1; r->normalized = 0; - secp256k1_fe_verify(r); + + SECP256K1_FE_VERIFY(r); } static int secp256k1_fe_impl_set_b32_limit(secp256k1_fe *r, const unsigned char *a); @@ -267,7 +281,7 @@ SECP256K1_INLINE static int secp256k1_fe_set_b32_limit(secp256k1_fe *r, const un if (secp256k1_fe_impl_set_b32_limit(r, a)) { r->magnitude = 1; r->normalized = 1; - secp256k1_fe_verify(r); + SECP256K1_FE_VERIFY(r); return 1; } else { /* Mark the output field element as invalid. */ @@ -278,83 +292,97 @@ SECP256K1_INLINE static int secp256k1_fe_set_b32_limit(secp256k1_fe *r, const un static void secp256k1_fe_impl_get_b32(unsigned char *r, const secp256k1_fe *a); SECP256K1_INLINE static void secp256k1_fe_get_b32(unsigned char *r, const secp256k1_fe *a) { - secp256k1_fe_verify(a); + SECP256K1_FE_VERIFY(a); VERIFY_CHECK(a->normalized); + secp256k1_fe_impl_get_b32(r, a); } static void secp256k1_fe_impl_negate_unchecked(secp256k1_fe *r, const secp256k1_fe *a, int m); SECP256K1_INLINE static void secp256k1_fe_negate_unchecked(secp256k1_fe *r, const secp256k1_fe *a, int m) { - secp256k1_fe_verify(a); + SECP256K1_FE_VERIFY(a); VERIFY_CHECK(m >= 0 && m <= 31); - secp256k1_fe_verify_magnitude(a, m); + SECP256K1_FE_VERIFY_MAGNITUDE(a, m); + secp256k1_fe_impl_negate_unchecked(r, a, m); r->magnitude = m + 1; r->normalized = 0; - secp256k1_fe_verify(r); + + SECP256K1_FE_VERIFY(r); } static void secp256k1_fe_impl_mul_int_unchecked(secp256k1_fe *r, int a); SECP256K1_INLINE static void secp256k1_fe_mul_int_unchecked(secp256k1_fe *r, int a) { - secp256k1_fe_verify(r); + SECP256K1_FE_VERIFY(r); + VERIFY_CHECK(a >= 0 && a <= 32); VERIFY_CHECK(a*r->magnitude <= 32); secp256k1_fe_impl_mul_int_unchecked(r, a); r->magnitude *= a; r->normalized = 0; - secp256k1_fe_verify(r); + + SECP256K1_FE_VERIFY(r); } static void secp256k1_fe_impl_add(secp256k1_fe *r, const secp256k1_fe *a); SECP256K1_INLINE static void secp256k1_fe_add(secp256k1_fe *r, const secp256k1_fe *a) { - secp256k1_fe_verify(r); - secp256k1_fe_verify(a); + SECP256K1_FE_VERIFY(r); + SECP256K1_FE_VERIFY(a); VERIFY_CHECK(r->magnitude + a->magnitude <= 32); + secp256k1_fe_impl_add(r, a); r->magnitude += a->magnitude; r->normalized = 0; - secp256k1_fe_verify(r); + + SECP256K1_FE_VERIFY(r); } static void secp256k1_fe_impl_mul(secp256k1_fe *r, const secp256k1_fe *a, const secp256k1_fe * SECP256K1_RESTRICT b); SECP256K1_INLINE static void secp256k1_fe_mul(secp256k1_fe *r, const secp256k1_fe *a, const secp256k1_fe * SECP256K1_RESTRICT b) { - secp256k1_fe_verify(a); - secp256k1_fe_verify(b); - secp256k1_fe_verify_magnitude(a, 8); - secp256k1_fe_verify_magnitude(b, 8); + SECP256K1_FE_VERIFY(a); + SECP256K1_FE_VERIFY(b); + SECP256K1_FE_VERIFY_MAGNITUDE(a, 8); + SECP256K1_FE_VERIFY_MAGNITUDE(b, 8); VERIFY_CHECK(r != b); VERIFY_CHECK(a != b); + secp256k1_fe_impl_mul(r, a, b); r->magnitude = 1; r->normalized = 0; - secp256k1_fe_verify(r); + + SECP256K1_FE_VERIFY(r); } static void secp256k1_fe_impl_sqr(secp256k1_fe *r, const secp256k1_fe *a); SECP256K1_INLINE static void secp256k1_fe_sqr(secp256k1_fe *r, const secp256k1_fe *a) { - secp256k1_fe_verify(a); - secp256k1_fe_verify_magnitude(a, 8); + SECP256K1_FE_VERIFY(a); + SECP256K1_FE_VERIFY_MAGNITUDE(a, 8); + secp256k1_fe_impl_sqr(r, a); r->magnitude = 1; r->normalized = 0; - secp256k1_fe_verify(r); + + SECP256K1_FE_VERIFY(r); } static void secp256k1_fe_impl_cmov(secp256k1_fe *r, const secp256k1_fe *a, int flag); SECP256K1_INLINE static void secp256k1_fe_cmov(secp256k1_fe *r, const secp256k1_fe *a, int flag) { VERIFY_CHECK(flag == 0 || flag == 1); - secp256k1_fe_verify(a); - secp256k1_fe_verify(r); + SECP256K1_FE_VERIFY(a); + SECP256K1_FE_VERIFY(r); + secp256k1_fe_impl_cmov(r, a, flag); if (a->magnitude > r->magnitude) r->magnitude = a->magnitude; if (!a->normalized) r->normalized = 0; - secp256k1_fe_verify(r); + + SECP256K1_FE_VERIFY(r); } static void secp256k1_fe_impl_to_storage(secp256k1_fe_storage *r, const secp256k1_fe *a); SECP256K1_INLINE static void secp256k1_fe_to_storage(secp256k1_fe_storage *r, const secp256k1_fe *a) { - secp256k1_fe_verify(a); + SECP256K1_FE_VERIFY(a); VERIFY_CHECK(a->normalized); + secp256k1_fe_impl_to_storage(r, a); } @@ -363,36 +391,42 @@ SECP256K1_INLINE static void secp256k1_fe_from_storage(secp256k1_fe *r, const se secp256k1_fe_impl_from_storage(r, a); r->magnitude = 1; r->normalized = 1; - secp256k1_fe_verify(r); + + SECP256K1_FE_VERIFY(r); } static void secp256k1_fe_impl_inv(secp256k1_fe *r, const secp256k1_fe *x); SECP256K1_INLINE static void secp256k1_fe_inv(secp256k1_fe *r, const secp256k1_fe *x) { int input_is_zero = secp256k1_fe_normalizes_to_zero(x); - secp256k1_fe_verify(x); + SECP256K1_FE_VERIFY(x); + secp256k1_fe_impl_inv(r, x); r->magnitude = x->magnitude > 0; r->normalized = 1; + VERIFY_CHECK(secp256k1_fe_normalizes_to_zero(r) == input_is_zero); - secp256k1_fe_verify(r); + SECP256K1_FE_VERIFY(r); } static void secp256k1_fe_impl_inv_var(secp256k1_fe *r, const secp256k1_fe *x); SECP256K1_INLINE static void secp256k1_fe_inv_var(secp256k1_fe *r, const secp256k1_fe *x) { int input_is_zero = secp256k1_fe_normalizes_to_zero(x); - secp256k1_fe_verify(x); + SECP256K1_FE_VERIFY(x); + secp256k1_fe_impl_inv_var(r, x); r->magnitude = x->magnitude > 0; r->normalized = 1; + VERIFY_CHECK(secp256k1_fe_normalizes_to_zero(r) == input_is_zero); - secp256k1_fe_verify(r); + SECP256K1_FE_VERIFY(r); } static int secp256k1_fe_impl_is_square_var(const secp256k1_fe *x); SECP256K1_INLINE static int secp256k1_fe_is_square_var(const secp256k1_fe *x) { int ret; secp256k1_fe tmp = *x, sqrt; - secp256k1_fe_verify(x); + SECP256K1_FE_VERIFY(x); + ret = secp256k1_fe_impl_is_square_var(x); secp256k1_fe_normalize_weak(&tmp); VERIFY_CHECK(ret == secp256k1_fe_sqrt(&sqrt, &tmp)); @@ -403,20 +437,24 @@ static void secp256k1_fe_impl_get_bounds(secp256k1_fe* r, int m); SECP256K1_INLINE static void secp256k1_fe_get_bounds(secp256k1_fe* r, int m) { VERIFY_CHECK(m >= 0); VERIFY_CHECK(m <= 32); + secp256k1_fe_impl_get_bounds(r, m); r->magnitude = m; r->normalized = (m == 0); - secp256k1_fe_verify(r); + + SECP256K1_FE_VERIFY(r); } static void secp256k1_fe_impl_half(secp256k1_fe *r); SECP256K1_INLINE static void secp256k1_fe_half(secp256k1_fe *r) { - secp256k1_fe_verify(r); - secp256k1_fe_verify_magnitude(r, 31); + SECP256K1_FE_VERIFY(r); + SECP256K1_FE_VERIFY_MAGNITUDE(r, 31); + secp256k1_fe_impl_half(r); r->magnitude = (r->magnitude >> 1) + 1; r->normalized = 0; - secp256k1_fe_verify(r); + + SECP256K1_FE_VERIFY(r); } #endif /* defined(VERIFY) */ diff --git a/src/secp256k1/src/group.h b/src/secp256k1/src/group.h index 86eb9e1f82..d81deb4264 100644 --- a/src/secp256k1/src/group.h +++ b/src/secp256k1/src/group.h @@ -102,6 +102,9 @@ static void secp256k1_ge_set_all_gej_var(secp256k1_ge *r, const secp256k1_gej *a */ static void secp256k1_ge_table_set_globalz(size_t len, secp256k1_ge *a, const secp256k1_fe *zr); +/** Check two group elements (affine) for equality in variable time. */ +static int secp256k1_ge_eq_var(const secp256k1_ge *a, const secp256k1_ge *b); + /** Set a group element (affine) equal to the point at infinity. */ static void secp256k1_ge_set_infinity(secp256k1_ge *r); @@ -114,6 +117,9 @@ static void secp256k1_gej_set_ge(secp256k1_gej *r, const secp256k1_ge *a); /** Check two group elements (jacobian) for equality in variable time. */ static int secp256k1_gej_eq_var(const secp256k1_gej *a, const secp256k1_gej *b); +/** Check two group elements (jacobian and affine) for equality in variable time. */ +static int secp256k1_gej_eq_ge_var(const secp256k1_gej *a, const secp256k1_ge *b); + /** Compare the X coordinate of a group element (jacobian). * The magnitude of the group element's X coordinate must not exceed 31. */ static int secp256k1_gej_eq_x_var(const secp256k1_fe *x, const secp256k1_gej *a); @@ -181,8 +187,10 @@ static int secp256k1_ge_is_in_correct_subgroup(const secp256k1_ge* ge); /** Check invariants on an affine group element (no-op unless VERIFY is enabled). */ static void secp256k1_ge_verify(const secp256k1_ge *a); +#define SECP256K1_GE_VERIFY(a) secp256k1_ge_verify(a) /** Check invariants on a Jacobian group element (no-op unless VERIFY is enabled). */ static void secp256k1_gej_verify(const secp256k1_gej *a); +#define SECP256K1_GEJ_VERIFY(a) secp256k1_gej_verify(a) #endif /* SECP256K1_GROUP_H */ diff --git a/src/secp256k1/src/group_impl.h b/src/secp256k1/src/group_impl.h index b9542ce8ae..537be32ff6 100644 --- a/src/secp256k1/src/group_impl.h +++ b/src/secp256k1/src/group_impl.h @@ -74,26 +74,22 @@ static const secp256k1_ge secp256k1_ge_const_g = SECP256K1_G; /* End of section generated by sage/gen_exhaustive_groups.sage. */ static void secp256k1_ge_verify(const secp256k1_ge *a) { -#ifdef VERIFY - secp256k1_fe_verify(&a->x); - secp256k1_fe_verify(&a->y); - secp256k1_fe_verify_magnitude(&a->x, SECP256K1_GE_X_MAGNITUDE_MAX); - secp256k1_fe_verify_magnitude(&a->y, SECP256K1_GE_Y_MAGNITUDE_MAX); + SECP256K1_FE_VERIFY(&a->x); + SECP256K1_FE_VERIFY(&a->y); + SECP256K1_FE_VERIFY_MAGNITUDE(&a->x, SECP256K1_GE_X_MAGNITUDE_MAX); + SECP256K1_FE_VERIFY_MAGNITUDE(&a->y, SECP256K1_GE_Y_MAGNITUDE_MAX); VERIFY_CHECK(a->infinity == 0 || a->infinity == 1); -#endif (void)a; } static void secp256k1_gej_verify(const secp256k1_gej *a) { -#ifdef VERIFY - secp256k1_fe_verify(&a->x); - secp256k1_fe_verify(&a->y); - secp256k1_fe_verify(&a->z); - secp256k1_fe_verify_magnitude(&a->x, SECP256K1_GEJ_X_MAGNITUDE_MAX); - secp256k1_fe_verify_magnitude(&a->y, SECP256K1_GEJ_Y_MAGNITUDE_MAX); - secp256k1_fe_verify_magnitude(&a->z, SECP256K1_GEJ_Z_MAGNITUDE_MAX); + SECP256K1_FE_VERIFY(&a->x); + SECP256K1_FE_VERIFY(&a->y); + SECP256K1_FE_VERIFY(&a->z); + SECP256K1_FE_VERIFY_MAGNITUDE(&a->x, SECP256K1_GEJ_X_MAGNITUDE_MAX); + SECP256K1_FE_VERIFY_MAGNITUDE(&a->y, SECP256K1_GEJ_Y_MAGNITUDE_MAX); + SECP256K1_FE_VERIFY_MAGNITUDE(&a->z, SECP256K1_GEJ_Z_MAGNITUDE_MAX); VERIFY_CHECK(a->infinity == 0 || a->infinity == 1); -#endif (void)a; } @@ -101,8 +97,8 @@ static void secp256k1_gej_verify(const secp256k1_gej *a) { static void secp256k1_ge_set_gej_zinv(secp256k1_ge *r, const secp256k1_gej *a, const secp256k1_fe *zi) { secp256k1_fe zi2; secp256k1_fe zi3; - secp256k1_gej_verify(a); - secp256k1_fe_verify(zi); + SECP256K1_GEJ_VERIFY(a); + SECP256K1_FE_VERIFY(zi); VERIFY_CHECK(!a->infinity); secp256k1_fe_sqr(&zi2, zi); @@ -111,15 +107,15 @@ static void secp256k1_ge_set_gej_zinv(secp256k1_ge *r, const secp256k1_gej *a, c secp256k1_fe_mul(&r->y, &a->y, &zi3); r->infinity = a->infinity; - secp256k1_ge_verify(r); + SECP256K1_GE_VERIFY(r); } /* Set r to the affine coordinates of Jacobian point (a.x, a.y, 1/zi). */ static void secp256k1_ge_set_ge_zinv(secp256k1_ge *r, const secp256k1_ge *a, const secp256k1_fe *zi) { secp256k1_fe zi2; secp256k1_fe zi3; - secp256k1_ge_verify(a); - secp256k1_fe_verify(zi); + SECP256K1_GE_VERIFY(a); + SECP256K1_FE_VERIFY(zi); VERIFY_CHECK(!a->infinity); secp256k1_fe_sqr(&zi2, zi); @@ -128,39 +124,39 @@ static void secp256k1_ge_set_ge_zinv(secp256k1_ge *r, const secp256k1_ge *a, con secp256k1_fe_mul(&r->y, &a->y, &zi3); r->infinity = a->infinity; - secp256k1_ge_verify(r); + SECP256K1_GE_VERIFY(r); } static void secp256k1_ge_set_xy(secp256k1_ge *r, const secp256k1_fe *x, const secp256k1_fe *y) { - secp256k1_fe_verify(x); - secp256k1_fe_verify(y); + SECP256K1_FE_VERIFY(x); + SECP256K1_FE_VERIFY(y); r->infinity = 0; r->x = *x; r->y = *y; - secp256k1_ge_verify(r); + SECP256K1_GE_VERIFY(r); } static int secp256k1_ge_is_infinity(const secp256k1_ge *a) { - secp256k1_ge_verify(a); + SECP256K1_GE_VERIFY(a); return a->infinity; } static void secp256k1_ge_neg(secp256k1_ge *r, const secp256k1_ge *a) { - secp256k1_ge_verify(a); + SECP256K1_GE_VERIFY(a); *r = *a; secp256k1_fe_normalize_weak(&r->y); secp256k1_fe_negate(&r->y, &r->y, 1); - secp256k1_ge_verify(r); + SECP256K1_GE_VERIFY(r); } static void secp256k1_ge_set_gej(secp256k1_ge *r, secp256k1_gej *a) { secp256k1_fe z2, z3; - secp256k1_gej_verify(a); + SECP256K1_GEJ_VERIFY(a); r->infinity = a->infinity; secp256k1_fe_inv(&a->z, &a->z); @@ -172,13 +168,13 @@ static void secp256k1_ge_set_gej(secp256k1_ge *r, secp256k1_gej *a) { r->x = a->x; r->y = a->y; - secp256k1_gej_verify(a); - secp256k1_ge_verify(r); + SECP256K1_GEJ_VERIFY(a); + SECP256K1_GE_VERIFY(r); } static void secp256k1_ge_set_gej_var(secp256k1_ge *r, secp256k1_gej *a) { secp256k1_fe z2, z3; - secp256k1_gej_verify(a); + SECP256K1_GEJ_VERIFY(a); if (secp256k1_gej_is_infinity(a)) { secp256k1_ge_set_infinity(r); @@ -193,8 +189,8 @@ static void secp256k1_ge_set_gej_var(secp256k1_ge *r, secp256k1_gej *a) { secp256k1_fe_set_int(&a->z, 1); secp256k1_ge_set_xy(r, &a->x, &a->y); - secp256k1_gej_verify(a); - secp256k1_ge_verify(r); + SECP256K1_GEJ_VERIFY(a); + SECP256K1_GE_VERIFY(r); } static void secp256k1_ge_set_all_gej_var(secp256k1_ge *r, const secp256k1_gej *a, size_t len) { @@ -203,7 +199,7 @@ static void secp256k1_ge_set_all_gej_var(secp256k1_ge *r, const secp256k1_gej *a size_t last_i = SIZE_MAX; #ifdef VERIFY for (i = 0; i < len; i++) { - secp256k1_gej_verify(&a[i]); + SECP256K1_GEJ_VERIFY(&a[i]); } #endif @@ -245,7 +241,7 @@ static void secp256k1_ge_set_all_gej_var(secp256k1_ge *r, const secp256k1_gej *a #ifdef VERIFY for (i = 0; i < len; i++) { - secp256k1_ge_verify(&r[i]); + SECP256K1_GE_VERIFY(&r[i]); } #endif } @@ -255,8 +251,8 @@ static void secp256k1_ge_table_set_globalz(size_t len, secp256k1_ge *a, const se secp256k1_fe zs; #ifdef VERIFY for (i = 0; i < len; i++) { - secp256k1_ge_verify(&a[i]); - secp256k1_fe_verify(&zr[i]); + SECP256K1_GE_VERIFY(&a[i]); + SECP256K1_FE_VERIFY(&zr[i]); } #endif @@ -278,7 +274,7 @@ static void secp256k1_ge_table_set_globalz(size_t len, secp256k1_ge *a, const se #ifdef VERIFY for (i = 0; i < len; i++) { - secp256k1_ge_verify(&a[i]); + SECP256K1_GE_VERIFY(&a[i]); } #endif } @@ -289,7 +285,7 @@ static void secp256k1_gej_set_infinity(secp256k1_gej *r) { secp256k1_fe_clear(&r->y); secp256k1_fe_clear(&r->z); - secp256k1_gej_verify(r); + SECP256K1_GEJ_VERIFY(r); } static void secp256k1_ge_set_infinity(secp256k1_ge *r) { @@ -297,7 +293,7 @@ static void secp256k1_ge_set_infinity(secp256k1_ge *r) { secp256k1_fe_clear(&r->x); secp256k1_fe_clear(&r->y); - secp256k1_ge_verify(r); + SECP256K1_GE_VERIFY(r); } static void secp256k1_gej_clear(secp256k1_gej *r) { @@ -306,7 +302,7 @@ static void secp256k1_gej_clear(secp256k1_gej *r) { secp256k1_fe_clear(&r->y); secp256k1_fe_clear(&r->z); - secp256k1_gej_verify(r); + SECP256K1_GEJ_VERIFY(r); } static void secp256k1_ge_clear(secp256k1_ge *r) { @@ -314,13 +310,13 @@ static void secp256k1_ge_clear(secp256k1_ge *r) { secp256k1_fe_clear(&r->x); secp256k1_fe_clear(&r->y); - secp256k1_ge_verify(r); + SECP256K1_GE_VERIFY(r); } static int secp256k1_ge_set_xo_var(secp256k1_ge *r, const secp256k1_fe *x, int odd) { secp256k1_fe x2, x3; int ret; - secp256k1_fe_verify(x); + SECP256K1_FE_VERIFY(x); r->x = *x; secp256k1_fe_sqr(&x2, x); @@ -333,45 +329,72 @@ static int secp256k1_ge_set_xo_var(secp256k1_ge *r, const secp256k1_fe *x, int o secp256k1_fe_negate(&r->y, &r->y, 1); } - secp256k1_ge_verify(r); + SECP256K1_GE_VERIFY(r); return ret; } static void secp256k1_gej_set_ge(secp256k1_gej *r, const secp256k1_ge *a) { - secp256k1_ge_verify(a); + SECP256K1_GE_VERIFY(a); r->infinity = a->infinity; r->x = a->x; r->y = a->y; secp256k1_fe_set_int(&r->z, 1); - secp256k1_gej_verify(r); + SECP256K1_GEJ_VERIFY(r); } static int secp256k1_gej_eq_var(const secp256k1_gej *a, const secp256k1_gej *b) { secp256k1_gej tmp; - secp256k1_gej_verify(b); - secp256k1_gej_verify(a); + SECP256K1_GEJ_VERIFY(b); + SECP256K1_GEJ_VERIFY(a); secp256k1_gej_neg(&tmp, a); secp256k1_gej_add_var(&tmp, &tmp, b, NULL); return secp256k1_gej_is_infinity(&tmp); } +static int secp256k1_gej_eq_ge_var(const secp256k1_gej *a, const secp256k1_ge *b) { + secp256k1_gej tmp; + SECP256K1_GEJ_VERIFY(a); + SECP256K1_GE_VERIFY(b); + + secp256k1_gej_neg(&tmp, a); + secp256k1_gej_add_ge_var(&tmp, &tmp, b, NULL); + return secp256k1_gej_is_infinity(&tmp); +} + +static int secp256k1_ge_eq_var(const secp256k1_ge *a, const secp256k1_ge *b) { + secp256k1_fe tmp; + SECP256K1_GE_VERIFY(a); + SECP256K1_GE_VERIFY(b); + + if (a->infinity != b->infinity) return 0; + if (a->infinity) return 1; + + tmp = a->x; + secp256k1_fe_normalize_weak(&tmp); + if (!secp256k1_fe_equal(&tmp, &b->x)) return 0; + + tmp = a->y; + secp256k1_fe_normalize_weak(&tmp); + if (!secp256k1_fe_equal(&tmp, &b->y)) return 0; + + return 1; +} + static int secp256k1_gej_eq_x_var(const secp256k1_fe *x, const secp256k1_gej *a) { secp256k1_fe r; - secp256k1_fe_verify(x); - secp256k1_gej_verify(a); -#ifdef VERIFY + SECP256K1_FE_VERIFY(x); + SECP256K1_GEJ_VERIFY(a); VERIFY_CHECK(!a->infinity); -#endif secp256k1_fe_sqr(&r, &a->z); secp256k1_fe_mul(&r, &r, x); return secp256k1_fe_equal(&r, &a->x); } static void secp256k1_gej_neg(secp256k1_gej *r, const secp256k1_gej *a) { - secp256k1_gej_verify(a); + SECP256K1_GEJ_VERIFY(a); r->infinity = a->infinity; r->x = a->x; @@ -380,18 +403,18 @@ static void secp256k1_gej_neg(secp256k1_gej *r, const secp256k1_gej *a) { secp256k1_fe_normalize_weak(&r->y); secp256k1_fe_negate(&r->y, &r->y, 1); - secp256k1_gej_verify(r); + SECP256K1_GEJ_VERIFY(r); } static int secp256k1_gej_is_infinity(const secp256k1_gej *a) { - secp256k1_gej_verify(a); + SECP256K1_GEJ_VERIFY(a); return a->infinity; } static int secp256k1_ge_is_valid_var(const secp256k1_ge *a) { secp256k1_fe y2, x3; - secp256k1_ge_verify(a); + SECP256K1_GE_VERIFY(a); if (a->infinity) { return 0; @@ -406,7 +429,7 @@ static int secp256k1_ge_is_valid_var(const secp256k1_ge *a) { static SECP256K1_INLINE void secp256k1_gej_double(secp256k1_gej *r, const secp256k1_gej *a) { /* Operations: 3 mul, 4 sqr, 8 add/half/mul_int/negate */ secp256k1_fe l, s, t; - secp256k1_gej_verify(a); + SECP256K1_GEJ_VERIFY(a); r->infinity = a->infinity; @@ -435,11 +458,11 @@ static SECP256K1_INLINE void secp256k1_gej_double(secp256k1_gej *r, const secp25 secp256k1_fe_add(&r->y, &s); /* Y3 = L*(X3 + T) + S^2 (2) */ secp256k1_fe_negate(&r->y, &r->y, 2); /* Y3 = -(L*(X3 + T) + S^2) (3) */ - secp256k1_gej_verify(r); + SECP256K1_GEJ_VERIFY(r); } static void secp256k1_gej_double_var(secp256k1_gej *r, const secp256k1_gej *a, secp256k1_fe *rzr) { - secp256k1_gej_verify(a); + SECP256K1_GEJ_VERIFY(a); /** For secp256k1, 2Q is infinity if and only if Q is infinity. This is because if 2Q = infinity, * Q must equal -Q, or that Q.y == -(Q.y), or Q.y is 0. For a point on y^2 = x^3 + 7 to have @@ -466,14 +489,14 @@ static void secp256k1_gej_double_var(secp256k1_gej *r, const secp256k1_gej *a, s secp256k1_gej_double(r, a); - secp256k1_gej_verify(r); + SECP256K1_GEJ_VERIFY(r); } static void secp256k1_gej_add_var(secp256k1_gej *r, const secp256k1_gej *a, const secp256k1_gej *b, secp256k1_fe *rzr) { /* 12 mul, 4 sqr, 11 add/negate/normalizes_to_zero (ignoring special cases) */ secp256k1_fe z22, z12, u1, u2, s1, s2, h, i, h2, h3, t; - secp256k1_gej_verify(a); - secp256k1_gej_verify(b); + SECP256K1_GEJ_VERIFY(a); + SECP256K1_GEJ_VERIFY(b); if (a->infinity) { VERIFY_CHECK(rzr == NULL); @@ -530,14 +553,14 @@ static void secp256k1_gej_add_var(secp256k1_gej *r, const secp256k1_gej *a, cons secp256k1_fe_mul(&h3, &h3, &s1); secp256k1_fe_add(&r->y, &h3); - secp256k1_gej_verify(r); + SECP256K1_GEJ_VERIFY(r); } static void secp256k1_gej_add_ge_var(secp256k1_gej *r, const secp256k1_gej *a, const secp256k1_ge *b, secp256k1_fe *rzr) { /* Operations: 8 mul, 3 sqr, 11 add/negate/normalizes_to_zero (ignoring special cases) */ secp256k1_fe z12, u1, u2, s1, s2, h, i, h2, h3, t; - secp256k1_gej_verify(a); - secp256k1_ge_verify(b); + SECP256K1_GEJ_VERIFY(a); + SECP256K1_GE_VERIFY(b); if (a->infinity) { VERIFY_CHECK(rzr == NULL); @@ -592,16 +615,16 @@ static void secp256k1_gej_add_ge_var(secp256k1_gej *r, const secp256k1_gej *a, c secp256k1_fe_mul(&h3, &h3, &s1); secp256k1_fe_add(&r->y, &h3); - secp256k1_gej_verify(r); - if (rzr != NULL) secp256k1_fe_verify(rzr); + SECP256K1_GEJ_VERIFY(r); + if (rzr != NULL) SECP256K1_FE_VERIFY(rzr); } static void secp256k1_gej_add_zinv_var(secp256k1_gej *r, const secp256k1_gej *a, const secp256k1_ge *b, const secp256k1_fe *bzinv) { /* Operations: 9 mul, 3 sqr, 11 add/negate/normalizes_to_zero (ignoring special cases) */ secp256k1_fe az, z12, u1, u2, s1, s2, h, i, h2, h3, t; - secp256k1_gej_verify(a); - secp256k1_ge_verify(b); - secp256k1_fe_verify(bzinv); + SECP256K1_GEJ_VERIFY(a); + SECP256K1_GE_VERIFY(b); + SECP256K1_FE_VERIFY(bzinv); if (a->infinity) { secp256k1_fe bzinv2, bzinv3; @@ -611,7 +634,7 @@ static void secp256k1_gej_add_zinv_var(secp256k1_gej *r, const secp256k1_gej *a, secp256k1_fe_mul(&r->x, &b->x, &bzinv2); secp256k1_fe_mul(&r->y, &b->y, &bzinv3); secp256k1_fe_set_int(&r->z, 1); - secp256k1_gej_verify(r); + SECP256K1_GEJ_VERIFY(r); return; } if (b->infinity) { @@ -663,7 +686,7 @@ static void secp256k1_gej_add_zinv_var(secp256k1_gej *r, const secp256k1_gej *a, secp256k1_fe_mul(&h3, &h3, &s1); secp256k1_fe_add(&r->y, &h3); - secp256k1_gej_verify(r); + SECP256K1_GEJ_VERIFY(r); } @@ -672,8 +695,8 @@ static void secp256k1_gej_add_ge(secp256k1_gej *r, const secp256k1_gej *a, const secp256k1_fe zz, u1, u2, s1, s2, t, tt, m, n, q, rr; secp256k1_fe m_alt, rr_alt; int degenerate; - secp256k1_gej_verify(a); - secp256k1_ge_verify(b); + SECP256K1_GEJ_VERIFY(a); + SECP256K1_GE_VERIFY(b); VERIFY_CHECK(!b->infinity); /* In: @@ -801,17 +824,15 @@ static void secp256k1_gej_add_ge(secp256k1_gej *r, const secp256k1_gej *a, const * Then r->infinity = ((y1 + y2)Z == 0) = (y1 == -y2) = false. */ r->infinity = secp256k1_fe_normalizes_to_zero(&r->z); - secp256k1_gej_verify(r); + SECP256K1_GEJ_VERIFY(r); } static void secp256k1_gej_rescale(secp256k1_gej *r, const secp256k1_fe *s) { /* Operations: 4 mul, 1 sqr */ secp256k1_fe zz; - secp256k1_gej_verify(r); - secp256k1_fe_verify(s); -#ifdef VERIFY + SECP256K1_GEJ_VERIFY(r); + SECP256K1_FE_VERIFY(s); VERIFY_CHECK(!secp256k1_fe_normalizes_to_zero_var(s)); -#endif secp256k1_fe_sqr(&zz, s); secp256k1_fe_mul(&r->x, &r->x, &zz); /* r->x *= s^2 */ @@ -819,12 +840,12 @@ static void secp256k1_gej_rescale(secp256k1_gej *r, const secp256k1_fe *s) { secp256k1_fe_mul(&r->y, &r->y, s); /* r->y *= s^3 */ secp256k1_fe_mul(&r->z, &r->z, s); /* r->z *= s */ - secp256k1_gej_verify(r); + SECP256K1_GEJ_VERIFY(r); } static void secp256k1_ge_to_storage(secp256k1_ge_storage *r, const secp256k1_ge *a) { secp256k1_fe x, y; - secp256k1_ge_verify(a); + SECP256K1_GE_VERIFY(a); VERIFY_CHECK(!a->infinity); x = a->x; @@ -840,19 +861,19 @@ static void secp256k1_ge_from_storage(secp256k1_ge *r, const secp256k1_ge_storag secp256k1_fe_from_storage(&r->y, &a->y); r->infinity = 0; - secp256k1_ge_verify(r); + SECP256K1_GE_VERIFY(r); } static SECP256K1_INLINE void secp256k1_gej_cmov(secp256k1_gej *r, const secp256k1_gej *a, int flag) { - secp256k1_gej_verify(r); - secp256k1_gej_verify(a); + SECP256K1_GEJ_VERIFY(r); + SECP256K1_GEJ_VERIFY(a); secp256k1_fe_cmov(&r->x, &a->x, flag); secp256k1_fe_cmov(&r->y, &a->y, flag); secp256k1_fe_cmov(&r->z, &a->z, flag); r->infinity ^= (r->infinity ^ a->infinity) & flag; - secp256k1_gej_verify(r); + SECP256K1_GEJ_VERIFY(r); } static SECP256K1_INLINE void secp256k1_ge_storage_cmov(secp256k1_ge_storage *r, const secp256k1_ge_storage *a, int flag) { @@ -861,19 +882,19 @@ static SECP256K1_INLINE void secp256k1_ge_storage_cmov(secp256k1_ge_storage *r, } static void secp256k1_ge_mul_lambda(secp256k1_ge *r, const secp256k1_ge *a) { - secp256k1_ge_verify(a); + SECP256K1_GE_VERIFY(a); *r = *a; secp256k1_fe_mul(&r->x, &r->x, &secp256k1_const_beta); - secp256k1_ge_verify(r); + SECP256K1_GE_VERIFY(r); } static int secp256k1_ge_is_in_correct_subgroup(const secp256k1_ge* ge) { #ifdef EXHAUSTIVE_TEST_ORDER secp256k1_gej out; int i; - secp256k1_ge_verify(ge); + SECP256K1_GE_VERIFY(ge); /* A very simple EC multiplication ladder that avoids a dependency on ecmult. */ secp256k1_gej_set_infinity(&out); @@ -885,7 +906,7 @@ static int secp256k1_ge_is_in_correct_subgroup(const secp256k1_ge* ge) { } return secp256k1_gej_is_infinity(&out); #else - secp256k1_ge_verify(ge); + SECP256K1_GE_VERIFY(ge); (void)ge; /* The real secp256k1 group has cofactor 1, so the subgroup is the entire curve. */ @@ -907,9 +928,8 @@ static int secp256k1_ge_x_frac_on_curve_var(const secp256k1_fe *xn, const secp25 * (xn/xd)^3 + 7 is square <=> xd*xn^3 + 7*xd^4 is square (multiplying by xd^4, a square). */ secp256k1_fe r, t; -#ifdef VERIFY VERIFY_CHECK(!secp256k1_fe_normalizes_to_zero_var(xd)); -#endif + secp256k1_fe_mul(&r, xd, xn); /* r = xd*xn */ secp256k1_fe_sqr(&t, xn); /* t = xn^2 */ secp256k1_fe_mul(&r, &r, &t); /* r = xd*xn^3 */ diff --git a/src/secp256k1/src/modinv32_impl.h b/src/secp256k1/src/modinv32_impl.h index 0ea2699863..75eb354ff0 100644 --- a/src/secp256k1/src/modinv32_impl.h +++ b/src/secp256k1/src/modinv32_impl.h @@ -144,7 +144,6 @@ static void secp256k1_modinv32_normalize_30(secp256k1_modinv32_signed30 *r, int3 r->v[7] = r7; r->v[8] = r8; -#ifdef VERIFY VERIFY_CHECK(r0 >> 30 == 0); VERIFY_CHECK(r1 >> 30 == 0); VERIFY_CHECK(r2 >> 30 == 0); @@ -156,7 +155,6 @@ static void secp256k1_modinv32_normalize_30(secp256k1_modinv32_signed30 *r, int3 VERIFY_CHECK(r8 >> 30 == 0); VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(r, 9, &modinfo->modulus, 0) >= 0); /* r >= 0 */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(r, 9, &modinfo->modulus, 1) < 0); /* r < modulus */ -#endif } /* Data type for transition matrices (see section 3 of explanation). @@ -413,14 +411,13 @@ static void secp256k1_modinv32_update_de_30(secp256k1_modinv32_signed30 *d, secp int32_t di, ei, md, me, sd, se; int64_t cd, ce; int i; -#ifdef VERIFY VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(d, 9, &modinfo->modulus, -2) > 0); /* d > -2*modulus */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(d, 9, &modinfo->modulus, 1) < 0); /* d < modulus */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(e, 9, &modinfo->modulus, -2) > 0); /* e > -2*modulus */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(e, 9, &modinfo->modulus, 1) < 0); /* e < modulus */ VERIFY_CHECK(labs(u) <= (M30 + 1 - labs(v))); /* |u|+|v| <= 2^30 */ VERIFY_CHECK(labs(q) <= (M30 + 1 - labs(r))); /* |q|+|r| <= 2^30 */ -#endif + /* [md,me] start as zero; plus [u,q] if d is negative; plus [v,r] if e is negative. */ sd = d->v[8] >> 31; se = e->v[8] >> 31; @@ -455,12 +452,11 @@ static void secp256k1_modinv32_update_de_30(secp256k1_modinv32_signed30 *d, secp /* What remains is limb 9 of t*[d,e]+modulus*[md,me]; store it as output limb 8. */ d->v[8] = (int32_t)cd; e->v[8] = (int32_t)ce; -#ifdef VERIFY + VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(d, 9, &modinfo->modulus, -2) > 0); /* d > -2*modulus */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(d, 9, &modinfo->modulus, 1) < 0); /* d < modulus */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(e, 9, &modinfo->modulus, -2) > 0); /* e > -2*modulus */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(e, 9, &modinfo->modulus, 1) < 0); /* e < modulus */ -#endif } /* Compute (t/2^30) * [f, g], where t is a transition matrix for 30 divsteps. @@ -550,25 +546,23 @@ static void secp256k1_modinv32(secp256k1_modinv32_signed30 *x, const secp256k1_m /* Update d,e using that transition matrix. */ secp256k1_modinv32_update_de_30(&d, &e, &t, modinfo); /* Update f,g using that transition matrix. */ -#ifdef VERIFY VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&f, 9, &modinfo->modulus, -1) > 0); /* f > -modulus */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&f, 9, &modinfo->modulus, 1) <= 0); /* f <= modulus */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&g, 9, &modinfo->modulus, -1) > 0); /* g > -modulus */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&g, 9, &modinfo->modulus, 1) < 0); /* g < modulus */ -#endif + secp256k1_modinv32_update_fg_30(&f, &g, &t); -#ifdef VERIFY + VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&f, 9, &modinfo->modulus, -1) > 0); /* f > -modulus */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&f, 9, &modinfo->modulus, 1) <= 0); /* f <= modulus */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&g, 9, &modinfo->modulus, -1) > 0); /* g > -modulus */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&g, 9, &modinfo->modulus, 1) < 0); /* g < modulus */ -#endif } /* At this point sufficient iterations have been performed that g must have reached 0 * and (if g was not originally 0) f must now equal +/- GCD of the initial f, g * values i.e. +/- 1, and d now contains +/- the modular inverse. */ -#ifdef VERIFY + /* g == 0 */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&g, 9, &SECP256K1_SIGNED30_ONE, 0) == 0); /* |f| == 1, or (x == 0 and d == 0 and |f|=modulus) */ @@ -578,7 +572,6 @@ static void secp256k1_modinv32(secp256k1_modinv32_signed30 *x, const secp256k1_m secp256k1_modinv32_mul_cmp_30(&d, 9, &SECP256K1_SIGNED30_ONE, 0) == 0 && (secp256k1_modinv32_mul_cmp_30(&f, 9, &modinfo->modulus, 1) == 0 || secp256k1_modinv32_mul_cmp_30(&f, 9, &modinfo->modulus, -1) == 0))); -#endif /* Optionally negate d, normalize to [0,modulus), and return it. */ secp256k1_modinv32_normalize_30(&d, f.v[8], modinfo); @@ -607,12 +600,12 @@ static void secp256k1_modinv32_var(secp256k1_modinv32_signed30 *x, const secp256 /* Update d,e using that transition matrix. */ secp256k1_modinv32_update_de_30(&d, &e, &t, modinfo); /* Update f,g using that transition matrix. */ -#ifdef VERIFY + VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&f, len, &modinfo->modulus, -1) > 0); /* f > -modulus */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&f, len, &modinfo->modulus, 1) <= 0); /* f <= modulus */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&g, len, &modinfo->modulus, -1) > 0); /* g > -modulus */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&g, len, &modinfo->modulus, 1) < 0); /* g < modulus */ -#endif + secp256k1_modinv32_update_fg_30_var(len, &f, &g, &t); /* If the bottom limb of g is 0, there is a chance g=0. */ if (g.v[0] == 0) { @@ -637,18 +630,17 @@ static void secp256k1_modinv32_var(secp256k1_modinv32_signed30 *x, const secp256 g.v[len - 2] |= (uint32_t)gn << 30; --len; } -#ifdef VERIFY + VERIFY_CHECK(++i < 25); /* We should never need more than 25*30 = 750 divsteps */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&f, len, &modinfo->modulus, -1) > 0); /* f > -modulus */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&f, len, &modinfo->modulus, 1) <= 0); /* f <= modulus */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&g, len, &modinfo->modulus, -1) > 0); /* g > -modulus */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&g, len, &modinfo->modulus, 1) < 0); /* g < modulus */ -#endif } /* At this point g is 0 and (if g was not originally 0) f must now equal +/- GCD of * the initial f, g values i.e. +/- 1, and d now contains +/- the modular inverse. */ -#ifdef VERIFY + /* g == 0 */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&g, len, &SECP256K1_SIGNED30_ONE, 0) == 0); /* |f| == 1, or (x == 0 and d == 0 and |f|=modulus) */ @@ -658,7 +650,6 @@ static void secp256k1_modinv32_var(secp256k1_modinv32_signed30 *x, const secp256 secp256k1_modinv32_mul_cmp_30(&d, 9, &SECP256K1_SIGNED30_ONE, 0) == 0 && (secp256k1_modinv32_mul_cmp_30(&f, len, &modinfo->modulus, 1) == 0 || secp256k1_modinv32_mul_cmp_30(&f, len, &modinfo->modulus, -1) == 0))); -#endif /* Optionally negate d, normalize to [0,modulus), and return it. */ secp256k1_modinv32_normalize_30(&d, f.v[len - 1], modinfo); @@ -697,12 +688,11 @@ static int secp256k1_jacobi32_maybe_var(const secp256k1_modinv32_signed30 *x, co secp256k1_modinv32_trans2x2 t; eta = secp256k1_modinv32_posdivsteps_30_var(eta, f.v[0] | ((uint32_t)f.v[1] << 30), g.v[0] | ((uint32_t)g.v[1] << 30), &t, &jac); /* Update f,g using that transition matrix. */ -#ifdef VERIFY VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&f, len, &modinfo->modulus, 0) > 0); /* f > 0 */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&f, len, &modinfo->modulus, 1) <= 0); /* f <= modulus */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&g, len, &modinfo->modulus, 0) > 0); /* g > 0 */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&g, len, &modinfo->modulus, 1) < 0); /* g < modulus */ -#endif + secp256k1_modinv32_update_fg_30_var(len, &f, &g, &t); /* If the bottom limb of f is 1, there is a chance that f=1. */ if (f.v[0] == 1) { @@ -723,12 +713,11 @@ static int secp256k1_jacobi32_maybe_var(const secp256k1_modinv32_signed30 *x, co cond |= gn; /* If so, reduce length. */ if (cond == 0) --len; -#ifdef VERIFY + VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&f, len, &modinfo->modulus, 0) > 0); /* f > 0 */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&f, len, &modinfo->modulus, 1) <= 0); /* f <= modulus */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&g, len, &modinfo->modulus, 0) > 0); /* g > 0 */ VERIFY_CHECK(secp256k1_modinv32_mul_cmp_30(&g, len, &modinfo->modulus, 1) < 0); /* g < modulus */ -#endif } /* The loop failed to converge to f=g after 1500 iterations. Return 0, indicating unknown result. */ diff --git a/src/secp256k1/src/modinv64_impl.h b/src/secp256k1/src/modinv64_impl.h index c7cef872a4..0dc1e80696 100644 --- a/src/secp256k1/src/modinv64_impl.h +++ b/src/secp256k1/src/modinv64_impl.h @@ -144,7 +144,6 @@ static void secp256k1_modinv64_normalize_62(secp256k1_modinv64_signed62 *r, int6 r->v[3] = r3; r->v[4] = r4; -#ifdef VERIFY VERIFY_CHECK(r0 >> 62 == 0); VERIFY_CHECK(r1 >> 62 == 0); VERIFY_CHECK(r2 >> 62 == 0); @@ -152,7 +151,6 @@ static void secp256k1_modinv64_normalize_62(secp256k1_modinv64_signed62 *r, int6 VERIFY_CHECK(r4 >> 62 == 0); VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(r, 5, &modinfo->modulus, 0) >= 0); /* r >= 0 */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(r, 5, &modinfo->modulus, 1) < 0); /* r < modulus */ -#endif } /* Compute the transition matrix and eta for 59 divsteps (where zeta=-(delta+1/2)). @@ -216,7 +214,7 @@ static int64_t secp256k1_modinv64_divsteps_59(int64_t zeta, uint64_t f0, uint64_ t->v = (int64_t)v; t->q = (int64_t)q; t->r = (int64_t)r; -#ifdef VERIFY + /* The determinant of t must be a power of two. This guarantees that multiplication with t * does not change the gcd of f and g, apart from adding a power-of-2 factor to it (which * will be divided out again). As each divstep's individual matrix has determinant 2, the @@ -224,7 +222,7 @@ static int64_t secp256k1_modinv64_divsteps_59(int64_t zeta, uint64_t f0, uint64_ * 8*identity (which has determinant 2^6) means the overall outputs has determinant * 2^65. */ VERIFY_CHECK(secp256k1_modinv64_det_check_pow2(t, 65, 0)); -#endif + return zeta; } @@ -301,13 +299,13 @@ static int64_t secp256k1_modinv64_divsteps_62_var(int64_t eta, uint64_t f0, uint t->v = (int64_t)v; t->q = (int64_t)q; t->r = (int64_t)r; -#ifdef VERIFY + /* The determinant of t must be a power of two. This guarantees that multiplication with t * does not change the gcd of f and g, apart from adding a power-of-2 factor to it (which * will be divided out again). As each divstep's individual matrix has determinant 2, the * aggregate of 62 of them will have determinant 2^62. */ VERIFY_CHECK(secp256k1_modinv64_det_check_pow2(t, 62, 0)); -#endif + return eta; } @@ -392,13 +390,13 @@ static int64_t secp256k1_modinv64_posdivsteps_62_var(int64_t eta, uint64_t f0, u t->v = (int64_t)v; t->q = (int64_t)q; t->r = (int64_t)r; -#ifdef VERIFY + /* The determinant of t must be a power of two. This guarantees that multiplication with t * does not change the gcd of f and g, apart from adding a power-of-2 factor to it (which * will be divided out again). As each divstep's individual matrix has determinant 2 or -2, * the aggregate of 62 of them will have determinant 2^62 or -2^62. */ VERIFY_CHECK(secp256k1_modinv64_det_check_pow2(t, 62, 1)); -#endif + *jacp = jac; return eta; } @@ -417,14 +415,13 @@ static void secp256k1_modinv64_update_de_62(secp256k1_modinv64_signed62 *d, secp const int64_t u = t->u, v = t->v, q = t->q, r = t->r; int64_t md, me, sd, se; secp256k1_int128 cd, ce; -#ifdef VERIFY VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(d, 5, &modinfo->modulus, -2) > 0); /* d > -2*modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(d, 5, &modinfo->modulus, 1) < 0); /* d < modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(e, 5, &modinfo->modulus, -2) > 0); /* e > -2*modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(e, 5, &modinfo->modulus, 1) < 0); /* e < modulus */ VERIFY_CHECK(secp256k1_modinv64_abs(u) <= (((int64_t)1 << 62) - secp256k1_modinv64_abs(v))); /* |u|+|v| <= 2^62 */ VERIFY_CHECK(secp256k1_modinv64_abs(q) <= (((int64_t)1 << 62) - secp256k1_modinv64_abs(r))); /* |q|+|r| <= 2^62 */ -#endif + /* [md,me] start as zero; plus [u,q] if d is negative; plus [v,r] if e is negative. */ sd = d4 >> 63; se = e4 >> 63; @@ -489,12 +486,11 @@ static void secp256k1_modinv64_update_de_62(secp256k1_modinv64_signed62 *d, secp /* What remains is limb 5 of t*[d,e]+modulus*[md,me]; store it as output limb 4. */ d->v[4] = secp256k1_i128_to_i64(&cd); e->v[4] = secp256k1_i128_to_i64(&ce); -#ifdef VERIFY + VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(d, 5, &modinfo->modulus, -2) > 0); /* d > -2*modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(d, 5, &modinfo->modulus, 1) < 0); /* d < modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(e, 5, &modinfo->modulus, -2) > 0); /* e > -2*modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(e, 5, &modinfo->modulus, 1) < 0); /* e < modulus */ -#endif } /* Compute (t/2^62) * [f, g], where t is a transition matrix scaled by 2^62. @@ -606,25 +602,23 @@ static void secp256k1_modinv64(secp256k1_modinv64_signed62 *x, const secp256k1_m /* Update d,e using that transition matrix. */ secp256k1_modinv64_update_de_62(&d, &e, &t, modinfo); /* Update f,g using that transition matrix. */ -#ifdef VERIFY VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&f, 5, &modinfo->modulus, -1) > 0); /* f > -modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&f, 5, &modinfo->modulus, 1) <= 0); /* f <= modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&g, 5, &modinfo->modulus, -1) > 0); /* g > -modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&g, 5, &modinfo->modulus, 1) < 0); /* g < modulus */ -#endif + secp256k1_modinv64_update_fg_62(&f, &g, &t); -#ifdef VERIFY + VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&f, 5, &modinfo->modulus, -1) > 0); /* f > -modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&f, 5, &modinfo->modulus, 1) <= 0); /* f <= modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&g, 5, &modinfo->modulus, -1) > 0); /* g > -modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&g, 5, &modinfo->modulus, 1) < 0); /* g < modulus */ -#endif } /* At this point sufficient iterations have been performed that g must have reached 0 * and (if g was not originally 0) f must now equal +/- GCD of the initial f, g * values i.e. +/- 1, and d now contains +/- the modular inverse. */ -#ifdef VERIFY + /* g == 0 */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&g, 5, &SECP256K1_SIGNED62_ONE, 0) == 0); /* |f| == 1, or (x == 0 and d == 0 and |f|=modulus) */ @@ -634,7 +628,6 @@ static void secp256k1_modinv64(secp256k1_modinv64_signed62 *x, const secp256k1_m secp256k1_modinv64_mul_cmp_62(&d, 5, &SECP256K1_SIGNED62_ONE, 0) == 0 && (secp256k1_modinv64_mul_cmp_62(&f, 5, &modinfo->modulus, 1) == 0 || secp256k1_modinv64_mul_cmp_62(&f, 5, &modinfo->modulus, -1) == 0))); -#endif /* Optionally negate d, normalize to [0,modulus), and return it. */ secp256k1_modinv64_normalize_62(&d, f.v[4], modinfo); @@ -663,12 +656,11 @@ static void secp256k1_modinv64_var(secp256k1_modinv64_signed62 *x, const secp256 /* Update d,e using that transition matrix. */ secp256k1_modinv64_update_de_62(&d, &e, &t, modinfo); /* Update f,g using that transition matrix. */ -#ifdef VERIFY VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&f, len, &modinfo->modulus, -1) > 0); /* f > -modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&f, len, &modinfo->modulus, 1) <= 0); /* f <= modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&g, len, &modinfo->modulus, -1) > 0); /* g > -modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&g, len, &modinfo->modulus, 1) < 0); /* g < modulus */ -#endif + secp256k1_modinv64_update_fg_62_var(len, &f, &g, &t); /* If the bottom limb of g is zero, there is a chance that g=0. */ if (g.v[0] == 0) { @@ -693,18 +685,17 @@ static void secp256k1_modinv64_var(secp256k1_modinv64_signed62 *x, const secp256 g.v[len - 2] |= (uint64_t)gn << 62; --len; } -#ifdef VERIFY + VERIFY_CHECK(++i < 12); /* We should never need more than 12*62 = 744 divsteps */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&f, len, &modinfo->modulus, -1) > 0); /* f > -modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&f, len, &modinfo->modulus, 1) <= 0); /* f <= modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&g, len, &modinfo->modulus, -1) > 0); /* g > -modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&g, len, &modinfo->modulus, 1) < 0); /* g < modulus */ -#endif } /* At this point g is 0 and (if g was not originally 0) f must now equal +/- GCD of * the initial f, g values i.e. +/- 1, and d now contains +/- the modular inverse. */ -#ifdef VERIFY + /* g == 0 */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&g, len, &SECP256K1_SIGNED62_ONE, 0) == 0); /* |f| == 1, or (x == 0 and d == 0 and |f|=modulus) */ @@ -714,7 +705,6 @@ static void secp256k1_modinv64_var(secp256k1_modinv64_signed62 *x, const secp256 secp256k1_modinv64_mul_cmp_62(&d, 5, &SECP256K1_SIGNED62_ONE, 0) == 0 && (secp256k1_modinv64_mul_cmp_62(&f, len, &modinfo->modulus, 1) == 0 || secp256k1_modinv64_mul_cmp_62(&f, len, &modinfo->modulus, -1) == 0))); -#endif /* Optionally negate d, normalize to [0,modulus), and return it. */ secp256k1_modinv64_normalize_62(&d, f.v[len - 1], modinfo); @@ -753,12 +743,11 @@ static int secp256k1_jacobi64_maybe_var(const secp256k1_modinv64_signed62 *x, co secp256k1_modinv64_trans2x2 t; eta = secp256k1_modinv64_posdivsteps_62_var(eta, f.v[0] | ((uint64_t)f.v[1] << 62), g.v[0] | ((uint64_t)g.v[1] << 62), &t, &jac); /* Update f,g using that transition matrix. */ -#ifdef VERIFY VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&f, len, &modinfo->modulus, 0) > 0); /* f > 0 */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&f, len, &modinfo->modulus, 1) <= 0); /* f <= modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&g, len, &modinfo->modulus, 0) > 0); /* g > 0 */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&g, len, &modinfo->modulus, 1) < 0); /* g < modulus */ -#endif + secp256k1_modinv64_update_fg_62_var(len, &f, &g, &t); /* If the bottom limb of f is 1, there is a chance that f=1. */ if (f.v[0] == 1) { @@ -779,12 +768,11 @@ static int secp256k1_jacobi64_maybe_var(const secp256k1_modinv64_signed62 *x, co cond |= gn; /* If so, reduce length. */ if (cond == 0) --len; -#ifdef VERIFY + VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&f, len, &modinfo->modulus, 0) > 0); /* f > 0 */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&f, len, &modinfo->modulus, 1) <= 0); /* f <= modulus */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&g, len, &modinfo->modulus, 0) > 0); /* g > 0 */ VERIFY_CHECK(secp256k1_modinv64_mul_cmp_62(&g, len, &modinfo->modulus, 1) < 0); /* g < modulus */ -#endif } /* The loop failed to converge to f=g after 1550 iterations. Return 0, indicating unknown result. */ diff --git a/src/secp256k1/src/modules/ecdh/tests_impl.h b/src/secp256k1/src/modules/ecdh/tests_impl.h index fa6f232227..6be96eacbe 100644 --- a/src/secp256k1/src/modules/ecdh/tests_impl.h +++ b/src/secp256k1/src/modules/ecdh/tests_impl.h @@ -25,32 +25,19 @@ static int ecdh_hash_function_custom(unsigned char *output, const unsigned char } static void test_ecdh_api(void) { - /* Setup context that just counts errors */ - secp256k1_context *tctx = secp256k1_context_create(SECP256K1_CONTEXT_NONE); secp256k1_pubkey point; unsigned char res[32]; unsigned char s_one[32] = { 0 }; - int32_t ecount = 0; s_one[31] = 1; - secp256k1_context_set_error_callback(tctx, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_illegal_callback(tctx, counting_illegal_callback_fn, &ecount); - CHECK(secp256k1_ec_pubkey_create(tctx, &point, s_one) == 1); + CHECK(secp256k1_ec_pubkey_create(CTX, &point, s_one) == 1); /* Check all NULLs are detected */ - CHECK(secp256k1_ecdh(tctx, res, &point, s_one, NULL, NULL) == 1); - CHECK(ecount == 0); - CHECK(secp256k1_ecdh(tctx, NULL, &point, s_one, NULL, NULL) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_ecdh(tctx, res, NULL, s_one, NULL, NULL) == 0); - CHECK(ecount == 2); - CHECK(secp256k1_ecdh(tctx, res, &point, NULL, NULL, NULL) == 0); - CHECK(ecount == 3); - CHECK(secp256k1_ecdh(tctx, res, &point, s_one, NULL, NULL) == 1); - CHECK(ecount == 3); - - /* Cleanup */ - secp256k1_context_destroy(tctx); + CHECK(secp256k1_ecdh(CTX, res, &point, s_one, NULL, NULL) == 1); + CHECK_ILLEGAL(CTX, secp256k1_ecdh(CTX, NULL, &point, s_one, NULL, NULL)); + CHECK_ILLEGAL(CTX, secp256k1_ecdh(CTX, res, NULL, s_one, NULL, NULL)); + CHECK_ILLEGAL(CTX, secp256k1_ecdh(CTX, res, &point, NULL, NULL, NULL)); + CHECK(secp256k1_ecdh(CTX, res, &point, s_one, NULL, NULL) == 1); } static void test_ecdh_generator_basepoint(void) { diff --git a/src/secp256k1/src/modules/ellswift/main_impl.h b/src/secp256k1/src/modules/ellswift/main_impl.h index 00bb8a3da5..b54ec08a22 100644 --- a/src/secp256k1/src/modules/ellswift/main_impl.h +++ b/src/secp256k1/src/modules/ellswift/main_impl.h @@ -126,9 +126,8 @@ static void secp256k1_ellswift_xswiftec_frac_var(secp256k1_fe *xn, secp256k1_fe secp256k1_fe_mul(&l, &p, &u1); /* l = u*(g+s) */ secp256k1_fe_add(&n, &l); /* n = u*(c1*s+c2*g)+u*(g+s) */ secp256k1_fe_negate(xn, &n, 2); /* n = -u*(c1*s+c2*g)-u*(g+s) */ -#ifdef VERIFY + VERIFY_CHECK(secp256k1_ge_x_frac_on_curve_var(xn, &p)); -#endif /* Return x3 = n/p = -(u*(c1*s+c2*g)/(g+s)+u) */ } @@ -193,10 +192,8 @@ static int secp256k1_ellswift_xswiftec_inv_var(secp256k1_fe *t, const secp256k1_ secp256k1_fe_normalize_weak(&x); secp256k1_fe_normalize_weak(&u); -#ifdef VERIFY VERIFY_CHECK(c >= 0 && c < 8); VERIFY_CHECK(secp256k1_ge_x_on_curve_var(&x)); -#endif if (!(c & 2)) { /* c is in {0, 1, 4, 5}. In this case we look for an inverse under the x1 (if c=0 or @@ -230,9 +227,7 @@ static int secp256k1_ellswift_xswiftec_inv_var(secp256k1_fe *t, const secp256k1_ * that (-u-x)^3 + B is not square (the secp256k1_ge_x_on_curve_var(&m) * test above would have failed). This is a contradiction, and thus the * assumption s=0 is false. */ -#ifdef VERIFY VERIFY_CHECK(!secp256k1_fe_normalizes_to_zero_var(&s)); -#endif /* If s is not square, fail. We have not fully computed s yet, but s is square iff * -(u^3+7)*(u^2+u*x+x^2) is square (because a/b is square iff a*b is square and b is @@ -272,7 +267,11 @@ static int secp256k1_ellswift_xswiftec_inv_var(secp256k1_fe *t, const secp256k1_ secp256k1_fe_negate(&q, &q, 1); /* q = -s*(4*(u^3+7)+3*u^2*s) */ if (!secp256k1_fe_is_square_var(&q)) return 0; ret = secp256k1_fe_sqrt(&r, &q); /* r = sqrt(-s*(4*(u^3+7)+3*u^2*s)) */ +#ifdef VERIFY VERIFY_CHECK(ret); +#else + (void)ret; +#endif /* If (c & 1) = 1 and r = 0, fail. */ if (EXPECT((c & 1) && secp256k1_fe_normalizes_to_zero_var(&r), 0)) return 0; @@ -320,10 +319,9 @@ static void secp256k1_ellswift_prng(unsigned char* out32, const secp256k1_sha256 buf4[3] = cnt >> 24; secp256k1_sha256_write(&hash, buf4, 4); secp256k1_sha256_finalize(&hash, out32); -#ifdef VERIFY + /* Writing and finalizing together should trigger exactly one SHA256 compression. */ VERIFY_CHECK(((hash.bytes) >> 6) == (blocks + 1)); -#endif } /** Find an ElligatorSwift encoding (u, t) for X coordinate x, and random Y coordinate. @@ -361,9 +359,8 @@ static void secp256k1_ellswift_xelligatorswift_var(unsigned char *u32, secp256k1 /* Since u is the output of a hash, it should practically never be 0. We could apply the * u=0 to u=1 correction here too to deal with that case still, but it's such a low * probability event that we do not bother. */ -#ifdef VERIFY VERIFY_CHECK(!secp256k1_fe_normalizes_to_zero_var(&u)); -#endif + /* Find a remainder t, and return it if found. */ if (EXPECT(secp256k1_ellswift_xswiftec_inv_var(t, x, &u, branch), 0)) break; } @@ -417,7 +414,11 @@ int secp256k1_ellswift_encode(const secp256k1_context *ctx, unsigned char *ell64 * BIP340 tagged hash with tag "secp256k1_ellswift_encode". */ secp256k1_ellswift_sha256_init_encode(&hash); ser_ret = secp256k1_eckey_pubkey_serialize(&p, p64, &ser_size, 1); +#ifdef VERIFY VERIFY_CHECK(ser_ret && ser_size == 33); +#else + (void)ser_ret; +#endif secp256k1_sha256_write(&hash, p64, sizeof(p64)); secp256k1_sha256_write(&hash, rnd32, 32); diff --git a/src/secp256k1/src/modules/ellswift/tests_exhaustive_impl.h b/src/secp256k1/src/modules/ellswift/tests_exhaustive_impl.h index e002a8c008..839c24aee4 100644 --- a/src/secp256k1/src/modules/ellswift/tests_exhaustive_impl.h +++ b/src/secp256k1/src/modules/ellswift/tests_exhaustive_impl.h @@ -32,7 +32,7 @@ static void test_exhaustive_ellswift(const secp256k1_context *ctx, const secp256 /* Decode ellswift pubkey and check that it matches the precomputed group element. */ secp256k1_ellswift_decode(ctx, &pub_decoded, ell64); secp256k1_pubkey_load(ctx, &ge_decoded, &pub_decoded); - ge_equals_ge(&ge_decoded, &group[i]); + CHECK(secp256k1_ge_eq_var(&ge_decoded, &group[i])); } } diff --git a/src/secp256k1/src/modules/ellswift/tests_impl.h b/src/secp256k1/src/modules/ellswift/tests_impl.h index 47f443d980..7d1efbc492 100644 --- a/src/secp256k1/src/modules/ellswift/tests_impl.h +++ b/src/secp256k1/src/modules/ellswift/tests_impl.h @@ -237,7 +237,7 @@ void run_ellswift_tests(void) { secp256k1_ellswift_decode(CTX, &pubkey2, ell64); secp256k1_pubkey_load(CTX, &g2, &pubkey2); /* Compare with original. */ - ge_equals_ge(&g, &g2); + CHECK(secp256k1_ge_eq_var(&g, &g2)); } /* Verify the behavior of secp256k1_ellswift_create */ for (i = 0; i < 400 * COUNT; i++) { @@ -259,7 +259,7 @@ void run_ellswift_tests(void) { secp256k1_ellswift_decode(CTX, &pub, ell64); secp256k1_pubkey_load(CTX, &dec, &pub); secp256k1_ecmult(&res, NULL, &secp256k1_scalar_zero, &sec); - ge_equals_gej(&dec, &res); + CHECK(secp256k1_gej_eq_ge_var(&res, &dec)); } /* Verify that secp256k1_ellswift_xdh computes the right shared X coordinate. */ for (i = 0; i < 800 * COUNT; i++) { @@ -285,7 +285,7 @@ void run_ellswift_tests(void) { ret = secp256k1_ellswift_xdh(CTX, share32, ell64, ell64, sec32, i & 1, &ellswift_xdh_hash_x32, NULL); CHECK(ret); (void)secp256k1_fe_set_b32_limit(&share_x, share32); /* no overflow is possible */ - secp256k1_fe_verify(&share_x); + SECP256K1_FE_VERIFY(&share_x); /* Compute seckey*pubkey directly. */ secp256k1_ecmult(&resj, &decj, &sec, NULL); secp256k1_ge_set_gej(&res, &resj); diff --git a/src/secp256k1/src/modules/extrakeys/tests_impl.h b/src/secp256k1/src/modules/extrakeys/tests_impl.h index ae1655923b..45521d1742 100644 --- a/src/secp256k1/src/modules/extrakeys/tests_impl.h +++ b/src/secp256k1/src/modules/extrakeys/tests_impl.h @@ -9,11 +9,6 @@ #include "../../../include/secp256k1_extrakeys.h" -static void set_counting_callbacks(secp256k1_context *ctx0, int *ecount) { - secp256k1_context_set_error_callback(ctx0, counting_illegal_callback_fn, ecount); - secp256k1_context_set_illegal_callback(ctx0, counting_illegal_callback_fn, ecount); -} - static void test_xonly_pubkey(void) { secp256k1_pubkey pk; secp256k1_xonly_pubkey xonly_pk, xonly_pk_tmp; @@ -28,10 +23,6 @@ static void test_xonly_pubkey(void) { int pk_parity; int i; - int ecount; - - set_counting_callbacks(CTX, &ecount); - secp256k1_testrand256(sk); memset(ones32, 0xFF, 32); secp256k1_testrand256(xy_sk); @@ -39,16 +30,12 @@ static void test_xonly_pubkey(void) { CHECK(secp256k1_xonly_pubkey_from_pubkey(CTX, &xonly_pk, &pk_parity, &pk) == 1); /* Test xonly_pubkey_from_pubkey */ - ecount = 0; CHECK(secp256k1_xonly_pubkey_from_pubkey(CTX, &xonly_pk, &pk_parity, &pk) == 1); - CHECK(secp256k1_xonly_pubkey_from_pubkey(CTX, NULL, &pk_parity, &pk) == 0); - CHECK(ecount == 1); + CHECK_ILLEGAL(CTX, secp256k1_xonly_pubkey_from_pubkey(CTX, NULL, &pk_parity, &pk)); CHECK(secp256k1_xonly_pubkey_from_pubkey(CTX, &xonly_pk, NULL, &pk) == 1); - CHECK(secp256k1_xonly_pubkey_from_pubkey(CTX, &xonly_pk, &pk_parity, NULL) == 0); - CHECK(ecount == 2); + CHECK_ILLEGAL(CTX, secp256k1_xonly_pubkey_from_pubkey(CTX, &xonly_pk, &pk_parity, NULL)); memset(&pk, 0, sizeof(pk)); - CHECK(secp256k1_xonly_pubkey_from_pubkey(CTX, &xonly_pk, &pk_parity, &pk) == 0); - CHECK(ecount == 3); + CHECK_ILLEGAL(CTX, secp256k1_xonly_pubkey_from_pubkey(CTX, &xonly_pk, &pk_parity, &pk)); /* Choose a secret key such that the resulting pubkey and xonly_pubkey match. */ memset(sk, 0, sizeof(sk)); @@ -72,28 +59,21 @@ static void test_xonly_pubkey(void) { CHECK(secp256k1_fe_equal(&pk1.y, &y) == 1); /* Test xonly_pubkey_serialize and xonly_pubkey_parse */ - ecount = 0; - CHECK(secp256k1_xonly_pubkey_serialize(CTX, NULL, &xonly_pk) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_xonly_pubkey_serialize(CTX, buf32, NULL) == 0); + CHECK_ILLEGAL(CTX, secp256k1_xonly_pubkey_serialize(CTX, NULL, &xonly_pk)); + CHECK_ILLEGAL(CTX, secp256k1_xonly_pubkey_serialize(CTX, buf32, NULL)); CHECK(secp256k1_memcmp_var(buf32, zeros64, 32) == 0); - CHECK(ecount == 2); { /* A pubkey filled with 0s will fail to serialize due to pubkey_load * special casing. */ secp256k1_xonly_pubkey pk_tmp; memset(&pk_tmp, 0, sizeof(pk_tmp)); - CHECK(secp256k1_xonly_pubkey_serialize(CTX, buf32, &pk_tmp) == 0); + /* pubkey_load calls illegal callback */ + CHECK_ILLEGAL(CTX, secp256k1_xonly_pubkey_serialize(CTX, buf32, &pk_tmp)); } - /* pubkey_load called illegal callback */ - CHECK(ecount == 3); CHECK(secp256k1_xonly_pubkey_serialize(CTX, buf32, &xonly_pk) == 1); - ecount = 0; - CHECK(secp256k1_xonly_pubkey_parse(CTX, NULL, buf32) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_xonly_pubkey_parse(CTX, &xonly_pk, NULL) == 0); - CHECK(ecount == 2); + CHECK_ILLEGAL(CTX, secp256k1_xonly_pubkey_parse(CTX, NULL, buf32)); + CHECK_ILLEGAL(CTX, secp256k1_xonly_pubkey_parse(CTX, &xonly_pk, NULL)); /* Serialization and parse roundtrip */ CHECK(secp256k1_xonly_pubkey_from_pubkey(CTX, &xonly_pk, NULL, &pk) == 1); @@ -125,7 +105,6 @@ static void test_xonly_pubkey(void) { CHECK(secp256k1_xonly_pubkey_parse(CTX, &xonly_pk, &rand33[1]) == 1); } } - CHECK(ecount == 2); } static void test_xonly_pubkey_comparison(void) { @@ -139,29 +118,26 @@ static void test_xonly_pubkey_comparison(void) { }; secp256k1_xonly_pubkey pk1; secp256k1_xonly_pubkey pk2; - int ecount = 0; - - set_counting_callbacks(CTX, &ecount); CHECK(secp256k1_xonly_pubkey_parse(CTX, &pk1, pk1_ser) == 1); CHECK(secp256k1_xonly_pubkey_parse(CTX, &pk2, pk2_ser) == 1); - CHECK(secp256k1_xonly_pubkey_cmp(CTX, NULL, &pk2) < 0); - CHECK(ecount == 1); - CHECK(secp256k1_xonly_pubkey_cmp(CTX, &pk1, NULL) > 0); - CHECK(ecount == 2); + CHECK_ILLEGAL_VOID(CTX, CHECK(secp256k1_xonly_pubkey_cmp(CTX, NULL, &pk2) < 0)); + CHECK_ILLEGAL_VOID(CTX, CHECK(secp256k1_xonly_pubkey_cmp(CTX, &pk1, NULL) > 0)); CHECK(secp256k1_xonly_pubkey_cmp(CTX, &pk1, &pk2) < 0); CHECK(secp256k1_xonly_pubkey_cmp(CTX, &pk2, &pk1) > 0); CHECK(secp256k1_xonly_pubkey_cmp(CTX, &pk1, &pk1) == 0); CHECK(secp256k1_xonly_pubkey_cmp(CTX, &pk2, &pk2) == 0); - CHECK(ecount == 2); memset(&pk1, 0, sizeof(pk1)); /* illegal pubkey */ - CHECK(secp256k1_xonly_pubkey_cmp(CTX, &pk1, &pk2) < 0); - CHECK(ecount == 3); - CHECK(secp256k1_xonly_pubkey_cmp(CTX, &pk1, &pk1) == 0); - CHECK(ecount == 5); - CHECK(secp256k1_xonly_pubkey_cmp(CTX, &pk2, &pk1) > 0); - CHECK(ecount == 6); + CHECK_ILLEGAL_VOID(CTX, CHECK(secp256k1_xonly_pubkey_cmp(CTX, &pk1, &pk2) < 0)); + { + int32_t ecount = 0; + secp256k1_context_set_illegal_callback(CTX, counting_callback_fn, &ecount); + CHECK(secp256k1_xonly_pubkey_cmp(CTX, &pk1, &pk1) == 0); + CHECK(ecount == 2); + secp256k1_context_set_illegal_callback(CTX, NULL, NULL); + } + CHECK_ILLEGAL_VOID(CTX, CHECK(secp256k1_xonly_pubkey_cmp(CTX, &pk2, &pk1) > 0)); } static void test_xonly_pubkey_tweak(void) { @@ -175,30 +151,20 @@ static void test_xonly_pubkey_tweak(void) { unsigned char tweak[32]; int i; - int ecount; - - set_counting_callbacks(CTX, &ecount); - memset(overflows, 0xff, sizeof(overflows)); secp256k1_testrand256(tweak); secp256k1_testrand256(sk); CHECK(secp256k1_ec_pubkey_create(CTX, &internal_pk, sk) == 1); CHECK(secp256k1_xonly_pubkey_from_pubkey(CTX, &internal_xonly_pk, &pk_parity, &internal_pk) == 1); - ecount = 0; CHECK(secp256k1_xonly_pubkey_tweak_add(CTX, &output_pk, &internal_xonly_pk, tweak) == 1); - CHECK(ecount == 0); CHECK(secp256k1_xonly_pubkey_tweak_add(CTX, &output_pk, &internal_xonly_pk, tweak) == 1); - CHECK(ecount == 0); CHECK(secp256k1_xonly_pubkey_tweak_add(CTX, &output_pk, &internal_xonly_pk, tweak) == 1); - CHECK(secp256k1_xonly_pubkey_tweak_add(CTX, NULL, &internal_xonly_pk, tweak) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_xonly_pubkey_tweak_add(CTX, &output_pk, NULL, tweak) == 0); - CHECK(ecount == 2); + CHECK_ILLEGAL(CTX, secp256k1_xonly_pubkey_tweak_add(CTX, NULL, &internal_xonly_pk, tweak)); + CHECK_ILLEGAL(CTX, secp256k1_xonly_pubkey_tweak_add(CTX, &output_pk, NULL, tweak)); /* NULL internal_xonly_pk zeroes the output_pk */ CHECK(secp256k1_memcmp_var(&output_pk, zeros64, sizeof(output_pk)) == 0); - CHECK(secp256k1_xonly_pubkey_tweak_add(CTX, &output_pk, &internal_xonly_pk, NULL) == 0); - CHECK(ecount == 3); + CHECK_ILLEGAL(CTX, secp256k1_xonly_pubkey_tweak_add(CTX, &output_pk, &internal_xonly_pk, NULL)); /* NULL tweak zeroes the output_pk */ CHECK(secp256k1_memcmp_var(&output_pk, zeros64, sizeof(output_pk)) == 0); @@ -225,9 +191,7 @@ static void test_xonly_pubkey_tweak(void) { /* Invalid pk with a valid tweak */ memset(&internal_xonly_pk, 0, sizeof(internal_xonly_pk)); secp256k1_testrand256(tweak); - ecount = 0; - CHECK(secp256k1_xonly_pubkey_tweak_add(CTX, &output_pk, &internal_xonly_pk, tweak) == 0); - CHECK(ecount == 1); + CHECK_ILLEGAL(CTX, secp256k1_xonly_pubkey_tweak_add(CTX, &output_pk, &internal_xonly_pk, tweak)); CHECK(secp256k1_memcmp_var(&output_pk, zeros64, sizeof(output_pk)) == 0); } @@ -244,34 +208,23 @@ static void test_xonly_pubkey_tweak_check(void) { int pk_parity; unsigned char tweak[32]; - int ecount; - - set_counting_callbacks(CTX, &ecount); - memset(overflows, 0xff, sizeof(overflows)); secp256k1_testrand256(tweak); secp256k1_testrand256(sk); CHECK(secp256k1_ec_pubkey_create(CTX, &internal_pk, sk) == 1); CHECK(secp256k1_xonly_pubkey_from_pubkey(CTX, &internal_xonly_pk, &pk_parity, &internal_pk) == 1); - ecount = 0; CHECK(secp256k1_xonly_pubkey_tweak_add(CTX, &output_pk, &internal_xonly_pk, tweak) == 1); CHECK(secp256k1_xonly_pubkey_from_pubkey(CTX, &output_xonly_pk, &pk_parity, &output_pk) == 1); CHECK(secp256k1_xonly_pubkey_serialize(CTX, buf32, &output_xonly_pk) == 1); CHECK(secp256k1_xonly_pubkey_tweak_add_check(CTX, buf32, pk_parity, &internal_xonly_pk, tweak) == 1); - CHECK(ecount == 0); CHECK(secp256k1_xonly_pubkey_tweak_add_check(CTX, buf32, pk_parity, &internal_xonly_pk, tweak) == 1); - CHECK(ecount == 0); CHECK(secp256k1_xonly_pubkey_tweak_add_check(CTX, buf32, pk_parity, &internal_xonly_pk, tweak) == 1); - CHECK(secp256k1_xonly_pubkey_tweak_add_check(CTX, NULL, pk_parity, &internal_xonly_pk, tweak) == 0); - CHECK(ecount == 1); + CHECK_ILLEGAL(CTX, secp256k1_xonly_pubkey_tweak_add_check(CTX, NULL, pk_parity, &internal_xonly_pk, tweak)); /* invalid pk_parity value */ CHECK(secp256k1_xonly_pubkey_tweak_add_check(CTX, buf32, 2, &internal_xonly_pk, tweak) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_xonly_pubkey_tweak_add_check(CTX, buf32, pk_parity, NULL, tweak) == 0); - CHECK(ecount == 2); - CHECK(secp256k1_xonly_pubkey_tweak_add_check(CTX, buf32, pk_parity, &internal_xonly_pk, NULL) == 0); - CHECK(ecount == 3); + CHECK_ILLEGAL(CTX, secp256k1_xonly_pubkey_tweak_add_check(CTX, buf32, pk_parity, NULL, tweak)); + CHECK_ILLEGAL(CTX, secp256k1_xonly_pubkey_tweak_add_check(CTX, buf32, pk_parity, &internal_xonly_pk, NULL)); memset(tweak, 1, sizeof(tweak)); CHECK(secp256k1_xonly_pubkey_from_pubkey(CTX, &internal_xonly_pk, NULL, &internal_pk) == 1); @@ -290,7 +243,6 @@ static void test_xonly_pubkey_tweak_check(void) { CHECK(secp256k1_xonly_pubkey_tweak_add_check(CTX, output_pk32, pk_parity, &internal_xonly_pk, overflows) == 0); CHECK(secp256k1_xonly_pubkey_tweak_add(CTX, &output_pk, &internal_xonly_pk, overflows) == 0); CHECK(secp256k1_memcmp_var(&output_pk, zeros64, sizeof(output_pk)) == 0); - CHECK(ecount == 3); } /* Starts with an initial pubkey and recursively creates N_PUBKEYS - 1 @@ -335,33 +287,22 @@ static void test_keypair(void) { secp256k1_pubkey pk, pk_tmp; secp256k1_xonly_pubkey xonly_pk, xonly_pk_tmp; int pk_parity, pk_parity_tmp; - int ecount; - - set_counting_callbacks(CTX, &ecount); - set_counting_callbacks(STATIC_CTX, &ecount); CHECK(sizeof(zeros96) == sizeof(keypair)); memset(overflows, 0xFF, sizeof(overflows)); /* Test keypair_create */ - ecount = 0; secp256k1_testrand256(sk); CHECK(secp256k1_keypair_create(CTX, &keypair, sk) == 1); CHECK(secp256k1_memcmp_var(zeros96, &keypair, sizeof(keypair)) != 0); - CHECK(ecount == 0); CHECK(secp256k1_keypair_create(CTX, &keypair, sk) == 1); CHECK(secp256k1_memcmp_var(zeros96, &keypair, sizeof(keypair)) != 0); - CHECK(ecount == 0); - CHECK(secp256k1_keypair_create(CTX, NULL, sk) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_keypair_create(CTX, &keypair, NULL) == 0); + CHECK_ILLEGAL(CTX, secp256k1_keypair_create(CTX, NULL, sk)); + CHECK_ILLEGAL(CTX, secp256k1_keypair_create(CTX, &keypair, NULL)); CHECK(secp256k1_memcmp_var(zeros96, &keypair, sizeof(keypair)) == 0); - CHECK(ecount == 2); CHECK(secp256k1_keypair_create(CTX, &keypair, sk) == 1); - CHECK(ecount == 2); - CHECK(secp256k1_keypair_create(STATIC_CTX, &keypair, sk) == 0); + CHECK_ILLEGAL(STATIC_CTX, secp256k1_keypair_create(STATIC_CTX, &keypair, sk)); CHECK(secp256k1_memcmp_var(zeros96, &keypair, sizeof(keypair)) == 0); - CHECK(ecount == 3); /* Invalid secret key */ CHECK(secp256k1_keypair_create(CTX, &keypair, zeros96) == 0); @@ -370,14 +311,11 @@ static void test_keypair(void) { CHECK(secp256k1_memcmp_var(zeros96, &keypair, sizeof(keypair)) == 0); /* Test keypair_pub */ - ecount = 0; secp256k1_testrand256(sk); CHECK(secp256k1_keypair_create(CTX, &keypair, sk) == 1); CHECK(secp256k1_keypair_pub(CTX, &pk, &keypair) == 1); - CHECK(secp256k1_keypair_pub(CTX, NULL, &keypair) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_keypair_pub(CTX, &pk, NULL) == 0); - CHECK(ecount == 2); + CHECK_ILLEGAL(CTX, secp256k1_keypair_pub(CTX, NULL, &keypair)); + CHECK_ILLEGAL(CTX, secp256k1_keypair_pub(CTX, &pk, NULL)); CHECK(secp256k1_memcmp_var(zeros96, &pk, sizeof(pk)) == 0); /* Using an invalid keypair is fine for keypair_pub */ @@ -392,23 +330,19 @@ static void test_keypair(void) { CHECK(secp256k1_memcmp_var(&pk, &pk_tmp, sizeof(pk)) == 0); /** Test keypair_xonly_pub **/ - ecount = 0; secp256k1_testrand256(sk); CHECK(secp256k1_keypair_create(CTX, &keypair, sk) == 1); CHECK(secp256k1_keypair_xonly_pub(CTX, &xonly_pk, &pk_parity, &keypair) == 1); - CHECK(secp256k1_keypair_xonly_pub(CTX, NULL, &pk_parity, &keypair) == 0); - CHECK(ecount == 1); + CHECK_ILLEGAL(CTX, secp256k1_keypair_xonly_pub(CTX, NULL, &pk_parity, &keypair)); CHECK(secp256k1_keypair_xonly_pub(CTX, &xonly_pk, NULL, &keypair) == 1); - CHECK(secp256k1_keypair_xonly_pub(CTX, &xonly_pk, &pk_parity, NULL) == 0); - CHECK(ecount == 2); + CHECK_ILLEGAL(CTX, secp256k1_keypair_xonly_pub(CTX, &xonly_pk, &pk_parity, NULL)); CHECK(secp256k1_memcmp_var(zeros96, &xonly_pk, sizeof(xonly_pk)) == 0); /* Using an invalid keypair will set the xonly_pk to 0 (first reset * xonly_pk). */ CHECK(secp256k1_keypair_xonly_pub(CTX, &xonly_pk, &pk_parity, &keypair) == 1); memset(&keypair, 0, sizeof(keypair)); - CHECK(secp256k1_keypair_xonly_pub(CTX, &xonly_pk, &pk_parity, &keypair) == 0); + CHECK_ILLEGAL(CTX, secp256k1_keypair_xonly_pub(CTX, &xonly_pk, &pk_parity, &keypair)); CHECK(secp256k1_memcmp_var(zeros96, &xonly_pk, sizeof(xonly_pk)) == 0); - CHECK(ecount == 3); /** keypair holds the same xonly pubkey as pubkey_create **/ CHECK(secp256k1_ec_pubkey_create(CTX, &pk, sk) == 1); @@ -419,14 +353,11 @@ static void test_keypair(void) { CHECK(pk_parity == pk_parity_tmp); /* Test keypair_seckey */ - ecount = 0; secp256k1_testrand256(sk); CHECK(secp256k1_keypair_create(CTX, &keypair, sk) == 1); CHECK(secp256k1_keypair_sec(CTX, sk_tmp, &keypair) == 1); - CHECK(secp256k1_keypair_sec(CTX, NULL, &keypair) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_keypair_sec(CTX, sk_tmp, NULL) == 0); - CHECK(ecount == 2); + CHECK_ILLEGAL(CTX, secp256k1_keypair_sec(CTX, NULL, &keypair)); + CHECK_ILLEGAL(CTX, secp256k1_keypair_sec(CTX, sk_tmp, NULL)); CHECK(secp256k1_memcmp_var(zeros96, sk_tmp, sizeof(sk_tmp)) == 0); /* keypair returns the same seckey it got */ @@ -439,9 +370,6 @@ static void test_keypair(void) { memset(&keypair, 0, sizeof(keypair)); CHECK(secp256k1_keypair_sec(CTX, sk_tmp, &keypair) == 1); CHECK(secp256k1_memcmp_var(zeros96, sk_tmp, sizeof(sk_tmp)) == 0); - - secp256k1_context_set_error_callback(STATIC_CTX, NULL, NULL); - secp256k1_context_set_illegal_callback(STATIC_CTX, NULL, NULL); } static void test_keypair_add(void) { @@ -451,9 +379,6 @@ static void test_keypair_add(void) { unsigned char zeros96[96] = { 0 }; unsigned char tweak[32]; int i; - int ecount = 0; - - set_counting_callbacks(CTX, &ecount); CHECK(sizeof(zeros96) == sizeof(keypair)); secp256k1_testrand256(sk); @@ -462,14 +387,10 @@ static void test_keypair_add(void) { CHECK(secp256k1_keypair_create(CTX, &keypair, sk) == 1); CHECK(secp256k1_keypair_xonly_tweak_add(CTX, &keypair, tweak) == 1); - CHECK(ecount == 0); CHECK(secp256k1_keypair_xonly_tweak_add(CTX, &keypair, tweak) == 1); - CHECK(ecount == 0); CHECK(secp256k1_keypair_xonly_tweak_add(CTX, &keypair, tweak) == 1); - CHECK(secp256k1_keypair_xonly_tweak_add(CTX, NULL, tweak) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_keypair_xonly_tweak_add(CTX, &keypair, NULL) == 0); - CHECK(ecount == 2); + CHECK_ILLEGAL(CTX, secp256k1_keypair_xonly_tweak_add(CTX, NULL, tweak)); + CHECK_ILLEGAL(CTX, secp256k1_keypair_xonly_tweak_add(CTX, &keypair, NULL)); /* This does not set the keypair to zeroes */ CHECK(secp256k1_memcmp_var(&keypair, zeros96, sizeof(keypair)) != 0); @@ -503,20 +424,16 @@ static void test_keypair_add(void) { /* Invalid keypair with a valid tweak */ memset(&keypair, 0, sizeof(keypair)); secp256k1_testrand256(tweak); - ecount = 0; - CHECK(secp256k1_keypair_xonly_tweak_add(CTX, &keypair, tweak) == 0); - CHECK(ecount == 1); + CHECK_ILLEGAL(CTX, secp256k1_keypair_xonly_tweak_add(CTX, &keypair, tweak)); CHECK(secp256k1_memcmp_var(&keypair, zeros96, sizeof(keypair)) == 0); /* Only seckey part of keypair invalid */ CHECK(secp256k1_keypair_create(CTX, &keypair, sk) == 1); memset(&keypair, 0, 32); - CHECK(secp256k1_keypair_xonly_tweak_add(CTX, &keypair, tweak) == 0); - CHECK(ecount == 2); + CHECK_ILLEGAL(CTX, secp256k1_keypair_xonly_tweak_add(CTX, &keypair, tweak)); /* Only pubkey part of keypair invalid */ CHECK(secp256k1_keypair_create(CTX, &keypair, sk) == 1); memset(&keypair.data[32], 0, 64); - CHECK(secp256k1_keypair_xonly_tweak_add(CTX, &keypair, tweak) == 0); - CHECK(ecount == 3); + CHECK_ILLEGAL(CTX, secp256k1_keypair_xonly_tweak_add(CTX, &keypair, tweak)); /* Check that the keypair_tweak_add implementation is correct */ CHECK(secp256k1_keypair_create(CTX, &keypair, sk) == 1); diff --git a/src/secp256k1/src/modules/recovery/tests_impl.h b/src/secp256k1/src/modules/recovery/tests_impl.h index 3502c71ffe..728ccfed8d 100644 --- a/src/secp256k1/src/modules/recovery/tests_impl.h +++ b/src/secp256k1/src/modules/recovery/tests_impl.h @@ -36,7 +36,6 @@ static void test_ecdsa_recovery_api(void) { secp256k1_ecdsa_recoverable_signature recsig; unsigned char privkey[32] = { 1 }; unsigned char message[32] = { 2 }; - int32_t ecount = 0; int recid = 0; unsigned char sig[74]; unsigned char zero_privkey[32] = { 0 }; @@ -45,86 +44,52 @@ static void test_ecdsa_recovery_api(void) { 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff, 0xff }; - secp256k1_context_set_error_callback(CTX, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_illegal_callback(CTX, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_error_callback(STATIC_CTX, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_illegal_callback(STATIC_CTX, counting_illegal_callback_fn, &ecount); - /* Construct and verify corresponding public key. */ CHECK(secp256k1_ec_seckey_verify(CTX, privkey) == 1); CHECK(secp256k1_ec_pubkey_create(CTX, &pubkey, privkey) == 1); /* Check bad contexts and NULLs for signing */ - ecount = 0; CHECK(secp256k1_ecdsa_sign_recoverable(CTX, &recsig, message, privkey, NULL, NULL) == 1); - CHECK(ecount == 0); - CHECK(secp256k1_ecdsa_sign_recoverable(CTX, NULL, message, privkey, NULL, NULL) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_ecdsa_sign_recoverable(CTX, &recsig, NULL, privkey, NULL, NULL) == 0); - CHECK(ecount == 2); - CHECK(secp256k1_ecdsa_sign_recoverable(CTX, &recsig, message, NULL, NULL, NULL) == 0); - CHECK(ecount == 3); - CHECK(secp256k1_ecdsa_sign_recoverable(STATIC_CTX, &recsig, message, privkey, NULL, NULL) == 0); - CHECK(ecount == 4); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_sign_recoverable(CTX, NULL, message, privkey, NULL, NULL)); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_sign_recoverable(CTX, &recsig, NULL, privkey, NULL, NULL)); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_sign_recoverable(CTX, &recsig, message, NULL, NULL, NULL)); + CHECK_ILLEGAL(STATIC_CTX, secp256k1_ecdsa_sign_recoverable(STATIC_CTX, &recsig, message, privkey, NULL, NULL)); /* This will fail or succeed randomly, and in either case will not ARG_CHECK failure */ secp256k1_ecdsa_sign_recoverable(CTX, &recsig, message, privkey, recovery_test_nonce_function, NULL); - CHECK(ecount == 4); /* These will all fail, but not in ARG_CHECK way */ CHECK(secp256k1_ecdsa_sign_recoverable(CTX, &recsig, message, zero_privkey, NULL, NULL) == 0); CHECK(secp256k1_ecdsa_sign_recoverable(CTX, &recsig, message, over_privkey, NULL, NULL) == 0); /* This one will succeed. */ CHECK(secp256k1_ecdsa_sign_recoverable(CTX, &recsig, message, privkey, NULL, NULL) == 1); - CHECK(ecount == 4); /* Check signing with a goofy nonce function */ /* Check bad contexts and NULLs for recovery */ - ecount = 0; CHECK(secp256k1_ecdsa_recover(CTX, &recpubkey, &recsig, message) == 1); - CHECK(ecount == 0); - CHECK(secp256k1_ecdsa_recover(CTX, NULL, &recsig, message) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_ecdsa_recover(CTX, &recpubkey, NULL, message) == 0); - CHECK(ecount == 2); - CHECK(secp256k1_ecdsa_recover(CTX, &recpubkey, &recsig, NULL) == 0); - CHECK(ecount == 3); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_recover(CTX, NULL, &recsig, message)); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_recover(CTX, &recpubkey, NULL, message)); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_recover(CTX, &recpubkey, &recsig, NULL)); /* Check NULLs for conversion */ CHECK(secp256k1_ecdsa_sign(CTX, &normal_sig, message, privkey, NULL, NULL) == 1); - ecount = 0; - CHECK(secp256k1_ecdsa_recoverable_signature_convert(CTX, NULL, &recsig) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_ecdsa_recoverable_signature_convert(CTX, &normal_sig, NULL) == 0); - CHECK(ecount == 2); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_recoverable_signature_convert(CTX, NULL, &recsig)); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_recoverable_signature_convert(CTX, &normal_sig, NULL)); CHECK(secp256k1_ecdsa_recoverable_signature_convert(CTX, &normal_sig, &recsig) == 1); /* Check NULLs for de/serialization */ CHECK(secp256k1_ecdsa_sign_recoverable(CTX, &recsig, message, privkey, NULL, NULL) == 1); - ecount = 0; - CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(CTX, NULL, &recid, &recsig) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(CTX, sig, NULL, &recsig) == 0); - CHECK(ecount == 2); - CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(CTX, sig, &recid, NULL) == 0); - CHECK(ecount == 3); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_recoverable_signature_serialize_compact(CTX, NULL, &recid, &recsig)); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_recoverable_signature_serialize_compact(CTX, sig, NULL, &recsig)); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_recoverable_signature_serialize_compact(CTX, sig, &recid, NULL)); CHECK(secp256k1_ecdsa_recoverable_signature_serialize_compact(CTX, sig, &recid, &recsig) == 1); - CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(CTX, NULL, sig, recid) == 0); - CHECK(ecount == 4); - CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(CTX, &recsig, NULL, recid) == 0); - CHECK(ecount == 5); - CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(CTX, &recsig, sig, -1) == 0); - CHECK(ecount == 6); - CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(CTX, &recsig, sig, 5) == 0); - CHECK(ecount == 7); - /* overflow in signature will fail but not affect ecount */ + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_recoverable_signature_parse_compact(CTX, NULL, sig, recid)); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_recoverable_signature_parse_compact(CTX, &recsig, NULL, recid)); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_recoverable_signature_parse_compact(CTX, &recsig, sig, -1)); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_recoverable_signature_parse_compact(CTX, &recsig, sig, 5)); + /* overflow in signature will not result in calling illegal_callback */ memcpy(sig, over_privkey, 32); CHECK(secp256k1_ecdsa_recoverable_signature_parse_compact(CTX, &recsig, sig, recid) == 0); - CHECK(ecount == 7); - - /* cleanup */ - secp256k1_context_set_error_callback(STATIC_CTX, NULL, NULL); - secp256k1_context_set_illegal_callback(STATIC_CTX, NULL, NULL); } static void test_ecdsa_recovery_end_to_end(void) { diff --git a/src/secp256k1/src/modules/schnorrsig/tests_impl.h b/src/secp256k1/src/modules/schnorrsig/tests_impl.h index 90337ff03e..8ada90a87b 100644 --- a/src/secp256k1/src/modules/schnorrsig/tests_impl.h +++ b/src/secp256k1/src/modules/schnorrsig/tests_impl.h @@ -116,14 +116,6 @@ static void test_schnorrsig_api(void) { secp256k1_schnorrsig_extraparams extraparams = SECP256K1_SCHNORRSIG_EXTRAPARAMS_INIT; secp256k1_schnorrsig_extraparams invalid_extraparams = {{ 0 }, NULL, NULL}; - /** setup **/ - int ecount = 0; - - secp256k1_context_set_error_callback(CTX, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_illegal_callback(CTX, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_error_callback(STATIC_CTX, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_illegal_callback(STATIC_CTX, counting_illegal_callback_fn, &ecount); - secp256k1_testrand256(sk1); secp256k1_testrand256(sk2); secp256k1_testrand256(sk3); @@ -137,57 +129,30 @@ static void test_schnorrsig_api(void) { memset(&zero_pk, 0, sizeof(zero_pk)); /** main test body **/ - ecount = 0; CHECK(secp256k1_schnorrsig_sign32(CTX, sig, msg, &keypairs[0], NULL) == 1); - CHECK(ecount == 0); - CHECK(secp256k1_schnorrsig_sign32(CTX, NULL, msg, &keypairs[0], NULL) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_schnorrsig_sign32(CTX, sig, NULL, &keypairs[0], NULL) == 0); - CHECK(ecount == 2); - CHECK(secp256k1_schnorrsig_sign32(CTX, sig, msg, NULL, NULL) == 0); - CHECK(ecount == 3); - CHECK(secp256k1_schnorrsig_sign32(CTX, sig, msg, &invalid_keypair, NULL) == 0); - CHECK(ecount == 4); - CHECK(secp256k1_schnorrsig_sign32(STATIC_CTX, sig, msg, &keypairs[0], NULL) == 0); - CHECK(ecount == 5); - - ecount = 0; + CHECK_ILLEGAL(CTX, secp256k1_schnorrsig_sign32(CTX, NULL, msg, &keypairs[0], NULL)); + CHECK_ILLEGAL(CTX, secp256k1_schnorrsig_sign32(CTX, sig, NULL, &keypairs[0], NULL)); + CHECK_ILLEGAL(CTX, secp256k1_schnorrsig_sign32(CTX, sig, msg, NULL, NULL)); + CHECK_ILLEGAL(CTX, secp256k1_schnorrsig_sign32(CTX, sig, msg, &invalid_keypair, NULL)); + CHECK_ILLEGAL(STATIC_CTX, secp256k1_schnorrsig_sign32(STATIC_CTX, sig, msg, &keypairs[0], NULL)); + CHECK(secp256k1_schnorrsig_sign_custom(CTX, sig, msg, sizeof(msg), &keypairs[0], &extraparams) == 1); - CHECK(ecount == 0); - CHECK(secp256k1_schnorrsig_sign_custom(CTX, NULL, msg, sizeof(msg), &keypairs[0], &extraparams) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_schnorrsig_sign_custom(CTX, sig, NULL, sizeof(msg), &keypairs[0], &extraparams) == 0); - CHECK(ecount == 2); + CHECK_ILLEGAL(CTX, secp256k1_schnorrsig_sign_custom(CTX, NULL, msg, sizeof(msg), &keypairs[0], &extraparams)); + CHECK_ILLEGAL(CTX, secp256k1_schnorrsig_sign_custom(CTX, sig, NULL, sizeof(msg), &keypairs[0], &extraparams)); CHECK(secp256k1_schnorrsig_sign_custom(CTX, sig, NULL, 0, &keypairs[0], &extraparams) == 1); - CHECK(ecount == 2); - CHECK(secp256k1_schnorrsig_sign_custom(CTX, sig, msg, sizeof(msg), NULL, &extraparams) == 0); - CHECK(ecount == 3); - CHECK(secp256k1_schnorrsig_sign_custom(CTX, sig, msg, sizeof(msg), &invalid_keypair, &extraparams) == 0); - CHECK(ecount == 4); + CHECK_ILLEGAL(CTX, secp256k1_schnorrsig_sign_custom(CTX, sig, msg, sizeof(msg), NULL, &extraparams)); + CHECK_ILLEGAL(CTX, secp256k1_schnorrsig_sign_custom(CTX, sig, msg, sizeof(msg), &invalid_keypair, &extraparams)); CHECK(secp256k1_schnorrsig_sign_custom(CTX, sig, msg, sizeof(msg), &keypairs[0], NULL) == 1); - CHECK(ecount == 4); - CHECK(secp256k1_schnorrsig_sign_custom(CTX, sig, msg, sizeof(msg), &keypairs[0], &invalid_extraparams) == 0); - CHECK(ecount == 5); - CHECK(secp256k1_schnorrsig_sign_custom(STATIC_CTX, sig, msg, sizeof(msg), &keypairs[0], &extraparams) == 0); - CHECK(ecount == 6); + CHECK_ILLEGAL(CTX, secp256k1_schnorrsig_sign_custom(CTX, sig, msg, sizeof(msg), &keypairs[0], &invalid_extraparams)); + CHECK_ILLEGAL(STATIC_CTX, secp256k1_schnorrsig_sign_custom(STATIC_CTX, sig, msg, sizeof(msg), &keypairs[0], &extraparams)); - ecount = 0; CHECK(secp256k1_schnorrsig_sign32(CTX, sig, msg, &keypairs[0], NULL) == 1); CHECK(secp256k1_schnorrsig_verify(CTX, sig, msg, sizeof(msg), &pk[0]) == 1); - CHECK(ecount == 0); - CHECK(secp256k1_schnorrsig_verify(CTX, NULL, msg, sizeof(msg), &pk[0]) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_schnorrsig_verify(CTX, sig, NULL, sizeof(msg), &pk[0]) == 0); - CHECK(ecount == 2); + CHECK_ILLEGAL(CTX, secp256k1_schnorrsig_verify(CTX, NULL, msg, sizeof(msg), &pk[0])); + CHECK_ILLEGAL(CTX, secp256k1_schnorrsig_verify(CTX, sig, NULL, sizeof(msg), &pk[0])); CHECK(secp256k1_schnorrsig_verify(CTX, sig, NULL, 0, &pk[0]) == 0); - CHECK(ecount == 2); - CHECK(secp256k1_schnorrsig_verify(CTX, sig, msg, sizeof(msg), NULL) == 0); - CHECK(ecount == 3); - CHECK(secp256k1_schnorrsig_verify(CTX, sig, msg, sizeof(msg), &zero_pk) == 0); - CHECK(ecount == 4); - - secp256k1_context_set_error_callback(STATIC_CTX, NULL, NULL); - secp256k1_context_set_illegal_callback(STATIC_CTX, NULL, NULL); + CHECK_ILLEGAL(CTX, secp256k1_schnorrsig_verify(CTX, sig, msg, sizeof(msg), NULL)); + CHECK_ILLEGAL(CTX, secp256k1_schnorrsig_verify(CTX, sig, msg, sizeof(msg), &zero_pk)); } /* Checks that hash initialized by secp256k1_schnorrsig_sha256_tagged has the diff --git a/src/secp256k1/src/scalar.h b/src/secp256k1/src/scalar.h index 4b3c2998bb..98b1287bb5 100644 --- a/src/secp256k1/src/scalar.h +++ b/src/secp256k1/src/scalar.h @@ -25,7 +25,7 @@ static void secp256k1_scalar_clear(secp256k1_scalar *r); /** Access bits from a scalar. All requested bits must belong to the same 32-bit limb. */ static unsigned int secp256k1_scalar_get_bits(const secp256k1_scalar *a, unsigned int offset, unsigned int count); -/** Access bits from a scalar. Not constant time. */ +/** Access bits from a scalar. Not constant time in offset and count. */ static unsigned int secp256k1_scalar_get_bits_var(const secp256k1_scalar *a, unsigned int offset, unsigned int count); /** Set a scalar from a big endian byte array. The scalar will be reduced modulo group order `n`. @@ -54,10 +54,6 @@ static void secp256k1_scalar_cadd_bit(secp256k1_scalar *r, unsigned int bit, int /** Multiply two scalars (modulo the group order). */ static void secp256k1_scalar_mul(secp256k1_scalar *r, const secp256k1_scalar *a, const secp256k1_scalar *b); -/** Shift a scalar right by some amount strictly between 0 and 16, returning - * the low bits that were shifted off */ -static int secp256k1_scalar_shr_int(secp256k1_scalar *r, int n); - /** Compute the inverse of a scalar (modulo the group order). */ static void secp256k1_scalar_inverse(secp256k1_scalar *r, const secp256k1_scalar *a); @@ -67,6 +63,9 @@ static void secp256k1_scalar_inverse_var(secp256k1_scalar *r, const secp256k1_sc /** Compute the complement of a scalar (modulo the group order). */ static void secp256k1_scalar_negate(secp256k1_scalar *r, const secp256k1_scalar *a); +/** Multiply a scalar with the multiplicative inverse of 2. */ +static void secp256k1_scalar_half(secp256k1_scalar *r, const secp256k1_scalar *a); + /** Check whether a scalar equals zero. */ static int secp256k1_scalar_is_zero(const secp256k1_scalar *a); @@ -101,5 +100,6 @@ static void secp256k1_scalar_cmov(secp256k1_scalar *r, const secp256k1_scalar *a /** Check invariants on a scalar (no-op unless VERIFY is enabled). */ static void secp256k1_scalar_verify(const secp256k1_scalar *r); +#define SECP256K1_SCALAR_VERIFY(r) secp256k1_scalar_verify(r) #endif /* SECP256K1_SCALAR_H */ diff --git a/src/secp256k1/src/scalar_4x64_impl.h b/src/secp256k1/src/scalar_4x64_impl.h index 715cc12ee5..7b9c542f07 100644 --- a/src/secp256k1/src/scalar_4x64_impl.h +++ b/src/secp256k1/src/scalar_4x64_impl.h @@ -42,18 +42,18 @@ SECP256K1_INLINE static void secp256k1_scalar_set_int(secp256k1_scalar *r, unsig r->d[2] = 0; r->d[3] = 0; - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); } SECP256K1_INLINE static unsigned int secp256k1_scalar_get_bits(const secp256k1_scalar *a, unsigned int offset, unsigned int count) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); VERIFY_CHECK((offset + count - 1) >> 6 == offset >> 6); return (a->d[offset >> 6] >> (offset & 0x3F)) & ((((uint64_t)1) << count) - 1); } SECP256K1_INLINE static unsigned int secp256k1_scalar_get_bits_var(const secp256k1_scalar *a, unsigned int offset, unsigned int count) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); VERIFY_CHECK(count < 32); VERIFY_CHECK(offset + count <= 256); @@ -93,15 +93,15 @@ SECP256K1_INLINE static int secp256k1_scalar_reduce(secp256k1_scalar *r, unsigne secp256k1_u128_accum_u64(&t, r->d[3]); r->d[3] = secp256k1_u128_to_u64(&t); - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); return overflow; } static int secp256k1_scalar_add(secp256k1_scalar *r, const secp256k1_scalar *a, const secp256k1_scalar *b) { int overflow; secp256k1_uint128 t; - secp256k1_scalar_verify(a); - secp256k1_scalar_verify(b); + SECP256K1_SCALAR_VERIFY(a); + SECP256K1_SCALAR_VERIFY(b); secp256k1_u128_from_u64(&t, a->d[0]); secp256k1_u128_accum_u64(&t, b->d[0]); @@ -119,14 +119,14 @@ static int secp256k1_scalar_add(secp256k1_scalar *r, const secp256k1_scalar *a, VERIFY_CHECK(overflow == 0 || overflow == 1); secp256k1_scalar_reduce(r, overflow); - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); return overflow; } static void secp256k1_scalar_cadd_bit(secp256k1_scalar *r, unsigned int bit, int flag) { secp256k1_uint128 t; volatile int vflag = flag; - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); VERIFY_CHECK(bit < 256); bit += ((uint32_t) vflag - 1) & 0x100; /* forcing (bit >> 6) > 3 makes this a noop */ @@ -143,10 +143,8 @@ static void secp256k1_scalar_cadd_bit(secp256k1_scalar *r, unsigned int bit, int secp256k1_u128_accum_u64(&t, ((uint64_t)((bit >> 6) == 3)) << (bit & 0x3F)); r->d[3] = secp256k1_u128_to_u64(&t); - secp256k1_scalar_verify(r); -#ifdef VERIFY + SECP256K1_SCALAR_VERIFY(r); VERIFY_CHECK(secp256k1_u128_hi_u64(&t) == 0); -#endif } static void secp256k1_scalar_set_b32(secp256k1_scalar *r, const unsigned char *b32, int *overflow) { @@ -160,11 +158,11 @@ static void secp256k1_scalar_set_b32(secp256k1_scalar *r, const unsigned char *b *overflow = over; } - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); } static void secp256k1_scalar_get_b32(unsigned char *bin, const secp256k1_scalar* a) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); secp256k1_write_be64(&bin[0], a->d[3]); secp256k1_write_be64(&bin[8], a->d[2]); @@ -173,7 +171,7 @@ static void secp256k1_scalar_get_b32(unsigned char *bin, const secp256k1_scalar* } SECP256K1_INLINE static int secp256k1_scalar_is_zero(const secp256k1_scalar *a) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); return (a->d[0] | a->d[1] | a->d[2] | a->d[3]) == 0; } @@ -181,7 +179,7 @@ SECP256K1_INLINE static int secp256k1_scalar_is_zero(const secp256k1_scalar *a) static void secp256k1_scalar_negate(secp256k1_scalar *r, const secp256k1_scalar *a) { uint64_t nonzero = 0xFFFFFFFFFFFFFFFFULL * (secp256k1_scalar_is_zero(a) == 0); secp256k1_uint128 t; - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); secp256k1_u128_from_u64(&t, ~a->d[0]); secp256k1_u128_accum_u64(&t, SECP256K1_N_0 + 1); @@ -196,11 +194,52 @@ static void secp256k1_scalar_negate(secp256k1_scalar *r, const secp256k1_scalar secp256k1_u128_accum_u64(&t, SECP256K1_N_3); r->d[3] = secp256k1_u128_to_u64(&t) & nonzero; - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); +} + +static void secp256k1_scalar_half(secp256k1_scalar *r, const secp256k1_scalar *a) { + /* Writing `/` for field division and `//` for integer division, we compute + * + * a/2 = (a - (a&1))/2 + (a&1)/2 + * = (a >> 1) + (a&1 ? 1/2 : 0) + * = (a >> 1) + (a&1 ? n//2+1 : 0), + * + * where n is the group order and in the last equality we have used 1/2 = n//2+1 (mod n). + * For n//2, we have the constants SECP256K1_N_H_0, ... + * + * This sum does not overflow. The most extreme case is a = -2, the largest odd scalar. Here: + * - the left summand is: a >> 1 = (a - a&1)/2 = (n-2-1)//2 = (n-3)//2 + * - the right summand is: a&1 ? n//2+1 : 0 = n//2+1 = (n-1)//2 + 2//2 = (n+1)//2 + * Together they sum to (n-3)//2 + (n+1)//2 = (2n-2)//2 = n - 1, which is less than n. + */ + uint64_t mask = -(uint64_t)(a->d[0] & 1U); + secp256k1_uint128 t; + SECP256K1_SCALAR_VERIFY(a); + + secp256k1_u128_from_u64(&t, (a->d[0] >> 1) | (a->d[1] << 63)); + secp256k1_u128_accum_u64(&t, (SECP256K1_N_H_0 + 1U) & mask); + r->d[0] = secp256k1_u128_to_u64(&t); secp256k1_u128_rshift(&t, 64); + secp256k1_u128_accum_u64(&t, (a->d[1] >> 1) | (a->d[2] << 63)); + secp256k1_u128_accum_u64(&t, SECP256K1_N_H_1 & mask); + r->d[1] = secp256k1_u128_to_u64(&t); secp256k1_u128_rshift(&t, 64); + secp256k1_u128_accum_u64(&t, (a->d[2] >> 1) | (a->d[3] << 63)); + secp256k1_u128_accum_u64(&t, SECP256K1_N_H_2 & mask); + r->d[2] = secp256k1_u128_to_u64(&t); secp256k1_u128_rshift(&t, 64); + r->d[3] = secp256k1_u128_to_u64(&t) + (a->d[3] >> 1) + (SECP256K1_N_H_3 & mask); +#ifdef VERIFY + /* The line above only computed the bottom 64 bits of r->d[3]; redo the computation + * in full 128 bits to make sure the top 64 bits are indeed zero. */ + secp256k1_u128_accum_u64(&t, a->d[3] >> 1); + secp256k1_u128_accum_u64(&t, SECP256K1_N_H_3 & mask); + secp256k1_u128_rshift(&t, 64); + VERIFY_CHECK(secp256k1_u128_to_u64(&t) == 0); + + SECP256K1_SCALAR_VERIFY(r); +#endif } SECP256K1_INLINE static int secp256k1_scalar_is_one(const secp256k1_scalar *a) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); return ((a->d[0] ^ 1) | a->d[1] | a->d[2] | a->d[3]) == 0; } @@ -208,7 +247,7 @@ SECP256K1_INLINE static int secp256k1_scalar_is_one(const secp256k1_scalar *a) { static int secp256k1_scalar_is_high(const secp256k1_scalar *a) { int yes = 0; int no = 0; - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); no |= (a->d[3] < SECP256K1_N_H_3); yes |= (a->d[3] > SECP256K1_N_H_3) & ~no; @@ -226,7 +265,7 @@ static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) { uint64_t mask = -vflag; uint64_t nonzero = (secp256k1_scalar_is_zero(r) != 0) - 1; secp256k1_uint128 t; - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); secp256k1_u128_from_u64(&t, r->d[0] ^ mask); secp256k1_u128_accum_u64(&t, (SECP256K1_N_0 + 1) & mask); @@ -241,7 +280,7 @@ static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) { secp256k1_u128_accum_u64(&t, SECP256K1_N_3 & mask); r->d[3] = secp256k1_u128_to_u64(&t) & nonzero; - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); return 2 * (mask == 0) - 1; } @@ -800,33 +839,17 @@ static void secp256k1_scalar_mul_512(uint64_t l[8], const secp256k1_scalar *a, c static void secp256k1_scalar_mul(secp256k1_scalar *r, const secp256k1_scalar *a, const secp256k1_scalar *b) { uint64_t l[8]; - secp256k1_scalar_verify(a); - secp256k1_scalar_verify(b); + SECP256K1_SCALAR_VERIFY(a); + SECP256K1_SCALAR_VERIFY(b); secp256k1_scalar_mul_512(l, a, b); secp256k1_scalar_reduce_512(r, l); - secp256k1_scalar_verify(r); -} - -static int secp256k1_scalar_shr_int(secp256k1_scalar *r, int n) { - int ret; - secp256k1_scalar_verify(r); - VERIFY_CHECK(n > 0); - VERIFY_CHECK(n < 16); - - ret = r->d[0] & ((1 << n) - 1); - r->d[0] = (r->d[0] >> n) + (r->d[1] << (64 - n)); - r->d[1] = (r->d[1] >> n) + (r->d[2] << (64 - n)); - r->d[2] = (r->d[2] >> n) + (r->d[3] << (64 - n)); - r->d[3] = (r->d[3] >> n); - - secp256k1_scalar_verify(r); - return ret; + SECP256K1_SCALAR_VERIFY(r); } static void secp256k1_scalar_split_128(secp256k1_scalar *r1, secp256k1_scalar *r2, const secp256k1_scalar *k) { - secp256k1_scalar_verify(k); + SECP256K1_SCALAR_VERIFY(k); r1->d[0] = k->d[0]; r1->d[1] = k->d[1]; @@ -837,13 +860,13 @@ static void secp256k1_scalar_split_128(secp256k1_scalar *r1, secp256k1_scalar *r r2->d[2] = 0; r2->d[3] = 0; - secp256k1_scalar_verify(r1); - secp256k1_scalar_verify(r2); + SECP256K1_SCALAR_VERIFY(r1); + SECP256K1_SCALAR_VERIFY(r2); } SECP256K1_INLINE static int secp256k1_scalar_eq(const secp256k1_scalar *a, const secp256k1_scalar *b) { - secp256k1_scalar_verify(a); - secp256k1_scalar_verify(b); + SECP256K1_SCALAR_VERIFY(a); + SECP256K1_SCALAR_VERIFY(b); return ((a->d[0] ^ b->d[0]) | (a->d[1] ^ b->d[1]) | (a->d[2] ^ b->d[2]) | (a->d[3] ^ b->d[3])) == 0; } @@ -853,8 +876,8 @@ SECP256K1_INLINE static void secp256k1_scalar_mul_shift_var(secp256k1_scalar *r, unsigned int shiftlimbs; unsigned int shiftlow; unsigned int shifthigh; - secp256k1_scalar_verify(a); - secp256k1_scalar_verify(b); + SECP256K1_SCALAR_VERIFY(a); + SECP256K1_SCALAR_VERIFY(b); VERIFY_CHECK(shift >= 256); secp256k1_scalar_mul_512(l, a, b); @@ -867,13 +890,13 @@ SECP256K1_INLINE static void secp256k1_scalar_mul_shift_var(secp256k1_scalar *r, r->d[3] = shift < 320 ? (l[3 + shiftlimbs] >> shiftlow) : 0; secp256k1_scalar_cadd_bit(r, 0, (l[(shift - 1) >> 6] >> ((shift - 1) & 0x3f)) & 1); - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); } static SECP256K1_INLINE void secp256k1_scalar_cmov(secp256k1_scalar *r, const secp256k1_scalar *a, int flag) { uint64_t mask0, mask1; volatile int vflag = flag; - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); SECP256K1_CHECKMEM_CHECK_VERIFY(r->d, sizeof(r->d)); mask0 = vflag + ~((uint64_t)0); @@ -883,7 +906,7 @@ static SECP256K1_INLINE void secp256k1_scalar_cmov(secp256k1_scalar *r, const se r->d[2] = (r->d[2] & mask0) | (a->d[2] & mask1); r->d[3] = (r->d[3] & mask0) | (a->d[3] & mask1); - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); } static void secp256k1_scalar_from_signed62(secp256k1_scalar *r, const secp256k1_modinv64_signed62 *a) { @@ -903,13 +926,13 @@ static void secp256k1_scalar_from_signed62(secp256k1_scalar *r, const secp256k1_ r->d[2] = a2 >> 4 | a3 << 58; r->d[3] = a3 >> 6 | a4 << 56; - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); } static void secp256k1_scalar_to_signed62(secp256k1_modinv64_signed62 *r, const secp256k1_scalar *a) { const uint64_t M62 = UINT64_MAX >> 2; const uint64_t a0 = a->d[0], a1 = a->d[1], a2 = a->d[2], a3 = a->d[3]; - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); r->v[0] = a0 & M62; r->v[1] = (a0 >> 62 | a1 << 2) & M62; @@ -928,16 +951,14 @@ static void secp256k1_scalar_inverse(secp256k1_scalar *r, const secp256k1_scalar #ifdef VERIFY int zero_in = secp256k1_scalar_is_zero(x); #endif - secp256k1_scalar_verify(x); + SECP256K1_SCALAR_VERIFY(x); secp256k1_scalar_to_signed62(&s, x); secp256k1_modinv64(&s, &secp256k1_const_modinfo_scalar); secp256k1_scalar_from_signed62(r, &s); - secp256k1_scalar_verify(r); -#ifdef VERIFY + SECP256K1_SCALAR_VERIFY(r); VERIFY_CHECK(secp256k1_scalar_is_zero(r) == zero_in); -#endif } static void secp256k1_scalar_inverse_var(secp256k1_scalar *r, const secp256k1_scalar *x) { @@ -945,20 +966,18 @@ static void secp256k1_scalar_inverse_var(secp256k1_scalar *r, const secp256k1_sc #ifdef VERIFY int zero_in = secp256k1_scalar_is_zero(x); #endif - secp256k1_scalar_verify(x); + SECP256K1_SCALAR_VERIFY(x); secp256k1_scalar_to_signed62(&s, x); secp256k1_modinv64_var(&s, &secp256k1_const_modinfo_scalar); secp256k1_scalar_from_signed62(r, &s); - secp256k1_scalar_verify(r); -#ifdef VERIFY + SECP256K1_SCALAR_VERIFY(r); VERIFY_CHECK(secp256k1_scalar_is_zero(r) == zero_in); -#endif } SECP256K1_INLINE static int secp256k1_scalar_is_even(const secp256k1_scalar *a) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); return !(a->d[0] & 1); } diff --git a/src/secp256k1/src/scalar_8x32_impl.h b/src/secp256k1/src/scalar_8x32_impl.h index 5ca1342273..58ae51bc02 100644 --- a/src/secp256k1/src/scalar_8x32_impl.h +++ b/src/secp256k1/src/scalar_8x32_impl.h @@ -59,18 +59,18 @@ SECP256K1_INLINE static void secp256k1_scalar_set_int(secp256k1_scalar *r, unsig r->d[6] = 0; r->d[7] = 0; - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); } SECP256K1_INLINE static unsigned int secp256k1_scalar_get_bits(const secp256k1_scalar *a, unsigned int offset, unsigned int count) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); VERIFY_CHECK((offset + count - 1) >> 5 == offset >> 5); return (a->d[offset >> 5] >> (offset & 0x1F)) & ((1 << count) - 1); } SECP256K1_INLINE static unsigned int secp256k1_scalar_get_bits_var(const secp256k1_scalar *a, unsigned int offset, unsigned int count) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); VERIFY_CHECK(count < 32); VERIFY_CHECK(offset + count <= 256); @@ -121,15 +121,15 @@ SECP256K1_INLINE static int secp256k1_scalar_reduce(secp256k1_scalar *r, uint32_ t += (uint64_t)r->d[7]; r->d[7] = t & 0xFFFFFFFFUL; - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); return overflow; } static int secp256k1_scalar_add(secp256k1_scalar *r, const secp256k1_scalar *a, const secp256k1_scalar *b) { int overflow; uint64_t t = (uint64_t)a->d[0] + b->d[0]; - secp256k1_scalar_verify(a); - secp256k1_scalar_verify(b); + SECP256K1_SCALAR_VERIFY(a); + SECP256K1_SCALAR_VERIFY(b); r->d[0] = t & 0xFFFFFFFFULL; t >>= 32; t += (uint64_t)a->d[1] + b->d[1]; @@ -150,14 +150,14 @@ static int secp256k1_scalar_add(secp256k1_scalar *r, const secp256k1_scalar *a, VERIFY_CHECK(overflow == 0 || overflow == 1); secp256k1_scalar_reduce(r, overflow); - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); return overflow; } static void secp256k1_scalar_cadd_bit(secp256k1_scalar *r, unsigned int bit, int flag) { uint64_t t; volatile int vflag = flag; - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); VERIFY_CHECK(bit < 256); bit += ((uint32_t) vflag - 1) & 0x100; /* forcing (bit >> 5) > 7 makes this a noop */ @@ -178,10 +178,8 @@ static void secp256k1_scalar_cadd_bit(secp256k1_scalar *r, unsigned int bit, int t += (uint64_t)r->d[7] + (((uint32_t)((bit >> 5) == 7)) << (bit & 0x1F)); r->d[7] = t & 0xFFFFFFFFULL; - secp256k1_scalar_verify(r); -#ifdef VERIFY + SECP256K1_SCALAR_VERIFY(r); VERIFY_CHECK((t >> 32) == 0); -#endif } static void secp256k1_scalar_set_b32(secp256k1_scalar *r, const unsigned char *b32, int *overflow) { @@ -199,11 +197,11 @@ static void secp256k1_scalar_set_b32(secp256k1_scalar *r, const unsigned char *b *overflow = over; } - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); } static void secp256k1_scalar_get_b32(unsigned char *bin, const secp256k1_scalar* a) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); secp256k1_write_be32(&bin[0], a->d[7]); secp256k1_write_be32(&bin[4], a->d[6]); @@ -216,7 +214,7 @@ static void secp256k1_scalar_get_b32(unsigned char *bin, const secp256k1_scalar* } SECP256K1_INLINE static int secp256k1_scalar_is_zero(const secp256k1_scalar *a) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); return (a->d[0] | a->d[1] | a->d[2] | a->d[3] | a->d[4] | a->d[5] | a->d[6] | a->d[7]) == 0; } @@ -224,7 +222,7 @@ SECP256K1_INLINE static int secp256k1_scalar_is_zero(const secp256k1_scalar *a) static void secp256k1_scalar_negate(secp256k1_scalar *r, const secp256k1_scalar *a) { uint32_t nonzero = 0xFFFFFFFFUL * (secp256k1_scalar_is_zero(a) == 0); uint64_t t = (uint64_t)(~a->d[0]) + SECP256K1_N_0 + 1; - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); r->d[0] = t & nonzero; t >>= 32; t += (uint64_t)(~a->d[1]) + SECP256K1_N_1; @@ -242,11 +240,59 @@ static void secp256k1_scalar_negate(secp256k1_scalar *r, const secp256k1_scalar t += (uint64_t)(~a->d[7]) + SECP256K1_N_7; r->d[7] = t & nonzero; - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); +} + +static void secp256k1_scalar_half(secp256k1_scalar *r, const secp256k1_scalar *a) { + /* Writing `/` for field division and `//` for integer division, we compute + * + * a/2 = (a - (a&1))/2 + (a&1)/2 + * = (a >> 1) + (a&1 ? 1/2 : 0) + * = (a >> 1) + (a&1 ? n//2+1 : 0), + * + * where n is the group order and in the last equality we have used 1/2 = n//2+1 (mod n). + * For n//2, we have the constants SECP256K1_N_H_0, ... + * + * This sum does not overflow. The most extreme case is a = -2, the largest odd scalar. Here: + * - the left summand is: a >> 1 = (a - a&1)/2 = (n-2-1)//2 = (n-3)//2 + * - the right summand is: a&1 ? n//2+1 : 0 = n//2+1 = (n-1)//2 + 2//2 = (n+1)//2 + * Together they sum to (n-3)//2 + (n+1)//2 = (2n-2)//2 = n - 1, which is less than n. + */ + uint32_t mask = -(uint32_t)(a->d[0] & 1U); + uint64_t t = (uint32_t)((a->d[0] >> 1) | (a->d[1] << 31)); + SECP256K1_SCALAR_VERIFY(a); + + t += (SECP256K1_N_H_0 + 1U) & mask; + r->d[0] = t; t >>= 32; + t += (uint32_t)((a->d[1] >> 1) | (a->d[2] << 31)); + t += SECP256K1_N_H_1 & mask; + r->d[1] = t; t >>= 32; + t += (uint32_t)((a->d[2] >> 1) | (a->d[3] << 31)); + t += SECP256K1_N_H_2 & mask; + r->d[2] = t; t >>= 32; + t += (uint32_t)((a->d[3] >> 1) | (a->d[4] << 31)); + t += SECP256K1_N_H_3 & mask; + r->d[3] = t; t >>= 32; + t += (uint32_t)((a->d[4] >> 1) | (a->d[5] << 31)); + t += SECP256K1_N_H_4 & mask; + r->d[4] = t; t >>= 32; + t += (uint32_t)((a->d[5] >> 1) | (a->d[6] << 31)); + t += SECP256K1_N_H_5 & mask; + r->d[5] = t; t >>= 32; + t += (uint32_t)((a->d[6] >> 1) | (a->d[7] << 31)); + t += SECP256K1_N_H_6 & mask; + r->d[6] = t; t >>= 32; + r->d[7] = (uint32_t)t + (uint32_t)(a->d[7] >> 1) + (SECP256K1_N_H_7 & mask); + + /* The line above only computed the bottom 32 bits of r->d[7]. Redo the computation + * in full 64 bits to make sure the top 32 bits are indeed zero. */ + VERIFY_CHECK((t + (a->d[7] >> 1) + (SECP256K1_N_H_7 & mask)) >> 32 == 0); + + SECP256K1_SCALAR_VERIFY(r); } SECP256K1_INLINE static int secp256k1_scalar_is_one(const secp256k1_scalar *a) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); return ((a->d[0] ^ 1) | a->d[1] | a->d[2] | a->d[3] | a->d[4] | a->d[5] | a->d[6] | a->d[7]) == 0; } @@ -254,7 +300,7 @@ SECP256K1_INLINE static int secp256k1_scalar_is_one(const secp256k1_scalar *a) { static int secp256k1_scalar_is_high(const secp256k1_scalar *a) { int yes = 0; int no = 0; - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); no |= (a->d[7] < SECP256K1_N_H_7); yes |= (a->d[7] > SECP256K1_N_H_7) & ~no; @@ -278,7 +324,7 @@ static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) { uint32_t mask = -vflag; uint32_t nonzero = 0xFFFFFFFFUL * (secp256k1_scalar_is_zero(r) == 0); uint64_t t = (uint64_t)(r->d[0] ^ mask) + ((SECP256K1_N_0 + 1) & mask); - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); r->d[0] = t & nonzero; t >>= 32; t += (uint64_t)(r->d[1] ^ mask) + (SECP256K1_N_1 & mask); @@ -296,7 +342,7 @@ static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) { t += (uint64_t)(r->d[7] ^ mask) + (SECP256K1_N_7 & mask); r->d[7] = t & nonzero; - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); return 2 * (mask == 0) - 1; } @@ -604,37 +650,17 @@ static void secp256k1_scalar_mul_512(uint32_t *l, const secp256k1_scalar *a, con static void secp256k1_scalar_mul(secp256k1_scalar *r, const secp256k1_scalar *a, const secp256k1_scalar *b) { uint32_t l[16]; - secp256k1_scalar_verify(a); - secp256k1_scalar_verify(b); + SECP256K1_SCALAR_VERIFY(a); + SECP256K1_SCALAR_VERIFY(b); secp256k1_scalar_mul_512(l, a, b); secp256k1_scalar_reduce_512(r, l); - secp256k1_scalar_verify(r); -} - -static int secp256k1_scalar_shr_int(secp256k1_scalar *r, int n) { - int ret; - secp256k1_scalar_verify(r); - VERIFY_CHECK(n > 0); - VERIFY_CHECK(n < 16); - - ret = r->d[0] & ((1 << n) - 1); - r->d[0] = (r->d[0] >> n) + (r->d[1] << (32 - n)); - r->d[1] = (r->d[1] >> n) + (r->d[2] << (32 - n)); - r->d[2] = (r->d[2] >> n) + (r->d[3] << (32 - n)); - r->d[3] = (r->d[3] >> n) + (r->d[4] << (32 - n)); - r->d[4] = (r->d[4] >> n) + (r->d[5] << (32 - n)); - r->d[5] = (r->d[5] >> n) + (r->d[6] << (32 - n)); - r->d[6] = (r->d[6] >> n) + (r->d[7] << (32 - n)); - r->d[7] = (r->d[7] >> n); - - secp256k1_scalar_verify(r); - return ret; + SECP256K1_SCALAR_VERIFY(r); } static void secp256k1_scalar_split_128(secp256k1_scalar *r1, secp256k1_scalar *r2, const secp256k1_scalar *k) { - secp256k1_scalar_verify(k); + SECP256K1_SCALAR_VERIFY(k); r1->d[0] = k->d[0]; r1->d[1] = k->d[1]; @@ -653,13 +679,13 @@ static void secp256k1_scalar_split_128(secp256k1_scalar *r1, secp256k1_scalar *r r2->d[6] = 0; r2->d[7] = 0; - secp256k1_scalar_verify(r1); - secp256k1_scalar_verify(r2); + SECP256K1_SCALAR_VERIFY(r1); + SECP256K1_SCALAR_VERIFY(r2); } SECP256K1_INLINE static int secp256k1_scalar_eq(const secp256k1_scalar *a, const secp256k1_scalar *b) { - secp256k1_scalar_verify(a); - secp256k1_scalar_verify(b); + SECP256K1_SCALAR_VERIFY(a); + SECP256K1_SCALAR_VERIFY(b); return ((a->d[0] ^ b->d[0]) | (a->d[1] ^ b->d[1]) | (a->d[2] ^ b->d[2]) | (a->d[3] ^ b->d[3]) | (a->d[4] ^ b->d[4]) | (a->d[5] ^ b->d[5]) | (a->d[6] ^ b->d[6]) | (a->d[7] ^ b->d[7])) == 0; } @@ -669,8 +695,8 @@ SECP256K1_INLINE static void secp256k1_scalar_mul_shift_var(secp256k1_scalar *r, unsigned int shiftlimbs; unsigned int shiftlow; unsigned int shifthigh; - secp256k1_scalar_verify(a); - secp256k1_scalar_verify(b); + SECP256K1_SCALAR_VERIFY(a); + SECP256K1_SCALAR_VERIFY(b); VERIFY_CHECK(shift >= 256); secp256k1_scalar_mul_512(l, a, b); @@ -687,13 +713,13 @@ SECP256K1_INLINE static void secp256k1_scalar_mul_shift_var(secp256k1_scalar *r, r->d[7] = shift < 288 ? (l[7 + shiftlimbs] >> shiftlow) : 0; secp256k1_scalar_cadd_bit(r, 0, (l[(shift - 1) >> 5] >> ((shift - 1) & 0x1f)) & 1); - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); } static SECP256K1_INLINE void secp256k1_scalar_cmov(secp256k1_scalar *r, const secp256k1_scalar *a, int flag) { uint32_t mask0, mask1; volatile int vflag = flag; - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); SECP256K1_CHECKMEM_CHECK_VERIFY(r->d, sizeof(r->d)); mask0 = vflag + ~((uint32_t)0); @@ -707,7 +733,7 @@ static SECP256K1_INLINE void secp256k1_scalar_cmov(secp256k1_scalar *r, const se r->d[6] = (r->d[6] & mask0) | (a->d[6] & mask1); r->d[7] = (r->d[7] & mask0) | (a->d[7] & mask1); - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); } static void secp256k1_scalar_from_signed30(secp256k1_scalar *r, const secp256k1_modinv32_signed30 *a) { @@ -736,14 +762,14 @@ static void secp256k1_scalar_from_signed30(secp256k1_scalar *r, const secp256k1_ r->d[6] = a6 >> 12 | a7 << 18; r->d[7] = a7 >> 14 | a8 << 16; - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); } static void secp256k1_scalar_to_signed30(secp256k1_modinv32_signed30 *r, const secp256k1_scalar *a) { const uint32_t M30 = UINT32_MAX >> 2; const uint32_t a0 = a->d[0], a1 = a->d[1], a2 = a->d[2], a3 = a->d[3], a4 = a->d[4], a5 = a->d[5], a6 = a->d[6], a7 = a->d[7]; - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); r->v[0] = a0 & M30; r->v[1] = (a0 >> 30 | a1 << 2) & M30; @@ -766,16 +792,14 @@ static void secp256k1_scalar_inverse(secp256k1_scalar *r, const secp256k1_scalar #ifdef VERIFY int zero_in = secp256k1_scalar_is_zero(x); #endif - secp256k1_scalar_verify(x); + SECP256K1_SCALAR_VERIFY(x); secp256k1_scalar_to_signed30(&s, x); secp256k1_modinv32(&s, &secp256k1_const_modinfo_scalar); secp256k1_scalar_from_signed30(r, &s); - secp256k1_scalar_verify(r); -#ifdef VERIFY + SECP256K1_SCALAR_VERIFY(r); VERIFY_CHECK(secp256k1_scalar_is_zero(r) == zero_in); -#endif } static void secp256k1_scalar_inverse_var(secp256k1_scalar *r, const secp256k1_scalar *x) { @@ -783,20 +807,18 @@ static void secp256k1_scalar_inverse_var(secp256k1_scalar *r, const secp256k1_sc #ifdef VERIFY int zero_in = secp256k1_scalar_is_zero(x); #endif - secp256k1_scalar_verify(x); + SECP256K1_SCALAR_VERIFY(x); secp256k1_scalar_to_signed30(&s, x); secp256k1_modinv32_var(&s, &secp256k1_const_modinfo_scalar); secp256k1_scalar_from_signed30(r, &s); - secp256k1_scalar_verify(r); -#ifdef VERIFY + SECP256K1_SCALAR_VERIFY(r); VERIFY_CHECK(secp256k1_scalar_is_zero(r) == zero_in); -#endif } SECP256K1_INLINE static int secp256k1_scalar_is_even(const secp256k1_scalar *a) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); return !(a->d[0] & 1); } diff --git a/src/secp256k1/src/scalar_impl.h b/src/secp256k1/src/scalar_impl.h index 3eca23b4f9..bbba83e937 100644 --- a/src/secp256k1/src/scalar_impl.h +++ b/src/secp256k1/src/scalar_impl.h @@ -31,14 +31,12 @@ static int secp256k1_scalar_set_b32_seckey(secp256k1_scalar *r, const unsigned c int overflow; secp256k1_scalar_set_b32(r, bin, &overflow); - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); return (!overflow) & (!secp256k1_scalar_is_zero(r)); } static void secp256k1_scalar_verify(const secp256k1_scalar *r) { -#ifdef VERIFY VERIFY_CHECK(secp256k1_scalar_check_overflow(r) == 0); -#endif (void)r; } @@ -63,7 +61,7 @@ static void secp256k1_scalar_verify(const secp256k1_scalar *r) { * (arbitrarily) set r2 = k + 5 (mod n) and r1 = k - r2 * lambda (mod n). */ static void secp256k1_scalar_split_lambda(secp256k1_scalar * SECP256K1_RESTRICT r1, secp256k1_scalar * SECP256K1_RESTRICT r2, const secp256k1_scalar * SECP256K1_RESTRICT k) { - secp256k1_scalar_verify(k); + SECP256K1_SCALAR_VERIFY(k); VERIFY_CHECK(r1 != k); VERIFY_CHECK(r2 != k); VERIFY_CHECK(r1 != r2); @@ -71,8 +69,8 @@ static void secp256k1_scalar_split_lambda(secp256k1_scalar * SECP256K1_RESTRICT *r2 = (*k + 5) % EXHAUSTIVE_TEST_ORDER; *r1 = (*k + (EXHAUSTIVE_TEST_ORDER - *r2) * EXHAUSTIVE_TEST_LAMBDA) % EXHAUSTIVE_TEST_ORDER; - secp256k1_scalar_verify(r1); - secp256k1_scalar_verify(r2); + SECP256K1_SCALAR_VERIFY(r1); + SECP256K1_SCALAR_VERIFY(r2); } #else /** @@ -155,7 +153,7 @@ static void secp256k1_scalar_split_lambda(secp256k1_scalar * SECP256K1_RESTRICT 0xE4437ED6UL, 0x010E8828UL, 0x6F547FA9UL, 0x0ABFE4C4UL, 0x221208ACUL, 0x9DF506C6UL, 0x1571B4AEUL, 0x8AC47F71UL ); - secp256k1_scalar_verify(k); + SECP256K1_SCALAR_VERIFY(k); VERIFY_CHECK(r1 != k); VERIFY_CHECK(r2 != k); VERIFY_CHECK(r1 != r2); @@ -170,8 +168,8 @@ static void secp256k1_scalar_split_lambda(secp256k1_scalar * SECP256K1_RESTRICT secp256k1_scalar_negate(r1, r1); secp256k1_scalar_add(r1, r1, k); - secp256k1_scalar_verify(r1); - secp256k1_scalar_verify(r2); + SECP256K1_SCALAR_VERIFY(r1); + SECP256K1_SCALAR_VERIFY(r2); #ifdef VERIFY secp256k1_scalar_split_lambda_verify(r1, r2, k); #endif diff --git a/src/secp256k1/src/scalar_low.h b/src/secp256k1/src/scalar_low.h index 67051bd30b..2711eb932d 100644 --- a/src/secp256k1/src/scalar_low.h +++ b/src/secp256k1/src/scalar_low.h @@ -1,5 +1,5 @@ /*********************************************************************** - * Copyright (c) 2015 Andrew Poelstra * + * Copyright (c) 2015, 2022 Andrew Poelstra, Pieter Wuille * * Distributed under the MIT software license, see the accompanying * * file COPYING or https://www.opensource.org/licenses/mit-license.php.* ***********************************************************************/ @@ -12,6 +12,13 @@ /** A scalar modulo the group order of the secp256k1 curve. */ typedef uint32_t secp256k1_scalar; -#define SECP256K1_SCALAR_CONST(d7, d6, d5, d4, d3, d2, d1, d0) (d0) +/* A compile-time constant equal to 2^32 (modulo order). */ +#define SCALAR_2P32 ((0xffffffffUL % EXHAUSTIVE_TEST_ORDER) + 1U) + +/* Compute a*2^32 + b (modulo order). */ +#define SCALAR_HORNER(a, b) (((uint64_t)(a) * SCALAR_2P32 + (b)) % EXHAUSTIVE_TEST_ORDER) + +/* Evaluates to the provided 256-bit constant reduced modulo order. */ +#define SECP256K1_SCALAR_CONST(d7, d6, d5, d4, d3, d2, d1, d0) SCALAR_HORNER(SCALAR_HORNER(SCALAR_HORNER(SCALAR_HORNER(SCALAR_HORNER(SCALAR_HORNER(SCALAR_HORNER((d7), (d6)), (d5)), (d4)), (d3)), (d2)), (d1)), (d0)) #endif /* SECP256K1_SCALAR_REPR_H */ diff --git a/src/secp256k1/src/scalar_low_impl.h b/src/secp256k1/src/scalar_low_impl.h index e2356a5be1..0895db6a17 100644 --- a/src/secp256k1/src/scalar_low_impl.h +++ b/src/secp256k1/src/scalar_low_impl.h @@ -14,7 +14,7 @@ #include <string.h> SECP256K1_INLINE static int secp256k1_scalar_is_even(const secp256k1_scalar *a) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); return !(*a & 1); } @@ -24,11 +24,11 @@ SECP256K1_INLINE static void secp256k1_scalar_clear(secp256k1_scalar *r) { *r = SECP256K1_INLINE static void secp256k1_scalar_set_int(secp256k1_scalar *r, unsigned int v) { *r = v % EXHAUSTIVE_TEST_ORDER; - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); } SECP256K1_INLINE static unsigned int secp256k1_scalar_get_bits(const secp256k1_scalar *a, unsigned int offset, unsigned int count) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); if (offset < 32) return ((*a >> offset) & ((((uint32_t)1) << count) - 1)); @@ -37,7 +37,7 @@ SECP256K1_INLINE static unsigned int secp256k1_scalar_get_bits(const secp256k1_s } SECP256K1_INLINE static unsigned int secp256k1_scalar_get_bits_var(const secp256k1_scalar *a, unsigned int offset, unsigned int count) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); return secp256k1_scalar_get_bits(a, offset, count); } @@ -45,27 +45,25 @@ SECP256K1_INLINE static unsigned int secp256k1_scalar_get_bits_var(const secp256 SECP256K1_INLINE static int secp256k1_scalar_check_overflow(const secp256k1_scalar *a) { return *a >= EXHAUSTIVE_TEST_ORDER; } static int secp256k1_scalar_add(secp256k1_scalar *r, const secp256k1_scalar *a, const secp256k1_scalar *b) { - secp256k1_scalar_verify(a); - secp256k1_scalar_verify(b); + SECP256K1_SCALAR_VERIFY(a); + SECP256K1_SCALAR_VERIFY(b); *r = (*a + *b) % EXHAUSTIVE_TEST_ORDER; - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); return *r < *b; } static void secp256k1_scalar_cadd_bit(secp256k1_scalar *r, unsigned int bit, int flag) { - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); if (flag && bit < 32) *r += ((uint32_t)1 << bit); - secp256k1_scalar_verify(r); -#ifdef VERIFY + SECP256K1_SCALAR_VERIFY(r); VERIFY_CHECK(bit < 32); /* Verify that adding (1 << bit) will not overflow any in-range scalar *r by overflowing the underlying uint32_t. */ VERIFY_CHECK(((uint32_t)1 << bit) - 1 <= UINT32_MAX - EXHAUSTIVE_TEST_ORDER); -#endif } static void secp256k1_scalar_set_b32(secp256k1_scalar *r, const unsigned char *b32, int *overflow) { @@ -81,24 +79,24 @@ static void secp256k1_scalar_set_b32(secp256k1_scalar *r, const unsigned char *b } if (overflow) *overflow = over; - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); } static void secp256k1_scalar_get_b32(unsigned char *bin, const secp256k1_scalar* a) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); memset(bin, 0, 32); bin[28] = *a >> 24; bin[29] = *a >> 16; bin[30] = *a >> 8; bin[31] = *a; } SECP256K1_INLINE static int secp256k1_scalar_is_zero(const secp256k1_scalar *a) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); return *a == 0; } static void secp256k1_scalar_negate(secp256k1_scalar *r, const secp256k1_scalar *a) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); if (*a == 0) { *r = 0; @@ -106,65 +104,52 @@ static void secp256k1_scalar_negate(secp256k1_scalar *r, const secp256k1_scalar *r = EXHAUSTIVE_TEST_ORDER - *a; } - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); } SECP256K1_INLINE static int secp256k1_scalar_is_one(const secp256k1_scalar *a) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); return *a == 1; } static int secp256k1_scalar_is_high(const secp256k1_scalar *a) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); return *a > EXHAUSTIVE_TEST_ORDER / 2; } static int secp256k1_scalar_cond_negate(secp256k1_scalar *r, int flag) { - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); if (flag) secp256k1_scalar_negate(r, r); - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); return flag ? -1 : 1; } static void secp256k1_scalar_mul(secp256k1_scalar *r, const secp256k1_scalar *a, const secp256k1_scalar *b) { - secp256k1_scalar_verify(a); - secp256k1_scalar_verify(b); + SECP256K1_SCALAR_VERIFY(a); + SECP256K1_SCALAR_VERIFY(b); *r = (*a * *b) % EXHAUSTIVE_TEST_ORDER; - secp256k1_scalar_verify(r); -} - -static int secp256k1_scalar_shr_int(secp256k1_scalar *r, int n) { - int ret; - secp256k1_scalar_verify(r); - VERIFY_CHECK(n > 0); - VERIFY_CHECK(n < 16); - - ret = *r & ((1 << n) - 1); - *r >>= n; - - secp256k1_scalar_verify(r); - return ret; + SECP256K1_SCALAR_VERIFY(r); } static void secp256k1_scalar_split_128(secp256k1_scalar *r1, secp256k1_scalar *r2, const secp256k1_scalar *a) { - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); *r1 = *a; *r2 = 0; - secp256k1_scalar_verify(r1); - secp256k1_scalar_verify(r2); + SECP256K1_SCALAR_VERIFY(r1); + SECP256K1_SCALAR_VERIFY(r2); } SECP256K1_INLINE static int secp256k1_scalar_eq(const secp256k1_scalar *a, const secp256k1_scalar *b) { - secp256k1_scalar_verify(a); - secp256k1_scalar_verify(b); + SECP256K1_SCALAR_VERIFY(a); + SECP256K1_SCALAR_VERIFY(b); return *a == *b; } @@ -172,37 +157,45 @@ SECP256K1_INLINE static int secp256k1_scalar_eq(const secp256k1_scalar *a, const static SECP256K1_INLINE void secp256k1_scalar_cmov(secp256k1_scalar *r, const secp256k1_scalar *a, int flag) { uint32_t mask0, mask1; volatile int vflag = flag; - secp256k1_scalar_verify(a); + SECP256K1_SCALAR_VERIFY(a); SECP256K1_CHECKMEM_CHECK_VERIFY(r, sizeof(*r)); mask0 = vflag + ~((uint32_t)0); mask1 = ~mask0; *r = (*r & mask0) | (*a & mask1); - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); } static void secp256k1_scalar_inverse(secp256k1_scalar *r, const secp256k1_scalar *x) { int i; *r = 0; - secp256k1_scalar_verify(x); + SECP256K1_SCALAR_VERIFY(x); for (i = 0; i < EXHAUSTIVE_TEST_ORDER; i++) if ((i * *x) % EXHAUSTIVE_TEST_ORDER == 1) *r = i; - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); /* If this VERIFY_CHECK triggers we were given a noninvertible scalar (and thus * have a composite group order; fix it in exhaustive_tests.c). */ VERIFY_CHECK(*r != 0); } static void secp256k1_scalar_inverse_var(secp256k1_scalar *r, const secp256k1_scalar *x) { - secp256k1_scalar_verify(x); + SECP256K1_SCALAR_VERIFY(x); secp256k1_scalar_inverse(r, x); - secp256k1_scalar_verify(r); + SECP256K1_SCALAR_VERIFY(r); +} + +static void secp256k1_scalar_half(secp256k1_scalar *r, const secp256k1_scalar *a) { + SECP256K1_SCALAR_VERIFY(a); + + *r = (*a + ((-(uint32_t)(*a & 1)) & EXHAUSTIVE_TEST_ORDER)) >> 1; + + SECP256K1_SCALAR_VERIFY(r); } #endif /* SECP256K1_SCALAR_REPR_IMPL_H */ diff --git a/src/secp256k1/src/tests.c b/src/secp256k1/src/tests.c index d3959406c7..bec1c45585 100644 --- a/src/secp256k1/src/tests.c +++ b/src/secp256k1/src/tests.c @@ -23,6 +23,7 @@ #include "../include/secp256k1_preallocated.h" #include "testrand_impl.h" #include "checkmem.h" +#include "testutil.h" #include "util.h" #include "../contrib/lax_der_parsing.c" @@ -52,26 +53,32 @@ static int all_bytes_equal(const void* s, unsigned char value, size_t n) { return 1; } -/* TODO Use CHECK_ILLEGAL(_VOID) everywhere and get rid of the uncounting callback */ -/* CHECK that expr_or_stmt calls the illegal callback of ctx exactly once - * - * For checking functions that use ARG_CHECK_VOID */ -#define CHECK_ILLEGAL_VOID(ctx, expr_or_stmt) do { \ - int32_t _calls_to_illegal_callback = 0; \ - secp256k1_callback _saved_illegal_cb = ctx->illegal_callback; \ - secp256k1_context_set_illegal_callback(ctx, \ - counting_illegal_callback_fn, &_calls_to_illegal_callback); \ +#define CHECK_COUNTING_CALLBACK_VOID(ctx, expr_or_stmt, callback, callback_setter) do { \ + int32_t _calls_to_callback = 0; \ + secp256k1_callback _saved_callback = ctx->callback; \ + callback_setter(ctx, counting_callback_fn, &_calls_to_callback); \ { expr_or_stmt; } \ - ctx->illegal_callback = _saved_illegal_cb; \ - CHECK(_calls_to_illegal_callback == 1); \ + ctx->callback = _saved_callback; \ + CHECK(_calls_to_callback == 1); \ } while(0); -/* CHECK that expr calls the illegal callback of ctx exactly once and that expr == 0 +/* CHECK that expr_or_stmt calls the error or illegal callback of ctx exactly once + * + * Useful for checking functions that return void (e.g., API functions that use ARG_CHECK_VOID) */ +#define CHECK_ERROR_VOID(ctx, expr_or_stmt) \ + CHECK_COUNTING_CALLBACK_VOID(ctx, expr_or_stmt, error_callback, secp256k1_context_set_error_callback) +#define CHECK_ILLEGAL_VOID(ctx, expr_or_stmt) \ + CHECK_COUNTING_CALLBACK_VOID(ctx, expr_or_stmt, illegal_callback, secp256k1_context_set_illegal_callback) + +/* CHECK that + * - expr calls the illegal callback of ctx exactly once and, + * - expr == 0 (or equivalently, expr == NULL) * - * For checking functions that use ARG_CHECK */ + * Useful for checking functions that return an integer or a pointer. */ #define CHECK_ILLEGAL(ctx, expr) CHECK_ILLEGAL_VOID(ctx, CHECK((expr) == 0)) +#define CHECK_ERROR(ctx, expr) CHECK_ERROR_VOID(ctx, CHECK((expr) == 0)) -static void counting_illegal_callback_fn(const char* str, void* data) { +static void counting_callback_fn(const char* str, void* data) { /* Dummy callback function that just counts. */ int32_t *p; (void)str; @@ -273,55 +280,34 @@ static void run_deprecated_context_flags_test(void) { } static void run_ec_illegal_argument_tests(void) { - int ecount = 0; - int ecount2 = 10; secp256k1_pubkey pubkey; secp256k1_pubkey zero_pubkey; secp256k1_ecdsa_signature sig; unsigned char ctmp[32]; /* Setup */ - secp256k1_context_set_illegal_callback(STATIC_CTX, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_illegal_callback(CTX, counting_illegal_callback_fn, &ecount2); memset(ctmp, 1, 32); memset(&zero_pubkey, 0, sizeof(zero_pubkey)); /* Verify context-type checking illegal-argument errors. */ - CHECK(secp256k1_ec_pubkey_create(STATIC_CTX, &pubkey, ctmp) == 0); - CHECK(ecount == 1); + CHECK_ILLEGAL(STATIC_CTX, secp256k1_ec_pubkey_create(STATIC_CTX, &pubkey, ctmp)); SECP256K1_CHECKMEM_UNDEFINE(&pubkey, sizeof(pubkey)); CHECK(secp256k1_ec_pubkey_create(CTX, &pubkey, ctmp) == 1); SECP256K1_CHECKMEM_CHECK(&pubkey, sizeof(pubkey)); - CHECK(secp256k1_ecdsa_sign(STATIC_CTX, &sig, ctmp, ctmp, NULL, NULL) == 0); - CHECK(ecount == 2); + CHECK_ILLEGAL(STATIC_CTX, secp256k1_ecdsa_sign(STATIC_CTX, &sig, ctmp, ctmp, NULL, NULL)); SECP256K1_CHECKMEM_UNDEFINE(&sig, sizeof(sig)); CHECK(secp256k1_ecdsa_sign(CTX, &sig, ctmp, ctmp, NULL, NULL) == 1); SECP256K1_CHECKMEM_CHECK(&sig, sizeof(sig)); - CHECK(ecount2 == 10); CHECK(secp256k1_ecdsa_verify(CTX, &sig, ctmp, &pubkey) == 1); - CHECK(ecount2 == 10); CHECK(secp256k1_ecdsa_verify(STATIC_CTX, &sig, ctmp, &pubkey) == 1); - CHECK(ecount == 2); CHECK(secp256k1_ec_pubkey_tweak_add(CTX, &pubkey, ctmp) == 1); - CHECK(ecount2 == 10); CHECK(secp256k1_ec_pubkey_tweak_add(STATIC_CTX, &pubkey, ctmp) == 1); - CHECK(ecount == 2); CHECK(secp256k1_ec_pubkey_tweak_mul(CTX, &pubkey, ctmp) == 1); - CHECK(ecount2 == 10); CHECK(secp256k1_ec_pubkey_negate(STATIC_CTX, &pubkey) == 1); - CHECK(ecount == 2); CHECK(secp256k1_ec_pubkey_negate(CTX, &pubkey) == 1); - CHECK(ecount == 2); - CHECK(secp256k1_ec_pubkey_negate(STATIC_CTX, &zero_pubkey) == 0); - CHECK(ecount == 3); - CHECK(secp256k1_ec_pubkey_negate(CTX, NULL) == 0); - CHECK(ecount2 == 11); + CHECK_ILLEGAL(STATIC_CTX, secp256k1_ec_pubkey_negate(STATIC_CTX, &zero_pubkey)); + CHECK_ILLEGAL(CTX, secp256k1_ec_pubkey_negate(CTX, NULL)); CHECK(secp256k1_ec_pubkey_tweak_mul(STATIC_CTX, &pubkey, ctmp) == 1); - CHECK(ecount == 3); - - /* Clean up */ - secp256k1_context_set_illegal_callback(STATIC_CTX, NULL, NULL); - secp256k1_context_set_illegal_callback(CTX, NULL, NULL); } static void run_static_context_tests(int use_prealloc) { @@ -356,8 +342,8 @@ static void run_static_context_tests(int use_prealloc) { { /* Verify that setting and resetting illegal callback works */ int32_t dummy = 0; - secp256k1_context_set_illegal_callback(STATIC_CTX, counting_illegal_callback_fn, &dummy); - CHECK(STATIC_CTX->illegal_callback.fn == counting_illegal_callback_fn); + secp256k1_context_set_illegal_callback(STATIC_CTX, counting_callback_fn, &dummy); + CHECK(STATIC_CTX->illegal_callback.fn == counting_callback_fn); CHECK(STATIC_CTX->illegal_callback.data == &dummy); secp256k1_context_set_illegal_callback(STATIC_CTX, NULL, NULL); CHECK(STATIC_CTX->illegal_callback.fn == secp256k1_default_illegal_callback_fn); @@ -448,8 +434,8 @@ static void run_proper_context_tests(int use_prealloc) { CHECK(context_eq(my_ctx, my_ctx_fresh)); /* Verify that setting and resetting illegal callback works */ - secp256k1_context_set_illegal_callback(my_ctx, counting_illegal_callback_fn, &dummy); - CHECK(my_ctx->illegal_callback.fn == counting_illegal_callback_fn); + secp256k1_context_set_illegal_callback(my_ctx, counting_callback_fn, &dummy); + CHECK(my_ctx->illegal_callback.fn == counting_callback_fn); CHECK(my_ctx->illegal_callback.data == &dummy); secp256k1_context_set_illegal_callback(my_ctx, NULL, NULL); CHECK(my_ctx->illegal_callback.fn == secp256k1_default_illegal_callback_fn); @@ -490,19 +476,14 @@ static void run_proper_context_tests(int use_prealloc) { static void run_scratch_tests(void) { const size_t adj_alloc = ((500 + ALIGNMENT - 1) / ALIGNMENT) * ALIGNMENT; - int32_t ecount = 0; size_t checkpoint; size_t checkpoint_2; secp256k1_scratch_space *scratch; secp256k1_scratch_space local_scratch; - secp256k1_context_set_illegal_callback(CTX, counting_illegal_callback_fn, &ecount); - secp256k1_context_set_error_callback(CTX, counting_illegal_callback_fn, &ecount); - /* Test public API */ scratch = secp256k1_scratch_space_create(CTX, 1000); CHECK(scratch != NULL); - CHECK(ecount == 0); /* Test internal API */ CHECK(secp256k1_scratch_max_allocation(&CTX->error_callback, scratch, 0) == 1000); @@ -535,22 +516,16 @@ static void run_scratch_tests(void) { /* try to apply a bad checkpoint */ checkpoint_2 = secp256k1_scratch_checkpoint(&CTX->error_callback, scratch); secp256k1_scratch_apply_checkpoint(&CTX->error_callback, scratch, checkpoint); - CHECK(ecount == 0); - secp256k1_scratch_apply_checkpoint(&CTX->error_callback, scratch, checkpoint_2); /* checkpoint_2 is after checkpoint */ - CHECK(ecount == 1); - secp256k1_scratch_apply_checkpoint(&CTX->error_callback, scratch, (size_t) -1); /* this is just wildly invalid */ - CHECK(ecount == 2); + CHECK_ERROR_VOID(CTX, secp256k1_scratch_apply_checkpoint(&CTX->error_callback, scratch, checkpoint_2)); /* checkpoint_2 is after checkpoint */ + CHECK_ERROR_VOID(CTX, secp256k1_scratch_apply_checkpoint(&CTX->error_callback, scratch, (size_t) -1)); /* this is just wildly invalid */ /* try to use badly initialized scratch space */ secp256k1_scratch_space_destroy(CTX, scratch); memset(&local_scratch, 0, sizeof(local_scratch)); scratch = &local_scratch; - CHECK(!secp256k1_scratch_max_allocation(&CTX->error_callback, scratch, 0)); - CHECK(ecount == 3); - CHECK(secp256k1_scratch_alloc(&CTX->error_callback, scratch, 500) == NULL); - CHECK(ecount == 4); - secp256k1_scratch_space_destroy(CTX, scratch); - CHECK(ecount == 5); + CHECK_ERROR(CTX, secp256k1_scratch_max_allocation(&CTX->error_callback, scratch, 0)); + CHECK_ERROR(CTX, secp256k1_scratch_alloc(&CTX->error_callback, scratch, 500)); + CHECK_ERROR_VOID(CTX, secp256k1_scratch_space_destroy(CTX, scratch)); /* Test that large integers do not wrap around in a bad way */ scratch = secp256k1_scratch_space_create(CTX, 1000); @@ -566,9 +541,6 @@ static void run_scratch_tests(void) { /* cleanup */ secp256k1_scratch_space_destroy(CTX, NULL); /* no-op */ - - secp256k1_context_set_illegal_callback(CTX, NULL, NULL); - secp256k1_context_set_error_callback(CTX, NULL, NULL); } static void run_ctz_tests(void) { @@ -848,7 +820,6 @@ static void run_rfc6979_hmac_sha256_tests(void) { } static void run_tagged_sha256_tests(void) { - int ecount = 0; unsigned char tag[32] = { 0 }; unsigned char msg[32] = { 0 }; unsigned char hash32[32]; @@ -859,16 +830,11 @@ static void run_tagged_sha256_tests(void) { 0xE2, 0x76, 0x55, 0x9A, 0x3B, 0xDE, 0x55, 0xB3 }; - secp256k1_context_set_illegal_callback(CTX, counting_illegal_callback_fn, &ecount); - /* API test */ CHECK(secp256k1_tagged_sha256(CTX, hash32, tag, sizeof(tag), msg, sizeof(msg)) == 1); - CHECK(secp256k1_tagged_sha256(CTX, NULL, tag, sizeof(tag), msg, sizeof(msg)) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_tagged_sha256(CTX, hash32, NULL, 0, msg, sizeof(msg)) == 0); - CHECK(ecount == 2); - CHECK(secp256k1_tagged_sha256(CTX, hash32, tag, sizeof(tag), NULL, 0) == 0); - CHECK(ecount == 3); + CHECK_ILLEGAL(CTX, secp256k1_tagged_sha256(CTX, NULL, tag, sizeof(tag), msg, sizeof(msg))); + CHECK_ILLEGAL(CTX, secp256k1_tagged_sha256(CTX, hash32, NULL, 0, msg, sizeof(msg))); + CHECK_ILLEGAL(CTX, secp256k1_tagged_sha256(CTX, hash32, tag, sizeof(tag), NULL, 0)); /* Static test vector */ memcpy(tag, "tag", 3); @@ -2215,20 +2181,6 @@ static void scalar_test(void) { } { - /* test secp256k1_scalar_shr_int */ - secp256k1_scalar r; - int i; - random_scalar_order_test(&r); - for (i = 0; i < 100; ++i) { - int low; - int shift = 1 + secp256k1_testrand_int(15); - int expected = r.d[0] % (1ULL << shift); - low = secp256k1_scalar_shr_int(&r, shift); - CHECK(expected == low); - } - } - - { /* Test commutativity of add. */ secp256k1_scalar r1, r2; secp256k1_scalar_add(&r1, &s1, &s2); @@ -2319,6 +2271,13 @@ static void scalar_test(void) { CHECK(secp256k1_scalar_eq(&r1, &secp256k1_scalar_zero)); } + { + /* Test halving. */ + secp256k1_scalar r; + secp256k1_scalar_add(&r, &s, &s); + secp256k1_scalar_half(&r, &r); + CHECK(secp256k1_scalar_eq(&r, &s)); + } } static void run_scalar_set_b32_seckey_tests(void) { @@ -2372,6 +2331,38 @@ static void run_scalar_tests(void) { } { + /* Test that halving and doubling roundtrips on some fixed values. */ + static const secp256k1_scalar HALF_TESTS[] = { + /* 0 */ + SECP256K1_SCALAR_CONST(0, 0, 0, 0, 0, 0, 0, 0), + /* 1 */ + SECP256K1_SCALAR_CONST(0, 0, 0, 0, 0, 0, 0, 1), + /* -1 */ + SECP256K1_SCALAR_CONST(0xfffffffful, 0xfffffffful, 0xfffffffful, 0xfffffffeul, 0xbaaedce6ul, 0xaf48a03bul, 0xbfd25e8cul, 0xd0364140ul), + /* -2 (largest odd value) */ + SECP256K1_SCALAR_CONST(0xfffffffful, 0xfffffffful, 0xfffffffful, 0xfffffffeul, 0xbaaedce6ul, 0xaf48a03bul, 0xbfd25e8cul, 0xd036413Ful), + /* Half the secp256k1 order */ + SECP256K1_SCALAR_CONST(0x7ffffffful, 0xfffffffful, 0xfffffffful, 0xfffffffful, 0x5d576e73ul, 0x57a4501dul, 0xdfe92f46ul, 0x681b20a0ul), + /* Half the secp256k1 order + 1 */ + SECP256K1_SCALAR_CONST(0x7ffffffful, 0xfffffffful, 0xfffffffful, 0xfffffffful, 0x5d576e73ul, 0x57a4501dul, 0xdfe92f46ul, 0x681b20a1ul), + /* 2^255 */ + SECP256K1_SCALAR_CONST(0x80000000ul, 0, 0, 0, 0, 0, 0, 0), + /* 2^255 - 1 */ + SECP256K1_SCALAR_CONST(0x7ffffffful, 0xfffffffful, 0xfffffffful, 0xfffffffful, 0xfffffffful, 0xfffffffful, 0xfffffffful, 0xfffffffful), + }; + unsigned n; + for (n = 0; n < sizeof(HALF_TESTS) / sizeof(HALF_TESTS[0]); ++n) { + secp256k1_scalar s; + secp256k1_scalar_half(&s, &HALF_TESTS[n]); + secp256k1_scalar_add(&s, &s, &s); + CHECK(secp256k1_scalar_eq(&s, &HALF_TESTS[n])); + secp256k1_scalar_add(&s, &s, &s); + secp256k1_scalar_half(&s, &s); + CHECK(secp256k1_scalar_eq(&s, &HALF_TESTS[n])); + } + } + + { /* Does check_overflow check catch all ones? */ static const secp256k1_scalar overflowed = SECP256K1_SCALAR_CONST( 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL, 0xFFFFFFFFUL, @@ -2956,29 +2947,6 @@ static void run_scalar_tests(void) { /***** FIELD TESTS *****/ -static void random_fe(secp256k1_fe *x) { - unsigned char bin[32]; - do { - secp256k1_testrand256(bin); - if (secp256k1_fe_set_b32_limit(x, bin)) { - return; - } - } while(1); -} - -static void random_fe_non_zero(secp256k1_fe *nz) { - int tries = 10; - while (--tries >= 0) { - random_fe(nz); - secp256k1_fe_normalize(nz); - if (!secp256k1_fe_is_zero(nz)) { - break; - } - } - /* Infinitesimal probability of spurious failure here */ - CHECK(tries >= 0); -} - static void random_fe_non_square(secp256k1_fe *ns) { secp256k1_fe r; random_fe_non_zero(ns); @@ -3698,15 +3666,6 @@ static void run_inverse_tests(void) /***** GROUP TESTS *****/ -static void ge_equals_ge(const secp256k1_ge *a, const secp256k1_ge *b) { - CHECK(a->infinity == b->infinity); - if (a->infinity) { - return; - } - CHECK(secp256k1_fe_equal(&a->x, &b->x)); - CHECK(secp256k1_fe_equal(&a->y, &b->y)); -} - /* This compares jacobian points including their Z, not just their geometric meaning. */ static int gej_xyz_equals_gej(const secp256k1_gej *a, const secp256k1_gej *b) { secp256k1_gej a2; @@ -3729,23 +3688,6 @@ static int gej_xyz_equals_gej(const secp256k1_gej *a, const secp256k1_gej *b) { return ret; } -static void ge_equals_gej(const secp256k1_ge *a, const secp256k1_gej *b) { - secp256k1_fe z2s; - secp256k1_fe u1, u2, s1, s2; - CHECK(a->infinity == b->infinity); - if (a->infinity) { - return; - } - /* Check a.x * b.z^2 == b.x && a.y * b.z^3 == b.y, to avoid inverses. */ - secp256k1_fe_sqr(&z2s, &b->z); - secp256k1_fe_mul(&u1, &a->x, &z2s); - u2 = b->x; - secp256k1_fe_mul(&s1, &a->y, &z2s); secp256k1_fe_mul(&s1, &s1, &b->z); - s2 = b->y; - CHECK(secp256k1_fe_equal(&u1, &u2)); - CHECK(secp256k1_fe_equal(&s1, &s2)); -} - static void test_ge(void) { int i, i1; int runs = 6; @@ -3764,11 +3706,12 @@ static void test_ge(void) { secp256k1_ge_clear(&ge[0]); secp256k1_ge_set_gej_var(&ge[0], &gej[0]); for (i = 0; i < runs; i++) { - int j; + int j, k; secp256k1_ge g; random_group_element_test(&g); if (i >= runs - 2) { secp256k1_ge_mul_lambda(&g, &ge[1]); + CHECK(!secp256k1_ge_eq_var(&g, &ge[1])); } if (i >= runs - 1) { secp256k1_ge_mul_lambda(&g, &g); @@ -3788,6 +3731,16 @@ static void test_ge(void) { random_gej_y_magnitude(&gej[1 + j + 4 * i]); random_gej_z_magnitude(&gej[1 + j + 4 * i]); } + + for (j = 0; j < 4; ++j) { + for (k = 0; k < 4; ++k) { + int expect_equal = (j >> 1) == (k >> 1); + CHECK(secp256k1_ge_eq_var(&ge[1 + j + 4 * i], &ge[1 + k + 4 * i]) == expect_equal); + CHECK(secp256k1_gej_eq_var(&gej[1 + j + 4 * i], &gej[1 + k + 4 * i]) == expect_equal); + CHECK(secp256k1_gej_eq_ge_var(&gej[1 + j + 4 * i], &ge[1 + k + 4 * i]) == expect_equal); + CHECK(secp256k1_gej_eq_ge_var(&gej[1 + k + 4 * i], &ge[1 + j + 4 * i]) == expect_equal); + } + } } /* Generate random zf, and zfi2 = 1/zf^2, zfi3 = 1/zf^3 */ @@ -3817,7 +3770,7 @@ static void test_ge(void) { /* Test gej + ge with Z ratio result (var). */ secp256k1_gej_add_ge_var(&resj, &gej[i1], &ge[i2], secp256k1_gej_is_infinity(&gej[i1]) ? NULL : &zr); - ge_equals_gej(&ref, &resj); + CHECK(secp256k1_gej_eq_ge_var(&resj, &ref)); if (!secp256k1_gej_is_infinity(&gej[i1]) && !secp256k1_gej_is_infinity(&resj)) { secp256k1_fe zrz; secp256k1_fe_mul(&zrz, &zr, &gej[i1].z); CHECK(secp256k1_fe_equal(&zrz, &resj.z)); @@ -3831,14 +3784,14 @@ static void test_ge(void) { random_ge_x_magnitude(&ge2_zfi); random_ge_y_magnitude(&ge2_zfi); secp256k1_gej_add_zinv_var(&resj, &gej[i1], &ge2_zfi, &zf); - ge_equals_gej(&ref, &resj); + CHECK(secp256k1_gej_eq_ge_var(&resj, &ref)); } /* Test gej + ge (const). */ if (i2 != 0) { /* secp256k1_gej_add_ge does not support its second argument being infinity. */ secp256k1_gej_add_ge(&resj, &gej[i1], &ge[i2]); - ge_equals_gej(&ref, &resj); + CHECK(secp256k1_gej_eq_ge_var(&resj, &ref)); } /* Test doubling (var). */ @@ -3846,16 +3799,16 @@ static void test_ge(void) { secp256k1_fe zr2; /* Normal doubling with Z ratio result. */ secp256k1_gej_double_var(&resj, &gej[i1], &zr2); - ge_equals_gej(&ref, &resj); + CHECK(secp256k1_gej_eq_ge_var(&resj, &ref)); /* Check Z ratio. */ secp256k1_fe_mul(&zr2, &zr2, &gej[i1].z); CHECK(secp256k1_fe_equal(&zr2, &resj.z)); /* Normal doubling. */ secp256k1_gej_double_var(&resj, &gej[i2], NULL); - ge_equals_gej(&ref, &resj); + CHECK(secp256k1_gej_eq_ge_var(&resj, &ref)); /* Constant-time doubling. */ secp256k1_gej_double(&resj, &gej[i2]); - ge_equals_gej(&ref, &resj); + CHECK(secp256k1_gej_eq_ge_var(&resj, &ref)); } /* Test adding opposites. */ @@ -3867,12 +3820,12 @@ static void test_ge(void) { if (i1 == 0) { CHECK(secp256k1_ge_is_infinity(&ge[i1])); CHECK(secp256k1_gej_is_infinity(&gej[i1])); - ge_equals_gej(&ref, &gej[i2]); + CHECK(secp256k1_gej_eq_ge_var(&gej[i2], &ref)); } if (i2 == 0) { CHECK(secp256k1_ge_is_infinity(&ge[i2])); CHECK(secp256k1_gej_is_infinity(&gej[i2])); - ge_equals_gej(&ref, &gej[i1]); + CHECK(secp256k1_gej_eq_ge_var(&gej[i1], &ref)); } } } @@ -3907,7 +3860,7 @@ static void test_ge(void) { secp256k1_fe s; random_fe_non_zero(&s); secp256k1_gej_rescale(&gej[i], &s); - ge_equals_gej(&ge_set_all[i], &gej[i]); + CHECK(secp256k1_gej_eq_ge_var(&gej[i], &ge_set_all[i])); } free(ge_set_all); } @@ -3951,7 +3904,7 @@ static void test_ge(void) { secp256k1_ge_set_all_gej_var(ge, gej, 4 * runs + 1); /* check result */ for (i = 0; i < 4 * runs + 1; i++) { - ge_equals_gej(&ge[i], &gej[i]); + CHECK(secp256k1_gej_eq_ge_var(&gej[i], &ge[i])); } /* Test batch gej -> ge conversion with all infinities. */ @@ -4050,15 +4003,15 @@ static void test_add_neg_y_diff_x(void) { secp256k1_gej_add_var(&resj, &aj, &bj, NULL); secp256k1_ge_set_gej(&res, &resj); - ge_equals_gej(&res, &sumj); + CHECK(secp256k1_gej_eq_ge_var(&sumj, &res)); secp256k1_gej_add_ge(&resj, &aj, &b); secp256k1_ge_set_gej(&res, &resj); - ge_equals_gej(&res, &sumj); + CHECK(secp256k1_gej_eq_ge_var(&sumj, &res)); secp256k1_gej_add_ge_var(&resj, &aj, &b, NULL); secp256k1_ge_set_gej(&res, &resj); - ge_equals_gej(&res, &sumj); + CHECK(secp256k1_gej_eq_ge_var(&sumj, &res)); } static void run_ge(void) { @@ -4351,10 +4304,10 @@ static void test_point_times_order(const secp256k1_gej *point) { CHECK(secp256k1_ge_is_infinity(&res3)); secp256k1_ecmult(&res1, point, &secp256k1_scalar_one, &secp256k1_scalar_zero); secp256k1_ge_set_gej(&res3, &res1); - ge_equals_gej(&res3, point); + CHECK(secp256k1_gej_eq_ge_var(point, &res3)); secp256k1_ecmult(&res1, point, &secp256k1_scalar_zero, &secp256k1_scalar_one); secp256k1_ge_set_gej(&res3, &res1); - ge_equals_ge(&res3, &secp256k1_ge_const_g); + CHECK(secp256k1_ge_eq_var(&secp256k1_ge_const_g, &res3)); } /* These scalars reach large (in absolute value) outputs when fed to secp256k1_scalar_split_lambda. @@ -4482,7 +4435,7 @@ static void ecmult_const_random_mult(void) { secp256k1_ecmult_const(&b, &a, &xn); CHECK(secp256k1_ge_is_valid_var(&a)); - ge_equals_gej(&expected_b, &b); + CHECK(secp256k1_gej_eq_ge_var(&b, &expected_b)); } static void ecmult_const_commutativity(void) { @@ -4503,27 +4456,76 @@ static void ecmult_const_commutativity(void) { secp256k1_ecmult_const(&res2, &mid2, &a); secp256k1_ge_set_gej(&mid1, &res1); secp256k1_ge_set_gej(&mid2, &res2); - ge_equals_ge(&mid1, &mid2); + CHECK(secp256k1_ge_eq_var(&mid1, &mid2)); } static void ecmult_const_mult_zero_one(void) { + secp256k1_scalar s; secp256k1_scalar negone; secp256k1_gej res1; secp256k1_ge res2; secp256k1_ge point; - secp256k1_scalar_negate(&negone, &secp256k1_scalar_one); + secp256k1_ge inf; + random_scalar_order_test(&s); + secp256k1_scalar_negate(&negone, &secp256k1_scalar_one); random_group_element_test(&point); + secp256k1_ge_set_infinity(&inf); + + /* 0*point */ secp256k1_ecmult_const(&res1, &point, &secp256k1_scalar_zero); - secp256k1_ge_set_gej(&res2, &res1); - CHECK(secp256k1_ge_is_infinity(&res2)); + CHECK(secp256k1_gej_is_infinity(&res1)); + + /* s*inf */ + secp256k1_ecmult_const(&res1, &inf, &s); + CHECK(secp256k1_gej_is_infinity(&res1)); + + /* 1*point */ secp256k1_ecmult_const(&res1, &point, &secp256k1_scalar_one); secp256k1_ge_set_gej(&res2, &res1); - ge_equals_ge(&res2, &point); + CHECK(secp256k1_ge_eq_var(&res2, &point)); + + /* -1*point */ secp256k1_ecmult_const(&res1, &point, &negone); secp256k1_gej_neg(&res1, &res1); secp256k1_ge_set_gej(&res2, &res1); - ge_equals_ge(&res2, &point); + CHECK(secp256k1_ge_eq_var(&res2, &point)); +} + +static void ecmult_const_check_result(const secp256k1_ge *A, const secp256k1_scalar* q, const secp256k1_gej *res) { + secp256k1_gej pointj, res2j; + secp256k1_ge res2; + secp256k1_gej_set_ge(&pointj, A); + secp256k1_ecmult(&res2j, &pointj, q, &secp256k1_scalar_zero); + secp256k1_ge_set_gej(&res2, &res2j); + CHECK(secp256k1_gej_eq_ge_var(res, &res2)); +} + +static void ecmult_const_edges(void) { + secp256k1_scalar q; + secp256k1_ge point; + secp256k1_gej res; + size_t i; + size_t cases = 1 + sizeof(scalars_near_split_bounds) / sizeof(scalars_near_split_bounds[0]); + + /* We are trying to reach the following edge cases (variables are defined as + * in ecmult_const_impl.h): + * 1. i = 0: s = 0 <=> q = -K + * 2. i > 0: v1, v2 large values + * <=> s1, s2 large values + * <=> s = scalars_near_split_bounds[i] + * <=> q = 2*scalars_near_split_bounds[i] - K + */ + for (i = 0; i < cases; ++i) { + secp256k1_scalar_negate(&q, &secp256k1_ecmult_const_K); + if (i > 0) { + secp256k1_scalar_add(&q, &q, &scalars_near_split_bounds[i - 1]); + secp256k1_scalar_add(&q, &q, &scalars_near_split_bounds[i - 1]); + } + random_group_element_test(&point); + secp256k1_ecmult_const(&res, &point, &q); + ecmult_const_check_result(&point, &q, &res); + } } static void ecmult_const_mult_xonly(void) { @@ -4604,11 +4606,12 @@ static void ecmult_const_chain_multiply(void) { secp256k1_ecmult_const(&point, &tmp, &scalar); } secp256k1_ge_set_gej(&res, &point); - ge_equals_gej(&res, &expected_point); + CHECK(secp256k1_gej_eq_ge_var(&expected_point, &res)); } static void run_ecmult_const_tests(void) { ecmult_const_mult_zero_one(); + ecmult_const_edges(); ecmult_const_random_mult(); ecmult_const_commutativity(); ecmult_const_chain_multiply(); @@ -5269,73 +5272,17 @@ static void test_wnaf(const secp256k1_scalar *number, int w) { CHECK(secp256k1_scalar_eq(&x, number)); /* check that wnaf represents number */ } -static void test_constant_wnaf_negate(const secp256k1_scalar *number) { - secp256k1_scalar neg1 = *number; - secp256k1_scalar neg2 = *number; - int sign1 = 1; - int sign2 = 1; - - if (!secp256k1_scalar_get_bits(&neg1, 0, 1)) { - secp256k1_scalar_negate(&neg1, &neg1); - sign1 = -1; - } - sign2 = secp256k1_scalar_cond_negate(&neg2, secp256k1_scalar_is_even(&neg2)); - CHECK(sign1 == sign2); - CHECK(secp256k1_scalar_eq(&neg1, &neg2)); -} - -static void test_constant_wnaf(const secp256k1_scalar *number, int w) { - secp256k1_scalar x, shift; - int wnaf[256] = {0}; - int i; - int skew; - int bits = 256; - secp256k1_scalar num = *number; - secp256k1_scalar scalar_skew; - - secp256k1_scalar_set_int(&x, 0); - secp256k1_scalar_set_int(&shift, 1 << w); - for (i = 0; i < 16; ++i) { - secp256k1_scalar_shr_int(&num, 8); - } - bits = 128; - skew = secp256k1_wnaf_const(wnaf, &num, w, bits); - - for (i = WNAF_SIZE_BITS(bits, w); i >= 0; --i) { - secp256k1_scalar t; - int v = wnaf[i]; - CHECK(v != 0); /* check nonzero */ - CHECK(v & 1); /* check parity */ - CHECK(v > -(1 << w)); /* check range above */ - CHECK(v < (1 << w)); /* check range below */ - - secp256k1_scalar_mul(&x, &x, &shift); - if (v >= 0) { - secp256k1_scalar_set_int(&t, v); - } else { - secp256k1_scalar_set_int(&t, -v); - secp256k1_scalar_negate(&t, &t); - } - secp256k1_scalar_add(&x, &x, &t); - } - /* Skew num because when encoding numbers as odd we use an offset */ - secp256k1_scalar_set_int(&scalar_skew, skew); - secp256k1_scalar_add(&num, &num, &scalar_skew); - CHECK(secp256k1_scalar_eq(&x, &num)); -} - static void test_fixed_wnaf(const secp256k1_scalar *number, int w) { secp256k1_scalar x, shift; int wnaf[256] = {0}; int i; int skew; - secp256k1_scalar num = *number; + secp256k1_scalar num, unused; secp256k1_scalar_set_int(&x, 0); secp256k1_scalar_set_int(&shift, 1 << w); - for (i = 0; i < 16; ++i) { - secp256k1_scalar_shr_int(&num, 8); - } + /* Make num a 128-bit scalar. */ + secp256k1_scalar_split_128(&num, &unused, number); skew = secp256k1_wnaf_fixed(wnaf, &num, w); for (i = WNAF_SIZE(w)-1; i >= 0; --i) { @@ -5427,32 +5374,7 @@ static void test_fixed_wnaf_small(void) { static void run_wnaf(void) { int i; - secp256k1_scalar n = {{0}}; - - test_constant_wnaf(&n, 4); - /* Sanity check: 1 and 2 are the smallest odd and even numbers and should - * have easier-to-diagnose failure modes */ - n.d[0] = 1; - test_constant_wnaf(&n, 4); - n.d[0] = 2; - test_constant_wnaf(&n, 4); - /* Test -1, because it's a special case in wnaf_const */ - n = secp256k1_scalar_one; - secp256k1_scalar_negate(&n, &n); - test_constant_wnaf(&n, 4); - - /* Test -2, which may not lead to overflows in wnaf_const */ - secp256k1_scalar_add(&n, &secp256k1_scalar_one, &secp256k1_scalar_one); - secp256k1_scalar_negate(&n, &n); - test_constant_wnaf(&n, 4); - - /* Test (1/2) - 1 = 1/-2 and 1/2 = (1/-2) + 1 - as corner cases of negation handling in wnaf_const */ - secp256k1_scalar_inverse(&n, &n); - test_constant_wnaf(&n, 4); - - secp256k1_scalar_add(&n, &n, &secp256k1_scalar_one); - test_constant_wnaf(&n, 4); + secp256k1_scalar n; /* Test 0 for fixed wnaf */ test_fixed_wnaf_small(); @@ -5460,8 +5382,6 @@ static void run_wnaf(void) { for (i = 0; i < COUNT; i++) { random_scalar_order(&n); test_wnaf(&n, 4+(i%10)); - test_constant_wnaf_negate(&n); - test_constant_wnaf(&n, 4 + (i % 10)); test_fixed_wnaf(&n, 4 + (i % 10)); } secp256k1_scalar_set_int(&n, 0); @@ -5494,11 +5414,11 @@ static void test_ecmult_accumulate(secp256k1_sha256* acc, const secp256k1_scalar secp256k1_ecmult_multi_var(NULL, scratch, &rj5, &secp256k1_scalar_zero, test_ecmult_accumulate_cb, (void*)x, 1); secp256k1_ecmult_const(&rj6, &secp256k1_ge_const_g, x); secp256k1_ge_set_gej_var(&r, &rj1); - ge_equals_gej(&r, &rj2); - ge_equals_gej(&r, &rj3); - ge_equals_gej(&r, &rj4); - ge_equals_gej(&r, &rj5); - ge_equals_gej(&r, &rj6); + CHECK(secp256k1_gej_eq_ge_var(&rj2, &r)); + CHECK(secp256k1_gej_eq_ge_var(&rj3, &r)); + CHECK(secp256k1_gej_eq_ge_var(&rj4, &r)); + CHECK(secp256k1_gej_eq_ge_var(&rj5, &r)); + CHECK(secp256k1_gej_eq_ge_var(&rj6, &r)); if (secp256k1_ge_is_infinity(&r)) { /* Store infinity as 0x00 */ const unsigned char zerobyte[1] = {0}; @@ -5652,7 +5572,7 @@ static void test_ecmult_gen_blind(void) { CHECK(!gej_xyz_equals_gej(&pgej, &pgej2)); CHECK(!gej_xyz_equals_gej(&i, &CTX->ecmult_gen_ctx.initial)); secp256k1_ge_set_gej(&pge, &pgej); - ge_equals_gej(&pge, &pgej2); + CHECK(secp256k1_gej_eq_ge_var(&pgej2, &pge)); } static void test_ecmult_gen_blind_reset(void) { @@ -5729,9 +5649,7 @@ static void ec_pubkey_parse_pointtest(const unsigned char *input, int xvalid, in secp256k1_pubkey pubkey; secp256k1_ge ge; size_t pubkeyclen; - int32_t ecount; - ecount = 0; - secp256k1_context_set_illegal_callback(CTX, counting_illegal_callback_fn, &ecount); + for (pubkeyclen = 3; pubkeyclen <= 65; pubkeyclen++) { /* Smaller sizes are tested exhaustively elsewhere. */ int32_t i; @@ -5756,7 +5674,6 @@ static void ec_pubkey_parse_pointtest(const unsigned char *input, int xvalid, in size_t outl; memset(&pubkey, 0, sizeof(pubkey)); SECP256K1_CHECKMEM_UNDEFINE(&pubkey, sizeof(pubkey)); - ecount = 0; CHECK(secp256k1_ec_pubkey_parse(CTX, &pubkey, pubkeyc, pubkeyclen) == 1); SECP256K1_CHECKMEM_CHECK(&pubkey, sizeof(pubkey)); outl = 65; @@ -5782,21 +5699,16 @@ static void ec_pubkey_parse_pointtest(const unsigned char *input, int xvalid, in CHECK(pubkeyo[0] == 4); CHECK(secp256k1_memcmp_var(&pubkeyo[1], input, 64) == 0); } - CHECK(ecount == 0); } else { /* These cases must fail to parse. */ memset(&pubkey, 0xfe, sizeof(pubkey)); - ecount = 0; SECP256K1_CHECKMEM_UNDEFINE(&pubkey, sizeof(pubkey)); CHECK(secp256k1_ec_pubkey_parse(CTX, &pubkey, pubkeyc, pubkeyclen) == 0); SECP256K1_CHECKMEM_CHECK(&pubkey, sizeof(pubkey)); - CHECK(ecount == 0); - CHECK(secp256k1_pubkey_load(CTX, &ge, &pubkey) == 0); - CHECK(ecount == 1); + CHECK_ILLEGAL(CTX, secp256k1_pubkey_load(CTX, &ge, &pubkey)); } } } - secp256k1_context_set_illegal_callback(CTX, NULL, NULL); } static void run_ec_pubkey_parse_test(void) { @@ -5979,142 +5891,99 @@ static void run_ec_pubkey_parse_test(void) { 0xB8, 0x00 }; unsigned char sout[65]; - unsigned char shortkey[2]; + unsigned char shortkey[2] = { 0 }; secp256k1_ge ge; secp256k1_pubkey pubkey; size_t len; int32_t i; - int32_t ecount; - int32_t ecount2; - ecount = 0; + /* Nothing should be reading this far into pubkeyc. */ SECP256K1_CHECKMEM_UNDEFINE(&pubkeyc[65], 1); - secp256k1_context_set_illegal_callback(CTX, counting_illegal_callback_fn, &ecount); /* Zero length claimed, fail, zeroize, no illegal arg error. */ memset(&pubkey, 0xfe, sizeof(pubkey)); - ecount = 0; SECP256K1_CHECKMEM_UNDEFINE(shortkey, 2); SECP256K1_CHECKMEM_UNDEFINE(&pubkey, sizeof(pubkey)); CHECK(secp256k1_ec_pubkey_parse(CTX, &pubkey, shortkey, 0) == 0); SECP256K1_CHECKMEM_CHECK(&pubkey, sizeof(pubkey)); - CHECK(ecount == 0); - CHECK(secp256k1_pubkey_load(CTX, &ge, &pubkey) == 0); - CHECK(ecount == 1); + CHECK_ILLEGAL(CTX, secp256k1_pubkey_load(CTX, &ge, &pubkey)); /* Length one claimed, fail, zeroize, no illegal arg error. */ for (i = 0; i < 256 ; i++) { memset(&pubkey, 0xfe, sizeof(pubkey)); - ecount = 0; shortkey[0] = i; SECP256K1_CHECKMEM_UNDEFINE(&shortkey[1], 1); SECP256K1_CHECKMEM_UNDEFINE(&pubkey, sizeof(pubkey)); CHECK(secp256k1_ec_pubkey_parse(CTX, &pubkey, shortkey, 1) == 0); SECP256K1_CHECKMEM_CHECK(&pubkey, sizeof(pubkey)); - CHECK(ecount == 0); - CHECK(secp256k1_pubkey_load(CTX, &ge, &pubkey) == 0); - CHECK(ecount == 1); + CHECK_ILLEGAL(CTX, secp256k1_pubkey_load(CTX, &ge, &pubkey)); } /* Length two claimed, fail, zeroize, no illegal arg error. */ for (i = 0; i < 65536 ; i++) { memset(&pubkey, 0xfe, sizeof(pubkey)); - ecount = 0; shortkey[0] = i & 255; shortkey[1] = i >> 8; SECP256K1_CHECKMEM_UNDEFINE(&pubkey, sizeof(pubkey)); CHECK(secp256k1_ec_pubkey_parse(CTX, &pubkey, shortkey, 2) == 0); SECP256K1_CHECKMEM_CHECK(&pubkey, sizeof(pubkey)); - CHECK(ecount == 0); - CHECK(secp256k1_pubkey_load(CTX, &ge, &pubkey) == 0); - CHECK(ecount == 1); + CHECK_ILLEGAL(CTX, secp256k1_pubkey_load(CTX, &ge, &pubkey)); } memset(&pubkey, 0xfe, sizeof(pubkey)); - ecount = 0; SECP256K1_CHECKMEM_UNDEFINE(&pubkey, sizeof(pubkey)); /* 33 bytes claimed on otherwise valid input starting with 0x04, fail, zeroize output, no illegal arg error. */ CHECK(secp256k1_ec_pubkey_parse(CTX, &pubkey, pubkeyc, 33) == 0); SECP256K1_CHECKMEM_CHECK(&pubkey, sizeof(pubkey)); - CHECK(ecount == 0); - CHECK(secp256k1_pubkey_load(CTX, &ge, &pubkey) == 0); - CHECK(ecount == 1); + CHECK_ILLEGAL(CTX, secp256k1_pubkey_load(CTX, &ge, &pubkey)); /* NULL pubkey, illegal arg error. Pubkey isn't rewritten before this step, since it's NULL into the parser. */ - CHECK(secp256k1_ec_pubkey_parse(CTX, NULL, pubkeyc, 65) == 0); - CHECK(ecount == 2); + CHECK_ILLEGAL(CTX, secp256k1_ec_pubkey_parse(CTX, NULL, pubkeyc, 65)); /* NULL input string. Illegal arg and zeroize output. */ memset(&pubkey, 0xfe, sizeof(pubkey)); - ecount = 0; SECP256K1_CHECKMEM_UNDEFINE(&pubkey, sizeof(pubkey)); - CHECK(secp256k1_ec_pubkey_parse(CTX, &pubkey, NULL, 65) == 0); + CHECK_ILLEGAL(CTX, secp256k1_ec_pubkey_parse(CTX, &pubkey, NULL, 65)); SECP256K1_CHECKMEM_CHECK(&pubkey, sizeof(pubkey)); - CHECK(ecount == 1); - CHECK(secp256k1_pubkey_load(CTX, &ge, &pubkey) == 0); - CHECK(ecount == 2); + CHECK_ILLEGAL(CTX, secp256k1_pubkey_load(CTX, &ge, &pubkey)); /* 64 bytes claimed on input starting with 0x04, fail, zeroize output, no illegal arg error. */ memset(&pubkey, 0xfe, sizeof(pubkey)); - ecount = 0; SECP256K1_CHECKMEM_UNDEFINE(&pubkey, sizeof(pubkey)); CHECK(secp256k1_ec_pubkey_parse(CTX, &pubkey, pubkeyc, 64) == 0); SECP256K1_CHECKMEM_CHECK(&pubkey, sizeof(pubkey)); - CHECK(ecount == 0); - CHECK(secp256k1_pubkey_load(CTX, &ge, &pubkey) == 0); - CHECK(ecount == 1); + CHECK_ILLEGAL(CTX, secp256k1_pubkey_load(CTX, &ge, &pubkey)); /* 66 bytes claimed, fail, zeroize output, no illegal arg error. */ memset(&pubkey, 0xfe, sizeof(pubkey)); - ecount = 0; SECP256K1_CHECKMEM_UNDEFINE(&pubkey, sizeof(pubkey)); CHECK(secp256k1_ec_pubkey_parse(CTX, &pubkey, pubkeyc, 66) == 0); SECP256K1_CHECKMEM_CHECK(&pubkey, sizeof(pubkey)); - CHECK(ecount == 0); - CHECK(secp256k1_pubkey_load(CTX, &ge, &pubkey) == 0); - CHECK(ecount == 1); + CHECK_ILLEGAL(CTX, secp256k1_pubkey_load(CTX, &ge, &pubkey)); /* Valid parse. */ memset(&pubkey, 0, sizeof(pubkey)); - ecount = 0; SECP256K1_CHECKMEM_UNDEFINE(&pubkey, sizeof(pubkey)); CHECK(secp256k1_ec_pubkey_parse(CTX, &pubkey, pubkeyc, 65) == 1); CHECK(secp256k1_ec_pubkey_parse(secp256k1_context_static, &pubkey, pubkeyc, 65) == 1); SECP256K1_CHECKMEM_CHECK(&pubkey, sizeof(pubkey)); - CHECK(ecount == 0); SECP256K1_CHECKMEM_UNDEFINE(&ge, sizeof(ge)); CHECK(secp256k1_pubkey_load(CTX, &ge, &pubkey) == 1); SECP256K1_CHECKMEM_CHECK(&ge.x, sizeof(ge.x)); SECP256K1_CHECKMEM_CHECK(&ge.y, sizeof(ge.y)); SECP256K1_CHECKMEM_CHECK(&ge.infinity, sizeof(ge.infinity)); - ge_equals_ge(&secp256k1_ge_const_g, &ge); - CHECK(ecount == 0); + CHECK(secp256k1_ge_eq_var(&ge, &secp256k1_ge_const_g)); /* secp256k1_ec_pubkey_serialize illegal args. */ - ecount = 0; len = 65; - CHECK(secp256k1_ec_pubkey_serialize(CTX, NULL, &len, &pubkey, SECP256K1_EC_UNCOMPRESSED) == 0); - CHECK(ecount == 1); + CHECK_ILLEGAL(CTX, secp256k1_ec_pubkey_serialize(CTX, NULL, &len, &pubkey, SECP256K1_EC_UNCOMPRESSED)); CHECK(len == 0); - CHECK(secp256k1_ec_pubkey_serialize(CTX, sout, NULL, &pubkey, SECP256K1_EC_UNCOMPRESSED) == 0); - CHECK(ecount == 2); + CHECK_ILLEGAL(CTX, secp256k1_ec_pubkey_serialize(CTX, sout, NULL, &pubkey, SECP256K1_EC_UNCOMPRESSED)); len = 65; SECP256K1_CHECKMEM_UNDEFINE(sout, 65); - CHECK(secp256k1_ec_pubkey_serialize(CTX, sout, &len, NULL, SECP256K1_EC_UNCOMPRESSED) == 0); + CHECK_ILLEGAL(CTX, secp256k1_ec_pubkey_serialize(CTX, sout, &len, NULL, SECP256K1_EC_UNCOMPRESSED)); SECP256K1_CHECKMEM_CHECK(sout, 65); - CHECK(ecount == 3); CHECK(len == 0); len = 65; - CHECK(secp256k1_ec_pubkey_serialize(CTX, sout, &len, &pubkey, ~0) == 0); - CHECK(ecount == 4); + CHECK_ILLEGAL(CTX, secp256k1_ec_pubkey_serialize(CTX, sout, &len, &pubkey, ~0)); CHECK(len == 0); len = 65; SECP256K1_CHECKMEM_UNDEFINE(sout, 65); CHECK(secp256k1_ec_pubkey_serialize(CTX, sout, &len, &pubkey, SECP256K1_EC_UNCOMPRESSED) == 1); SECP256K1_CHECKMEM_CHECK(sout, 65); - CHECK(ecount == 4); CHECK(len == 65); /* Multiple illegal args. Should still set arg error only once. */ - ecount = 0; - ecount2 = 11; - CHECK(secp256k1_ec_pubkey_parse(CTX, NULL, NULL, 65) == 0); - CHECK(ecount == 1); - /* Does the illegal arg callback actually change the behavior? */ - secp256k1_context_set_illegal_callback(CTX, uncounting_illegal_callback_fn, &ecount2); - CHECK(secp256k1_ec_pubkey_parse(CTX, NULL, NULL, 65) == 0); - CHECK(ecount == 1); - CHECK(ecount2 == 10); - secp256k1_context_set_illegal_callback(CTX, NULL, NULL); + CHECK_ILLEGAL(CTX, secp256k1_ec_pubkey_parse(CTX, NULL, NULL, 65)); /* Try a bunch of prefabbed points with all possible encodings. */ for (i = 0; i < SECP256K1_EC_PARSE_TEST_NVALID; i++) { ec_pubkey_parse_pointtest(valid[i], 1, 1); @@ -6143,7 +6012,6 @@ static void run_eckey_edge_case_test(void) { secp256k1_pubkey pubkey_negone; const secp256k1_pubkey *pubkeys[3]; size_t len; - int32_t ecount; /* Group order is too large, reject. */ CHECK(secp256k1_ec_seckey_verify(CTX, orderc) == 0); SECP256K1_CHECKMEM_UNDEFINE(&pubkey, sizeof(pubkey)); @@ -6265,88 +6133,59 @@ static void run_eckey_edge_case_test(void) { ctmp2[31] = 2; CHECK(secp256k1_ec_pubkey_tweak_mul(CTX, &pubkey2, ctmp2) == 1); CHECK(secp256k1_memcmp_var(&pubkey, &pubkey2, sizeof(pubkey)) == 0); - /* Test argument errors. */ - ecount = 0; - secp256k1_context_set_illegal_callback(CTX, counting_illegal_callback_fn, &ecount); - CHECK(ecount == 0); /* Zeroize pubkey on parse error. */ memset(&pubkey, 0, 32); - CHECK(secp256k1_ec_pubkey_tweak_add(CTX, &pubkey, ctmp2) == 0); - CHECK(ecount == 1); + CHECK_ILLEGAL(CTX, secp256k1_ec_pubkey_tweak_add(CTX, &pubkey, ctmp2)); CHECK(secp256k1_memcmp_var(&pubkey, zeros, sizeof(pubkey)) == 0); memcpy(&pubkey, &pubkey2, sizeof(pubkey)); memset(&pubkey2, 0, 32); - CHECK(secp256k1_ec_pubkey_tweak_mul(CTX, &pubkey2, ctmp2) == 0); - CHECK(ecount == 2); + CHECK_ILLEGAL(CTX, secp256k1_ec_pubkey_tweak_mul(CTX, &pubkey2, ctmp2)); CHECK(secp256k1_memcmp_var(&pubkey2, zeros, sizeof(pubkey2)) == 0); /* Plain argument errors. */ - ecount = 0; CHECK(secp256k1_ec_seckey_verify(CTX, ctmp) == 1); - CHECK(ecount == 0); - CHECK(secp256k1_ec_seckey_verify(CTX, NULL) == 0); - CHECK(ecount == 1); - ecount = 0; + CHECK_ILLEGAL(CTX, secp256k1_ec_seckey_verify(CTX, NULL)); memset(ctmp2, 0, 32); ctmp2[31] = 4; - CHECK(secp256k1_ec_pubkey_tweak_add(CTX, NULL, ctmp2) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_ec_pubkey_tweak_add(CTX, &pubkey, NULL) == 0); - CHECK(ecount == 2); - ecount = 0; + CHECK_ILLEGAL(CTX, secp256k1_ec_pubkey_tweak_add(CTX, NULL, ctmp2)); + CHECK_ILLEGAL(CTX, secp256k1_ec_pubkey_tweak_add(CTX, &pubkey, NULL)); memset(ctmp2, 0, 32); ctmp2[31] = 4; - CHECK(secp256k1_ec_pubkey_tweak_mul(CTX, NULL, ctmp2) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_ec_pubkey_tweak_mul(CTX, &pubkey, NULL) == 0); - CHECK(ecount == 2); - ecount = 0; + CHECK_ILLEGAL(CTX, secp256k1_ec_pubkey_tweak_mul(CTX, NULL, ctmp2)); + CHECK_ILLEGAL(CTX, secp256k1_ec_pubkey_tweak_mul(CTX, &pubkey, NULL)); memset(ctmp2, 0, 32); - CHECK(secp256k1_ec_seckey_tweak_add(CTX, NULL, ctmp2) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_ec_seckey_tweak_add(CTX, ctmp, NULL) == 0); - CHECK(ecount == 2); - ecount = 0; + CHECK_ILLEGAL(CTX, secp256k1_ec_seckey_tweak_add(CTX, NULL, ctmp2)); + CHECK_ILLEGAL(CTX, secp256k1_ec_seckey_tweak_add(CTX, ctmp, NULL)); memset(ctmp2, 0, 32); ctmp2[31] = 1; - CHECK(secp256k1_ec_seckey_tweak_mul(CTX, NULL, ctmp2) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_ec_seckey_tweak_mul(CTX, ctmp, NULL) == 0); - CHECK(ecount == 2); - ecount = 0; - CHECK(secp256k1_ec_pubkey_create(CTX, NULL, ctmp) == 0); - CHECK(ecount == 1); + CHECK_ILLEGAL(CTX, secp256k1_ec_seckey_tweak_mul(CTX, NULL, ctmp2)); + CHECK_ILLEGAL(CTX, secp256k1_ec_seckey_tweak_mul(CTX, ctmp, NULL)); + CHECK_ILLEGAL(CTX, secp256k1_ec_pubkey_create(CTX, NULL, ctmp)); memset(&pubkey, 1, sizeof(pubkey)); - CHECK(secp256k1_ec_pubkey_create(CTX, &pubkey, NULL) == 0); - CHECK(ecount == 2); + CHECK_ILLEGAL(CTX, secp256k1_ec_pubkey_create(CTX, &pubkey, NULL)); CHECK(secp256k1_memcmp_var(&pubkey, zeros, sizeof(secp256k1_pubkey)) == 0); /* secp256k1_ec_pubkey_combine tests. */ - ecount = 0; pubkeys[0] = &pubkey_one; SECP256K1_CHECKMEM_UNDEFINE(&pubkeys[0], sizeof(secp256k1_pubkey *)); SECP256K1_CHECKMEM_UNDEFINE(&pubkeys[1], sizeof(secp256k1_pubkey *)); SECP256K1_CHECKMEM_UNDEFINE(&pubkeys[2], sizeof(secp256k1_pubkey *)); memset(&pubkey, 255, sizeof(secp256k1_pubkey)); SECP256K1_CHECKMEM_UNDEFINE(&pubkey, sizeof(secp256k1_pubkey)); - CHECK(secp256k1_ec_pubkey_combine(CTX, &pubkey, pubkeys, 0) == 0); + CHECK_ILLEGAL(CTX, secp256k1_ec_pubkey_combine(CTX, &pubkey, pubkeys, 0)); SECP256K1_CHECKMEM_CHECK(&pubkey, sizeof(secp256k1_pubkey)); CHECK(secp256k1_memcmp_var(&pubkey, zeros, sizeof(secp256k1_pubkey)) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_ec_pubkey_combine(CTX, NULL, pubkeys, 1) == 0); + CHECK_ILLEGAL(CTX, secp256k1_ec_pubkey_combine(CTX, NULL, pubkeys, 1)); CHECK(secp256k1_memcmp_var(&pubkey, zeros, sizeof(secp256k1_pubkey)) == 0); - CHECK(ecount == 2); memset(&pubkey, 255, sizeof(secp256k1_pubkey)); SECP256K1_CHECKMEM_UNDEFINE(&pubkey, sizeof(secp256k1_pubkey)); - CHECK(secp256k1_ec_pubkey_combine(CTX, &pubkey, NULL, 1) == 0); + CHECK_ILLEGAL(CTX, secp256k1_ec_pubkey_combine(CTX, &pubkey, NULL, 1)); SECP256K1_CHECKMEM_CHECK(&pubkey, sizeof(secp256k1_pubkey)); CHECK(secp256k1_memcmp_var(&pubkey, zeros, sizeof(secp256k1_pubkey)) == 0); - CHECK(ecount == 3); pubkeys[0] = &pubkey_negone; memset(&pubkey, 255, sizeof(secp256k1_pubkey)); SECP256K1_CHECKMEM_UNDEFINE(&pubkey, sizeof(secp256k1_pubkey)); CHECK(secp256k1_ec_pubkey_combine(CTX, &pubkey, pubkeys, 1) == 1); SECP256K1_CHECKMEM_CHECK(&pubkey, sizeof(secp256k1_pubkey)); CHECK(secp256k1_memcmp_var(&pubkey, zeros, sizeof(secp256k1_pubkey)) > 0); - CHECK(ecount == 3); len = 33; CHECK(secp256k1_ec_pubkey_serialize(CTX, ctmp, &len, &pubkey, SECP256K1_EC_COMPRESSED) == 1); CHECK(secp256k1_ec_pubkey_serialize(CTX, ctmp2, &len, &pubkey_negone, SECP256K1_EC_COMPRESSED) == 1); @@ -6359,7 +6198,6 @@ static void run_eckey_edge_case_test(void) { CHECK(secp256k1_ec_pubkey_combine(CTX, &pubkey, pubkeys, 2) == 0); SECP256K1_CHECKMEM_CHECK(&pubkey, sizeof(secp256k1_pubkey)); CHECK(secp256k1_memcmp_var(&pubkey, zeros, sizeof(secp256k1_pubkey)) == 0); - CHECK(ecount == 3); /* Passes through infinity but comes out one. */ pubkeys[2] = &pubkey_one; memset(&pubkey, 255, sizeof(secp256k1_pubkey)); @@ -6367,7 +6205,6 @@ static void run_eckey_edge_case_test(void) { CHECK(secp256k1_ec_pubkey_combine(CTX, &pubkey, pubkeys, 3) == 1); SECP256K1_CHECKMEM_CHECK(&pubkey, sizeof(secp256k1_pubkey)); CHECK(secp256k1_memcmp_var(&pubkey, zeros, sizeof(secp256k1_pubkey)) > 0); - CHECK(ecount == 3); len = 33; CHECK(secp256k1_ec_pubkey_serialize(CTX, ctmp, &len, &pubkey, SECP256K1_EC_COMPRESSED) == 1); CHECK(secp256k1_ec_pubkey_serialize(CTX, ctmp2, &len, &pubkey_one, SECP256K1_EC_COMPRESSED) == 1); @@ -6379,8 +6216,6 @@ static void run_eckey_edge_case_test(void) { CHECK(secp256k1_ec_pubkey_combine(CTX, &pubkey, pubkeys, 2) == 1); SECP256K1_CHECKMEM_CHECK(&pubkey, sizeof(secp256k1_pubkey)); CHECK(secp256k1_memcmp_var(&pubkey, zeros, sizeof(secp256k1_pubkey)) > 0); - CHECK(ecount == 3); - secp256k1_context_set_illegal_callback(CTX, NULL, NULL); } static void run_eckey_negate_test(void) { @@ -6697,7 +6532,7 @@ static void test_random_pubkeys(void) { CHECK(secp256k1_eckey_pubkey_serialize(&elem, in, &size, 0)); CHECK(size == 65); CHECK(secp256k1_eckey_pubkey_parse(&elem2, in, size)); - ge_equals_ge(&elem,&elem2); + CHECK(secp256k1_ge_eq_var(&elem2, &elem)); /* Check that the X9.62 hybrid type is checked. */ in[0] = secp256k1_testrand_bits(1) ? 6 : 7; res = secp256k1_eckey_pubkey_parse(&elem2, in, size); @@ -6709,7 +6544,7 @@ static void test_random_pubkeys(void) { } } if (res) { - ge_equals_ge(&elem,&elem2); + CHECK(secp256k1_ge_eq_var(&elem, &elem2)); CHECK(secp256k1_eckey_pubkey_serialize(&elem, out, &size, 0)); CHECK(secp256k1_memcmp_var(&in[1], &out[1], 64) == 0); } @@ -6729,34 +6564,30 @@ static void run_pubkey_comparison(void) { }; secp256k1_pubkey pk1; secp256k1_pubkey pk2; - int32_t ecount = 0; CHECK(secp256k1_ec_pubkey_parse(CTX, &pk1, pk1_ser, sizeof(pk1_ser)) == 1); CHECK(secp256k1_ec_pubkey_parse(CTX, &pk2, pk2_ser, sizeof(pk2_ser)) == 1); - secp256k1_context_set_illegal_callback(CTX, counting_illegal_callback_fn, &ecount); - CHECK(secp256k1_ec_pubkey_cmp(CTX, NULL, &pk2) < 0); - CHECK(ecount == 1); - CHECK(secp256k1_ec_pubkey_cmp(CTX, &pk1, NULL) > 0); - CHECK(ecount == 2); + CHECK_ILLEGAL_VOID(CTX, CHECK(secp256k1_ec_pubkey_cmp(CTX, NULL, &pk2) < 0)); + CHECK_ILLEGAL_VOID(CTX, CHECK(secp256k1_ec_pubkey_cmp(CTX, &pk1, NULL) > 0)); CHECK(secp256k1_ec_pubkey_cmp(CTX, &pk1, &pk2) < 0); CHECK(secp256k1_ec_pubkey_cmp(CTX, &pk2, &pk1) > 0); CHECK(secp256k1_ec_pubkey_cmp(CTX, &pk1, &pk1) == 0); CHECK(secp256k1_ec_pubkey_cmp(CTX, &pk2, &pk2) == 0); - CHECK(ecount == 2); { secp256k1_pubkey pk_tmp; memset(&pk_tmp, 0, sizeof(pk_tmp)); /* illegal pubkey */ - CHECK(secp256k1_ec_pubkey_cmp(CTX, &pk_tmp, &pk2) < 0); - CHECK(ecount == 3); - CHECK(secp256k1_ec_pubkey_cmp(CTX, &pk_tmp, &pk_tmp) == 0); - CHECK(ecount == 5); - CHECK(secp256k1_ec_pubkey_cmp(CTX, &pk2, &pk_tmp) > 0); - CHECK(ecount == 6); + CHECK_ILLEGAL_VOID(CTX, CHECK(secp256k1_ec_pubkey_cmp(CTX, &pk_tmp, &pk2) < 0)); + { + int32_t ecount = 0; + secp256k1_context_set_illegal_callback(CTX, counting_callback_fn, &ecount); + CHECK(secp256k1_ec_pubkey_cmp(CTX, &pk_tmp, &pk_tmp) == 0); + CHECK(ecount == 2); + secp256k1_context_set_illegal_callback(CTX, NULL, NULL); + } + CHECK_ILLEGAL_VOID(CTX, CHECK(secp256k1_ec_pubkey_cmp(CTX, &pk2, &pk_tmp) > 0)); } - secp256k1_context_set_illegal_callback(CTX, NULL, NULL); - /* Make pk2 the same as pk1 but with 3 rather than 2. Note that in * an uncompressed encoding, these would have the opposite ordering */ pk1_ser[0] = 3; @@ -7226,7 +7057,6 @@ static void test_ecdsa_edge_cases(void) { { secp256k1_pubkey pubkey; size_t siglen; - int32_t ecount; unsigned char signature[72]; static const unsigned char nonce[32] = { 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, @@ -7252,72 +7082,42 @@ static void test_ecdsa_edge_cases(void) { 0xb8, 0x12, 0xe0, 0x0b, 0x81, 0x7a, 0x77, 0x62, 0x65, 0xdf, 0xdd, 0x31, 0xb9, 0x3e, 0x29, 0xa9, }; - ecount = 0; - secp256k1_context_set_illegal_callback(CTX, counting_illegal_callback_fn, &ecount); CHECK(secp256k1_ecdsa_sign(CTX, &sig, msg, key, precomputed_nonce_function, nonce) == 0); CHECK(secp256k1_ecdsa_sign(CTX, &sig, msg, key, precomputed_nonce_function, nonce2) == 0); msg[31] = 0xaa; CHECK(secp256k1_ecdsa_sign(CTX, &sig, msg, key, precomputed_nonce_function, nonce) == 1); - CHECK(ecount == 0); - CHECK(secp256k1_ecdsa_sign(CTX, NULL, msg, key, precomputed_nonce_function, nonce2) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_ecdsa_sign(CTX, &sig, NULL, key, precomputed_nonce_function, nonce2) == 0); - CHECK(ecount == 2); - CHECK(secp256k1_ecdsa_sign(CTX, &sig, msg, NULL, precomputed_nonce_function, nonce2) == 0); - CHECK(ecount == 3); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_sign(CTX, NULL, msg, key, precomputed_nonce_function, nonce2)); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_sign(CTX, &sig, NULL, key, precomputed_nonce_function, nonce2)); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_sign(CTX, &sig, msg, NULL, precomputed_nonce_function, nonce2)); CHECK(secp256k1_ecdsa_sign(CTX, &sig, msg, key, precomputed_nonce_function, nonce2) == 1); CHECK(secp256k1_ec_pubkey_create(CTX, &pubkey, key) == 1); - CHECK(secp256k1_ecdsa_verify(CTX, NULL, msg, &pubkey) == 0); - CHECK(ecount == 4); - CHECK(secp256k1_ecdsa_verify(CTX, &sig, NULL, &pubkey) == 0); - CHECK(ecount == 5); - CHECK(secp256k1_ecdsa_verify(CTX, &sig, msg, NULL) == 0); - CHECK(ecount == 6); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_verify(CTX, NULL, msg, &pubkey)); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_verify(CTX, &sig, NULL, &pubkey)); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_verify(CTX, &sig, msg, NULL)); CHECK(secp256k1_ecdsa_verify(CTX, &sig, msg, &pubkey) == 1); - CHECK(ecount == 6); - CHECK(secp256k1_ec_pubkey_create(CTX, &pubkey, NULL) == 0); - CHECK(ecount == 7); + CHECK_ILLEGAL(CTX, secp256k1_ec_pubkey_create(CTX, &pubkey, NULL)); /* That pubkeyload fails via an ARGCHECK is a little odd but makes sense because pubkeys are an opaque data type. */ - CHECK(secp256k1_ecdsa_verify(CTX, &sig, msg, &pubkey) == 0); - CHECK(ecount == 8); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_verify(CTX, &sig, msg, &pubkey)); siglen = 72; - CHECK(secp256k1_ecdsa_signature_serialize_der(CTX, NULL, &siglen, &sig) == 0); - CHECK(ecount == 9); - CHECK(secp256k1_ecdsa_signature_serialize_der(CTX, signature, NULL, &sig) == 0); - CHECK(ecount == 10); - CHECK(secp256k1_ecdsa_signature_serialize_der(CTX, signature, &siglen, NULL) == 0); - CHECK(ecount == 11); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_signature_serialize_der(CTX, NULL, &siglen, &sig)); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_signature_serialize_der(CTX, signature, NULL, &sig)); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_signature_serialize_der(CTX, signature, &siglen, NULL)); CHECK(secp256k1_ecdsa_signature_serialize_der(CTX, signature, &siglen, &sig) == 1); - CHECK(ecount == 11); - CHECK(secp256k1_ecdsa_signature_parse_der(CTX, NULL, signature, siglen) == 0); - CHECK(ecount == 12); - CHECK(secp256k1_ecdsa_signature_parse_der(CTX, &sig, NULL, siglen) == 0); - CHECK(ecount == 13); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_signature_parse_der(CTX, NULL, signature, siglen)); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_signature_parse_der(CTX, &sig, NULL, siglen)); CHECK(secp256k1_ecdsa_signature_parse_der(CTX, &sig, signature, siglen) == 1); - CHECK(ecount == 13); siglen = 10; /* Too little room for a signature does not fail via ARGCHECK. */ CHECK(secp256k1_ecdsa_signature_serialize_der(CTX, signature, &siglen, &sig) == 0); - CHECK(ecount == 13); - ecount = 0; - CHECK(secp256k1_ecdsa_signature_normalize(CTX, NULL, NULL) == 0); - CHECK(ecount == 1); - CHECK(secp256k1_ecdsa_signature_serialize_compact(CTX, NULL, &sig) == 0); - CHECK(ecount == 2); - CHECK(secp256k1_ecdsa_signature_serialize_compact(CTX, signature, NULL) == 0); - CHECK(ecount == 3); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_signature_normalize(CTX, NULL, NULL)); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_signature_serialize_compact(CTX, NULL, &sig)); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_signature_serialize_compact(CTX, signature, NULL)); CHECK(secp256k1_ecdsa_signature_serialize_compact(CTX, signature, &sig) == 1); - CHECK(ecount == 3); - CHECK(secp256k1_ecdsa_signature_parse_compact(CTX, NULL, signature) == 0); - CHECK(ecount == 4); - CHECK(secp256k1_ecdsa_signature_parse_compact(CTX, &sig, NULL) == 0); - CHECK(ecount == 5); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_signature_parse_compact(CTX, NULL, signature)); + CHECK_ILLEGAL(CTX, secp256k1_ecdsa_signature_parse_compact(CTX, &sig, NULL)); CHECK(secp256k1_ecdsa_signature_parse_compact(CTX, &sig, signature) == 1); - CHECK(ecount == 5); memset(signature, 255, 64); CHECK(secp256k1_ecdsa_signature_parse_compact(CTX, &sig, signature) == 0); - CHECK(ecount == 5); - secp256k1_context_set_illegal_callback(CTX, NULL, NULL); } /* Nonce function corner cases. */ diff --git a/src/secp256k1/src/tests_exhaustive.c b/src/secp256k1/src/tests_exhaustive.c index 3af8ec1ee5..1a8be57d07 100644 --- a/src/secp256k1/src/tests_exhaustive.c +++ b/src/secp256k1/src/tests_exhaustive.c @@ -28,61 +28,11 @@ #include "testrand_impl.h" #include "ecmult_compute_table_impl.h" #include "ecmult_gen_compute_table_impl.h" +#include "testutil.h" #include "util.h" static int count = 2; -/** stolen from tests.c */ -static void ge_equals_ge(const secp256k1_ge *a, const secp256k1_ge *b) { - CHECK(a->infinity == b->infinity); - if (a->infinity) { - return; - } - CHECK(secp256k1_fe_equal(&a->x, &b->x)); - CHECK(secp256k1_fe_equal(&a->y, &b->y)); -} - -static void ge_equals_gej(const secp256k1_ge *a, const secp256k1_gej *b) { - secp256k1_fe z2s; - secp256k1_fe u1, u2, s1, s2; - CHECK(a->infinity == b->infinity); - if (a->infinity) { - return; - } - /* Check a.x * b.z^2 == b.x && a.y * b.z^3 == b.y, to avoid inverses. */ - secp256k1_fe_sqr(&z2s, &b->z); - secp256k1_fe_mul(&u1, &a->x, &z2s); - u2 = b->x; - secp256k1_fe_mul(&s1, &a->y, &z2s); secp256k1_fe_mul(&s1, &s1, &b->z); - s2 = b->y; - CHECK(secp256k1_fe_equal(&u1, &u2)); - CHECK(secp256k1_fe_equal(&s1, &s2)); -} - -static void random_fe(secp256k1_fe *x) { - unsigned char bin[32]; - do { - secp256k1_testrand256(bin); - if (secp256k1_fe_set_b32_limit(x, bin)) { - return; - } - } while(1); -} - -static void random_fe_non_zero(secp256k1_fe *nz) { - int tries = 10; - while (--tries >= 0) { - random_fe(nz); - secp256k1_fe_normalize(nz); - if (!secp256k1_fe_is_zero(nz)) { - break; - } - } - /* Infinitesimal probability of spurious failure here */ - CHECK(tries >= 0); -} -/** END stolen from tests.c */ - static uint32_t num_cores = 1; static uint32_t this_core = 0; @@ -117,7 +67,7 @@ static void test_exhaustive_endomorphism(const secp256k1_ge *group) { for (i = 0; i < EXHAUSTIVE_TEST_ORDER; i++) { secp256k1_ge res; secp256k1_ge_mul_lambda(&res, &group[i]); - ge_equals_ge(&group[i * EXHAUSTIVE_TEST_LAMBDA % EXHAUSTIVE_TEST_ORDER], &res); + CHECK(secp256k1_ge_eq_var(&group[i * EXHAUSTIVE_TEST_LAMBDA % EXHAUSTIVE_TEST_ORDER], &res)); } } @@ -143,21 +93,21 @@ static void test_exhaustive_addition(const secp256k1_ge *group, const secp256k1_ secp256k1_gej tmp; /* add_var */ secp256k1_gej_add_var(&tmp, &groupj[i], &groupj[j], NULL); - ge_equals_gej(&group[(i + j) % EXHAUSTIVE_TEST_ORDER], &tmp); + CHECK(secp256k1_gej_eq_ge_var(&tmp, &group[(i + j) % EXHAUSTIVE_TEST_ORDER])); /* add_ge */ if (j > 0) { secp256k1_gej_add_ge(&tmp, &groupj[i], &group[j]); - ge_equals_gej(&group[(i + j) % EXHAUSTIVE_TEST_ORDER], &tmp); + CHECK(secp256k1_gej_eq_ge_var(&tmp, &group[(i + j) % EXHAUSTIVE_TEST_ORDER])); } /* add_ge_var */ secp256k1_gej_add_ge_var(&tmp, &groupj[i], &group[j], NULL); - ge_equals_gej(&group[(i + j) % EXHAUSTIVE_TEST_ORDER], &tmp); + CHECK(secp256k1_gej_eq_ge_var(&tmp, &group[(i + j) % EXHAUSTIVE_TEST_ORDER])); /* add_zinv_var */ zless_gej.infinity = groupj[j].infinity; zless_gej.x = groupj[j].x; zless_gej.y = groupj[j].y; secp256k1_gej_add_zinv_var(&tmp, &groupj[i], &zless_gej, &fe_inv); - ge_equals_gej(&group[(i + j) % EXHAUSTIVE_TEST_ORDER], &tmp); + CHECK(secp256k1_gej_eq_ge_var(&tmp, &group[(i + j) % EXHAUSTIVE_TEST_ORDER])); } } @@ -165,9 +115,9 @@ static void test_exhaustive_addition(const secp256k1_ge *group, const secp256k1_ for (i = 0; i < EXHAUSTIVE_TEST_ORDER; i++) { secp256k1_gej tmp; secp256k1_gej_double(&tmp, &groupj[i]); - ge_equals_gej(&group[(2 * i) % EXHAUSTIVE_TEST_ORDER], &tmp); + CHECK(secp256k1_gej_eq_ge_var(&tmp, &group[(2 * i) % EXHAUSTIVE_TEST_ORDER])); secp256k1_gej_double_var(&tmp, &groupj[i], NULL); - ge_equals_gej(&group[(2 * i) % EXHAUSTIVE_TEST_ORDER], &tmp); + CHECK(secp256k1_gej_eq_ge_var(&tmp, &group[(2 * i) % EXHAUSTIVE_TEST_ORDER])); } /* Check negation */ @@ -175,9 +125,9 @@ static void test_exhaustive_addition(const secp256k1_ge *group, const secp256k1_ secp256k1_ge tmp; secp256k1_gej tmpj; secp256k1_ge_neg(&tmp, &group[i]); - ge_equals_ge(&group[EXHAUSTIVE_TEST_ORDER - i], &tmp); + CHECK(secp256k1_ge_eq_var(&tmp, &group[EXHAUSTIVE_TEST_ORDER - i])); secp256k1_gej_neg(&tmpj, &groupj[i]); - ge_equals_gej(&group[EXHAUSTIVE_TEST_ORDER - i], &tmpj); + CHECK(secp256k1_gej_eq_ge_var(&tmpj, &group[EXHAUSTIVE_TEST_ORDER - i])); } } @@ -194,8 +144,7 @@ static void test_exhaustive_ecmult(const secp256k1_ge *group, const secp256k1_ge secp256k1_scalar_set_int(&ng, j); secp256k1_ecmult(&tmp, &groupj[r_log], &na, &ng); - ge_equals_gej(&group[(i * r_log + j) % EXHAUSTIVE_TEST_ORDER], &tmp); - + CHECK(secp256k1_gej_eq_ge_var(&tmp, &group[(i * r_log + j) % EXHAUSTIVE_TEST_ORDER])); } } } @@ -213,7 +162,7 @@ static void test_exhaustive_ecmult(const secp256k1_ge *group, const secp256k1_ge /* Test secp256k1_ecmult_const. */ secp256k1_ecmult_const(&tmp, &group[i], &ng); - ge_equals_gej(&group[(i * j) % EXHAUSTIVE_TEST_ORDER], &tmp); + CHECK(secp256k1_gej_eq_ge_var(&tmp, &group[(i * j) % EXHAUSTIVE_TEST_ORDER])); if (i != 0 && j != 0) { /* Test secp256k1_ecmult_const_xonly with all curve X coordinates, and xd=NULL. */ @@ -265,7 +214,7 @@ static void test_exhaustive_ecmult_multi(const secp256k1_context *ctx, const sec data.pt[1] = group[y]; secp256k1_ecmult_multi_var(&ctx->error_callback, scratch, &tmp, &g_sc, ecmult_multi_callback, &data, 2); - ge_equals_gej(&group[(i * x + j * y + k) % EXHAUSTIVE_TEST_ORDER], &tmp); + CHECK(secp256k1_gej_eq_ge_var(&tmp, &group[(i * x + j * y + k) % EXHAUSTIVE_TEST_ORDER])); } } } diff --git a/src/secp256k1/src/testutil.h b/src/secp256k1/src/testutil.h new file mode 100644 index 0000000000..4e2cb7d5b3 --- /dev/null +++ b/src/secp256k1/src/testutil.h @@ -0,0 +1,29 @@ +/*********************************************************************** + * Distributed under the MIT software license, see the accompanying * + * file COPYING or https://www.opensource.org/licenses/mit-license.php.* + ***********************************************************************/ + +#ifndef SECP256K1_TESTUTIL_H +#define SECP256K1_TESTUTIL_H + +#include "field.h" +#include "testrand.h" +#include "util.h" + +static void random_fe(secp256k1_fe *x) { + unsigned char bin[32]; + do { + secp256k1_testrand256(bin); + if (secp256k1_fe_set_b32_limit(x, bin)) { + return; + } + } while(1); +} + +static void random_fe_non_zero(secp256k1_fe *nz) { + do { + random_fe(nz); + } while (secp256k1_fe_is_zero(nz)); +} + +#endif /* SECP256K1_TESTUTIL_H */ diff --git a/src/secp256k1/src/util.h b/src/secp256k1/src/util.h index cf7e5d1af5..187bf1c5e0 100644 --- a/src/secp256k1/src/util.h +++ b/src/secp256k1/src/util.h @@ -132,16 +132,11 @@ static const secp256k1_callback default_error_callback = { } while(0) #endif -/* Like assert(), but when VERIFY is defined, and side-effect safe. */ -#if defined(COVERAGE) -#define VERIFY_CHECK(check) -#define VERIFY_SETUP(stmt) -#elif defined(VERIFY) +/* Like assert(), but when VERIFY is defined. */ +#if defined(VERIFY) #define VERIFY_CHECK CHECK -#define VERIFY_SETUP(stmt) do { stmt; } while(0) #else -#define VERIFY_CHECK(cond) do { (void)(cond); } while(0) -#define VERIFY_SETUP(stmt) +#define VERIFY_CHECK(cond) #endif static SECP256K1_INLINE void *checked_malloc(const secp256k1_callback* cb, size_t size) { diff --git a/src/secp256k1/tools/check-abi.sh b/src/secp256k1/tools/check-abi.sh new file mode 100755 index 0000000000..8f6119cd8e --- /dev/null +++ b/src/secp256k1/tools/check-abi.sh @@ -0,0 +1,64 @@ +#!/bin/sh + +set -eu + +default_base_version="$(git describe --match "v*.*.*" --abbrev=0)" +default_new_version="master" + +display_help_and_exit() { + echo "Usage: $0 <base_ver> <new_ver>" + echo "" + echo "Description: This script uses the ABI Compliance Checker tool to determine if the ABI" + echo " of a new version of libsecp256k1 has changed in a backward-incompatible way." + echo "" + echo "Options:" + echo " base_ver Specify the base version (default: $default_base_version)" + echo " new_ver Specify the new version (default: $default_new_version)" + echo " -h, --help Display this help message" + exit 0 +} + +if [ "$#" -eq 0 ]; then + base_version="$default_base_version" + new_version="$default_new_version" +elif [ "$#" -eq 1 ] && { [ "$1" = "-h" ] || [ "$1" = "--help" ]; }; then + display_help_and_exit +elif [ "$#" -eq 2 ]; then + base_version="$1" + new_version="$2" +else + echo "Invalid usage. See help:" + echo "" + display_help_and_exit +fi + +checkout_and_build() { + git worktree add -d "$1" "$2" + cd "$1" + mkdir build && cd build + cmake -S .. --preset dev-mode \ + -DCMAKE_C_COMPILER=gcc -DCMAKE_BUILD_TYPE=None -DCMAKE_C_FLAGS="-g -Og -gdwarf-4" \ + -DSECP256K1_BUILD_BENCHMARK=OFF \ + -DSECP256K1_BUILD_TESTS=OFF \ + -DSECP256K1_BUILD_EXHAUSTIVE_TESTS=OFF \ + -DSECP256K1_BUILD_CTIME_TESTS=OFF \ + -DSECP256K1_BUILD_EXAMPLES=OFF + cmake --build . -j "$(nproc)" + abi-dumper src/libsecp256k1.so -o ABI.dump -lver "$2" +} + +echo "Comparing $base_version (base version) to $new_version (new version)" +echo + +original_dir="$(pwd)" + +base_source_dir=$(mktemp -d) +checkout_and_build "$base_source_dir" "$base_version" + +new_source_dir=$(mktemp -d) +checkout_and_build "$new_source_dir" "$new_version" + +cd "$original_dir" +abi-compliance-checker -lib libsecp256k1 -old "${base_source_dir}/build/ABI.dump" -new "${new_source_dir}/build/ABI.dump" +git worktree remove "$base_source_dir" +git worktree remove "$new_source_dir" diff --git a/src/serialize.h b/src/serialize.h index 8b15178ec0..19585c630a 100644 --- a/src/serialize.h +++ b/src/serialize.h @@ -7,7 +7,10 @@ #define BITCOIN_SERIALIZE_H #include <attributes.h> +#include <compat/assumptions.h> // IWYU pragma: keep #include <compat/endian.h> +#include <prevector.h> +#include <span.h> #include <algorithm> #include <cstdint> @@ -18,13 +21,9 @@ #include <memory> #include <set> #include <string> -#include <string.h> #include <utility> #include <vector> -#include <prevector.h> -#include <span.h> - /** * The maximum size of a serialized object in bytes or number of elements * (for eg vectors) when the size is encoded as CompactSize. @@ -118,20 +117,7 @@ template<typename Stream> inline uint64_t ser_readdata64(Stream &s) } -///////////////////////////////////////////////////////////////// -// -// Templates for serializing to anything that looks like a stream, -// i.e. anything that supports .read(Span<std::byte>) and .write(Span<const std::byte>) -// - -class CSizeComputer; - -enum -{ - // primary actions - SER_NETWORK = (1 << 0), - SER_DISK = (1 << 1), -}; +class SizeComputer; /** * Convert any argument to a reference to X, maintaining constness. @@ -214,11 +200,11 @@ const Out& AsBase(const In& x) * } * }; * which would then be invoked as - * READWRITE(WithParams(BarParameter{...}, Using<FooFormatter>(obj.foo))) + * READWRITE(BarParameter{...}(Using<FooFormatter>(obj.foo))) * - * WithParams(parameter, obj) can be invoked anywhere in the call stack; it is + * parameter(obj) can be invoked anywhere in the call stack; it is * passed down recursively into all serialization code, until another - * WithParams overrides it. + * serialization parameter overrides it. * * Parameters will be implicitly converted where appropriate. This means that * "parent" serialization code can use a parameter that derives from, or is @@ -269,6 +255,9 @@ const Out& AsBase(const In& x) BASE_SERIALIZE_METHODS(cls) \ FORMATTER_METHODS_PARAMS(cls, obj, paramcls, paramobj) +// Templates for serializing to anything that looks like a stream, +// i.e. anything that supports .read(Span<std::byte>) and .write(Span<const std::byte>) +// // clang-format off #ifndef CHAR_EQUALS_INT8 template <typename Stream> void Serialize(Stream&, char) = delete; // char serialization forbidden. Use uint8_t or int8_t @@ -282,10 +271,9 @@ template<typename Stream> inline void Serialize(Stream& s, int32_t a ) { ser_wri template<typename Stream> inline void Serialize(Stream& s, uint32_t a) { ser_writedata32(s, a); } template<typename Stream> inline void Serialize(Stream& s, int64_t a ) { ser_writedata64(s, a); } template<typename Stream> inline void Serialize(Stream& s, uint64_t a) { ser_writedata64(s, a); } -template<typename Stream, int N> inline void Serialize(Stream& s, const char (&a)[N]) { s.write(MakeByteSpan(a)); } -template<typename Stream, int N> inline void Serialize(Stream& s, const unsigned char (&a)[N]) { s.write(MakeByteSpan(a)); } -template <typename Stream, typename B, std::size_t N> void Serialize(Stream& s, const std::array<B, N>& a) { (void)/* force byte-type */UCharCast(a.data()); s.write(MakeByteSpan(a)); } -template <typename Stream, typename B> void Serialize(Stream& s, Span<B> span) { (void)/* force byte-type */UCharCast(span.data()); s.write(AsBytes(span)); } +template <typename Stream, BasicByte B, int N> void Serialize(Stream& s, const B (&a)[N]) { s.write(MakeByteSpan(a)); } +template <typename Stream, BasicByte B, std::size_t N> void Serialize(Stream& s, const std::array<B, N>& a) { s.write(MakeByteSpan(a)); } +template <typename Stream, BasicByte B> void Serialize(Stream& s, Span<B> span) { s.write(AsBytes(span)); } #ifndef CHAR_EQUALS_INT8 template <typename Stream> void Unserialize(Stream&, char) = delete; // char serialization forbidden. Use uint8_t or int8_t @@ -299,10 +287,9 @@ template<typename Stream> inline void Unserialize(Stream& s, int32_t& a ) { a = template<typename Stream> inline void Unserialize(Stream& s, uint32_t& a) { a = ser_readdata32(s); } template<typename Stream> inline void Unserialize(Stream& s, int64_t& a ) { a = ser_readdata64(s); } template<typename Stream> inline void Unserialize(Stream& s, uint64_t& a) { a = ser_readdata64(s); } -template<typename Stream, int N> inline void Unserialize(Stream& s, char (&a)[N]) { s.read(MakeWritableByteSpan(a)); } -template<typename Stream, int N> inline void Unserialize(Stream& s, unsigned char (&a)[N]) { s.read(MakeWritableByteSpan(a)); } -template <typename Stream, typename B, std::size_t N> void Unserialize(Stream& s, std::array<B, N>& a) { (void)/* force byte-type */UCharCast(a.data()); s.read(MakeWritableByteSpan(a)); } -template <typename Stream, typename B> void Unserialize(Stream& s, Span<B> span) { (void)/* force byte-type */UCharCast(span.data()); s.read(AsWritableBytes(span)); } +template <typename Stream, BasicByte B, int N> void Unserialize(Stream& s, B (&a)[N]) { s.read(MakeWritableByteSpan(a)); } +template <typename Stream, BasicByte B, std::size_t N> void Unserialize(Stream& s, std::array<B, N>& a) { s.read(MakeWritableByteSpan(a)); } +template <typename Stream, BasicByte B> void Unserialize(Stream& s, Span<B> span) { s.read(AsWritableBytes(span)); } template <typename Stream> inline void Serialize(Stream& s, bool a) { uint8_t f = a; ser_writedata8(s, f); } template <typename Stream> inline void Unserialize(Stream& s, bool& a) { uint8_t f = ser_readdata8(s); a = f; } @@ -324,7 +311,7 @@ constexpr inline unsigned int GetSizeOfCompactSize(uint64_t nSize) else return sizeof(unsigned char) + sizeof(uint64_t); } -inline void WriteCompactSize(CSizeComputer& os, uint64_t nSize); +inline void WriteCompactSize(SizeComputer& os, uint64_t nSize); template<typename Stream> void WriteCompactSize(Stream& os, uint64_t nSize) @@ -450,7 +437,7 @@ inline unsigned int GetSizeOfVarInt(I n) } template<typename I> -inline void WriteVarInt(CSizeComputer& os, I n); +inline void WriteVarInt(SizeComputer& os, I n); template<typename Stream, VarIntMode Mode, typename I> void WriteVarInt(Stream& os, I n) @@ -766,18 +753,23 @@ template<typename Stream, typename T> void Serialize(Stream& os, const std::uniq template<typename Stream, typename T> void Unserialize(Stream& os, std::unique_ptr<const T>& p); - /** * If none of the specialized versions above matched, default to calling member function. */ -template<typename Stream, typename T> -inline void Serialize(Stream& os, const T& a) +template <class T, class Stream> +concept Serializable = requires(T a, Stream s) { a.Serialize(s); }; +template <typename Stream, typename T> + requires Serializable<T, Stream> +void Serialize(Stream& os, const T& a) { a.Serialize(os); } -template<typename Stream, typename T> -inline void Unserialize(Stream& is, T&& a) +template <class T, class Stream> +concept Unserializable = requires(T a, Stream s) { a.Unserialize(s); }; +template <typename Stream, typename T> + requires Unserializable<T, Stream> +void Unserialize(Stream& is, T&& a) { a.Unserialize(is); } @@ -1070,22 +1062,21 @@ struct ActionUnserialize { /* ::GetSerializeSize implementations * * Computing the serialized size of objects is done through a special stream - * object of type CSizeComputer, which only records the number of bytes written + * object of type SizeComputer, which only records the number of bytes written * to it. * * If your Serialize or SerializationOp method has non-trivial overhead for * serialization, it may be worthwhile to implement a specialized version for - * CSizeComputer, which uses the s.seek() method to record bytes that would + * SizeComputer, which uses the s.seek() method to record bytes that would * be written instead. */ -class CSizeComputer +class SizeComputer { protected: size_t nSize{0}; - const int nVersion; public: - explicit CSizeComputer(int nVersionIn) : nVersion(nVersionIn) {} + SizeComputer() {} void write(Span<const std::byte> src) { @@ -1099,7 +1090,7 @@ public: } template<typename T> - CSizeComputer& operator<<(const T& obj) + SizeComputer& operator<<(const T& obj) { ::Serialize(*this, obj); return (*this); @@ -1108,33 +1099,23 @@ public: size_t size() const { return nSize; } - - int GetVersion() const { return nVersion; } }; template<typename I> -inline void WriteVarInt(CSizeComputer &s, I n) +inline void WriteVarInt(SizeComputer &s, I n) { s.seek(GetSizeOfVarInt<I>(n)); } -inline void WriteCompactSize(CSizeComputer &s, uint64_t nSize) +inline void WriteCompactSize(SizeComputer &s, uint64_t nSize) { s.seek(GetSizeOfCompactSize(nSize)); } template <typename T> -size_t GetSerializeSize(const T& t, int nVersion = 0) +size_t GetSerializeSize(const T& t) { - return (CSizeComputer(nVersion) << t).size(); -} - -template <typename... T> -size_t GetSerializeSizeMany(int nVersion, const T&... t) -{ - CSizeComputer sc(nVersion); - SerializeMany(sc, t...); - return sc.size(); + return (SizeComputer() << t).size(); } /** Wrapper that overrides the GetParams() function of a stream (and hides GetVersion/GetType). */ @@ -1183,17 +1164,6 @@ public: }; /** - * Return a wrapper around t that (de)serializes it with specified parameter params. - * - * See FORMATTER_METHODS_PARAMS for more information on serialization parameters. - */ -template <typename Params, typename T> -static auto WithParams(const Params& params, T&& t) -{ - return ParamsWrapper<Params, T>{params, t}; -} - -/** * Helper macro for SerParams structs * * Allows you define SerParams instances and then apply them directly @@ -1202,8 +1172,16 @@ static auto WithParams(const Params& params, T&& t) * constexpr SerParams FOO{....}; * ss << FOO(obj); */ -#define SER_PARAMS_OPFUNC \ - template <typename T> \ - auto operator()(T&& t) const { return WithParams(*this, t); } +#define SER_PARAMS_OPFUNC \ + /** \ + * Return a wrapper around t that (de)serializes it with specified parameter params. \ + * \ + * See FORMATTER_METHODS_PARAMS for more information on serialization parameters. \ + */ \ + template <typename T> \ + auto operator()(T&& t) const \ + { \ + return ParamsWrapper{*this, t}; \ + } #endif // BITCOIN_SERIALIZE_H diff --git a/src/shutdown.cpp b/src/shutdown.cpp deleted file mode 100644 index fc18ccd207..0000000000 --- a/src/shutdown.cpp +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <shutdown.h> - -#include <kernel/context.h> -#include <logging.h> -#include <util/check.h> -#include <util/signalinterrupt.h> - -#include <assert.h> -#include <system_error> - -void StartShutdown() -{ - try { - Assert(kernel::g_context)->interrupt(); - } catch (const std::system_error&) { - LogPrintf("Sending shutdown token failed\n"); - assert(0); - } -} - -void AbortShutdown() -{ - Assert(kernel::g_context)->interrupt.reset(); -} - -bool ShutdownRequested() -{ - return bool{Assert(kernel::g_context)->interrupt}; -} - -void WaitForShutdown() -{ - try { - Assert(kernel::g_context)->interrupt.wait(); - } catch (const std::system_error&) { - LogPrintf("Reading shutdown token failed\n"); - assert(0); - } -} diff --git a/src/shutdown.h b/src/shutdown.h deleted file mode 100644 index 2d6ace8d93..0000000000 --- a/src/shutdown.h +++ /dev/null @@ -1,25 +0,0 @@ -// Copyright (c) 2009-2010 Satoshi Nakamoto -// Copyright (c) 2009-2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_SHUTDOWN_H -#define BITCOIN_SHUTDOWN_H - -/** Request shutdown of the application. */ -void StartShutdown(); - -/** Clear shutdown flag. Only use this during init (before calling WaitForShutdown in any - * thread), or in the unit tests. Calling it in other circumstances will cause a race condition. - */ -void AbortShutdown(); - -/** Returns true if a shutdown is requested, false otherwise. */ -bool ShutdownRequested(); - -/** Wait for StartShutdown to be called in any thread. This can only be used - * from a single thread. - */ -void WaitForShutdown(); - -#endif // BITCOIN_SHUTDOWN_H diff --git a/src/signet.cpp b/src/signet.cpp index ef0faaa5f8..ebf0de09d3 100644 --- a/src/signet.cpp +++ b/src/signet.cpp @@ -98,7 +98,7 @@ std::optional<SignetTxs> SignetTxs::Create(const CBlock& block, const CScript& c // no signet solution -- allow this to support OP_TRUE as trivial block challenge } else { try { - SpanReader v{INIT_PROTO_VERSION, signet_solution}; + SpanReader v{signet_solution}; v >> tx_spending.vin[0].scriptSig; v >> tx_spending.vin[0].scriptWitness.stack; if (!v.empty()) return std::nullopt; // extraneous data encountered @@ -109,7 +109,7 @@ std::optional<SignetTxs> SignetTxs::Create(const CBlock& block, const CScript& c uint256 signet_merkle = ComputeModifiedMerkleRoot(modified_cb, block); std::vector<uint8_t> block_data; - CVectorWriter writer{INIT_PROTO_VERSION, block_data, 0}; + VectorWriter writer{block_data, 0}; writer << block.nVersion; writer << block.hashPrevBlock; writer << signet_merkle; diff --git a/src/span.h b/src/span.h index 7209d21a58..2c27a54fc7 100644 --- a/src/span.h +++ b/src/span.h @@ -8,6 +8,7 @@ #include <algorithm> #include <cassert> #include <cstddef> +#include <span> #include <type_traits> #ifdef DEBUG @@ -222,15 +223,32 @@ public: template <typename O> friend class Span; }; +// Return result of calling .data() method on type T. This is used to be able to +// write template deduction guides for the single-parameter Span constructor +// below that will work if the value that is passed has a .data() method, and if +// the data method does not return a void pointer. +// +// It is important to check for the void type specifically below, so the +// deduction guides can be used in SFINAE contexts to check whether objects can +// be converted to spans. If the deduction guides did not explicitly check for +// void, and an object was passed that returned void* from data (like +// std::vector<bool>), the template deduction would succeed, but the Span<void> +// object instantiation would fail, resulting in a hard error, rather than a +// SFINAE error. +// https://stackoverflow.com/questions/68759148/sfinae-to-detect-the-explicitness-of-a-ctad-deduction-guide +// https://stackoverflow.com/questions/16568986/what-happens-when-you-call-data-on-a-stdvectorbool +template<typename T> +using DataResult = std::remove_pointer_t<decltype(std::declval<T&>().data())>; + // Deduction guides for Span // For the pointer/size based and iterator based constructor: template <typename T, typename EndOrSize> Span(T*, EndOrSize) -> Span<T>; // For the array constructor: template <typename T, std::size_t N> Span(T (&)[N]) -> Span<T>; // For the temporaries/rvalue references constructor, only supporting const output. -template <typename T> Span(T&&) -> Span<std::enable_if_t<!std::is_lvalue_reference_v<T>, const std::remove_pointer_t<decltype(std::declval<T&&>().data())>>>; +template <typename T> Span(T&&) -> Span<std::enable_if_t<!std::is_lvalue_reference_v<T> && !std::is_void_v<DataResult<T&&>>, const DataResult<T&&>>>; // For (lvalue) references, supporting mutable output. -template <typename T> Span(T&) -> Span<std::remove_pointer_t<decltype(std::declval<T&>().data())>>; +template <typename T> Span(T&) -> Span<std::enable_if_t<!std::is_void_v<DataResult<T&>>, DataResult<T&>>>; /** Pop the last element off a span, and return a reference to that element. */ template <typename T> @@ -266,13 +284,16 @@ Span<std::byte> MakeWritableByteSpan(V&& v) noexcept return AsWritableBytes(Span{std::forward<V>(v)}); } -// Helper functions to safely cast to unsigned char pointers. +// Helper functions to safely cast basic byte pointers to unsigned char pointers. inline unsigned char* UCharCast(char* c) { return reinterpret_cast<unsigned char*>(c); } inline unsigned char* UCharCast(unsigned char* c) { return c; } inline unsigned char* UCharCast(std::byte* c) { return reinterpret_cast<unsigned char*>(c); } inline const unsigned char* UCharCast(const char* c) { return reinterpret_cast<const unsigned char*>(c); } inline const unsigned char* UCharCast(const unsigned char* c) { return c; } inline const unsigned char* UCharCast(const std::byte* c) { return reinterpret_cast<const unsigned char*>(c); } +// Helper concept for the basic byte types. +template <typename B> +concept BasicByte = requires { UCharCast(std::span<B>{}.data()); }; // Helper function to safely convert a Span to a Span<[const] unsigned char>. template <typename T> constexpr auto UCharSpanCast(Span<T> s) -> Span<typename std::remove_pointer<decltype(UCharCast(s.data()))>::type> { return {UCharCast(s.data()), s.size()}; } diff --git a/src/streams.h b/src/streams.h index d58de5233b..bc04a2babd 100644 --- a/src/streams.h +++ b/src/streams.h @@ -45,60 +45,19 @@ inline void Xor(Span<std::byte> write, Span<const std::byte> key, size_t key_off } } // namespace util -template<typename Stream> -class OverrideStream -{ - Stream* stream; - - const int nVersion; - -public: - OverrideStream(Stream* stream_, int nVersion_) : stream{stream_}, nVersion{nVersion_} {} - - template<typename T> - OverrideStream<Stream>& operator<<(const T& obj) - { - ::Serialize(*this, obj); - return (*this); - } - - template<typename T> - OverrideStream<Stream>& operator>>(T&& obj) - { - ::Unserialize(*this, obj); - return (*this); - } - - void write(Span<const std::byte> src) - { - stream->write(src); - } - - void read(Span<std::byte> dst) - { - stream->read(dst); - } - - int GetVersion() const { return nVersion; } - size_t size() const { return stream->size(); } - void ignore(size_t size) { return stream->ignore(size); } -}; - /* Minimal stream for overwriting and/or appending to an existing byte vector * * The referenced vector will grow as necessary */ -class CVectorWriter +class VectorWriter { - public: - +public: /* - * @param[in] nVersionIn Serialization Version (including any flags) * @param[in] vchDataIn Referenced byte vector to overwrite/append * @param[in] nPosIn Starting position. Vector index where writes should start. The vector will initially * grow as necessary to max(nPosIn, vec.size()). So to append, use vec.size(). */ - CVectorWriter(int nVersionIn, std::vector<unsigned char>& vchDataIn, size_t nPosIn) : nVersion{nVersionIn}, vchData{vchDataIn}, nPos{nPosIn} + VectorWriter(std::vector<unsigned char>& vchDataIn, size_t nPosIn) : vchData{vchDataIn}, nPos{nPosIn} { if(nPos > vchData.size()) vchData.resize(nPos); @@ -108,7 +67,7 @@ class CVectorWriter * @param[in] args A list of items to serialize starting at nPosIn. */ template <typename... Args> - CVectorWriter(int nVersionIn, std::vector<unsigned char>& vchDataIn, size_t nPosIn, Args&&... args) : CVectorWriter{nVersionIn, vchDataIn, nPosIn} + VectorWriter(std::vector<unsigned char>& vchDataIn, size_t nPosIn, Args&&... args) : VectorWriter{vchDataIn, nPosIn} { ::SerializeMany(*this, std::forward<Args>(args)...); } @@ -124,19 +83,14 @@ class CVectorWriter } nPos += src.size(); } - template<typename T> - CVectorWriter& operator<<(const T& obj) + template <typename T> + VectorWriter& operator<<(const T& obj) { ::Serialize(*this, obj); return (*this); } - int GetVersion() const - { - return nVersion; - } private: - const int nVersion; std::vector<unsigned char>& vchData; size_t nPos; }; @@ -146,16 +100,13 @@ private: class SpanReader { private: - const int m_version; Span<const unsigned char> m_data; public: /** - * @param[in] version Serialization Version (including any flags) * @param[in] data Referenced byte vector to overwrite/append */ - SpanReader(int version, Span<const unsigned char> data) - : m_version{version}, m_data{data} {} + explicit SpanReader(Span<const unsigned char> data) : m_data{data} {} template<typename T> SpanReader& operator>>(T&& obj) @@ -164,8 +115,6 @@ public: return (*this); } - int GetVersion() const { return m_version; } - size_t size() const { return m_data.size(); } bool empty() const { return m_data.empty(); } @@ -182,6 +131,11 @@ public: memcpy(dst.data(), m_data.data(), dst.size()); m_data = m_data.subspan(dst.size()); } + + void ignore(size_t n) + { + m_data = m_data.subspan(n); + } }; /** Double ended buffer combining vector and stream-like interfaces. @@ -325,42 +279,6 @@ public: } }; -class CDataStream : public DataStream -{ -private: - int nType; - int nVersion; - -public: - explicit CDataStream(int nTypeIn, int nVersionIn) - : nType{nTypeIn}, - nVersion{nVersionIn} {} - - explicit CDataStream(Span<const uint8_t> sp, int type, int version) : CDataStream{AsBytes(sp), type, version} {} - explicit CDataStream(Span<const value_type> sp, int nTypeIn, int nVersionIn) - : DataStream{sp}, - nType{nTypeIn}, - nVersion{nVersionIn} {} - - int GetType() const { return nType; } - void SetVersion(int n) { nVersion = n; } - int GetVersion() const { return nVersion; } - - template <typename T> - CDataStream& operator<<(const T& obj) - { - ::Serialize(*this, obj); - return *this; - } - - template <typename T> - CDataStream& operator>>(T&& obj) - { - ::Unserialize(*this, obj); - return *this; - } -}; - template <typename IStream> class BitStreamReader { @@ -471,7 +389,7 @@ class AutoFile { protected: std::FILE* m_file; - const std::vector<std::byte> m_xor; + std::vector<std::byte> m_xor; public: explicit AutoFile(std::FILE* file, std::vector<std::byte> data_xor={}) : m_file{file}, m_xor{std::move(data_xor)} {} @@ -511,6 +429,9 @@ public: */ bool IsNull() const { return m_file == nullptr; } + /** Continue with a different XOR key */ + void SetXor(std::vector<std::byte> data_xor) { m_xor = data_xor; } + /** Implementation detail, only used internally. */ std::size_t detail_fread(Span<std::byte> dst); @@ -536,31 +457,7 @@ public: } }; -class CAutoFile : public AutoFile -{ -private: - const int nVersion; - -public: - explicit CAutoFile(std::FILE* file, int version, std::vector<std::byte> data_xor = {}) : AutoFile{file, std::move(data_xor)}, nVersion{version} {} - int GetVersion() const { return nVersion; } - - template<typename T> - CAutoFile& operator<<(const T& obj) - { - ::Serialize(*this, obj); - return (*this); - } - - template<typename T> - CAutoFile& operator>>(T&& obj) - { - ::Unserialize(*this, obj); - return (*this); - } -}; - -/** Wrapper around a CAutoFile& that implements a ring buffer to +/** Wrapper around an AutoFile& that implements a ring buffer to * deserialize from. It guarantees the ability to rewind a given number of bytes. * * Will automatically close the file when it goes out of scope if not null. @@ -569,7 +466,7 @@ public: class BufferedFile { private: - CAutoFile& m_src; + AutoFile& m_src; uint64_t nSrcPos{0}; //!< how many bytes have been read from source uint64_t m_read_pos{0}; //!< how many bytes have been read from this uint64_t nReadLimit; //!< up to which position we're allowed to read @@ -616,15 +513,13 @@ private: } public: - BufferedFile(CAutoFile& file, uint64_t nBufSize, uint64_t nRewindIn) + BufferedFile(AutoFile& file, uint64_t nBufSize, uint64_t nRewindIn) : m_src{file}, nReadLimit{std::numeric_limits<uint64_t>::max()}, nRewind{nRewindIn}, vchBuf(nBufSize, std::byte{0}) { if (nRewindIn >= nBufSize) throw std::ios_base::failure("Rewind limit must be less than buffer size"); } - int GetVersion() const { return m_src.GetVersion(); } - //! check whether we're at the end of the source file bool eof() const { return m_read_pos == nSrcPos && m_src.feof(); diff --git a/src/support/allocators/pool.h b/src/support/allocators/pool.h index c8e70ebacf..873e260b65 100644 --- a/src/support/allocators/pool.h +++ b/src/support/allocators/pool.h @@ -272,7 +272,7 @@ public: /** * Forwards all allocations/deallocations to the PoolResource. */ -template <class T, std::size_t MAX_BLOCK_SIZE_BYTES, std::size_t ALIGN_BYTES> +template <class T, std::size_t MAX_BLOCK_SIZE_BYTES, std::size_t ALIGN_BYTES = alignof(T)> class PoolAllocator { PoolResource<MAX_BLOCK_SIZE_BYTES, ALIGN_BYTES>* m_resource; diff --git a/src/sync.cpp b/src/sync.cpp index 4621805653..58752a9f18 100644 --- a/src/sync.cpp +++ b/src/sync.cpp @@ -246,7 +246,7 @@ void LeaveCritical() pop_lock(); } -std::string LocksHeld() +static std::string LocksHeld() { LockData& lockdata = GetLockData(); std::lock_guard<std::mutex> lock(lockdata.dd_mutex); diff --git a/src/sync.h b/src/sync.h index 45d40b5fdc..dc63e3f2d0 100644 --- a/src/sync.h +++ b/src/sync.h @@ -57,7 +57,6 @@ template <typename MutexType> void EnterCritical(const char* pszName, const char* pszFile, int nLine, MutexType* cs, bool fTry = false); void LeaveCritical(); void CheckLastCritical(void* cs, std::string& lockname, const char* guardname, const char* file, int line); -std::string LocksHeld(); template <typename MutexType> void AssertLockHeldInternal(const char* pszName, const char* pszFile, int nLine, MutexType* cs) EXCLUSIVE_LOCKS_REQUIRED(cs); template <typename MutexType> diff --git a/src/test/addrman_tests.cpp b/src/test/addrman_tests.cpp index bfefc3ff97..5bd4f271cd 100644 --- a/src/test/addrman_tests.cpp +++ b/src/test/addrman_tests.cpp @@ -429,6 +429,24 @@ BOOST_AUTO_TEST_CASE(addrman_getaddr) BOOST_CHECK_EQUAL(addrman->Size(), 2006U); } +BOOST_AUTO_TEST_CASE(getaddr_unfiltered) +{ + auto addrman = std::make_unique<AddrMan>(EMPTY_NETGROUPMAN, DETERMINISTIC, GetCheckRatio(m_node)); + + // Set time on this addr so isTerrible = false + CAddress addr1 = CAddress(ResolveService("250.250.2.1", 8333), NODE_NONE); + addr1.nTime = Now<NodeSeconds>(); + // Not setting time so this addr should be isTerrible = true + CAddress addr2 = CAddress(ResolveService("250.251.2.2", 9999), NODE_NONE); + + CNetAddr source = ResolveIP("250.1.2.1"); + BOOST_CHECK(addrman->Add({addr1, addr2}, source)); + + // Filtered GetAddr should only return addr1 + BOOST_CHECK_EQUAL(addrman->GetAddr(/*max_addresses=*/0, /*max_pct=*/0, /*network=*/std::nullopt).size(), 1U); + // Unfiltered GetAddr should return addr1 and addr2 + BOOST_CHECK_EQUAL(addrman->GetAddr(/*max_addresses=*/0, /*max_pct=*/0, /*network=*/std::nullopt, /*filtered=*/false).size(), 2U); +} BOOST_AUTO_TEST_CASE(caddrinfo_get_tried_bucket_legacy) { diff --git a/src/test/amount_tests.cpp b/src/test/amount_tests.cpp index 3815a5bba6..e5ab1cfb90 100644 --- a/src/test/amount_tests.cpp +++ b/src/test/amount_tests.cpp @@ -85,6 +85,32 @@ BOOST_AUTO_TEST_CASE(GetFeeTest) BOOST_CHECK(CFeeRate(CAmount(27), 789) == CFeeRate(34)); // Maximum size in bytes, should not crash CFeeRate(MAX_MONEY, std::numeric_limits<uint32_t>::max()).GetFeePerK(); + + // check multiplication operator + // check multiplying by zero + feeRate = CFeeRate(1000); + BOOST_CHECK(0 * feeRate == CFeeRate(0)); + BOOST_CHECK(feeRate * 0 == CFeeRate(0)); + // check multiplying by a positive integer + BOOST_CHECK(3 * feeRate == CFeeRate(3000)); + BOOST_CHECK(feeRate * 3 == CFeeRate(3000)); + // check multiplying by a negative integer + BOOST_CHECK(-3 * feeRate == CFeeRate(-3000)); + BOOST_CHECK(feeRate * -3 == CFeeRate(-3000)); + // check commutativity + BOOST_CHECK(2 * feeRate == feeRate * 2); + // check with large numbers + int largeNumber = 1000000; + BOOST_CHECK(largeNumber * feeRate == feeRate * largeNumber); + // check boundary values + int maxInt = std::numeric_limits<int>::max(); + feeRate = CFeeRate(maxInt); + BOOST_CHECK(feeRate * 2 == CFeeRate(static_cast<int64_t>(maxInt) * 2)); + BOOST_CHECK(2 * feeRate == CFeeRate(static_cast<int64_t>(maxInt) * 2)); + // check with zero fee rate + feeRate = CFeeRate(0); + BOOST_CHECK(feeRate * 5 == CFeeRate(0)); + BOOST_CHECK(5 * feeRate == CFeeRate(0)); } BOOST_AUTO_TEST_CASE(BinaryOperatorTest) diff --git a/src/test/arith_uint256_tests.cpp b/src/test/arith_uint256_tests.cpp index 6a37b7d83b..10028c7c93 100644 --- a/src/test/arith_uint256_tests.cpp +++ b/src/test/arith_uint256_tests.cpp @@ -22,6 +22,7 @@ static inline arith_uint256 arith_uint256V(const std::vector<unsigned char>& vch { return UintToArith256(uint256(vch)); } +static inline arith_uint256 arith_uint256S(const std::string& str) { return UintToArith256(uint256S(str)); } const unsigned char R1Array[] = "\x9c\x52\x4a\xdb\xcf\x56\x11\x12\x2b\x29\x12\x5e\x5d\x35\xd2\xd2" @@ -95,25 +96,25 @@ BOOST_AUTO_TEST_CASE( basics ) // constructors, equality, inequality BOOST_CHECK(ZeroL == (OneL << 256)); // String Constructor and Copy Constructor - BOOST_CHECK(arith_uint256("0x"+R1L.ToString()) == R1L); - BOOST_CHECK(arith_uint256("0x"+R2L.ToString()) == R2L); - BOOST_CHECK(arith_uint256("0x"+ZeroL.ToString()) == ZeroL); - BOOST_CHECK(arith_uint256("0x"+OneL.ToString()) == OneL); - BOOST_CHECK(arith_uint256("0x"+MaxL.ToString()) == MaxL); - BOOST_CHECK(arith_uint256(R1L.ToString()) == R1L); - BOOST_CHECK(arith_uint256(" 0x"+R1L.ToString()+" ") == R1L); - BOOST_CHECK(arith_uint256("") == ZeroL); - BOOST_CHECK(R1L == arith_uint256(R1ArrayHex)); + BOOST_CHECK(arith_uint256S("0x" + R1L.ToString()) == R1L); + BOOST_CHECK(arith_uint256S("0x" + R2L.ToString()) == R2L); + BOOST_CHECK(arith_uint256S("0x" + ZeroL.ToString()) == ZeroL); + BOOST_CHECK(arith_uint256S("0x" + OneL.ToString()) == OneL); + BOOST_CHECK(arith_uint256S("0x" + MaxL.ToString()) == MaxL); + BOOST_CHECK(arith_uint256S(R1L.ToString()) == R1L); + BOOST_CHECK(arith_uint256S(" 0x" + R1L.ToString() + " ") == R1L); + BOOST_CHECK(arith_uint256S("") == ZeroL); + BOOST_CHECK(R1L == arith_uint256S(R1ArrayHex)); BOOST_CHECK(arith_uint256(R1L) == R1L); BOOST_CHECK((arith_uint256(R1L^R2L)^R2L) == R1L); BOOST_CHECK(arith_uint256(ZeroL) == ZeroL); BOOST_CHECK(arith_uint256(OneL) == OneL); // uint64_t constructor - BOOST_CHECK( (R1L & arith_uint256("0xffffffffffffffff")) == arith_uint256(R1LLow64)); + BOOST_CHECK((R1L & arith_uint256S("0xffffffffffffffff")) == arith_uint256(R1LLow64)); BOOST_CHECK(ZeroL == arith_uint256(0)); BOOST_CHECK(OneL == arith_uint256(1)); - BOOST_CHECK(arith_uint256("0xffffffffffffffff") == arith_uint256(0xffffffffffffffffULL)); + BOOST_CHECK(arith_uint256S("0xffffffffffffffff") == arith_uint256(0xffffffffffffffffULL)); // Assignment (from base_uint) arith_uint256 tmpL = ~ZeroL; BOOST_CHECK(tmpL == ~ZeroL); @@ -282,7 +283,7 @@ BOOST_AUTO_TEST_CASE( comparison ) // <= >= < > BOOST_AUTO_TEST_CASE( plusMinus ) { arith_uint256 TmpL = 0; - BOOST_CHECK(R1L+R2L == arith_uint256(R1LplusR2L)); + BOOST_CHECK(R1L + R2L == arith_uint256S(R1LplusR2L)); TmpL += R1L; BOOST_CHECK(TmpL == R1L); TmpL += R2L; @@ -346,8 +347,8 @@ BOOST_AUTO_TEST_CASE( multiply ) BOOST_AUTO_TEST_CASE( divide ) { - arith_uint256 D1L("AD7133AC1977FA2B7"); - arith_uint256 D2L("ECD751716"); + arith_uint256 D1L{arith_uint256S("AD7133AC1977FA2B7")}; + arith_uint256 D2L{arith_uint256S("ECD751716")}; BOOST_CHECK((R1L / D1L).ToString() == "00000000000000000b8ac01106981635d9ed112290f8895545a7654dde28fb3a"); BOOST_CHECK((R1L / D2L).ToString() == "000000000873ce8efec5b67150bad3aa8c5fcb70e947586153bf2cec7c37c57a"); BOOST_CHECK(R1L / OneL == R1L); @@ -368,7 +369,7 @@ static bool almostEqual(double d1, double d2) return fabs(d1-d2) <= 4*fabs(d1)*std::numeric_limits<double>::epsilon(); } -BOOST_AUTO_TEST_CASE( methods ) // GetHex SetHex size() GetLow64 GetSerializeSize, Serialize, Unserialize +BOOST_AUTO_TEST_CASE(methods) // GetHex operator= size() GetLow64 GetSerializeSize, Serialize, Unserialize { BOOST_CHECK(R1L.GetHex() == R1L.ToString()); BOOST_CHECK(R2L.GetHex() == R2L.ToString()); @@ -376,11 +377,14 @@ BOOST_AUTO_TEST_CASE( methods ) // GetHex SetHex size() GetLow64 GetSerializeSiz BOOST_CHECK(MaxL.GetHex() == MaxL.ToString()); arith_uint256 TmpL(R1L); BOOST_CHECK(TmpL == R1L); - TmpL.SetHex(R2L.ToString()); BOOST_CHECK(TmpL == R2L); - TmpL.SetHex(ZeroL.ToString()); BOOST_CHECK(TmpL == 0); - TmpL.SetHex(HalfL.ToString()); BOOST_CHECK(TmpL == HalfL); + TmpL = R2L; + BOOST_CHECK(TmpL == R2L); + TmpL = ZeroL; + BOOST_CHECK(TmpL == 0); + TmpL = HalfL; + BOOST_CHECK(TmpL == HalfL); - TmpL.SetHex(R1L.ToString()); + TmpL = R1L; BOOST_CHECK(R1L.size() == 32); BOOST_CHECK(R2L.size() == 32); BOOST_CHECK(ZeroL.size() == 32); diff --git a/src/test/blockchain_tests.cpp b/src/test/blockchain_tests.cpp index b590467a43..be515a9eac 100644 --- a/src/test/blockchain_tests.cpp +++ b/src/test/blockchain_tests.cpp @@ -41,7 +41,7 @@ static void RejectDifficultyMismatch(double difficulty, double expected_difficul static void TestDifficulty(uint32_t nbits, double expected_difficulty) { CBlockIndex* block_index = CreateBlockIndexWithNbits(nbits); - double difficulty = GetDifficulty(block_index); + double difficulty = GetDifficulty(*block_index); delete block_index; RejectDifficultyMismatch(difficulty, expected_difficulty); diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 4348a20886..763f0f897e 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -32,13 +32,13 @@ static CBlock BuildBlockTestCase() { block.hashPrevBlock = InsecureRand256(); block.nBits = 0x207fffff; - tx.vin[0].prevout.hash = InsecureRand256(); + tx.vin[0].prevout.hash = Txid::FromUint256(InsecureRand256()); tx.vin[0].prevout.n = 0; block.vtx[1] = MakeTransactionRef(tx); tx.vin.resize(10); for (size_t i = 0; i < tx.vin.size(); i++) { - tx.vin[i].prevout.hash = InsecureRand256(); + tx.vin[i].prevout.hash = Txid::FromUint256(InsecureRand256()); tx.vin[i].prevout.n = 0; } block.vtx[2] = MakeTransactionRef(tx); @@ -51,8 +51,8 @@ static CBlock BuildBlockTestCase() { } // Number of shared use_counts we expect for a tx we haven't touched -// (block + mempool + our copy from the GetSharedTx call) -constexpr long SHARED_TX_OFFSET{3}; +// (block + mempool entry + mempool txns_randomized + our copy from the GetSharedTx call) +constexpr long SHARED_TX_OFFSET{4}; BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) { @@ -62,13 +62,13 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) LOCK2(cs_main, pool.cs); pool.addUnchecked(entry.FromTx(block.vtx[2])); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 0); // Do a simple ShortTxIDs RT { CBlockHeaderAndShortTxIDs shortIDs{block}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + DataStream stream{}; stream << shortIDs; CBlockHeaderAndShortTxIDs shortIDs2; @@ -80,7 +80,7 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) BOOST_CHECK(!partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 1); size_t poolSize = pool.size(); pool.removeRecursive(*block.vtx[2], MemPoolRemovalReason::REPLACED); @@ -119,7 +119,7 @@ public: std::vector<PrefilledTransaction> prefilledtxn; explicit TestHeaderAndShortIDs(const CBlockHeaderAndShortTxIDs& orig) { - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + DataStream stream{}; stream << orig; stream >> *this; } @@ -127,7 +127,7 @@ public: TestHeaderAndShortIDs(CBlockHeaderAndShortTxIDs{block}) {} uint64_t GetShortID(const uint256& txhash) const { - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + DataStream stream{}; stream << *this; CBlockHeaderAndShortTxIDs base; stream >> base; @@ -145,7 +145,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) LOCK2(cs_main, pool.cs); pool.addUnchecked(entry.FromTx(block.vtx[2])); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 0); uint256 txhash; @@ -158,7 +158,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[0]->GetHash()); shortIDs.shorttxids[1] = shortIDs.GetShortID(block.vtx[2]->GetHash()); - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + DataStream stream{}; stream << shortIDs; CBlockHeaderAndShortTxIDs shortIDs2; @@ -170,7 +170,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) BOOST_CHECK( partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); // +1 because of partialBlock + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 1); // +1 because of partialBlock CBlock block2; { @@ -185,7 +185,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) partialBlock.FillBlock(block2, {block.vtx[1]}); // Current implementation doesn't check txn here, but don't require that partialBlock = tmp; } - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 2); // +2 because of partialBlock and block2 + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 2); // +2 because of partialBlock and block2 bool mutated; BOOST_CHECK(block.hashMerkleRoot != BlockMerkleRoot(block2, &mutated)); @@ -196,15 +196,15 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block3, &mutated).ToString()); BOOST_CHECK(!mutated); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 3); // +2 because of partialBlock and block2 and block3 + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 3); // +2 because of partialBlock and block2 and block3 txhash = block.vtx[2]->GetHash(); block.vtx.clear(); block2.vtx.clear(); block3.vtx.clear(); - BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1 - 1); // + 1 because of partialBlock; -1 because of block. + BOOST_CHECK_EQUAL(pool.get(txhash).use_count(), SHARED_TX_OFFSET + 1 - 1); // + 1 because of partialBlock; -1 because of block. } - BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET - 1); // -1 because of block + BOOST_CHECK_EQUAL(pool.get(txhash).use_count(), SHARED_TX_OFFSET - 1); // -1 because of block } BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) @@ -215,7 +215,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) LOCK2(cs_main, pool.cs); pool.addUnchecked(entry.FromTx(block.vtx[1])); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + BOOST_CHECK_EQUAL(pool.get(block.vtx[1]->GetHash()).use_count(), SHARED_TX_OFFSET + 0); uint256 txhash; @@ -228,7 +228,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) shortIDs.shorttxids.resize(1); shortIDs.shorttxids[0] = shortIDs.GetShortID(block.vtx[1]->GetHash()); - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + DataStream stream{}; stream << shortIDs; CBlockHeaderAndShortTxIDs shortIDs2; @@ -240,7 +240,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) BOOST_CHECK( partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + BOOST_CHECK_EQUAL(pool.get(block.vtx[1]->GetHash()).use_count(), SHARED_TX_OFFSET + 1); CBlock block2; PartiallyDownloadedBlock partialBlockCopy = partialBlock; @@ -253,9 +253,9 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) txhash = block.vtx[1]->GetHash(); block.vtx.clear(); block2.vtx.clear(); - BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1 - 1); // + 1 because of partialBlock; -1 because of block. + BOOST_CHECK_EQUAL(pool.get(txhash).use_count(), SHARED_TX_OFFSET + 1 - 1); // + 1 because of partialBlock; -1 because of block. } - BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET - 1); // -1 because of block + BOOST_CHECK_EQUAL(pool.get(txhash).use_count(), SHARED_TX_OFFSET - 1); // -1 because of block } BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest) @@ -283,7 +283,7 @@ BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest) { CBlockHeaderAndShortTxIDs shortIDs{block}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + DataStream stream{}; stream << shortIDs; CBlockHeaderAndShortTxIDs shortIDs2; diff --git a/src/test/blockfilter_index_tests.cpp b/src/test/blockfilter_index_tests.cpp index 97ea5cfbf3..d44d84af93 100644 --- a/src/test/blockfilter_index_tests.cpp +++ b/src/test/blockfilter_index_tests.cpp @@ -143,7 +143,7 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) BOOST_REQUIRE(filter_index.StartBackgroundSync()); // Allow filter index to catch up with the block index. - IndexWaitSynced(filter_index); + IndexWaitSynced(filter_index, *Assert(m_node.shutdown)); // Check that filter index has all blocks that were in the chain before it started. { @@ -162,9 +162,8 @@ BOOST_FIXTURE_TEST_CASE(blockfilter_index_initial_sync, BuildChainTestingSetup) LOCK(cs_main); tip = m_node.chainman->ActiveChain().Tip(); } - CKey coinbase_key_A, coinbase_key_B; - coinbase_key_A.MakeNewKey(true); - coinbase_key_B.MakeNewKey(true); + CKey coinbase_key_A = GenerateRandomKey(); + CKey coinbase_key_B = GenerateRandomKey(); CScript coinbase_script_pub_key_A = GetScriptForDestination(PKHash(coinbase_key_A.GetPubKey())); CScript coinbase_script_pub_key_B = GetScriptForDestination(PKHash(coinbase_key_B.GetPubKey())); std::vector<std::shared_ptr<CBlock>> chainA, chainB; diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp index c6800c498b..d7ac0bf823 100644 --- a/src/test/blockmanager_tests.cpp +++ b/src/test/blockmanager_tests.cpp @@ -27,13 +27,13 @@ BOOST_FIXTURE_TEST_SUITE(blockmanager_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos) { const auto params {CreateChainParams(ArgsManager{}, ChainType::MAIN)}; - KernelNotifications notifications{m_node.exit_status}; + KernelNotifications notifications{*Assert(m_node.shutdown), m_node.exit_status}; const BlockManager::Options blockman_opts{ .chainparams = *params, .blocks_dir = m_args.GetBlocksDirPath(), .notifications = notifications, }; - BlockManager blockman{m_node.kernel->interrupt, blockman_opts}; + BlockManager blockman{*Assert(m_node.shutdown), blockman_opts}; // simulate adding a genesis block normally BOOST_CHECK_EQUAL(blockman.SaveBlockToDisk(params->GenesisBlock(), 0, nullptr).nPos, BLOCK_SERIALIZATION_HEADER_SIZE); // simulate what happens during reindex @@ -49,7 +49,7 @@ BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos) // 8 bytes (for serialization header) + 285 (for serialized genesis block) = 293 // add another 8 bytes for the second block's serialization header and we get 293 + 8 = 301 FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1, nullptr)}; - BOOST_CHECK_EQUAL(actual.nPos, BLOCK_SERIALIZATION_HEADER_SIZE + ::GetSerializeSize(params->GenesisBlock(), CLIENT_VERSION) + BLOCK_SERIALIZATION_HEADER_SIZE); + BOOST_CHECK_EQUAL(actual.nPos, BLOCK_SERIALIZATION_HEADER_SIZE + ::GetSerializeSize(TX_WITH_WITNESS(params->GenesisBlock())) + BLOCK_SERIALIZATION_HEADER_SIZE); } BOOST_FIXTURE_TEST_CASE(blockmanager_scan_unlink_already_pruned_files, TestChain100Setup) @@ -134,13 +134,13 @@ BOOST_FIXTURE_TEST_CASE(blockmanager_block_data_availability, TestChain100Setup) BOOST_AUTO_TEST_CASE(blockmanager_flush_block_file) { - KernelNotifications notifications{m_node.exit_status}; + KernelNotifications notifications{*Assert(m_node.shutdown), m_node.exit_status}; node::BlockManager::Options blockman_opts{ .chainparams = Params(), .blocks_dir = m_args.GetBlocksDirPath(), .notifications = notifications, }; - BlockManager blockman{m_node.kernel->interrupt, blockman_opts}; + BlockManager blockman{*Assert(m_node.shutdown), blockman_opts}; // Test blocks with no transactions, not even a coinbase CBlock block1; diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 93c0412593..cbf85277a8 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -100,14 +100,16 @@ BOOST_AUTO_TEST_CASE(bloom_create_insert_key) BOOST_AUTO_TEST_CASE(bloom_match) { // Random real transaction (b4749f017444b051c44dfd2720e88f314ff94f3dd6d56d40ef65854fcd7fff6b) - CDataStream stream(ParseHex("01000000010b26e9b7735eb6aabdf358bab62f9816a21ba9ebdb719d5299e88607d722c190000000008b4830450220070aca44506c5cef3a16ed519d7c3c39f8aab192c4e1c90d065f37b8a4af6141022100a8e160b856c2d43d27d8fba71e5aef6405b8643ac4cb7cb3c462aced7f14711a0141046d11fee51b0e60666d5049a9101a72741df480b96ee26488a4d3466b95c9a40ac5eeef87e10a5cd336c19a84565f80fa6c547957b7700ff4dfbdefe76036c339ffffffff021bff3d11000000001976a91404943fdd508053c75000106d3bc6e2754dbcff1988ac2f15de00000000001976a914a266436d2965547608b9e15d9032a7b9d64fa43188ac00000000"), SER_DISK, CLIENT_VERSION); - CTransaction tx(deserialize, stream); + DataStream stream{ + ParseHex("01000000010b26e9b7735eb6aabdf358bab62f9816a21ba9ebdb719d5299e88607d722c190000000008b4830450220070aca44506c5cef3a16ed519d7c3c39f8aab192c4e1c90d065f37b8a4af6141022100a8e160b856c2d43d27d8fba71e5aef6405b8643ac4cb7cb3c462aced7f14711a0141046d11fee51b0e60666d5049a9101a72741df480b96ee26488a4d3466b95c9a40ac5eeef87e10a5cd336c19a84565f80fa6c547957b7700ff4dfbdefe76036c339ffffffff021bff3d11000000001976a91404943fdd508053c75000106d3bc6e2754dbcff1988ac2f15de00000000001976a914a266436d2965547608b9e15d9032a7b9d64fa43188ac00000000"), + }; + CTransaction tx(deserialize, TX_WITH_WITNESS, stream); // and one which spends it (e2769b09e784f32f62ef849763d4f45b98e07ba658647343b915ff832b110436) unsigned char ch[] = {0x01, 0x00, 0x00, 0x00, 0x01, 0x6b, 0xff, 0x7f, 0xcd, 0x4f, 0x85, 0x65, 0xef, 0x40, 0x6d, 0xd5, 0xd6, 0x3d, 0x4f, 0xf9, 0x4f, 0x31, 0x8f, 0xe8, 0x20, 0x27, 0xfd, 0x4d, 0xc4, 0x51, 0xb0, 0x44, 0x74, 0x01, 0x9f, 0x74, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x49, 0x30, 0x46, 0x02, 0x21, 0x00, 0xda, 0x0d, 0xc6, 0xae, 0xce, 0xfe, 0x1e, 0x06, 0xef, 0xdf, 0x05, 0x77, 0x37, 0x57, 0xde, 0xb1, 0x68, 0x82, 0x09, 0x30, 0xe3, 0xb0, 0xd0, 0x3f, 0x46, 0xf5, 0xfc, 0xf1, 0x50, 0xbf, 0x99, 0x0c, 0x02, 0x21, 0x00, 0xd2, 0x5b, 0x5c, 0x87, 0x04, 0x00, 0x76, 0xe4, 0xf2, 0x53, 0xf8, 0x26, 0x2e, 0x76, 0x3e, 0x2d, 0xd5, 0x1e, 0x7f, 0xf0, 0xbe, 0x15, 0x77, 0x27, 0xc4, 0xbc, 0x42, 0x80, 0x7f, 0x17, 0xbd, 0x39, 0x01, 0x41, 0x04, 0xe6, 0xc2, 0x6e, 0xf6, 0x7d, 0xc6, 0x10, 0xd2, 0xcd, 0x19, 0x24, 0x84, 0x78, 0x9a, 0x6c, 0xf9, 0xae, 0xa9, 0x93, 0x0b, 0x94, 0x4b, 0x7e, 0x2d, 0xb5, 0x34, 0x2b, 0x9d, 0x9e, 0x5b, 0x9f, 0xf7, 0x9a, 0xff, 0x9a, 0x2e, 0xe1, 0x97, 0x8d, 0xd7, 0xfd, 0x01, 0xdf, 0xc5, 0x22, 0xee, 0x02, 0x28, 0x3d, 0x3b, 0x06, 0xa9, 0xd0, 0x3a, 0xcf, 0x80, 0x96, 0x96, 0x8d, 0x7d, 0xbb, 0x0f, 0x91, 0x78, 0xff, 0xff, 0xff, 0xff, 0x02, 0x8b, 0xa7, 0x94, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, 0xa9, 0x14, 0xba, 0xde, 0xec, 0xfd, 0xef, 0x05, 0x07, 0x24, 0x7f, 0xc8, 0xf7, 0x42, 0x41, 0xd7, 0x3b, 0xc0, 0x39, 0x97, 0x2d, 0x7b, 0x88, 0xac, 0x40, 0x94, 0xa8, 0x02, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, 0xa9, 0x14, 0xc1, 0x09, 0x32, 0x48, 0x3f, 0xec, 0x93, 0xed, 0x51, 0xf5, 0xfe, 0x95, 0xe7, 0x25, 0x59, 0xf2, 0xcc, 0x70, 0x43, 0xf9, 0x88, 0xac, 0x00, 0x00, 0x00, 0x00, 0x00}; std::vector<unsigned char> vch(ch, ch + sizeof(ch) -1); - CDataStream spendStream(vch, SER_DISK, CLIENT_VERSION); - CTransaction spendingTx(deserialize, spendStream); + DataStream spendStream{vch}; + CTransaction spendingTx(deserialize, TX_WITH_WITNESS, spendStream); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_ALL); filter.insert(uint256S("0xb4749f017444b051c44dfd2720e88f314ff94f3dd6d56d40ef65854fcd7fff6b")); @@ -136,11 +138,11 @@ BOOST_AUTO_TEST_CASE(bloom_match) BOOST_CHECK_MESSAGE(filter.IsRelevantAndUpdate(tx), "Simple Bloom filter didn't match output address"); filter = CBloomFilter(10, 0.000001, 0, BLOOM_UPDATE_ALL); - filter.insert(COutPoint(uint256S("0x90c122d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b"), 0)); + filter.insert(COutPoint(TxidFromString("0x90c122d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b"), 0)); BOOST_CHECK_MESSAGE(filter.IsRelevantAndUpdate(tx), "Simple Bloom filter didn't match COutPoint"); filter = CBloomFilter(10, 0.000001, 0, BLOOM_UPDATE_ALL); - COutPoint prevOutPoint(uint256S("0x90c122d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b"), 0); + COutPoint prevOutPoint(TxidFromString("0x90c122d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b"), 0); { std::vector<unsigned char> data(32 + sizeof(unsigned int)); memcpy(data.data(), prevOutPoint.hash.begin(), 32); @@ -158,11 +160,11 @@ BOOST_AUTO_TEST_CASE(bloom_match) BOOST_CHECK_MESSAGE(!filter.IsRelevantAndUpdate(tx), "Simple Bloom filter matched random address"); filter = CBloomFilter(10, 0.000001, 0, BLOOM_UPDATE_ALL); - filter.insert(COutPoint(uint256S("0x90c122d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b"), 1)); + filter.insert(COutPoint(TxidFromString("0x90c122d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b"), 1)); BOOST_CHECK_MESSAGE(!filter.IsRelevantAndUpdate(tx), "Simple Bloom filter matched COutPoint for an output we didn't care about"); filter = CBloomFilter(10, 0.000001, 0, BLOOM_UPDATE_ALL); - filter.insert(COutPoint(uint256S("0x000000d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b"), 0)); + filter.insert(COutPoint(TxidFromString("0x000000d70786e899529d71dbeba91ba216982fb6ba58f3bdaab65e73b7e9260b"), 0)); BOOST_CHECK_MESSAGE(!filter.IsRelevantAndUpdate(tx), "Simple Bloom filter matched COutPoint for an output we didn't care about"); } @@ -212,8 +214,10 @@ BOOST_AUTO_TEST_CASE(merkle_block_2) // Random real block (000000005a4ded781e667e06ceefafb71410b511fe0d5adc3e5a27ecbec34ae6) // With 4 txes CBlock block; - CDataStream stream(ParseHex("0100000075616236cc2126035fadb38deb65b9102cc2c41c09cdf29fc051906800000000fe7d5e12ef0ff901f6050211249919b1c0653771832b3a80c66cea42847f0ae1d4d26e49ffff001d00f0a4410401000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0804ffff001d029105ffffffff0100f2052a010000004341046d8709a041d34357697dfcb30a9d05900a6294078012bf3bb09c6f9b525f1d16d5503d7905db1ada9501446ea00728668fc5719aa80be2fdfc8a858a4dbdd4fbac00000000010000000255605dc6f5c3dc148b6da58442b0b2cd422be385eab2ebea4119ee9c268d28350000000049483045022100aa46504baa86df8a33b1192b1b9367b4d729dc41e389f2c04f3e5c7f0559aae702205e82253a54bf5c4f65b7428551554b2045167d6d206dfe6a2e198127d3f7df1501ffffffff55605dc6f5c3dc148b6da58442b0b2cd422be385eab2ebea4119ee9c268d2835010000004847304402202329484c35fa9d6bb32a55a70c0982f606ce0e3634b69006138683bcd12cbb6602200c28feb1e2555c3210f1dddb299738b4ff8bbe9667b68cb8764b5ac17b7adf0001ffffffff0200e1f505000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00180d8f000000004341044a656f065871a353f216ca26cef8dde2f03e8c16202d2e8ad769f02032cb86a5eb5e56842e92e19141d60a01928f8dd2c875a390f67c1f6c94cfc617c0ea45afac0000000001000000025f9a06d3acdceb56be1bfeaa3e8a25e62d182fa24fefe899d1c17f1dad4c2028000000004847304402205d6058484157235b06028c30736c15613a28bdb768ee628094ca8b0030d4d6eb0220328789c9a2ec27ddaec0ad5ef58efded42e6ea17c2e1ce838f3d6913f5e95db601ffffffff5f9a06d3acdceb56be1bfeaa3e8a25e62d182fa24fefe899d1c17f1dad4c2028010000004a493046022100c45af050d3cea806cedd0ab22520c53ebe63b987b8954146cdca42487b84bdd6022100b9b027716a6b59e640da50a864d6dd8a0ef24c76ce62391fa3eabaf4d2886d2d01ffffffff0200e1f505000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00180d8f000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac000000000100000002e2274e5fea1bf29d963914bd301aa63b64daaf8a3e88f119b5046ca5738a0f6b0000000048473044022016e7a727a061ea2254a6c358376aaa617ac537eb836c77d646ebda4c748aac8b0220192ce28bf9f2c06a6467e6531e27648d2b3e2e2bae85159c9242939840295ba501ffffffffe2274e5fea1bf29d963914bd301aa63b64daaf8a3e88f119b5046ca5738a0f6b010000004a493046022100b7a1a755588d4190118936e15cd217d133b0e4a53c3c15924010d5648d8925c9022100aaef031874db2114f2d869ac2de4ae53908fbfea5b2b1862e181626bb9005c9f01ffffffff0200e1f505000000004341044a656f065871a353f216ca26cef8dde2f03e8c16202d2e8ad769f02032cb86a5eb5e56842e92e19141d60a01928f8dd2c875a390f67c1f6c94cfc617c0ea45afac00180d8f000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + DataStream stream{ + ParseHex("0100000075616236cc2126035fadb38deb65b9102cc2c41c09cdf29fc051906800000000fe7d5e12ef0ff901f6050211249919b1c0653771832b3a80c66cea42847f0ae1d4d26e49ffff001d00f0a4410401000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0804ffff001d029105ffffffff0100f2052a010000004341046d8709a041d34357697dfcb30a9d05900a6294078012bf3bb09c6f9b525f1d16d5503d7905db1ada9501446ea00728668fc5719aa80be2fdfc8a858a4dbdd4fbac00000000010000000255605dc6f5c3dc148b6da58442b0b2cd422be385eab2ebea4119ee9c268d28350000000049483045022100aa46504baa86df8a33b1192b1b9367b4d729dc41e389f2c04f3e5c7f0559aae702205e82253a54bf5c4f65b7428551554b2045167d6d206dfe6a2e198127d3f7df1501ffffffff55605dc6f5c3dc148b6da58442b0b2cd422be385eab2ebea4119ee9c268d2835010000004847304402202329484c35fa9d6bb32a55a70c0982f606ce0e3634b69006138683bcd12cbb6602200c28feb1e2555c3210f1dddb299738b4ff8bbe9667b68cb8764b5ac17b7adf0001ffffffff0200e1f505000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00180d8f000000004341044a656f065871a353f216ca26cef8dde2f03e8c16202d2e8ad769f02032cb86a5eb5e56842e92e19141d60a01928f8dd2c875a390f67c1f6c94cfc617c0ea45afac0000000001000000025f9a06d3acdceb56be1bfeaa3e8a25e62d182fa24fefe899d1c17f1dad4c2028000000004847304402205d6058484157235b06028c30736c15613a28bdb768ee628094ca8b0030d4d6eb0220328789c9a2ec27ddaec0ad5ef58efded42e6ea17c2e1ce838f3d6913f5e95db601ffffffff5f9a06d3acdceb56be1bfeaa3e8a25e62d182fa24fefe899d1c17f1dad4c2028010000004a493046022100c45af050d3cea806cedd0ab22520c53ebe63b987b8954146cdca42487b84bdd6022100b9b027716a6b59e640da50a864d6dd8a0ef24c76ce62391fa3eabaf4d2886d2d01ffffffff0200e1f505000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00180d8f000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac000000000100000002e2274e5fea1bf29d963914bd301aa63b64daaf8a3e88f119b5046ca5738a0f6b0000000048473044022016e7a727a061ea2254a6c358376aaa617ac537eb836c77d646ebda4c748aac8b0220192ce28bf9f2c06a6467e6531e27648d2b3e2e2bae85159c9242939840295ba501ffffffffe2274e5fea1bf29d963914bd301aa63b64daaf8a3e88f119b5046ca5738a0f6b010000004a493046022100b7a1a755588d4190118936e15cd217d133b0e4a53c3c15924010d5648d8925c9022100aaef031874db2114f2d869ac2de4ae53908fbfea5b2b1862e181626bb9005c9f01ffffffff0200e1f505000000004341044a656f065871a353f216ca26cef8dde2f03e8c16202d2e8ad769f02032cb86a5eb5e56842e92e19141d60a01928f8dd2c875a390f67c1f6c94cfc617c0ea45afac00180d8f000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00000000"), + }; + stream >> TX_WITH_WITNESS(block); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_ALL); // Match the first transaction @@ -267,8 +271,10 @@ BOOST_AUTO_TEST_CASE(merkle_block_2_with_update_none) // Random real block (000000005a4ded781e667e06ceefafb71410b511fe0d5adc3e5a27ecbec34ae6) // With 4 txes CBlock block; - CDataStream stream(ParseHex("0100000075616236cc2126035fadb38deb65b9102cc2c41c09cdf29fc051906800000000fe7d5e12ef0ff901f6050211249919b1c0653771832b3a80c66cea42847f0ae1d4d26e49ffff001d00f0a4410401000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0804ffff001d029105ffffffff0100f2052a010000004341046d8709a041d34357697dfcb30a9d05900a6294078012bf3bb09c6f9b525f1d16d5503d7905db1ada9501446ea00728668fc5719aa80be2fdfc8a858a4dbdd4fbac00000000010000000255605dc6f5c3dc148b6da58442b0b2cd422be385eab2ebea4119ee9c268d28350000000049483045022100aa46504baa86df8a33b1192b1b9367b4d729dc41e389f2c04f3e5c7f0559aae702205e82253a54bf5c4f65b7428551554b2045167d6d206dfe6a2e198127d3f7df1501ffffffff55605dc6f5c3dc148b6da58442b0b2cd422be385eab2ebea4119ee9c268d2835010000004847304402202329484c35fa9d6bb32a55a70c0982f606ce0e3634b69006138683bcd12cbb6602200c28feb1e2555c3210f1dddb299738b4ff8bbe9667b68cb8764b5ac17b7adf0001ffffffff0200e1f505000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00180d8f000000004341044a656f065871a353f216ca26cef8dde2f03e8c16202d2e8ad769f02032cb86a5eb5e56842e92e19141d60a01928f8dd2c875a390f67c1f6c94cfc617c0ea45afac0000000001000000025f9a06d3acdceb56be1bfeaa3e8a25e62d182fa24fefe899d1c17f1dad4c2028000000004847304402205d6058484157235b06028c30736c15613a28bdb768ee628094ca8b0030d4d6eb0220328789c9a2ec27ddaec0ad5ef58efded42e6ea17c2e1ce838f3d6913f5e95db601ffffffff5f9a06d3acdceb56be1bfeaa3e8a25e62d182fa24fefe899d1c17f1dad4c2028010000004a493046022100c45af050d3cea806cedd0ab22520c53ebe63b987b8954146cdca42487b84bdd6022100b9b027716a6b59e640da50a864d6dd8a0ef24c76ce62391fa3eabaf4d2886d2d01ffffffff0200e1f505000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00180d8f000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac000000000100000002e2274e5fea1bf29d963914bd301aa63b64daaf8a3e88f119b5046ca5738a0f6b0000000048473044022016e7a727a061ea2254a6c358376aaa617ac537eb836c77d646ebda4c748aac8b0220192ce28bf9f2c06a6467e6531e27648d2b3e2e2bae85159c9242939840295ba501ffffffffe2274e5fea1bf29d963914bd301aa63b64daaf8a3e88f119b5046ca5738a0f6b010000004a493046022100b7a1a755588d4190118936e15cd217d133b0e4a53c3c15924010d5648d8925c9022100aaef031874db2114f2d869ac2de4ae53908fbfea5b2b1862e181626bb9005c9f01ffffffff0200e1f505000000004341044a656f065871a353f216ca26cef8dde2f03e8c16202d2e8ad769f02032cb86a5eb5e56842e92e19141d60a01928f8dd2c875a390f67c1f6c94cfc617c0ea45afac00180d8f000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + DataStream stream{ + ParseHex("0100000075616236cc2126035fadb38deb65b9102cc2c41c09cdf29fc051906800000000fe7d5e12ef0ff901f6050211249919b1c0653771832b3a80c66cea42847f0ae1d4d26e49ffff001d00f0a4410401000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0804ffff001d029105ffffffff0100f2052a010000004341046d8709a041d34357697dfcb30a9d05900a6294078012bf3bb09c6f9b525f1d16d5503d7905db1ada9501446ea00728668fc5719aa80be2fdfc8a858a4dbdd4fbac00000000010000000255605dc6f5c3dc148b6da58442b0b2cd422be385eab2ebea4119ee9c268d28350000000049483045022100aa46504baa86df8a33b1192b1b9367b4d729dc41e389f2c04f3e5c7f0559aae702205e82253a54bf5c4f65b7428551554b2045167d6d206dfe6a2e198127d3f7df1501ffffffff55605dc6f5c3dc148b6da58442b0b2cd422be385eab2ebea4119ee9c268d2835010000004847304402202329484c35fa9d6bb32a55a70c0982f606ce0e3634b69006138683bcd12cbb6602200c28feb1e2555c3210f1dddb299738b4ff8bbe9667b68cb8764b5ac17b7adf0001ffffffff0200e1f505000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00180d8f000000004341044a656f065871a353f216ca26cef8dde2f03e8c16202d2e8ad769f02032cb86a5eb5e56842e92e19141d60a01928f8dd2c875a390f67c1f6c94cfc617c0ea45afac0000000001000000025f9a06d3acdceb56be1bfeaa3e8a25e62d182fa24fefe899d1c17f1dad4c2028000000004847304402205d6058484157235b06028c30736c15613a28bdb768ee628094ca8b0030d4d6eb0220328789c9a2ec27ddaec0ad5ef58efded42e6ea17c2e1ce838f3d6913f5e95db601ffffffff5f9a06d3acdceb56be1bfeaa3e8a25e62d182fa24fefe899d1c17f1dad4c2028010000004a493046022100c45af050d3cea806cedd0ab22520c53ebe63b987b8954146cdca42487b84bdd6022100b9b027716a6b59e640da50a864d6dd8a0ef24c76ce62391fa3eabaf4d2886d2d01ffffffff0200e1f505000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00180d8f000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac000000000100000002e2274e5fea1bf29d963914bd301aa63b64daaf8a3e88f119b5046ca5738a0f6b0000000048473044022016e7a727a061ea2254a6c358376aaa617ac537eb836c77d646ebda4c748aac8b0220192ce28bf9f2c06a6467e6531e27648d2b3e2e2bae85159c9242939840295ba501ffffffffe2274e5fea1bf29d963914bd301aa63b64daaf8a3e88f119b5046ca5738a0f6b010000004a493046022100b7a1a755588d4190118936e15cd217d133b0e4a53c3c15924010d5648d8925c9022100aaef031874db2114f2d869ac2de4ae53908fbfea5b2b1862e181626bb9005c9f01ffffffff0200e1f505000000004341044a656f065871a353f216ca26cef8dde2f03e8c16202d2e8ad769f02032cb86a5eb5e56842e92e19141d60a01928f8dd2c875a390f67c1f6c94cfc617c0ea45afac00180d8f000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00000000"), + }; + stream >> TX_WITH_WITNESS(block); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_NONE); // Match the first transaction @@ -319,8 +325,10 @@ BOOST_AUTO_TEST_CASE(merkle_block_3_and_serialize) // Random real block (000000000000dab0130bbcc991d3d7ae6b81aa6f50a798888dfe62337458dc45) // With one tx CBlock block; - CDataStream stream(ParseHex("0100000079cda856b143d9db2c1caff01d1aecc8630d30625d10e8b4b8b0000000000000b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc96b2c3ff60abe184f196367291b4d4c86041b8fa45d630101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff08044c86041b020a02ffffffff0100f2052a01000000434104ecd3229b0571c3be876feaac0442a9f13c5a572742927af1dc623353ecf8c202225f64868137a18cdd85cbbb4c74fbccfd4f49639cf1bdc94a5672bb15ad5d4cac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + DataStream stream{ + ParseHex("0100000079cda856b143d9db2c1caff01d1aecc8630d30625d10e8b4b8b0000000000000b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc96b2c3ff60abe184f196367291b4d4c86041b8fa45d630101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff08044c86041b020a02ffffffff0100f2052a01000000434104ecd3229b0571c3be876feaac0442a9f13c5a572742927af1dc623353ecf8c202225f64868137a18cdd85cbbb4c74fbccfd4f49639cf1bdc94a5672bb15ad5d4cac00000000"), + }; + stream >> TX_WITH_WITNESS(block); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_ALL); // Match the only transaction @@ -355,8 +363,10 @@ BOOST_AUTO_TEST_CASE(merkle_block_4) // Random real block (000000000000b731f2eef9e8c63173adfb07e41bd53eb0ef0a6b720d6cb6dea4) // With 7 txes CBlock block; - CDataStream stream(ParseHex("0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f093bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b85290701000000010000000000000000000000000000000000000000000000000000000000000000ffffffff07044c86041b0136ffffffff0100f2052a01000000434104eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c876f2c01ec0f0dd5b2e86e7168cefe0d81113c3807420ce13ad1357231a2252247d97a46a91ac000000000100000001bcad20a6a29827d1424f08989255120bf7f3e9e3cdaaa6bb31b0737fe048724300000000494830450220356e834b046cadc0f8ebb5a8a017b02de59c86305403dad52cd77b55af062ea10221009253cd6c119d4729b77c978e1e2aa19f5ea6e0e52b3f16e32fa608cd5bab753901ffffffff02008d380c010000001976a9142b4b8072ecbba129b6453c63e129e643207249ca88ac0065cd1d000000001976a9141b8dd13b994bcfc787b32aeadf58ccb3615cbd5488ac000000000100000003fdacf9b3eb077412e7a968d2e4f11b9a9dee312d666187ed77ee7d26af16cb0b000000008c493046022100ea1608e70911ca0de5af51ba57ad23b9a51db8d28f82c53563c56a05c20f5a87022100a8bdc8b4a8acc8634c6b420410150775eb7f2474f5615f7fccd65af30f310fbf01410465fdf49e29b06b9a1582287b6279014f834edc317695d125ef623c1cc3aaece245bd69fcad7508666e9c74a49dc9056d5fc14338ef38118dc4afae5fe2c585caffffffff309e1913634ecb50f3c4f83e96e70b2df071b497b8973a3e75429df397b5af83000000004948304502202bdb79c596a9ffc24e96f4386199aba386e9bc7b6071516e2b51dda942b3a1ed022100c53a857e76b724fc14d45311eac5019650d415c3abb5428f3aae16d8e69bec2301ffffffff2089e33491695080c9edc18a428f7d834db5b6d372df13ce2b1b0e0cbcb1e6c10000000049483045022100d4ce67c5896ee251c810ac1ff9ceccd328b497c8f553ab6e08431e7d40bad6b5022033119c0c2b7d792d31f1187779c7bd95aefd93d90a715586d73801d9b47471c601ffffffff0100714460030000001976a914c7b55141d097ea5df7a0ed330cf794376e53ec8d88ac0000000001000000045bf0e214aa4069a3e792ecee1e1bf0c1d397cde8dd08138f4b72a00681743447000000008b48304502200c45de8c4f3e2c1821f2fc878cba97b1e6f8807d94930713aa1c86a67b9bf1e40221008581abfef2e30f957815fc89978423746b2086375ca8ecf359c85c2a5b7c88ad01410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffffd669f7d7958d40fc59d2253d88e0f248e29b599c80bbcec344a83dda5f9aa72c000000008a473044022078124c8beeaa825f9e0b30bff96e564dd859432f2d0cb3b72d3d5d93d38d7e930220691d233b6c0f995be5acb03d70a7f7a65b6bc9bdd426260f38a1346669507a3601410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95fffffffff878af0d93f5229a68166cf051fd372bb7a537232946e0a46f53636b4dafdaa4000000008c493046022100c717d1714551663f69c3c5759bdbb3a0fcd3fab023abc0e522fe6440de35d8290221008d9cbe25bffc44af2b18e81c58eb37293fd7fe1c2e7b46fc37ee8c96c50ab1e201410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff27f2b668859cd7f2f894aa0fd2d9e60963bcd07c88973f425f999b8cbfd7a1e2000000008c493046022100e00847147cbf517bcc2f502f3ddc6d284358d102ed20d47a8aa788a62f0db780022100d17b2d6fa84dcaf1c95d88d7e7c30385aecf415588d749afd3ec81f6022cecd701410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff0100c817a8040000001976a914b6efd80d99179f4f4ff6f4dd0a007d018c385d2188ac000000000100000001834537b2f1ce8ef9373a258e10545ce5a50b758df616cd4356e0032554ebd3c4000000008b483045022100e68f422dd7c34fdce11eeb4509ddae38201773dd62f284e8aa9d96f85099d0b002202243bd399ff96b649a0fad05fa759d6a882f0af8c90cf7632c2840c29070aec20141045e58067e815c2f464c6a2a15f987758374203895710c2d452442e28496ff38ba8f5fd901dc20e29e88477167fe4fc299bf818fd0d9e1632d467b2a3d9503b1aaffffffff0280d7e636030000001976a914f34c3e10eb387efe872acb614c89e78bfca7815d88ac404b4c00000000001976a914a84e272933aaf87e1715d7786c51dfaeb5b65a6f88ac00000000010000000143ac81c8e6f6ef307dfe17f3d906d999e23e0189fda838c5510d850927e03ae7000000008c4930460221009c87c344760a64cb8ae6685a3eec2c1ac1bed5b88c87de51acd0e124f266c16602210082d07c037359c3a257b5c63ebd90f5a5edf97b2ac1c434b08ca998839f346dd40141040ba7e521fa7946d12edbb1d1e95a15c34bd4398195e86433c92b431cd315f455fe30032ede69cad9d1e1ed6c3c4ec0dbfced53438c625462afb792dcb098544bffffffff0240420f00000000001976a9144676d1b820d63ec272f1900d59d43bc6463d96f888ac40420f00000000001976a914648d04341d00d7968b3405c034adc38d4d8fb9bd88ac00000000010000000248cc917501ea5c55f4a8d2009c0567c40cfe037c2e71af017d0a452ff705e3f1000000008b483045022100bf5fdc86dc5f08a5d5c8e43a8c9d5b1ed8c65562e280007b52b133021acd9acc02205e325d613e555f772802bf413d36ba807892ed1a690a77811d3033b3de226e0a01410429fa713b124484cb2bd7b5557b2c0b9df7b2b1fee61825eadc5ae6c37a9920d38bfccdc7dc3cb0c47d7b173dbc9db8d37db0a33ae487982c59c6f8606e9d1791ffffffff41ed70551dd7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d068000000008b4830450221008513ad65187b903aed1102d1d0c47688127658c51106753fed0151ce9c16b80902201432b9ebcb87bd04ceb2de66035fbbaf4bf8b00d1cfe41f1a1f7338f9ad79d210141049d4cf80125bf50be1709f718c07ad15d0fc612b7da1f5570dddc35f2a352f0f27c978b06820edca9ef982c35fda2d255afba340068c5035552368bc7200c1488ffffffff0100093d00000000001976a9148edb68822f1ad580b043c7b3df2e400f8699eb4888ac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + DataStream stream{ + ParseHex("0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f093bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b85290701000000010000000000000000000000000000000000000000000000000000000000000000ffffffff07044c86041b0136ffffffff0100f2052a01000000434104eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c876f2c01ec0f0dd5b2e86e7168cefe0d81113c3807420ce13ad1357231a2252247d97a46a91ac000000000100000001bcad20a6a29827d1424f08989255120bf7f3e9e3cdaaa6bb31b0737fe048724300000000494830450220356e834b046cadc0f8ebb5a8a017b02de59c86305403dad52cd77b55af062ea10221009253cd6c119d4729b77c978e1e2aa19f5ea6e0e52b3f16e32fa608cd5bab753901ffffffff02008d380c010000001976a9142b4b8072ecbba129b6453c63e129e643207249ca88ac0065cd1d000000001976a9141b8dd13b994bcfc787b32aeadf58ccb3615cbd5488ac000000000100000003fdacf9b3eb077412e7a968d2e4f11b9a9dee312d666187ed77ee7d26af16cb0b000000008c493046022100ea1608e70911ca0de5af51ba57ad23b9a51db8d28f82c53563c56a05c20f5a87022100a8bdc8b4a8acc8634c6b420410150775eb7f2474f5615f7fccd65af30f310fbf01410465fdf49e29b06b9a1582287b6279014f834edc317695d125ef623c1cc3aaece245bd69fcad7508666e9c74a49dc9056d5fc14338ef38118dc4afae5fe2c585caffffffff309e1913634ecb50f3c4f83e96e70b2df071b497b8973a3e75429df397b5af83000000004948304502202bdb79c596a9ffc24e96f4386199aba386e9bc7b6071516e2b51dda942b3a1ed022100c53a857e76b724fc14d45311eac5019650d415c3abb5428f3aae16d8e69bec2301ffffffff2089e33491695080c9edc18a428f7d834db5b6d372df13ce2b1b0e0cbcb1e6c10000000049483045022100d4ce67c5896ee251c810ac1ff9ceccd328b497c8f553ab6e08431e7d40bad6b5022033119c0c2b7d792d31f1187779c7bd95aefd93d90a715586d73801d9b47471c601ffffffff0100714460030000001976a914c7b55141d097ea5df7a0ed330cf794376e53ec8d88ac0000000001000000045bf0e214aa4069a3e792ecee1e1bf0c1d397cde8dd08138f4b72a00681743447000000008b48304502200c45de8c4f3e2c1821f2fc878cba97b1e6f8807d94930713aa1c86a67b9bf1e40221008581abfef2e30f957815fc89978423746b2086375ca8ecf359c85c2a5b7c88ad01410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffffd669f7d7958d40fc59d2253d88e0f248e29b599c80bbcec344a83dda5f9aa72c000000008a473044022078124c8beeaa825f9e0b30bff96e564dd859432f2d0cb3b72d3d5d93d38d7e930220691d233b6c0f995be5acb03d70a7f7a65b6bc9bdd426260f38a1346669507a3601410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95fffffffff878af0d93f5229a68166cf051fd372bb7a537232946e0a46f53636b4dafdaa4000000008c493046022100c717d1714551663f69c3c5759bdbb3a0fcd3fab023abc0e522fe6440de35d8290221008d9cbe25bffc44af2b18e81c58eb37293fd7fe1c2e7b46fc37ee8c96c50ab1e201410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff27f2b668859cd7f2f894aa0fd2d9e60963bcd07c88973f425f999b8cbfd7a1e2000000008c493046022100e00847147cbf517bcc2f502f3ddc6d284358d102ed20d47a8aa788a62f0db780022100d17b2d6fa84dcaf1c95d88d7e7c30385aecf415588d749afd3ec81f6022cecd701410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff0100c817a8040000001976a914b6efd80d99179f4f4ff6f4dd0a007d018c385d2188ac000000000100000001834537b2f1ce8ef9373a258e10545ce5a50b758df616cd4356e0032554ebd3c4000000008b483045022100e68f422dd7c34fdce11eeb4509ddae38201773dd62f284e8aa9d96f85099d0b002202243bd399ff96b649a0fad05fa759d6a882f0af8c90cf7632c2840c29070aec20141045e58067e815c2f464c6a2a15f987758374203895710c2d452442e28496ff38ba8f5fd901dc20e29e88477167fe4fc299bf818fd0d9e1632d467b2a3d9503b1aaffffffff0280d7e636030000001976a914f34c3e10eb387efe872acb614c89e78bfca7815d88ac404b4c00000000001976a914a84e272933aaf87e1715d7786c51dfaeb5b65a6f88ac00000000010000000143ac81c8e6f6ef307dfe17f3d906d999e23e0189fda838c5510d850927e03ae7000000008c4930460221009c87c344760a64cb8ae6685a3eec2c1ac1bed5b88c87de51acd0e124f266c16602210082d07c037359c3a257b5c63ebd90f5a5edf97b2ac1c434b08ca998839f346dd40141040ba7e521fa7946d12edbb1d1e95a15c34bd4398195e86433c92b431cd315f455fe30032ede69cad9d1e1ed6c3c4ec0dbfced53438c625462afb792dcb098544bffffffff0240420f00000000001976a9144676d1b820d63ec272f1900d59d43bc6463d96f888ac40420f00000000001976a914648d04341d00d7968b3405c034adc38d4d8fb9bd88ac00000000010000000248cc917501ea5c55f4a8d2009c0567c40cfe037c2e71af017d0a452ff705e3f1000000008b483045022100bf5fdc86dc5f08a5d5c8e43a8c9d5b1ed8c65562e280007b52b133021acd9acc02205e325d613e555f772802bf413d36ba807892ed1a690a77811d3033b3de226e0a01410429fa713b124484cb2bd7b5557b2c0b9df7b2b1fee61825eadc5ae6c37a9920d38bfccdc7dc3cb0c47d7b173dbc9db8d37db0a33ae487982c59c6f8606e9d1791ffffffff41ed70551dd7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d068000000008b4830450221008513ad65187b903aed1102d1d0c47688127658c51106753fed0151ce9c16b80902201432b9ebcb87bd04ceb2de66035fbbaf4bf8b00d1cfe41f1a1f7338f9ad79d210141049d4cf80125bf50be1709f718c07ad15d0fc612b7da1f5570dddc35f2a352f0f27c978b06820edca9ef982c35fda2d255afba340068c5035552368bc7200c1488ffffffff0100093d00000000001976a9148edb68822f1ad580b043c7b3df2e400f8699eb4888ac00000000"), + }; + stream >> TX_WITH_WITNESS(block); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_ALL); // Match the last transaction @@ -401,8 +411,10 @@ BOOST_AUTO_TEST_CASE(merkle_block_4_test_p2pubkey_only) // Random real block (000000000000b731f2eef9e8c63173adfb07e41bd53eb0ef0a6b720d6cb6dea4) // With 7 txes CBlock block; - CDataStream stream(ParseHex("0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f093bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b85290701000000010000000000000000000000000000000000000000000000000000000000000000ffffffff07044c86041b0136ffffffff0100f2052a01000000434104eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c876f2c01ec0f0dd5b2e86e7168cefe0d81113c3807420ce13ad1357231a2252247d97a46a91ac000000000100000001bcad20a6a29827d1424f08989255120bf7f3e9e3cdaaa6bb31b0737fe048724300000000494830450220356e834b046cadc0f8ebb5a8a017b02de59c86305403dad52cd77b55af062ea10221009253cd6c119d4729b77c978e1e2aa19f5ea6e0e52b3f16e32fa608cd5bab753901ffffffff02008d380c010000001976a9142b4b8072ecbba129b6453c63e129e643207249ca88ac0065cd1d000000001976a9141b8dd13b994bcfc787b32aeadf58ccb3615cbd5488ac000000000100000003fdacf9b3eb077412e7a968d2e4f11b9a9dee312d666187ed77ee7d26af16cb0b000000008c493046022100ea1608e70911ca0de5af51ba57ad23b9a51db8d28f82c53563c56a05c20f5a87022100a8bdc8b4a8acc8634c6b420410150775eb7f2474f5615f7fccd65af30f310fbf01410465fdf49e29b06b9a1582287b6279014f834edc317695d125ef623c1cc3aaece245bd69fcad7508666e9c74a49dc9056d5fc14338ef38118dc4afae5fe2c585caffffffff309e1913634ecb50f3c4f83e96e70b2df071b497b8973a3e75429df397b5af83000000004948304502202bdb79c596a9ffc24e96f4386199aba386e9bc7b6071516e2b51dda942b3a1ed022100c53a857e76b724fc14d45311eac5019650d415c3abb5428f3aae16d8e69bec2301ffffffff2089e33491695080c9edc18a428f7d834db5b6d372df13ce2b1b0e0cbcb1e6c10000000049483045022100d4ce67c5896ee251c810ac1ff9ceccd328b497c8f553ab6e08431e7d40bad6b5022033119c0c2b7d792d31f1187779c7bd95aefd93d90a715586d73801d9b47471c601ffffffff0100714460030000001976a914c7b55141d097ea5df7a0ed330cf794376e53ec8d88ac0000000001000000045bf0e214aa4069a3e792ecee1e1bf0c1d397cde8dd08138f4b72a00681743447000000008b48304502200c45de8c4f3e2c1821f2fc878cba97b1e6f8807d94930713aa1c86a67b9bf1e40221008581abfef2e30f957815fc89978423746b2086375ca8ecf359c85c2a5b7c88ad01410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffffd669f7d7958d40fc59d2253d88e0f248e29b599c80bbcec344a83dda5f9aa72c000000008a473044022078124c8beeaa825f9e0b30bff96e564dd859432f2d0cb3b72d3d5d93d38d7e930220691d233b6c0f995be5acb03d70a7f7a65b6bc9bdd426260f38a1346669507a3601410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95fffffffff878af0d93f5229a68166cf051fd372bb7a537232946e0a46f53636b4dafdaa4000000008c493046022100c717d1714551663f69c3c5759bdbb3a0fcd3fab023abc0e522fe6440de35d8290221008d9cbe25bffc44af2b18e81c58eb37293fd7fe1c2e7b46fc37ee8c96c50ab1e201410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff27f2b668859cd7f2f894aa0fd2d9e60963bcd07c88973f425f999b8cbfd7a1e2000000008c493046022100e00847147cbf517bcc2f502f3ddc6d284358d102ed20d47a8aa788a62f0db780022100d17b2d6fa84dcaf1c95d88d7e7c30385aecf415588d749afd3ec81f6022cecd701410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff0100c817a8040000001976a914b6efd80d99179f4f4ff6f4dd0a007d018c385d2188ac000000000100000001834537b2f1ce8ef9373a258e10545ce5a50b758df616cd4356e0032554ebd3c4000000008b483045022100e68f422dd7c34fdce11eeb4509ddae38201773dd62f284e8aa9d96f85099d0b002202243bd399ff96b649a0fad05fa759d6a882f0af8c90cf7632c2840c29070aec20141045e58067e815c2f464c6a2a15f987758374203895710c2d452442e28496ff38ba8f5fd901dc20e29e88477167fe4fc299bf818fd0d9e1632d467b2a3d9503b1aaffffffff0280d7e636030000001976a914f34c3e10eb387efe872acb614c89e78bfca7815d88ac404b4c00000000001976a914a84e272933aaf87e1715d7786c51dfaeb5b65a6f88ac00000000010000000143ac81c8e6f6ef307dfe17f3d906d999e23e0189fda838c5510d850927e03ae7000000008c4930460221009c87c344760a64cb8ae6685a3eec2c1ac1bed5b88c87de51acd0e124f266c16602210082d07c037359c3a257b5c63ebd90f5a5edf97b2ac1c434b08ca998839f346dd40141040ba7e521fa7946d12edbb1d1e95a15c34bd4398195e86433c92b431cd315f455fe30032ede69cad9d1e1ed6c3c4ec0dbfced53438c625462afb792dcb098544bffffffff0240420f00000000001976a9144676d1b820d63ec272f1900d59d43bc6463d96f888ac40420f00000000001976a914648d04341d00d7968b3405c034adc38d4d8fb9bd88ac00000000010000000248cc917501ea5c55f4a8d2009c0567c40cfe037c2e71af017d0a452ff705e3f1000000008b483045022100bf5fdc86dc5f08a5d5c8e43a8c9d5b1ed8c65562e280007b52b133021acd9acc02205e325d613e555f772802bf413d36ba807892ed1a690a77811d3033b3de226e0a01410429fa713b124484cb2bd7b5557b2c0b9df7b2b1fee61825eadc5ae6c37a9920d38bfccdc7dc3cb0c47d7b173dbc9db8d37db0a33ae487982c59c6f8606e9d1791ffffffff41ed70551dd7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d068000000008b4830450221008513ad65187b903aed1102d1d0c47688127658c51106753fed0151ce9c16b80902201432b9ebcb87bd04ceb2de66035fbbaf4bf8b00d1cfe41f1a1f7338f9ad79d210141049d4cf80125bf50be1709f718c07ad15d0fc612b7da1f5570dddc35f2a352f0f27c978b06820edca9ef982c35fda2d255afba340068c5035552368bc7200c1488ffffffff0100093d00000000001976a9148edb68822f1ad580b043c7b3df2e400f8699eb4888ac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + DataStream stream{ + ParseHex("0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f093bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b85290701000000010000000000000000000000000000000000000000000000000000000000000000ffffffff07044c86041b0136ffffffff0100f2052a01000000434104eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c876f2c01ec0f0dd5b2e86e7168cefe0d81113c3807420ce13ad1357231a2252247d97a46a91ac000000000100000001bcad20a6a29827d1424f08989255120bf7f3e9e3cdaaa6bb31b0737fe048724300000000494830450220356e834b046cadc0f8ebb5a8a017b02de59c86305403dad52cd77b55af062ea10221009253cd6c119d4729b77c978e1e2aa19f5ea6e0e52b3f16e32fa608cd5bab753901ffffffff02008d380c010000001976a9142b4b8072ecbba129b6453c63e129e643207249ca88ac0065cd1d000000001976a9141b8dd13b994bcfc787b32aeadf58ccb3615cbd5488ac000000000100000003fdacf9b3eb077412e7a968d2e4f11b9a9dee312d666187ed77ee7d26af16cb0b000000008c493046022100ea1608e70911ca0de5af51ba57ad23b9a51db8d28f82c53563c56a05c20f5a87022100a8bdc8b4a8acc8634c6b420410150775eb7f2474f5615f7fccd65af30f310fbf01410465fdf49e29b06b9a1582287b6279014f834edc317695d125ef623c1cc3aaece245bd69fcad7508666e9c74a49dc9056d5fc14338ef38118dc4afae5fe2c585caffffffff309e1913634ecb50f3c4f83e96e70b2df071b497b8973a3e75429df397b5af83000000004948304502202bdb79c596a9ffc24e96f4386199aba386e9bc7b6071516e2b51dda942b3a1ed022100c53a857e76b724fc14d45311eac5019650d415c3abb5428f3aae16d8e69bec2301ffffffff2089e33491695080c9edc18a428f7d834db5b6d372df13ce2b1b0e0cbcb1e6c10000000049483045022100d4ce67c5896ee251c810ac1ff9ceccd328b497c8f553ab6e08431e7d40bad6b5022033119c0c2b7d792d31f1187779c7bd95aefd93d90a715586d73801d9b47471c601ffffffff0100714460030000001976a914c7b55141d097ea5df7a0ed330cf794376e53ec8d88ac0000000001000000045bf0e214aa4069a3e792ecee1e1bf0c1d397cde8dd08138f4b72a00681743447000000008b48304502200c45de8c4f3e2c1821f2fc878cba97b1e6f8807d94930713aa1c86a67b9bf1e40221008581abfef2e30f957815fc89978423746b2086375ca8ecf359c85c2a5b7c88ad01410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffffd669f7d7958d40fc59d2253d88e0f248e29b599c80bbcec344a83dda5f9aa72c000000008a473044022078124c8beeaa825f9e0b30bff96e564dd859432f2d0cb3b72d3d5d93d38d7e930220691d233b6c0f995be5acb03d70a7f7a65b6bc9bdd426260f38a1346669507a3601410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95fffffffff878af0d93f5229a68166cf051fd372bb7a537232946e0a46f53636b4dafdaa4000000008c493046022100c717d1714551663f69c3c5759bdbb3a0fcd3fab023abc0e522fe6440de35d8290221008d9cbe25bffc44af2b18e81c58eb37293fd7fe1c2e7b46fc37ee8c96c50ab1e201410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff27f2b668859cd7f2f894aa0fd2d9e60963bcd07c88973f425f999b8cbfd7a1e2000000008c493046022100e00847147cbf517bcc2f502f3ddc6d284358d102ed20d47a8aa788a62f0db780022100d17b2d6fa84dcaf1c95d88d7e7c30385aecf415588d749afd3ec81f6022cecd701410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff0100c817a8040000001976a914b6efd80d99179f4f4ff6f4dd0a007d018c385d2188ac000000000100000001834537b2f1ce8ef9373a258e10545ce5a50b758df616cd4356e0032554ebd3c4000000008b483045022100e68f422dd7c34fdce11eeb4509ddae38201773dd62f284e8aa9d96f85099d0b002202243bd399ff96b649a0fad05fa759d6a882f0af8c90cf7632c2840c29070aec20141045e58067e815c2f464c6a2a15f987758374203895710c2d452442e28496ff38ba8f5fd901dc20e29e88477167fe4fc299bf818fd0d9e1632d467b2a3d9503b1aaffffffff0280d7e636030000001976a914f34c3e10eb387efe872acb614c89e78bfca7815d88ac404b4c00000000001976a914a84e272933aaf87e1715d7786c51dfaeb5b65a6f88ac00000000010000000143ac81c8e6f6ef307dfe17f3d906d999e23e0189fda838c5510d850927e03ae7000000008c4930460221009c87c344760a64cb8ae6685a3eec2c1ac1bed5b88c87de51acd0e124f266c16602210082d07c037359c3a257b5c63ebd90f5a5edf97b2ac1c434b08ca998839f346dd40141040ba7e521fa7946d12edbb1d1e95a15c34bd4398195e86433c92b431cd315f455fe30032ede69cad9d1e1ed6c3c4ec0dbfced53438c625462afb792dcb098544bffffffff0240420f00000000001976a9144676d1b820d63ec272f1900d59d43bc6463d96f888ac40420f00000000001976a914648d04341d00d7968b3405c034adc38d4d8fb9bd88ac00000000010000000248cc917501ea5c55f4a8d2009c0567c40cfe037c2e71af017d0a452ff705e3f1000000008b483045022100bf5fdc86dc5f08a5d5c8e43a8c9d5b1ed8c65562e280007b52b133021acd9acc02205e325d613e555f772802bf413d36ba807892ed1a690a77811d3033b3de226e0a01410429fa713b124484cb2bd7b5557b2c0b9df7b2b1fee61825eadc5ae6c37a9920d38bfccdc7dc3cb0c47d7b173dbc9db8d37db0a33ae487982c59c6f8606e9d1791ffffffff41ed70551dd7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d068000000008b4830450221008513ad65187b903aed1102d1d0c47688127658c51106753fed0151ce9c16b80902201432b9ebcb87bd04ceb2de66035fbbaf4bf8b00d1cfe41f1a1f7338f9ad79d210141049d4cf80125bf50be1709f718c07ad15d0fc612b7da1f5570dddc35f2a352f0f27c978b06820edca9ef982c35fda2d255afba340068c5035552368bc7200c1488ffffffff0100093d00000000001976a9148edb68822f1ad580b043c7b3df2e400f8699eb4888ac00000000"), + }; + stream >> TX_WITH_WITNESS(block); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_P2PUBKEY_ONLY); // Match the generation pubkey @@ -414,9 +426,9 @@ BOOST_AUTO_TEST_CASE(merkle_block_4_test_p2pubkey_only) BOOST_CHECK(merkleBlock.header.GetHash() == block.GetHash()); // We should match the generation outpoint - BOOST_CHECK(filter.contains(COutPoint(uint256S("0x147caa76786596590baa4e98f5d9f48b86c7765e489f7a6ff3360fe5c674360b"), 0))); + BOOST_CHECK(filter.contains(COutPoint(TxidFromString("0x147caa76786596590baa4e98f5d9f48b86c7765e489f7a6ff3360fe5c674360b"), 0))); // ... but not the 4th transaction's output (its not pay-2-pubkey) - BOOST_CHECK(!filter.contains(COutPoint(uint256S("0x02981fa052f0481dbc5868f4fc2166035a10f27a03cfd2de67326471df5bc041"), 0))); + BOOST_CHECK(!filter.contains(COutPoint(TxidFromString("0x02981fa052f0481dbc5868f4fc2166035a10f27a03cfd2de67326471df5bc041"), 0))); } BOOST_AUTO_TEST_CASE(merkle_block_4_test_update_none) @@ -424,8 +436,10 @@ BOOST_AUTO_TEST_CASE(merkle_block_4_test_update_none) // Random real block (000000000000b731f2eef9e8c63173adfb07e41bd53eb0ef0a6b720d6cb6dea4) // With 7 txes CBlock block; - CDataStream stream(ParseHex("0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f093bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b85290701000000010000000000000000000000000000000000000000000000000000000000000000ffffffff07044c86041b0136ffffffff0100f2052a01000000434104eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c876f2c01ec0f0dd5b2e86e7168cefe0d81113c3807420ce13ad1357231a2252247d97a46a91ac000000000100000001bcad20a6a29827d1424f08989255120bf7f3e9e3cdaaa6bb31b0737fe048724300000000494830450220356e834b046cadc0f8ebb5a8a017b02de59c86305403dad52cd77b55af062ea10221009253cd6c119d4729b77c978e1e2aa19f5ea6e0e52b3f16e32fa608cd5bab753901ffffffff02008d380c010000001976a9142b4b8072ecbba129b6453c63e129e643207249ca88ac0065cd1d000000001976a9141b8dd13b994bcfc787b32aeadf58ccb3615cbd5488ac000000000100000003fdacf9b3eb077412e7a968d2e4f11b9a9dee312d666187ed77ee7d26af16cb0b000000008c493046022100ea1608e70911ca0de5af51ba57ad23b9a51db8d28f82c53563c56a05c20f5a87022100a8bdc8b4a8acc8634c6b420410150775eb7f2474f5615f7fccd65af30f310fbf01410465fdf49e29b06b9a1582287b6279014f834edc317695d125ef623c1cc3aaece245bd69fcad7508666e9c74a49dc9056d5fc14338ef38118dc4afae5fe2c585caffffffff309e1913634ecb50f3c4f83e96e70b2df071b497b8973a3e75429df397b5af83000000004948304502202bdb79c596a9ffc24e96f4386199aba386e9bc7b6071516e2b51dda942b3a1ed022100c53a857e76b724fc14d45311eac5019650d415c3abb5428f3aae16d8e69bec2301ffffffff2089e33491695080c9edc18a428f7d834db5b6d372df13ce2b1b0e0cbcb1e6c10000000049483045022100d4ce67c5896ee251c810ac1ff9ceccd328b497c8f553ab6e08431e7d40bad6b5022033119c0c2b7d792d31f1187779c7bd95aefd93d90a715586d73801d9b47471c601ffffffff0100714460030000001976a914c7b55141d097ea5df7a0ed330cf794376e53ec8d88ac0000000001000000045bf0e214aa4069a3e792ecee1e1bf0c1d397cde8dd08138f4b72a00681743447000000008b48304502200c45de8c4f3e2c1821f2fc878cba97b1e6f8807d94930713aa1c86a67b9bf1e40221008581abfef2e30f957815fc89978423746b2086375ca8ecf359c85c2a5b7c88ad01410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffffd669f7d7958d40fc59d2253d88e0f248e29b599c80bbcec344a83dda5f9aa72c000000008a473044022078124c8beeaa825f9e0b30bff96e564dd859432f2d0cb3b72d3d5d93d38d7e930220691d233b6c0f995be5acb03d70a7f7a65b6bc9bdd426260f38a1346669507a3601410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95fffffffff878af0d93f5229a68166cf051fd372bb7a537232946e0a46f53636b4dafdaa4000000008c493046022100c717d1714551663f69c3c5759bdbb3a0fcd3fab023abc0e522fe6440de35d8290221008d9cbe25bffc44af2b18e81c58eb37293fd7fe1c2e7b46fc37ee8c96c50ab1e201410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff27f2b668859cd7f2f894aa0fd2d9e60963bcd07c88973f425f999b8cbfd7a1e2000000008c493046022100e00847147cbf517bcc2f502f3ddc6d284358d102ed20d47a8aa788a62f0db780022100d17b2d6fa84dcaf1c95d88d7e7c30385aecf415588d749afd3ec81f6022cecd701410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff0100c817a8040000001976a914b6efd80d99179f4f4ff6f4dd0a007d018c385d2188ac000000000100000001834537b2f1ce8ef9373a258e10545ce5a50b758df616cd4356e0032554ebd3c4000000008b483045022100e68f422dd7c34fdce11eeb4509ddae38201773dd62f284e8aa9d96f85099d0b002202243bd399ff96b649a0fad05fa759d6a882f0af8c90cf7632c2840c29070aec20141045e58067e815c2f464c6a2a15f987758374203895710c2d452442e28496ff38ba8f5fd901dc20e29e88477167fe4fc299bf818fd0d9e1632d467b2a3d9503b1aaffffffff0280d7e636030000001976a914f34c3e10eb387efe872acb614c89e78bfca7815d88ac404b4c00000000001976a914a84e272933aaf87e1715d7786c51dfaeb5b65a6f88ac00000000010000000143ac81c8e6f6ef307dfe17f3d906d999e23e0189fda838c5510d850927e03ae7000000008c4930460221009c87c344760a64cb8ae6685a3eec2c1ac1bed5b88c87de51acd0e124f266c16602210082d07c037359c3a257b5c63ebd90f5a5edf97b2ac1c434b08ca998839f346dd40141040ba7e521fa7946d12edbb1d1e95a15c34bd4398195e86433c92b431cd315f455fe30032ede69cad9d1e1ed6c3c4ec0dbfced53438c625462afb792dcb098544bffffffff0240420f00000000001976a9144676d1b820d63ec272f1900d59d43bc6463d96f888ac40420f00000000001976a914648d04341d00d7968b3405c034adc38d4d8fb9bd88ac00000000010000000248cc917501ea5c55f4a8d2009c0567c40cfe037c2e71af017d0a452ff705e3f1000000008b483045022100bf5fdc86dc5f08a5d5c8e43a8c9d5b1ed8c65562e280007b52b133021acd9acc02205e325d613e555f772802bf413d36ba807892ed1a690a77811d3033b3de226e0a01410429fa713b124484cb2bd7b5557b2c0b9df7b2b1fee61825eadc5ae6c37a9920d38bfccdc7dc3cb0c47d7b173dbc9db8d37db0a33ae487982c59c6f8606e9d1791ffffffff41ed70551dd7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d068000000008b4830450221008513ad65187b903aed1102d1d0c47688127658c51106753fed0151ce9c16b80902201432b9ebcb87bd04ceb2de66035fbbaf4bf8b00d1cfe41f1a1f7338f9ad79d210141049d4cf80125bf50be1709f718c07ad15d0fc612b7da1f5570dddc35f2a352f0f27c978b06820edca9ef982c35fda2d255afba340068c5035552368bc7200c1488ffffffff0100093d00000000001976a9148edb68822f1ad580b043c7b3df2e400f8699eb4888ac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + DataStream stream{ + ParseHex("0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f093bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b85290701000000010000000000000000000000000000000000000000000000000000000000000000ffffffff07044c86041b0136ffffffff0100f2052a01000000434104eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c876f2c01ec0f0dd5b2e86e7168cefe0d81113c3807420ce13ad1357231a2252247d97a46a91ac000000000100000001bcad20a6a29827d1424f08989255120bf7f3e9e3cdaaa6bb31b0737fe048724300000000494830450220356e834b046cadc0f8ebb5a8a017b02de59c86305403dad52cd77b55af062ea10221009253cd6c119d4729b77c978e1e2aa19f5ea6e0e52b3f16e32fa608cd5bab753901ffffffff02008d380c010000001976a9142b4b8072ecbba129b6453c63e129e643207249ca88ac0065cd1d000000001976a9141b8dd13b994bcfc787b32aeadf58ccb3615cbd5488ac000000000100000003fdacf9b3eb077412e7a968d2e4f11b9a9dee312d666187ed77ee7d26af16cb0b000000008c493046022100ea1608e70911ca0de5af51ba57ad23b9a51db8d28f82c53563c56a05c20f5a87022100a8bdc8b4a8acc8634c6b420410150775eb7f2474f5615f7fccd65af30f310fbf01410465fdf49e29b06b9a1582287b6279014f834edc317695d125ef623c1cc3aaece245bd69fcad7508666e9c74a49dc9056d5fc14338ef38118dc4afae5fe2c585caffffffff309e1913634ecb50f3c4f83e96e70b2df071b497b8973a3e75429df397b5af83000000004948304502202bdb79c596a9ffc24e96f4386199aba386e9bc7b6071516e2b51dda942b3a1ed022100c53a857e76b724fc14d45311eac5019650d415c3abb5428f3aae16d8e69bec2301ffffffff2089e33491695080c9edc18a428f7d834db5b6d372df13ce2b1b0e0cbcb1e6c10000000049483045022100d4ce67c5896ee251c810ac1ff9ceccd328b497c8f553ab6e08431e7d40bad6b5022033119c0c2b7d792d31f1187779c7bd95aefd93d90a715586d73801d9b47471c601ffffffff0100714460030000001976a914c7b55141d097ea5df7a0ed330cf794376e53ec8d88ac0000000001000000045bf0e214aa4069a3e792ecee1e1bf0c1d397cde8dd08138f4b72a00681743447000000008b48304502200c45de8c4f3e2c1821f2fc878cba97b1e6f8807d94930713aa1c86a67b9bf1e40221008581abfef2e30f957815fc89978423746b2086375ca8ecf359c85c2a5b7c88ad01410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffffd669f7d7958d40fc59d2253d88e0f248e29b599c80bbcec344a83dda5f9aa72c000000008a473044022078124c8beeaa825f9e0b30bff96e564dd859432f2d0cb3b72d3d5d93d38d7e930220691d233b6c0f995be5acb03d70a7f7a65b6bc9bdd426260f38a1346669507a3601410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95fffffffff878af0d93f5229a68166cf051fd372bb7a537232946e0a46f53636b4dafdaa4000000008c493046022100c717d1714551663f69c3c5759bdbb3a0fcd3fab023abc0e522fe6440de35d8290221008d9cbe25bffc44af2b18e81c58eb37293fd7fe1c2e7b46fc37ee8c96c50ab1e201410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff27f2b668859cd7f2f894aa0fd2d9e60963bcd07c88973f425f999b8cbfd7a1e2000000008c493046022100e00847147cbf517bcc2f502f3ddc6d284358d102ed20d47a8aa788a62f0db780022100d17b2d6fa84dcaf1c95d88d7e7c30385aecf415588d749afd3ec81f6022cecd701410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff0100c817a8040000001976a914b6efd80d99179f4f4ff6f4dd0a007d018c385d2188ac000000000100000001834537b2f1ce8ef9373a258e10545ce5a50b758df616cd4356e0032554ebd3c4000000008b483045022100e68f422dd7c34fdce11eeb4509ddae38201773dd62f284e8aa9d96f85099d0b002202243bd399ff96b649a0fad05fa759d6a882f0af8c90cf7632c2840c29070aec20141045e58067e815c2f464c6a2a15f987758374203895710c2d452442e28496ff38ba8f5fd901dc20e29e88477167fe4fc299bf818fd0d9e1632d467b2a3d9503b1aaffffffff0280d7e636030000001976a914f34c3e10eb387efe872acb614c89e78bfca7815d88ac404b4c00000000001976a914a84e272933aaf87e1715d7786c51dfaeb5b65a6f88ac00000000010000000143ac81c8e6f6ef307dfe17f3d906d999e23e0189fda838c5510d850927e03ae7000000008c4930460221009c87c344760a64cb8ae6685a3eec2c1ac1bed5b88c87de51acd0e124f266c16602210082d07c037359c3a257b5c63ebd90f5a5edf97b2ac1c434b08ca998839f346dd40141040ba7e521fa7946d12edbb1d1e95a15c34bd4398195e86433c92b431cd315f455fe30032ede69cad9d1e1ed6c3c4ec0dbfced53438c625462afb792dcb098544bffffffff0240420f00000000001976a9144676d1b820d63ec272f1900d59d43bc6463d96f888ac40420f00000000001976a914648d04341d00d7968b3405c034adc38d4d8fb9bd88ac00000000010000000248cc917501ea5c55f4a8d2009c0567c40cfe037c2e71af017d0a452ff705e3f1000000008b483045022100bf5fdc86dc5f08a5d5c8e43a8c9d5b1ed8c65562e280007b52b133021acd9acc02205e325d613e555f772802bf413d36ba807892ed1a690a77811d3033b3de226e0a01410429fa713b124484cb2bd7b5557b2c0b9df7b2b1fee61825eadc5ae6c37a9920d38bfccdc7dc3cb0c47d7b173dbc9db8d37db0a33ae487982c59c6f8606e9d1791ffffffff41ed70551dd7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d068000000008b4830450221008513ad65187b903aed1102d1d0c47688127658c51106753fed0151ce9c16b80902201432b9ebcb87bd04ceb2de66035fbbaf4bf8b00d1cfe41f1a1f7338f9ad79d210141049d4cf80125bf50be1709f718c07ad15d0fc612b7da1f5570dddc35f2a352f0f27c978b06820edca9ef982c35fda2d255afba340068c5035552368bc7200c1488ffffffff0100093d00000000001976a9148edb68822f1ad580b043c7b3df2e400f8699eb4888ac00000000"), + }; + stream >> TX_WITH_WITNESS(block); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_NONE); // Match the generation pubkey @@ -437,8 +451,8 @@ BOOST_AUTO_TEST_CASE(merkle_block_4_test_update_none) BOOST_CHECK(merkleBlock.header.GetHash() == block.GetHash()); // We shouldn't match any outpoints (UPDATE_NONE) - BOOST_CHECK(!filter.contains(COutPoint(uint256S("0x147caa76786596590baa4e98f5d9f48b86c7765e489f7a6ff3360fe5c674360b"), 0))); - BOOST_CHECK(!filter.contains(COutPoint(uint256S("0x02981fa052f0481dbc5868f4fc2166035a10f27a03cfd2de67326471df5bc041"), 0))); + BOOST_CHECK(!filter.contains(COutPoint(TxidFromString("0x147caa76786596590baa4e98f5d9f48b86c7765e489f7a6ff3360fe5c674360b"), 0))); + BOOST_CHECK(!filter.contains(COutPoint(TxidFromString("0x02981fa052f0481dbc5868f4fc2166035a10f27a03cfd2de67326471df5bc041"), 0))); } static std::vector<unsigned char> RandomData() diff --git a/src/test/checkqueue_tests.cpp b/src/test/checkqueue_tests.cpp index cb3831071a..023a5e8e70 100644 --- a/src/test/checkqueue_tests.cpp +++ b/src/test/checkqueue_tests.cpp @@ -158,8 +158,7 @@ typedef CCheckQueue<FrozenCleanupCheck> FrozenCleanup_Queue; */ static void Correct_Queue_range(std::vector<size_t> range) { - auto small_queue = std::make_unique<Correct_Queue>(QUEUE_BATCH_SIZE); - small_queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); + auto small_queue = std::make_unique<Correct_Queue>(QUEUE_BATCH_SIZE, SCRIPT_CHECK_THREADS); // Make vChecks here to save on malloc (this test can be slow...) std::vector<FakeCheckCheckCompletion> vChecks; vChecks.reserve(9); @@ -176,7 +175,6 @@ static void Correct_Queue_range(std::vector<size_t> range) BOOST_REQUIRE(control.Wait()); BOOST_REQUIRE_EQUAL(FakeCheckCheckCompletion::n_calls, i); } - small_queue->StopWorkerThreads(); } /** Test that 0 checks is correct @@ -218,9 +216,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Correct_Random) /** Test that failing checks are caught */ BOOST_AUTO_TEST_CASE(test_CheckQueue_Catches_Failure) { - auto fail_queue = std::make_unique<Failing_Queue>(QUEUE_BATCH_SIZE); - fail_queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); - + auto fail_queue = std::make_unique<Failing_Queue>(QUEUE_BATCH_SIZE, SCRIPT_CHECK_THREADS); for (size_t i = 0; i < 1001; ++i) { CCheckQueueControl<FailingCheck> control(fail_queue.get()); size_t remaining = i; @@ -240,15 +236,12 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Catches_Failure) BOOST_REQUIRE(success); } } - fail_queue->StopWorkerThreads(); } // Test that a block validation which fails does not interfere with // future blocks, ie, the bad state is cleared. BOOST_AUTO_TEST_CASE(test_CheckQueue_Recovers_From_Failure) { - auto fail_queue = std::make_unique<Failing_Queue>(QUEUE_BATCH_SIZE); - fail_queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); - + auto fail_queue = std::make_unique<Failing_Queue>(QUEUE_BATCH_SIZE, SCRIPT_CHECK_THREADS); for (auto times = 0; times < 10; ++times) { for (const bool end_fails : {true, false}) { CCheckQueueControl<FailingCheck> control(fail_queue.get()); @@ -262,7 +255,6 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Recovers_From_Failure) BOOST_REQUIRE(r != end_fails); } } - fail_queue->StopWorkerThreads(); } // Test that unique checks are actually all called individually, rather than @@ -270,9 +262,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Recovers_From_Failure) // more than once as well BOOST_AUTO_TEST_CASE(test_CheckQueue_UniqueCheck) { - auto queue = std::make_unique<Unique_Queue>(QUEUE_BATCH_SIZE); - queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); - + auto queue = std::make_unique<Unique_Queue>(QUEUE_BATCH_SIZE, SCRIPT_CHECK_THREADS); size_t COUNT = 100000; size_t total = COUNT; { @@ -294,7 +284,6 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_UniqueCheck) } BOOST_REQUIRE(r); } - queue->StopWorkerThreads(); } @@ -305,8 +294,7 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_UniqueCheck) // time could leave the data hanging across a sequence of blocks. BOOST_AUTO_TEST_CASE(test_CheckQueue_Memory) { - auto queue = std::make_unique<Memory_Queue>(QUEUE_BATCH_SIZE); - queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); + auto queue = std::make_unique<Memory_Queue>(QUEUE_BATCH_SIZE, SCRIPT_CHECK_THREADS); for (size_t i = 0; i < 1000; ++i) { size_t total = i; { @@ -325,16 +313,14 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_Memory) } BOOST_REQUIRE_EQUAL(MemoryCheck::fake_allocated_memory, 0U); } - queue->StopWorkerThreads(); } // Test that a new verification cannot occur until all checks // have been destructed BOOST_AUTO_TEST_CASE(test_CheckQueue_FrozenCleanup) { - auto queue = std::make_unique<FrozenCleanup_Queue>(QUEUE_BATCH_SIZE); + auto queue = std::make_unique<FrozenCleanup_Queue>(QUEUE_BATCH_SIZE, SCRIPT_CHECK_THREADS); bool fails = false; - queue->StartWorkerThreads(SCRIPT_CHECK_THREADS); std::thread t0([&]() { CCheckQueueControl<FrozenCleanupCheck> control(queue.get()); std::vector<FrozenCleanupCheck> vChecks(1); @@ -361,14 +347,13 @@ BOOST_AUTO_TEST_CASE(test_CheckQueue_FrozenCleanup) // Wait for control to finish t0.join(); BOOST_REQUIRE(!fails); - queue->StopWorkerThreads(); } /** Test that CCheckQueueControl is threadsafe */ BOOST_AUTO_TEST_CASE(test_CheckQueueControl_Locks) { - auto queue = std::make_unique<Standard_Queue>(QUEUE_BATCH_SIZE); + auto queue = std::make_unique<Standard_Queue>(QUEUE_BATCH_SIZE, SCRIPT_CHECK_THREADS); { std::vector<std::thread> tg; std::atomic<int> nThreads {0}; diff --git a/src/test/coins_tests.cpp b/src/test/coins_tests.cpp index 12dc4d1ccc..b6d3e7d567 100644 --- a/src/test/coins_tests.cpp +++ b/src/test/coins_tests.cpp @@ -137,16 +137,16 @@ void SimulationTest(CCoinsView* base, bool fake_best_block) stack.push_back(std::make_unique<CCoinsViewCacheTest>(base)); // Start with one cache. // Use a limited set of random transaction ids, so we do test overwriting entries. - std::vector<uint256> txids; + std::vector<Txid> txids; txids.resize(NUM_SIMULATION_ITERATIONS / 8); for (unsigned int i = 0; i < txids.size(); i++) { - txids[i] = InsecureRand256(); + txids[i] = Txid::FromUint256(InsecureRand256()); } for (unsigned int i = 0; i < NUM_SIMULATION_ITERATIONS; i++) { // Do a random modification. { - uint256 txid = txids[InsecureRandRange(txids.size())]; // txid we're going to modify in this iteration. + auto txid = txids[InsecureRandRange(txids.size())]; // txid we're going to modify in this iteration. Coin& coin = result[COutPoint(txid, 0)]; // Determine whether to test HaveCoin before or after Access* (or both). As these functions @@ -290,7 +290,7 @@ UtxoData utxoData; UtxoData::iterator FindRandomFrom(const std::set<COutPoint> &utxoSet) { assert(utxoSet.size()); - auto utxoSetIt = utxoSet.lower_bound(COutPoint(InsecureRand256(), 0)); + auto utxoSetIt = utxoSet.lower_bound(COutPoint(Txid::FromUint256(InsecureRand256()), 0)); if (utxoSetIt == utxoSet.end()) { utxoSetIt = utxoSet.begin(); } @@ -926,7 +926,7 @@ void TestFlushBehavior( } }; - uint256 txid = InsecureRand256(); + Txid txid = Txid::FromUint256(InsecureRand256()); COutPoint outp = COutPoint(txid, 0); Coin coin = MakeCoin(); // Ensure the coins views haven't seen this coin before. @@ -1017,7 +1017,7 @@ void TestFlushBehavior( // --- Bonus check: ensure that a coin added to the base view via one cache // can be spent by another cache which has never seen it. // - txid = InsecureRand256(); + txid = Txid::FromUint256(InsecureRand256()); outp = COutPoint(txid, 0); coin = MakeCoin(); BOOST_CHECK(!base.HaveCoin(outp)); @@ -1040,7 +1040,7 @@ void TestFlushBehavior( // --- Bonus check 2: ensure that a FRESH, spent coin is deleted by Sync() // - txid = InsecureRand256(); + txid = Txid::FromUint256(InsecureRand256()); outp = COutPoint(txid, 0); coin = MakeCoin(); CAmount coin_val = coin.out.nValue; diff --git a/src/test/coinstatsindex_tests.cpp b/src/test/coinstatsindex_tests.cpp index 50f3f7d833..cc1ec49d41 100644 --- a/src/test/coinstatsindex_tests.cpp +++ b/src/test/coinstatsindex_tests.cpp @@ -35,7 +35,7 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_initial_sync, TestChain100Setup) BOOST_REQUIRE(coin_stats_index.StartBackgroundSync()); - IndexWaitSynced(coin_stats_index); + IndexWaitSynced(coin_stats_index, *Assert(m_node.shutdown)); // Check that CoinStatsIndex works for genesis block. const CBlockIndex* genesis_block_index; @@ -86,7 +86,7 @@ BOOST_FIXTURE_TEST_CASE(coinstatsindex_unclean_shutdown, TestChain100Setup) CoinStatsIndex index{interfaces::MakeChain(m_node), 1 << 20}; BOOST_REQUIRE(index.Init()); BOOST_REQUIRE(index.StartBackgroundSync()); - IndexWaitSynced(index); + IndexWaitSynced(index, *Assert(m_node.shutdown)); std::shared_ptr<const CBlock> new_block; CBlockIndex* new_block_index = nullptr; { diff --git a/src/test/compress_tests.cpp b/src/test/compress_tests.cpp index b56629ae40..264b47b07c 100644 --- a/src/test/compress_tests.cpp +++ b/src/test/compress_tests.cpp @@ -65,8 +65,7 @@ BOOST_AUTO_TEST_CASE(compress_amounts) BOOST_AUTO_TEST_CASE(compress_script_to_ckey_id) { // case CKeyID - CKey key; - key.MakeNewKey(true); + CKey key = GenerateRandomKey(); CPubKey pubkey = key.GetPubKey(); CScript script = CScript() << OP_DUP << OP_HASH160 << ToByteVector(pubkey.GetID()) << OP_EQUALVERIFY << OP_CHECKSIG; @@ -101,8 +100,7 @@ BOOST_AUTO_TEST_CASE(compress_script_to_cscript_id) BOOST_AUTO_TEST_CASE(compress_script_to_compressed_pubkey_id) { - CKey key; - key.MakeNewKey(true); // case compressed PubKeyID + CKey key = GenerateRandomKey(); // case compressed PubKeyID CScript script = CScript() << ToByteVector(key.GetPubKey()) << OP_CHECKSIG; // COMPRESSED_PUBLIC_KEY_SIZE (33) BOOST_CHECK_EQUAL(script.size(), 35U); @@ -119,8 +117,7 @@ BOOST_AUTO_TEST_CASE(compress_script_to_compressed_pubkey_id) BOOST_AUTO_TEST_CASE(compress_script_to_uncompressed_pubkey_id) { - CKey key; - key.MakeNewKey(false); // case uncompressed PubKeyID + CKey key = GenerateRandomKey(/*compressed=*/false); // case uncompressed PubKeyID CScript script = CScript() << ToByteVector(key.GetPubKey()) << OP_CHECKSIG; // PUBLIC_KEY_SIZE (65) BOOST_CHECK_EQUAL(script.size(), 67U); // 1 char code + 65 char pubkey + OP_CHECKSIG diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 6e740a4f53..0fef8c5906 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -147,9 +147,7 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) constexpr int max_outbound_full_relay = MAX_OUTBOUND_FULL_RELAY_CONNECTIONS; CConnman::Options options; - options.nMaxConnections = DEFAULT_MAX_PEER_CONNECTIONS; - options.m_max_outbound_full_relay = max_outbound_full_relay; - options.nMaxFeeler = MAX_FEELER_CONNECTIONS; + options.m_max_automatic_connections = DEFAULT_MAX_PEER_CONNECTIONS; const auto time_init{GetTime<std::chrono::seconds>()}; SetMockTime(time_init); @@ -248,9 +246,7 @@ BOOST_AUTO_TEST_CASE(block_relay_only_eviction) constexpr int max_outbound_block_relay{MAX_BLOCK_RELAY_ONLY_CONNECTIONS}; constexpr int64_t MINIMUM_CONNECT_TIME{30}; CConnman::Options options; - options.nMaxConnections = DEFAULT_MAX_PEER_CONNECTIONS; - options.m_max_outbound_full_relay = MAX_OUTBOUND_FULL_RELAY_CONNECTIONS; - options.m_max_outbound_block_relay = max_outbound_block_relay; + options.m_max_automatic_connections = DEFAULT_MAX_PEER_CONNECTIONS; connman->Init(options); std::vector<CNode*> vNodes; diff --git a/src/test/descriptor_tests.cpp b/src/test/descriptor_tests.cpp index f4f4e39f40..c779bf6f73 100644 --- a/src/test/descriptor_tests.cpp +++ b/src/test/descriptor_tests.cpp @@ -592,9 +592,9 @@ BOOST_AUTO_TEST_CASE(descriptor_test) CheckUnparsable("raw(and_v(vc:andor(pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),pk_k(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn),and_v(v:older(1),pk_k(L4o2kDvXXDRH2VS9uBnouScLduWt4dZnM25se7kvEjJeQ285en2A))),after(10)))", "sh(and_v(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)))", "Miniscript expressions can only be used in wsh or tr."); CheckUnparsable("", "tr(034D2224bbbbbbbbbbcbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb40,{{{{{{{{{{{{{{{{{{{{{{multi(1,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/967808'/9,xprvA1RpRA33e1JQ7ifknakTFNpgXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/968/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/3/4/5/58/55/2/5/58/58/2/5/5/5/8/5/2/8/5/85/2/8/2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/8/5/8/5/4/5/585/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/8/2/5/8/5/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/58/58/2/0/8/5/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/5/8/5/8/24/5/58/52/5/8/5/2/8/24/5/58/588/246/8/5/2/8/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/5/4/5/58/55/58/2/5/8/55/2/5/8/58/555/58/2/5/8/4//2/5/58/5w/2/5/8/5/2/4/5/58/5558'/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/8/2/5/8/5/5/8/58/2/5/58/58/2/5/8/9/588/2/58/2/5/8/5/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/82/5/8/5/5/58/52/6/8/5/2/8/{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{}{{{{{{{{{DDD2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8588/246/8/5/2DLDDDDDDDbbD3DDDD/8/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/3/4/5/58/55/2/5/58/58/2/5/5/5/8/5/2/8/5/85/2/8/2/5/8D)/5/2/5/58/58/2/5/58/58/58/588/2/58/2/5/8/5/25/58/58/2/5/58/58/2/5/8/9/588/2/58/2/6780,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFW/8/5/2/5/58678008')", "'multi(1,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/967808'/9,xprvA1RpRA33e1JQ7ifknakTFNpgXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFWc/968/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/3/4/5/58/55/2/5/58/58/2/5/5/5/8/5/2/8/5/85/2/8/2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/8/5/8/5/4/5/585/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/8/2/5/8/5/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/58/58/2/0/8/5/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/5/8/5/8/24/5/58/52/5/8/5/2/8/24/5/58/588/246/8/5/2/8/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/5/4/5/58/55/58/2/5/8/55/2/5/8/58/555/58/2/5/8/4//2/5/58/5w/2/5/8/5/2/4/5/58/5558'/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/8/2/5/8/5/5/8/58/2/5/58/58/2/5/8/9/588/2/58/2/5/8/5/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8/5/2/5/58/58/2/5/5/58/588/2/58/2/5/8/5/2/82/5/8/5/5/58/52/6/8/5/2/8/{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{{}{{{{{{{{{DDD2/5/8/5/2/5/58/58/2/5/58/58/588/2/58/2/8/5/8/5/4/5/58/588/2/6/8/5/2/8/2/5/8588/246/8/5/2DLDDDDDDDbbD3DDDD/8/2/5/8/5/2/5/58/58/2/5/5/5/58/588/2/6/8/5/2/8/2/5/8/2/58/2/5/8/5/2/8/5/8/3/4/5/58/55/2/5/58/58/2/5/5/5/8/5/2/8/5/85/2/8/2/5/8D)/5/2/5/58/58/2/5/58/58/58/588/2/58/2/5/8/5/25/58/58/2/5/58/58/2/5/8/9/588/2/58/2/6780,xprvA1RpRA33e1JQ7ifknakTFpgNXPmW2YvmhqLQYMmrj4xJXXWYpDPS3xz7iAxn8L39njGVyuoseXzU6rcxFLJ8HFsTjSyQbLYnMpCqE2VbFW/8/5/2/5/58678008'' is not a valid descriptor function"); // No uncompressed keys allowed - CheckUnparsable("", "wsh(and_v(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(049228de6902abb4f541791f6d7f925b10e2078ccb1298856e5ea5cc5fd667f930eac37a00cc07f9a91ef3c2d17bf7a17db04552ff90ac312a5b8b4caca6c97aa4))),after(10)))", "A function is needed within P2WSH"); + CheckUnparsable("", "wsh(and_v(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(049228de6902abb4f541791f6d7f925b10e2078ccb1298856e5ea5cc5fd667f930eac37a00cc07f9a91ef3c2d17bf7a17db04552ff90ac312a5b8b4caca6c97aa4))),after(10)))", "Uncompressed keys are not allowed"); // No hybrid keys allowed - CheckUnparsable("", "wsh(and_v(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(069228de6902abb4f541791f6d7f925b10e2078ccb1298856e5ea5cc5fd667f930eac37a00cc07f9a91ef3c2d17bf7a17db04552ff90ac312a5b8b4caca6c97aa4))),after(10)))", "A function is needed within P2WSH"); + CheckUnparsable("", "wsh(and_v(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(069228de6902abb4f541791f6d7f925b10e2078ccb1298856e5ea5cc5fd667f930eac37a00cc07f9a91ef3c2d17bf7a17db04552ff90ac312a5b8b4caca6c97aa4))),after(10)))", "Hybrid public keys are not allowed"); // Insane at top level CheckUnparsable("wsh(and_b(vc:andor(pk(L4gM1FBdyHNpkzsFh9ipnofLhpZRp2mwobpeULy1a6dBTvw8Ywtd),pk_k(Kx9HCDjGiwFcgVNhTrS5z5NeZdD6veeam61eDxLDCkGWujvL4Gnn),and_v(v:older(1),pk_k(L4o2kDvXXDRH2VS9uBnouScLduWt4dZnM25se7kvEjJeQ285en2A))),after(10)))", "wsh(and_b(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)))", "and_b(vc:andor(pk(03cdabb7f2dce7bfbd8a0b9570c6fd1e712e5d64045e9d6b517b3d5072251dc204),pk_k(032707170c71d8f75e4ca4e3fce870b9409dcaf12b051d3bcadff74747fa7619c0),and_v(v:older(1),pk_k(02aa27e5eb2c185e87cd1dbc3e0efc9cb1175235e0259df1713424941c3cb40402))),after(10)) is invalid"); // Invalid sub diff --git a/src/test/disconnected_transactions.cpp b/src/test/disconnected_transactions.cpp new file mode 100644 index 0000000000..d4dc124b7b --- /dev/null +++ b/src/test/disconnected_transactions.cpp @@ -0,0 +1,95 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. +// +#include <boost/test/unit_test.hpp> +#include <core_memusage.h> +#include <kernel/disconnected_transactions.h> +#include <test/util/setup_common.h> + +BOOST_FIXTURE_TEST_SUITE(disconnected_transactions, TestChain100Setup) + +//! Tests that DisconnectedBlockTransactions limits its own memory properly +BOOST_AUTO_TEST_CASE(disconnectpool_memory_limits) +{ + // Use the coinbase transactions from TestChain100Setup. It doesn't matter whether these + // transactions would realistically be in a block together, they just need distinct txids and + // uniform size for this test to work. + std::vector<CTransactionRef> block_vtx(m_coinbase_txns); + BOOST_CHECK_EQUAL(block_vtx.size(), 100); + + // Roughly estimate sizes to sanity check that DisconnectedBlockTransactions::DynamicMemoryUsage + // is within an expected range. + + // Overhead for the hashmap depends on number of buckets + std::unordered_map<uint256, CTransaction*, SaltedTxidHasher> temp_map; + temp_map.reserve(1); + const size_t MAP_1{memusage::DynamicUsage(temp_map)}; + temp_map.reserve(100); + const size_t MAP_100{memusage::DynamicUsage(temp_map)}; + + const size_t TX_USAGE{RecursiveDynamicUsage(block_vtx.front())}; + for (const auto& tx : block_vtx) + BOOST_CHECK_EQUAL(RecursiveDynamicUsage(tx), TX_USAGE); + + // Our overall formula is unordered map overhead + usage per entry. + // Implementations may vary, but we're trying to guess the usage of data structures. + const size_t ENTRY_USAGE_ESTIMATE{ + TX_USAGE + // list entry: 2 pointers (next pointer and prev pointer) + element itself + + memusage::MallocUsage((2 * sizeof(void*)) + sizeof(decltype(block_vtx)::value_type)) + // unordered map: 1 pointer for the hashtable + key and value + + memusage::MallocUsage(sizeof(void*) + sizeof(decltype(temp_map)::key_type) + + sizeof(decltype(temp_map)::value_type))}; + + // DisconnectedBlockTransactions that's just big enough for 1 transaction. + { + DisconnectedBlockTransactions disconnectpool{MAP_1 + ENTRY_USAGE_ESTIMATE}; + // Add just 2 (and not all 100) transactions to keep the unordered map's hashtable overhead + // to a minimum and avoid all (instead of all but 1) transactions getting evicted. + std::vector<CTransactionRef> two_txns({block_vtx.at(0), block_vtx.at(1)}); + auto evicted_txns{disconnectpool.AddTransactionsFromBlock(two_txns)}; + BOOST_CHECK(disconnectpool.DynamicMemoryUsage() <= MAP_1 + ENTRY_USAGE_ESTIMATE); + + // Only 1 transaction can be kept + BOOST_CHECK_EQUAL(1, evicted_txns.size()); + // Transactions are added from back to front and eviction is FIFO. + BOOST_CHECK_EQUAL(block_vtx.at(1), evicted_txns.front()); + + disconnectpool.clear(); + } + + // DisconnectedBlockTransactions with a comfortable maximum memory usage so that nothing is evicted. + // Record usage so we can check size limiting in the next test. + size_t usage_full{0}; + { + const size_t USAGE_100_OVERESTIMATE{MAP_100 + ENTRY_USAGE_ESTIMATE * 100}; + DisconnectedBlockTransactions disconnectpool{USAGE_100_OVERESTIMATE}; + auto evicted_txns{disconnectpool.AddTransactionsFromBlock(block_vtx)}; + BOOST_CHECK_EQUAL(evicted_txns.size(), 0); + BOOST_CHECK(disconnectpool.DynamicMemoryUsage() <= USAGE_100_OVERESTIMATE); + + usage_full = disconnectpool.DynamicMemoryUsage(); + + disconnectpool.clear(); + } + + // DisconnectedBlockTransactions that's just a little too small for all of the transactions. + { + const size_t MAX_MEMUSAGE_99{usage_full - sizeof(void*)}; + DisconnectedBlockTransactions disconnectpool{MAX_MEMUSAGE_99}; + auto evicted_txns{disconnectpool.AddTransactionsFromBlock(block_vtx)}; + BOOST_CHECK(disconnectpool.DynamicMemoryUsage() <= MAX_MEMUSAGE_99); + + // Only 1 transaction needed to be evicted + BOOST_CHECK_EQUAL(1, evicted_txns.size()); + + // Transactions are added from back to front and eviction is FIFO. + // The last transaction of block_vtx should be the first to be evicted. + BOOST_CHECK_EQUAL(block_vtx.back(), evicted_txns.front()); + + disconnectpool.clear(); + } +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/flatfile_tests.cpp b/src/test/flatfile_tests.cpp index 3874b38f61..21a36d9d43 100644 --- a/src/test/flatfile_tests.cpp +++ b/src/test/flatfile_tests.cpp @@ -40,7 +40,7 @@ BOOST_AUTO_TEST_CASE(flatfile_open) "lost if a trusted third party is still required to prevent double-spending."); size_t pos1 = 0; - size_t pos2 = pos1 + GetSerializeSize(line1, CLIENT_VERSION); + size_t pos2 = pos1 + GetSerializeSize(line1); // Write first line to file. { diff --git a/src/test/fs_tests.cpp b/src/test/fs_tests.cpp index 7cfecb2b22..c237963af3 100644 --- a/src/test/fs_tests.cpp +++ b/src/test/fs_tests.cpp @@ -5,7 +5,6 @@ #include <test/util/setup_common.h> #include <util/fs.h> #include <util/fs_helpers.h> -#include <util/getuniquepath.h> #include <boost/test/unit_test.hpp> @@ -18,9 +17,12 @@ BOOST_FIXTURE_TEST_SUITE(fs_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(fsbridge_pathtostring) { std::string u8_str = "fs_tests_₿_🏃"; + std::u8string str8{u8"fs_tests_₿_🏃"}; BOOST_CHECK_EQUAL(fs::PathToString(fs::PathFromString(u8_str)), u8_str); - BOOST_CHECK_EQUAL(fs::u8path(u8_str).u8string(), u8_str); - BOOST_CHECK_EQUAL(fs::PathFromString(u8_str).u8string(), u8_str); + BOOST_CHECK_EQUAL(fs::u8path(u8_str).utf8string(), u8_str); + BOOST_CHECK_EQUAL(fs::path(str8).utf8string(), u8_str); + BOOST_CHECK(fs::path(str8).u8string() == str8); + BOOST_CHECK_EQUAL(fs::PathFromString(u8_str).utf8string(), u8_str); BOOST_CHECK_EQUAL(fs::PathToString(fs::u8path(u8_str)), u8_str); #ifndef WIN32 // On non-windows systems, verify that arbitrary byte strings containing @@ -47,7 +49,7 @@ BOOST_AUTO_TEST_CASE(fsbridge_fstream) fs::path tmpfolder = m_args.GetDataDirBase(); // tmpfile1 should be the same as tmpfile2 fs::path tmpfile1 = tmpfolder / fs::u8path("fs_tests_₿_🏃"); - fs::path tmpfile2 = tmpfolder / fs::u8path("fs_tests_₿_🏃"); + fs::path tmpfile2 = tmpfolder / fs::path(u8"fs_tests_₿_🏃"); { std::ofstream file{tmpfile1}; file << "bitcoin"; @@ -101,29 +103,14 @@ BOOST_AUTO_TEST_CASE(fsbridge_fstream) BOOST_CHECK_EQUAL(tmpfile1, fsbridge::AbsPathJoin(tmpfile1, "")); BOOST_CHECK_EQUAL(tmpfile1, fsbridge::AbsPathJoin(tmpfile1, {})); } - { - fs::path p1 = GetUniquePath(tmpfolder); - fs::path p2 = GetUniquePath(tmpfolder); - fs::path p3 = GetUniquePath(tmpfolder); - - // Ensure that the parent path is always the same. - BOOST_CHECK_EQUAL(tmpfolder, p1.parent_path()); - BOOST_CHECK_EQUAL(tmpfolder, p2.parent_path()); - BOOST_CHECK_EQUAL(tmpfolder, p3.parent_path()); - - // Ensure that generated paths are actually different. - BOOST_CHECK(p1 != p2); - BOOST_CHECK(p2 != p3); - BOOST_CHECK(p1 != p3); - } } BOOST_AUTO_TEST_CASE(rename) { const fs::path tmpfolder{m_args.GetDataDirBase()}; - const fs::path path1{GetUniquePath(tmpfolder)}; - const fs::path path2{GetUniquePath(tmpfolder)}; + const fs::path path1{tmpfolder / "a"}; + const fs::path path2{tmpfolder / "b"}; const std::string path1_contents{"1111"}; const std::string path2_contents{"2222"}; @@ -158,13 +145,13 @@ BOOST_AUTO_TEST_CASE(create_directories) // Test fs::create_directories workaround. const fs::path tmpfolder{m_args.GetDataDirBase()}; - const fs::path dir{GetUniquePath(tmpfolder)}; + const fs::path dir{tmpfolder / "a"}; fs::create_directory(dir); BOOST_CHECK(fs::exists(dir)); BOOST_CHECK(fs::is_directory(dir)); BOOST_CHECK(!fs::create_directories(dir)); - const fs::path symlink{GetUniquePath(tmpfolder)}; + const fs::path symlink{tmpfolder / "b"}; fs::create_directory_symlink(dir, symlink); BOOST_CHECK(fs::exists(symlink)); BOOST_CHECK(fs::is_symlink(symlink)); diff --git a/src/test/fuzz/FuzzedDataProvider.h b/src/test/fuzz/FuzzedDataProvider.h index 8a8214bd99..5903ed8379 100644 --- a/src/test/fuzz/FuzzedDataProvider.h +++ b/src/test/fuzz/FuzzedDataProvider.h @@ -158,7 +158,7 @@ FuzzedDataProvider::ConsumeRandomLengthString(size_t max_length) { // picking its contents. std::string result; - // Reserve the anticipated capaticity to prevent several reallocations. + // Reserve the anticipated capacity to prevent several reallocations. result.reserve(std::min(max_length, remaining_bytes_)); for (size_t i = 0; i < max_length && remaining_bytes_ != 0; ++i) { char next = ConvertUnsignedToSigned<char>(data_ptr_[0]); diff --git a/src/test/fuzz/addrman.cpp b/src/test/fuzz/addrman.cpp index 1b11ff6fdf..ece396aadf 100644 --- a/src/test/fuzz/addrman.cpp +++ b/src/test/fuzz/addrman.cpp @@ -286,7 +286,8 @@ FUZZ_TARGET(addrman, .init = initialize_addrman) (void)const_addr_man.GetAddr( /*max_addresses=*/fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096), /*max_pct=*/fuzzed_data_provider.ConsumeIntegralInRange<size_t>(0, 4096), - network); + network, + /*filtered=*/fuzzed_data_provider.ConsumeBool()); (void)const_addr_man.Select(fuzzed_data_provider.ConsumeBool(), network); std::optional<bool> in_new; if (fuzzed_data_provider.ConsumeBool()) { diff --git a/src/test/fuzz/autofile.cpp b/src/test/fuzz/autofile.cpp index a7b41370a8..45316b6b21 100644 --- a/src/test/fuzz/autofile.cpp +++ b/src/test/fuzz/autofile.cpp @@ -2,24 +2,28 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <span.h> #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <array> -#include <cstdint> +#include <cstddef> +#include <cstdio> #include <iostream> -#include <optional> -#include <string> #include <vector> FUZZ_TARGET(autofile) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); - AutoFile auto_file{fuzzed_auto_file_provider.open()}; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { + FuzzedFileProvider fuzzed_file_provider{fuzzed_data_provider}; + AutoFile auto_file{ + fuzzed_file_provider.open(), + ConsumeRandomLengthByteVector<std::byte>(fuzzed_data_provider), + }; + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) + { CallOneOf( fuzzed_data_provider, [&] { diff --git a/src/test/fuzz/banman.cpp b/src/test/fuzz/banman.cpp index 3882e0e547..4a040c56de 100644 --- a/src/test/fuzz/banman.cpp +++ b/src/test/fuzz/banman.cpp @@ -63,17 +63,28 @@ FUZZ_TARGET(banman, .init = initialize_banman) // The complexity is O(N^2), where N is the input size, because each call // might call DumpBanlist (or other methods that are at least linear // complexity of the input size). + bool contains_invalid{false}; LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 300) { CallOneOf( fuzzed_data_provider, [&] { - ban_man.Ban(ConsumeNetAddr(fuzzed_data_provider), - ConsumeBanTimeOffset(fuzzed_data_provider), fuzzed_data_provider.ConsumeBool()); + CNetAddr net_addr{ConsumeNetAddr(fuzzed_data_provider)}; + const std::optional<CNetAddr>& addr{LookupHost(net_addr.ToStringAddr(), /*fAllowLookup=*/false)}; + if (addr.has_value() && addr->IsValid()) { + net_addr = *addr; + } else { + contains_invalid = true; + } + ban_man.Ban(net_addr, ConsumeBanTimeOffset(fuzzed_data_provider), fuzzed_data_provider.ConsumeBool()); }, [&] { - ban_man.Ban(ConsumeSubNet(fuzzed_data_provider), - ConsumeBanTimeOffset(fuzzed_data_provider), fuzzed_data_provider.ConsumeBool()); + CSubNet subnet{ConsumeSubNet(fuzzed_data_provider)}; + subnet = LookupSubNet(subnet.ToString()); + if (!subnet.IsValid()) { + contains_invalid = true; + } + ban_man.Ban(subnet, ConsumeBanTimeOffset(fuzzed_data_provider), fuzzed_data_provider.ConsumeBool()); }, [&] { ban_man.ClearBanned(); @@ -109,7 +120,9 @@ FUZZ_TARGET(banman, .init = initialize_banman) BanMan ban_man_read{banlist_file, /*client_interface=*/nullptr, /*default_ban_time=*/0}; banmap_t banmap_read; ban_man_read.GetBanned(banmap_read); - assert(banmap == banmap_read); + if (!contains_invalid) { + assert(banmap == banmap_read); + } } } fs::remove(fs::PathToString(banlist_file + ".json")); diff --git a/src/test/fuzz/bip324.cpp b/src/test/fuzz/bip324.cpp index e5ed9bfd52..37c41f3895 100644 --- a/src/test/fuzz/bip324.cpp +++ b/src/test/fuzz/bip324.cpp @@ -98,7 +98,7 @@ FUZZ_TARGET(bip324_cipher_roundtrip, .init=Initialize) unsigned damage_bit = provider.ConsumeIntegralInRange<unsigned>(0, (ciphertext.size() + aad.size()) * 8U - 1U); unsigned damage_pos = damage_bit >> 3; - std::byte damage_val{(uint8_t)(1U << (damage_bit & 3))}; + std::byte damage_val{(uint8_t)(1U << (damage_bit & 7))}; if (damage_pos >= ciphertext.size()) { aad[damage_pos - ciphertext.size()] ^= damage_val; } else { diff --git a/src/test/fuzz/bitdeque.cpp b/src/test/fuzz/bitdeque.cpp index 65f5cb3fd0..d5cc9cfd34 100644 --- a/src/test/fuzz/bitdeque.cpp +++ b/src/test/fuzz/bitdeque.cpp @@ -53,21 +53,11 @@ FUZZ_TARGET(bitdeque, .init = InitRandData) --initlen; } - LIMITED_WHILE(provider.remaining_bytes() > 0, 900) + const auto iter_limit{maxlen > 6000 ? 90U : 900U}; + LIMITED_WHILE(provider.remaining_bytes() > 0, iter_limit) { - { - assert(deq.size() == bitdeq.size()); - auto it = deq.begin(); - auto bitit = bitdeq.begin(); - auto itend = deq.end(); - while (it != itend) { - assert(*it == *bitit); - ++it; - ++bitit; - } - } - - CallOneOf(provider, + CallOneOf( + provider, [&] { // constructor() deq = std::deque<bool>{}; @@ -535,7 +525,17 @@ FUZZ_TARGET(bitdeque, .init = InitRandData) assert(it == deq.begin() + before); assert(bitit == bitdeq.begin() + before); } - } - ); + }); + } + { + assert(deq.size() == bitdeq.size()); + auto it = deq.begin(); + auto bitit = bitdeq.begin(); + auto itend = deq.end(); + while (it != itend) { + assert(*it == *bitit); + ++it; + ++bitit; + } } } diff --git a/src/test/fuzz/block.cpp b/src/test/fuzz/block.cpp index 8c97fba323..e38b871747 100644 --- a/src/test/fuzz/block.cpp +++ b/src/test/fuzz/block.cpp @@ -13,7 +13,6 @@ #include <test/fuzz/fuzz.h> #include <util/chaintype.h> #include <validation.h> -#include <version.h> #include <cassert> #include <string> @@ -25,13 +24,10 @@ void initialize_block() FUZZ_TARGET(block, .init = initialize_block) { - CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); + DataStream ds{buffer}; CBlock block; try { - int nVersion; - ds >> nVersion; - ds.SetVersion(nVersion); - ds >> block; + ds >> TX_WITH_WITNESS(block); } catch (const std::ios_base::failure&) { return; } diff --git a/src/test/fuzz/bloom_filter.cpp b/src/test/fuzz/bloom_filter.cpp index 3e303ecc0f..5dbb463a16 100644 --- a/src/test/fuzz/bloom_filter.cpp +++ b/src/test/fuzz/bloom_filter.cpp @@ -10,21 +10,22 @@ #include <uint256.h> #include <cassert> -#include <cstdint> +#include <limits> #include <optional> -#include <string> #include <vector> FUZZ_TARGET(bloom_filter) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + bool good_data{true}; CBloomFilter bloom_filter{ fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(1, 10000000), 1.0 / fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(1, std::numeric_limits<unsigned int>::max()), fuzzed_data_provider.ConsumeIntegral<unsigned int>(), static_cast<unsigned char>(fuzzed_data_provider.PickValueInArray({BLOOM_UPDATE_NONE, BLOOM_UPDATE_ALL, BLOOM_UPDATE_P2PUBKEY_ONLY, BLOOM_UPDATE_MASK}))}; - LIMITED_WHILE(fuzzed_data_provider.remaining_bytes() > 0, 10000) { + LIMITED_WHILE(good_data && fuzzed_data_provider.remaining_bytes() > 0, 10'000) + { CallOneOf( fuzzed_data_provider, [&] { @@ -37,6 +38,7 @@ FUZZ_TARGET(bloom_filter) [&] { const std::optional<COutPoint> out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); if (!out_point) { + good_data = false; return; } (void)bloom_filter.contains(*out_point); @@ -47,6 +49,7 @@ FUZZ_TARGET(bloom_filter) [&] { const std::optional<uint256> u256 = ConsumeDeserializable<uint256>(fuzzed_data_provider); if (!u256) { + good_data = false; return; } (void)bloom_filter.contains(*u256); @@ -55,8 +58,9 @@ FUZZ_TARGET(bloom_filter) assert(present); }, [&] { - const std::optional<CMutableTransaction> mut_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mut_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!mut_tx) { + good_data = false; return; } const CTransaction tx{*mut_tx}; diff --git a/src/test/fuzz/buffered_file.cpp b/src/test/fuzz/buffered_file.cpp index 636f11b381..e30c19b265 100644 --- a/src/test/fuzz/buffered_file.cpp +++ b/src/test/fuzz/buffered_file.cpp @@ -2,31 +2,36 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <span.h> #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <array> +#include <cstddef> #include <cstdint> #include <iostream> #include <optional> -#include <string> #include <vector> FUZZ_TARGET(buffered_file) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - FuzzedFileProvider fuzzed_file_provider = ConsumeFile(fuzzed_data_provider); + FuzzedFileProvider fuzzed_file_provider{fuzzed_data_provider}; std::optional<BufferedFile> opt_buffered_file; - CAutoFile fuzzed_file{fuzzed_file_provider.open(), 0}; + AutoFile fuzzed_file{ + fuzzed_file_provider.open(), + ConsumeRandomLengthByteVector<std::byte>(fuzzed_data_provider), + }; try { opt_buffered_file.emplace(fuzzed_file, fuzzed_data_provider.ConsumeIntegralInRange<uint64_t>(0, 4096), fuzzed_data_provider.ConsumeIntegralInRange<uint64_t>(0, 4096)); } catch (const std::ios_base::failure&) { } if (opt_buffered_file && !fuzzed_file.IsNull()) { bool setpos_fail = false; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) + { CallOneOf( fuzzed_data_provider, [&] { @@ -59,6 +64,5 @@ FUZZ_TARGET(buffered_file) }); } opt_buffered_file->GetPos(); - opt_buffered_file->GetVersion(); } } diff --git a/src/test/fuzz/checkqueue.cpp b/src/test/fuzz/checkqueue.cpp index 429570526f..6320b500b6 100644 --- a/src/test/fuzz/checkqueue.cpp +++ b/src/test/fuzz/checkqueue.cpp @@ -31,8 +31,8 @@ FUZZ_TARGET(checkqueue) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); const unsigned int batch_size = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, 1024); - CCheckQueue<DumbCheck> check_queue_1{batch_size}; - CCheckQueue<DumbCheck> check_queue_2{batch_size}; + CCheckQueue<DumbCheck> check_queue_1{batch_size, /*worker_threads_num=*/0}; + CCheckQueue<DumbCheck> check_queue_2{batch_size, /*worker_threads_num=*/0}; std::vector<DumbCheck> checks_1; std::vector<DumbCheck> checks_2; const int size = fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 1024); diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index b088aa0bd7..8f3e357a84 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -2,26 +2,28 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <chainparams.h> #include <coins.h> #include <consensus/amount.h> #include <consensus/tx_check.h> #include <consensus/tx_verify.h> #include <consensus/validation.h> -#include <key.h> #include <policy/policy.h> #include <primitives/transaction.h> -#include <pubkey.h> +#include <script/interpreter.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <test/util/setup_common.h> -#include <validation.h> +#include <util/hasher.h> +#include <cassert> #include <cstdint> #include <limits> +#include <memory> #include <optional> +#include <stdexcept> #include <string> +#include <utility> #include <vector> namespace { @@ -44,12 +46,15 @@ void initialize_coins_view() FUZZ_TARGET(coins_view, .init = initialize_coins_view) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + bool good_data{true}; + CCoinsView backend_coins_view; CCoinsViewCache coins_view_cache{&backend_coins_view, /*deterministic=*/true}; COutPoint random_out_point; Coin random_coin; CMutableTransaction random_mutable_transaction; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 10'000) + { CallOneOf( fuzzed_data_provider, [&] { @@ -95,6 +100,7 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) [&] { const std::optional<COutPoint> opt_out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); if (!opt_out_point) { + good_data = false; return; } random_out_point = *opt_out_point; @@ -102,13 +108,15 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) [&] { const std::optional<Coin> opt_coin = ConsumeDeserializable<Coin>(fuzzed_data_provider); if (!opt_coin) { + good_data = false; return; } random_coin = *opt_coin; }, [&] { - const std::optional<CMutableTransaction> opt_mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> opt_mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!opt_mutable_transaction) { + good_data = false; return; } random_mutable_transaction = *opt_mutable_transaction; @@ -116,7 +124,8 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) [&] { CCoinsMapMemoryResource resource; CCoinsMap coins_map{0, SaltedOutpointHasher{/*deterministic=*/true}, CCoinsMap::key_equal{}, &resource}; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 10'000) + { CCoinsCacheEntry coins_cache_entry; coins_cache_entry.flags = fuzzed_data_provider.ConsumeIntegral<unsigned char>(); if (fuzzed_data_provider.ConsumeBool()) { @@ -124,6 +133,7 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) } else { const std::optional<Coin> opt_coin = ConsumeDeserializable<Coin>(fuzzed_data_provider); if (!opt_coin) { + good_data = false; return; } coins_cache_entry.coin = *opt_coin; diff --git a/src/test/fuzz/coinscache_sim.cpp b/src/test/fuzz/coinscache_sim.cpp index f350c9d032..648e96b4a0 100644 --- a/src/test/fuzz/coinscache_sim.cpp +++ b/src/test/fuzz/coinscache_sim.cpp @@ -43,7 +43,9 @@ struct PrecomputedData for (uint32_t i = 0; i < NUM_OUTPOINTS; ++i) { uint32_t idx = (i * 1200U) >> 12; /* Map 3 or 4 entries to same txid. */ const uint8_t ser[4] = {uint8_t(idx), uint8_t(idx >> 8), uint8_t(idx >> 16), uint8_t(idx >> 24)}; - CSHA256().Write(PREFIX_O, 1).Write(ser, sizeof(ser)).Finalize(outpoints[i].hash.begin()); + uint256 txid; + CSHA256().Write(PREFIX_O, 1).Write(ser, sizeof(ser)).Finalize(txid.begin()); + outpoints[i].hash = Txid::FromUint256(txid); outpoints[i].n = i; } diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index 0dab2a2e97..24f91abd25 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -88,7 +88,8 @@ FUZZ_TARGET(connman, .init = initialize_connman) (void)connman.GetAddresses( /*max_addresses=*/fuzzed_data_provider.ConsumeIntegral<size_t>(), /*max_pct=*/fuzzed_data_provider.ConsumeIntegral<size_t>(), - /*network=*/std::nullopt); + /*network=*/std::nullopt, + /*filtered=*/fuzzed_data_provider.ConsumeBool()); }, [&] { (void)connman.GetAddresses( @@ -121,7 +122,7 @@ FUZZ_TARGET(connman, .init = initialize_connman) connman.SetTryNewOutboundPeer(fuzzed_data_provider.ConsumeBool()); }); } - (void)connman.GetAddedNodeInfo(); + (void)connman.GetAddedNodeInfo(fuzzed_data_provider.ConsumeBool()); (void)connman.GetExtraFullOutboundCount(); (void)connman.GetLocalServices(); (void)connman.GetMaxOutboundTarget(); diff --git a/src/test/fuzz/descriptor_parse.cpp b/src/test/fuzz/descriptor_parse.cpp index 57129a60b8..6ea315d4e2 100644 --- a/src/test/fuzz/descriptor_parse.cpp +++ b/src/test/fuzz/descriptor_parse.cpp @@ -7,104 +7,10 @@ #include <pubkey.h> #include <script/descriptor.h> #include <test/fuzz/fuzz.h> +#include <test/fuzz/util/descriptor.h> #include <util/chaintype.h> #include <util/strencodings.h> -//! Types are raw (un)compressed pubkeys, raw xonly pubkeys, raw privkeys (WIF), xpubs, xprvs. -static constexpr uint8_t KEY_TYPES_COUNT{6}; -//! How many keys we'll generate in total. -static constexpr size_t TOTAL_KEYS_GENERATED{std::numeric_limits<uint8_t>::max() + 1}; - -/** - * Converts a mocked descriptor string to a valid one. Every key in a mocked descriptor key is - * represented by 2 hex characters preceded by the '%' character. We parse the two hex characters - * as an index in a list of pre-generated keys. This list contains keys of the various types - * accepted in descriptor keys expressions. - */ -class MockedDescriptorConverter { - //! 256 keys of various types. - std::array<std::string, TOTAL_KEYS_GENERATED> keys_str; - -public: - // We derive the type of key to generate from the 1-byte id parsed from hex. - bool IdIsCompPubKey(uint8_t idx) const { return idx % KEY_TYPES_COUNT == 0; } - bool IdIsUnCompPubKey(uint8_t idx) const { return idx % KEY_TYPES_COUNT == 1; } - bool IdIsXOnlyPubKey(uint8_t idx) const { return idx % KEY_TYPES_COUNT == 2; } - bool IdIsConstPrivKey(uint8_t idx) const { return idx % KEY_TYPES_COUNT == 3; } - bool IdIsXpub(uint8_t idx) const { return idx % KEY_TYPES_COUNT == 4; } - bool IdIsXprv(uint8_t idx) const { return idx % KEY_TYPES_COUNT == 5; } - - //! When initializing the target, populate the list of keys. - void Init() { - // The data to use as a private key or a seed for an xprv. - std::array<std::byte, 32> key_data{std::byte{1}}; - // Generate keys of all kinds and store them in the keys array. - for (size_t i{0}; i < TOTAL_KEYS_GENERATED; i++) { - key_data[31] = std::byte(i); - - // If this is a "raw" key, generate a normal privkey. Otherwise generate - // an extended one. - if (IdIsCompPubKey(i) || IdIsUnCompPubKey(i) || IdIsXOnlyPubKey(i) || IdIsConstPrivKey(i)) { - CKey privkey; - privkey.Set(UCharCast(key_data.begin()), UCharCast(key_data.end()), !IdIsUnCompPubKey(i)); - if (IdIsCompPubKey(i) || IdIsUnCompPubKey(i)) { - CPubKey pubkey{privkey.GetPubKey()}; - keys_str[i] = HexStr(pubkey); - } else if (IdIsXOnlyPubKey(i)) { - const XOnlyPubKey pubkey{privkey.GetPubKey()}; - keys_str[i] = HexStr(pubkey); - } else { - keys_str[i] = EncodeSecret(privkey); - } - } else { - CExtKey ext_privkey; - ext_privkey.SetSeed(key_data); - if (IdIsXprv(i)) { - keys_str[i] = EncodeExtKey(ext_privkey); - } else { - const CExtPubKey ext_pubkey{ext_privkey.Neuter()}; - keys_str[i] = EncodeExtPubKey(ext_pubkey); - } - } - } - } - - //! Parse an id in the keys vectors from a 2-characters hex string. - std::optional<uint8_t> IdxFromHex(std::string_view hex_characters) const { - if (hex_characters.size() != 2) return {}; - auto idx = ParseHex(hex_characters); - if (idx.size() != 1) return {}; - return idx[0]; - } - - //! Get an actual descriptor string from a descriptor string whose keys were mocked. - std::optional<std::string> GetDescriptor(std::string_view mocked_desc) const { - // The smallest fragment would be "pk(%00)" - if (mocked_desc.size() < 7) return {}; - - // The actual descriptor string to be returned. - std::string desc; - desc.reserve(mocked_desc.size()); - - // Replace all occurrences of '%' followed by two hex characters with the corresponding key. - for (size_t i = 0; i < mocked_desc.size();) { - if (mocked_desc[i] == '%') { - if (i + 3 >= mocked_desc.size()) return {}; - if (const auto idx = IdxFromHex(mocked_desc.substr(i + 1, 2))) { - desc += keys_str[*idx]; - i += 3; - } else { - return {}; - } - } else { - desc += mocked_desc[i++]; - } - } - - return desc; - } -}; - //! The converter of mocked descriptors, needs to be initialized when the target is. MockedDescriptorConverter MOCKED_DESC_CONVERTER; @@ -161,6 +67,11 @@ void initialize_mocked_descriptor_parse() FUZZ_TARGET(mocked_descriptor_parse, .init = initialize_mocked_descriptor_parse) { + // Key derivation is expensive. Deriving deep derivation paths take a lot of compute and we'd + // rather spend time elsewhere in this target, like on the actual descriptor syntax. So rule + // out strings which could correspond to a descriptor containing a too large derivation path. + if (HasDeepDerivPath(buffer)) return; + const std::string mocked_descriptor{buffer.begin(), buffer.end()}; if (const auto descriptor = MOCKED_DESC_CONVERTER.GetDescriptor(mocked_descriptor)) { FlatSigningProvider signing_provider; @@ -172,6 +83,9 @@ FUZZ_TARGET(mocked_descriptor_parse, .init = initialize_mocked_descriptor_parse) FUZZ_TARGET(descriptor_parse, .init = initialize_descriptor_parse) { + // See comment above for rationale. + if (HasDeepDerivPath(buffer)) return; + const std::string descriptor(buffer.begin(), buffer.end()); FlatSigningProvider signing_provider; std::string error; diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index 510ee7fb5b..ebc5673e71 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -28,7 +28,6 @@ #include <test/fuzz/util.h> #include <test/util/setup_common.h> #include <undo.h> -#include <version.h> #include <exception> #include <optional> @@ -66,7 +65,7 @@ template <typename T, typename P> DataStream Serialize(const T& obj, const P& params) { DataStream ds{}; - ds << WithParams(params, obj); + ds << params(obj); return ds; } @@ -74,7 +73,7 @@ template <typename T, typename P> T Deserialize(DataStream&& ds, const P& params) { T obj; - ds >> WithParams(params, obj); + ds >> params(obj); return obj; } @@ -83,7 +82,7 @@ void DeserializeFromFuzzingInput(FuzzBufferType buffer, T&& obj, const P& params { DataStream ds{buffer}; try { - ds >> WithParams(params, obj); + ds >> params(obj); } catch (const std::ios_base::failure&) { throw invalid_fuzzing_input_exception(); } @@ -91,15 +90,15 @@ void DeserializeFromFuzzingInput(FuzzBufferType buffer, T&& obj, const P& params } template <typename T> -CDataStream Serialize(const T& obj) +DataStream Serialize(const T& obj) { - CDataStream ds{SER_NETWORK, INIT_PROTO_VERSION}; + DataStream ds{}; ds << obj; return ds; } template <typename T> -T Deserialize(CDataStream ds) +T Deserialize(DataStream ds) { T obj; ds >> obj; @@ -109,16 +108,7 @@ T Deserialize(CDataStream ds) template <typename T> void DeserializeFromFuzzingInput(FuzzBufferType buffer, T&& obj) { - CDataStream ds{buffer, SER_NETWORK, INIT_PROTO_VERSION}; - { - try { - int version; - ds >> version; - ds.SetVersion(version); - } catch (const std::ios_base::failure&) { - throw invalid_fuzzing_input_exception(); - } - } + DataStream ds{buffer}; try { ds >> obj; } catch (const std::ios_base::failure&) { @@ -217,7 +207,7 @@ FUZZ_TARGET_DESERIALIZE(psbt_output_deserialize, { }) FUZZ_TARGET_DESERIALIZE(block_deserialize, { CBlock block; - DeserializeFromFuzzingInput(buffer, block); + DeserializeFromFuzzingInput(buffer, TX_WITH_WITNESS(block)); }) FUZZ_TARGET_DESERIALIZE(blocklocator_deserialize, { CBlockLocator bl; @@ -225,7 +215,7 @@ FUZZ_TARGET_DESERIALIZE(blocklocator_deserialize, { }) FUZZ_TARGET_DESERIALIZE(blockmerkleroot, { CBlock block; - DeserializeFromFuzzingInput(buffer, block); + DeserializeFromFuzzingInput(buffer, TX_WITH_WITNESS(block)); bool mutated; BlockMerkleRoot(block, &mutated); }) diff --git a/src/test/fuzz/float.cpp b/src/test/fuzz/float.cpp index 8714ab8a04..6897e81494 100644 --- a/src/test/fuzz/float.cpp +++ b/src/test/fuzz/float.cpp @@ -7,7 +7,6 @@ #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <util/serfloat.h> -#include <version.h> #include <cassert> #include <cmath> diff --git a/src/test/fuzz/fuzz.cpp b/src/test/fuzz/fuzz.cpp index 5245b4607b..6de480ff15 100644 --- a/src/test/fuzz/fuzz.cpp +++ b/src/test/fuzz/fuzz.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2009-2022 The Bitcoin Core developers +// Copyright (c) 2009-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -71,7 +71,7 @@ auto& FuzzTargets() void FuzzFrameworkRegisterTarget(std::string_view name, TypeTestOneInput target, FuzzTargetOptions opts) { - const auto it_ins{FuzzTargets().try_emplace(name, FuzzTarget /* temporary can be dropped in C++20 */ {std::move(target), std::move(opts)})}; + const auto it_ins{FuzzTargets().try_emplace(name, FuzzTarget /* temporary can be dropped after clang-16 */ {std::move(target), std::move(opts)})}; Assert(it_ins.second); } diff --git a/src/test/fuzz/fuzz.h b/src/test/fuzz/fuzz.h index 0534f9bcf1..ca74d53de7 100644 --- a/src/test/fuzz/fuzz.h +++ b/src/test/fuzz/fuzz.h @@ -14,6 +14,11 @@ /** * Can be used to limit a theoretically unbounded loop. This caps the runtime * to avoid timeouts or OOMs. + * + * This can be used in combination with a check in the condition to confirm + * whether the fuzz engine provided "good" data. If the fuzz input contains + * invalid data, the loop aborts early. This will teach the fuzz engine to look + * for useful data and avoids bloating the fuzz input folder with useless data. */ #define LIMITED_WHILE(condition, limit) \ for (unsigned _count{limit}; (condition) && _count; --_count) @@ -28,11 +33,7 @@ struct FuzzTargetOptions { void FuzzFrameworkRegisterTarget(std::string_view name, TypeTestOneInput target, FuzzTargetOptions opts); -#if defined(__clang__) -#define FUZZ_TARGET(...) _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wgnu-zero-variadic-macro-arguments\"") DETAIL_FUZZ(__VA_ARGS__) _Pragma("clang diagnostic pop") -#else #define FUZZ_TARGET(...) DETAIL_FUZZ(__VA_ARGS__) -#endif #define DETAIL_FUZZ(name, ...) \ void name##_fuzz_target(FuzzBufferType); \ diff --git a/src/test/fuzz/golomb_rice.cpp b/src/test/fuzz/golomb_rice.cpp index f3073c5c97..92e49445ba 100644 --- a/src/test/fuzz/golomb_rice.cpp +++ b/src/test/fuzz/golomb_rice.cpp @@ -51,9 +51,9 @@ FUZZ_TARGET(golomb_rice) for (int i = 0; i < n; ++i) { elements.insert(ConsumeRandomLengthByteVector(fuzzed_data_provider, 16)); } - CVectorWriter stream{0, golomb_rice_data, 0}; + VectorWriter stream{golomb_rice_data, 0}; WriteCompactSize(stream, static_cast<uint32_t>(elements.size())); - BitStreamWriter<CVectorWriter> bitwriter(stream); + BitStreamWriter bitwriter{stream}; if (!elements.empty()) { uint64_t last_value = 0; for (const uint64_t value : BuildHashedSet(elements, static_cast<uint64_t>(elements.size()) * static_cast<uint64_t>(BASIC_FILTER_M))) { @@ -68,8 +68,8 @@ FUZZ_TARGET(golomb_rice) std::vector<uint64_t> decoded_deltas; { - SpanReader stream{0, golomb_rice_data}; - BitStreamReader<SpanReader> bitreader{stream}; + SpanReader stream{golomb_rice_data}; + BitStreamReader bitreader{stream}; const uint32_t n = static_cast<uint32_t>(ReadCompactSize(stream)); for (uint32_t i = 0; i < n; ++i) { decoded_deltas.push_back(GolombRiceDecode(bitreader, BASIC_FILTER_P)); @@ -80,14 +80,14 @@ FUZZ_TARGET(golomb_rice) { const std::vector<uint8_t> random_bytes = ConsumeRandomLengthByteVector(fuzzed_data_provider, 1024); - SpanReader stream{0, random_bytes}; + SpanReader stream{random_bytes}; uint32_t n; try { n = static_cast<uint32_t>(ReadCompactSize(stream)); } catch (const std::ios_base::failure&) { return; } - BitStreamReader<SpanReader> bitreader{stream}; + BitStreamReader bitreader{stream}; for (uint32_t i = 0; i < std::min<uint32_t>(n, 1024); ++i) { try { (void)GolombRiceDecode(bitreader, BASIC_FILTER_P); diff --git a/src/test/fuzz/http_request.cpp b/src/test/fuzz/http_request.cpp index 9928c4a1ab..f13f1c72a5 100644 --- a/src/test/fuzz/http_request.cpp +++ b/src/test/fuzz/http_request.cpp @@ -7,6 +7,7 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> +#include <util/signalinterrupt.h> #include <util/strencodings.h> #include <event2/buffer.h> @@ -47,7 +48,8 @@ FUZZ_TARGET(http_request) return; } - HTTPRequest http_request{evreq, true}; + util::SignalInterrupt interrupt; + HTTPRequest http_request{evreq, interrupt, true}; const HTTPRequest::RequestMethod request_method = http_request.GetRequestMethod(); (void)RequestMethodString(request_method); (void)http_request.GetURI(); diff --git a/src/test/fuzz/integer.cpp b/src/test/fuzz/integer.cpp index 849618d748..9b97958a5d 100644 --- a/src/test/fuzz/integer.cpp +++ b/src/test/fuzz/integer.cpp @@ -33,7 +33,6 @@ #include <util/overflow.h> #include <util/strencodings.h> #include <util/string.h> -#include <version.h> #include <cassert> #include <chrono> diff --git a/src/test/fuzz/load_external_block_file.cpp b/src/test/fuzz/load_external_block_file.cpp index fc903e5ec2..6460261f0f 100644 --- a/src/test/fuzz/load_external_block_file.cpp +++ b/src/test/fuzz/load_external_block_file.cpp @@ -27,8 +27,8 @@ void initialize_load_external_block_file() FUZZ_TARGET(load_external_block_file, .init = initialize_load_external_block_file) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - FuzzedFileProvider fuzzed_file_provider = ConsumeFile(fuzzed_data_provider); - CAutoFile fuzzed_block_file{fuzzed_file_provider.open(), CLIENT_VERSION}; + FuzzedFileProvider fuzzed_file_provider{fuzzed_data_provider}; + AutoFile fuzzed_block_file{fuzzed_file_provider.open()}; if (fuzzed_block_file.IsNull()) { return; } diff --git a/src/test/fuzz/merkleblock.cpp b/src/test/fuzz/merkleblock.cpp index 6271367a9c..e03e628444 100644 --- a/src/test/fuzz/merkleblock.cpp +++ b/src/test/fuzz/merkleblock.cpp @@ -27,15 +27,15 @@ FUZZ_TARGET(merkleblock) }, [&] { CMerkleBlock merkle_block; - const std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider); + const std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider, TX_WITH_WITNESS); CBloomFilter bloom_filter; - std::set<uint256> txids; + std::set<Txid> txids; if (opt_block && !opt_block->vtx.empty()) { if (fuzzed_data_provider.ConsumeBool()) { merkle_block = CMerkleBlock{*opt_block, bloom_filter}; } else if (fuzzed_data_provider.ConsumeBool()) { LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { - txids.insert(ConsumeUInt256(fuzzed_data_provider)); + txids.insert(Txid::FromUint256(ConsumeUInt256(fuzzed_data_provider))); } merkle_block = CMerkleBlock{*opt_block, txids}; } diff --git a/src/test/fuzz/mini_miner.cpp b/src/test/fuzz/mini_miner.cpp index 2f53943c31..84f9bb4ad0 100644 --- a/src/test/fuzz/mini_miner.cpp +++ b/src/test/fuzz/mini_miner.cpp @@ -25,7 +25,7 @@ void initialize_miner() static const auto testing_setup = MakeNoLogFileContext<const TestingSetup>(); g_setup = testing_setup.get(); for (uint32_t i = 0; i < uint32_t{100}; ++i) { - g_available_coins.emplace_back(uint256::ZERO, i); + g_available_coins.emplace_back(Txid::FromUint256(uint256::ZERO), i); } } diff --git a/src/test/fuzz/miniscript.cpp b/src/test/fuzz/miniscript.cpp index 8c73edfa9d..0d0ee59ab3 100644 --- a/src/test/fuzz/miniscript.cpp +++ b/src/test/fuzz/miniscript.cpp @@ -1121,7 +1121,7 @@ void TestNode(const MsCtx script_ctx, const NodeRef& node, FuzzedDataProvider& p assert(mal_success); assert(stack_nonmal == stack_mal); // Compute witness size (excluding script push, control block, and witness count encoding). - const size_t wit_size = GetSerializeSize(stack_nonmal, PROTOCOL_VERSION) - GetSizeOfCompactSize(stack_nonmal.size()); + const size_t wit_size = GetSerializeSize(stack_nonmal) - GetSizeOfCompactSize(stack_nonmal.size()); assert(wit_size <= *node->GetWitnessSize()); // Test non-malleable satisfaction. diff --git a/src/test/fuzz/minisketch.cpp b/src/test/fuzz/minisketch.cpp index a17be73f6c..698cb15fc9 100644 --- a/src/test/fuzz/minisketch.cpp +++ b/src/test/fuzz/minisketch.cpp @@ -12,14 +12,27 @@ #include <map> #include <numeric> -using node::MakeMinisketch32; +namespace { + +Minisketch MakeFuzzMinisketch32(size_t capacity, uint32_t impl) +{ + return Assert(Minisketch(32, impl, capacity)); +} + +} // namespace FUZZ_TARGET(minisketch) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + const auto capacity{fuzzed_data_provider.ConsumeIntegralInRange<size_t>(1, 200)}; - Minisketch sketch_a{Assert(MakeMinisketch32(capacity))}; - Minisketch sketch_b{Assert(MakeMinisketch32(capacity))}; + const uint32_t impl{fuzzed_data_provider.ConsumeIntegralInRange<uint32_t>(0, Minisketch::MaxImplementation())}; + if (!Minisketch::ImplementationSupported(32, impl)) return; + + Minisketch sketch_a{MakeFuzzMinisketch32(capacity, impl)}; + Minisketch sketch_b{MakeFuzzMinisketch32(capacity, impl)}; + sketch_a.SetSeed(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + sketch_b.SetSeed(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); // Fill two sets and keep the difference in a map std::map<uint32_t, bool> diff; @@ -47,8 +60,11 @@ FUZZ_TARGET(minisketch) } const auto num_diff{std::accumulate(diff.begin(), diff.end(), size_t{0}, [](auto n, const auto& e) { return n + e.second; })}; - Minisketch sketch_ar{MakeMinisketch32(capacity)}; - Minisketch sketch_br{MakeMinisketch32(capacity)}; + Minisketch sketch_ar{MakeFuzzMinisketch32(capacity, impl)}; + Minisketch sketch_br{MakeFuzzMinisketch32(capacity, impl)}; + sketch_ar.SetSeed(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + sketch_br.SetSeed(fuzzed_data_provider.ConsumeIntegral<uint64_t>()); + sketch_ar.Deserialize(sketch_a.Serialize()); sketch_br.Deserialize(sketch_b.Serialize()); diff --git a/src/test/fuzz/netbase_dns_lookup.cpp b/src/test/fuzz/netbase_dns_lookup.cpp index dcf500acc3..ba31315297 100644 --- a/src/test/fuzz/netbase_dns_lookup.cpp +++ b/src/test/fuzz/netbase_dns_lookup.cpp @@ -59,9 +59,6 @@ FUZZ_TARGET(netbase_dns_lookup) assert(!resolved_service.IsInternal()); } { - CSubNet resolved_subnet; - if (LookupSubNet(name, resolved_subnet)) { - assert(resolved_subnet.IsValid()); - } + (void)LookupSubNet(name); } } diff --git a/src/test/fuzz/p2p_transport_serialization.cpp b/src/test/fuzz/p2p_transport_serialization.cpp index 21d8dab536..a205ce19f4 100644 --- a/src/test/fuzz/p2p_transport_serialization.cpp +++ b/src/test/fuzz/p2p_transport_serialization.cpp @@ -36,8 +36,8 @@ void initialize_p2p_transport_serialization() FUZZ_TARGET(p2p_transport_serialization, .init = initialize_p2p_transport_serialization) { // Construct transports for both sides, with dummy NodeIds. - V1Transport recv_transport{NodeId{0}, SER_NETWORK, INIT_PROTO_VERSION}; - V1Transport send_transport{NodeId{1}, SER_NETWORK, INIT_PROTO_VERSION}; + V1Transport recv_transport{NodeId{0}}; + V1Transport send_transport{NodeId{1}}; FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; @@ -88,7 +88,7 @@ FUZZ_TARGET(p2p_transport_serialization, .init = initialize_p2p_transport_serial assert(msg.m_time == m_time); std::vector<unsigned char> header; - auto msg2 = CNetMsgMaker{msg.m_recv.GetVersion()}.Make(msg.m_type, Span{msg.m_recv}); + auto msg2 = NetMsg::Make(msg.m_type, Span{msg.m_recv}); bool queued = send_transport.SetMessageToSend(msg2); assert(queued); std::optional<bool> known_more; @@ -335,7 +335,7 @@ void SimulationTest(Transport& initiator, Transport& responder, R& rng, FuzzedDa std::unique_ptr<Transport> MakeV1Transport(NodeId nodeid) noexcept { - return std::make_unique<V1Transport>(nodeid, SER_NETWORK, INIT_PROTO_VERSION); + return std::make_unique<V1Transport>(nodeid); } template<typename RNG> @@ -369,7 +369,7 @@ std::unique_ptr<Transport> MakeV2Transport(NodeId nodeid, bool initiator, RNG& r .Write(garb.data(), garb.size()) .Finalize(UCharCast(ent.data())); - return std::make_unique<V2Transport>(nodeid, initiator, SER_NETWORK, INIT_PROTO_VERSION, key, ent, std::move(garb)); + return std::make_unique<V2Transport>(nodeid, initiator, key, ent, std::move(garb)); } } // namespace diff --git a/src/test/fuzz/package_eval.cpp b/src/test/fuzz/package_eval.cpp index 4c81c0b679..5a08d0ff44 100644 --- a/src/test/fuzz/package_eval.cpp +++ b/src/test/fuzz/package_eval.cpp @@ -40,7 +40,7 @@ void initialize_tx_pool() g_setup = testing_setup.get(); for (int i = 0; i < 2 * COINBASE_MATURITY; ++i) { - COutPoint prevout{MineBlock(g_setup->m_node, P2WSH_OP_TRUE)}; + COutPoint prevout{MineBlock(g_setup->m_node, P2WSH_EMPTY)}; if (i < COINBASE_MATURITY) { // Remember the txids to avoid expensive disk access later on g_outpoints_coinbase_init_mature.push_back(prevout); @@ -55,13 +55,13 @@ struct OutpointsUpdater final : public CValidationInterface { explicit OutpointsUpdater(std::set<COutPoint>& r) : m_mempool_outpoints{r} {} - void TransactionAddedToMempool(const CTransactionRef& tx, uint64_t /* mempool_sequence */) override + void TransactionAddedToMempool(const NewMempoolTransactionInfo& tx, uint64_t /* mempool_sequence */) override { // for coins spent we always want to be able to rbf so they're not removed // outputs from this tx can now be spent - for (uint32_t index{0}; index < tx->vout.size(); ++index) { - m_mempool_outpoints.insert(COutPoint{tx->GetHash(), index}); + for (uint32_t index{0}; index < tx.info.m_tx->vout.size(); ++index) { + m_mempool_outpoints.insert(COutPoint{tx.info.m_tx->GetHash(), index}); } } @@ -85,10 +85,10 @@ struct TransactionsDelta final : public CValidationInterface { explicit TransactionsDelta(std::set<CTransactionRef>& a) : m_added{a} {} - void TransactionAddedToMempool(const CTransactionRef& tx, uint64_t /* mempool_sequence */) override + void TransactionAddedToMempool(const NewMempoolTransactionInfo& tx, uint64_t /* mempool_sequence */) override { // Transactions may be entered and booted any number of times - m_added.insert(tx); + m_added.insert(tx.info.m_tx); } void TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason, uint64_t /* mempool_sequence */) override @@ -121,7 +121,6 @@ CTxMemPool MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeConte mempool_opts.expiry = std::chrono::hours{fuzzed_data_provider.ConsumeIntegralInRange<unsigned>(0, 999)}; nBytesPerSigOp = fuzzed_data_provider.ConsumeIntegralInRange<unsigned>(1, 999); - mempool_opts.estimator = nullptr; mempool_opts.check_ratio = 1; mempool_opts.require_standard = fuzzed_data_provider.ConsumeBool(); @@ -195,7 +194,8 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) // Create input const auto sequence = ConsumeSequence(fuzzed_data_provider); const auto script_sig = CScript{}; - const auto script_wit_stack = std::vector<std::vector<uint8_t>>{WITNESS_STACK_ELEM_OP_TRUE}; + const auto script_wit_stack = fuzzed_data_provider.ConsumeBool() ? P2WSH_EMPTY_TRUE_STACK : P2WSH_EMPTY_TWO_STACK; + CTxIn in; in.prevout = outpoint; in.nSequence = sequence; @@ -204,17 +204,30 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) tx_mut.vin.push_back(in); } + + // Duplicate an input + bool dup_input = fuzzed_data_provider.ConsumeBool(); + if (dup_input) { + tx_mut.vin.push_back(tx_mut.vin.back()); + } + + // Refer to a non-existant input + if (fuzzed_data_provider.ConsumeBool()) { + tx_mut.vin.emplace_back(); + } + const auto amount_fee = fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(0, amount_in); const auto amount_out = (amount_in - amount_fee) / num_out; for (int i = 0; i < num_out; ++i) { - tx_mut.vout.emplace_back(amount_out, P2WSH_OP_TRUE); + tx_mut.vout.emplace_back(amount_out, P2WSH_EMPTY); } // TODO vary transaction sizes to catch size-related issues auto tx = MakeTransactionRef(tx_mut); // Restore previously removed outpoints, except in-package outpoints if (!last_tx) { for (const auto& in : tx->vin) { - Assert(outpoints.insert(in.prevout).second); + // It's a fake input, or a new input, or a duplicate + Assert(in == CTxIn() || outpoints.insert(in.prevout).second || dup_input); } // Cache the in-package outpoints being made for (size_t i = 0; i < tx->vout.size(); ++i) { @@ -241,7 +254,7 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) txs.back()->GetHash() : PickValue(fuzzed_data_provider, mempool_outpoints).hash; const auto delta = fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(-50 * COIN, +50 * COIN); - tx_pool.PrioritiseTransaction(txid, delta); + tx_pool.PrioritiseTransaction(txid.ToUint256(), delta); } // Remember all added transactions @@ -257,15 +270,6 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) const auto result_package = WITH_LOCK(::cs_main, return ProcessNewPackage(chainstate, tx_pool, txs, /*test_accept=*/single_submit)); - // If something went wrong due to a package-specific policy, it might not return a - // validation result for the transaction. - if (result_package.m_state.GetResult() != PackageValidationResult::PCKG_POLICY) { - auto it = result_package.m_tx_results.find(txs.back()->GetWitnessHash()); - Assert(it != result_package.m_tx_results.end()); - Assert(it->second.m_result_type == MempoolAcceptResult::ResultType::VALID || - it->second.m_result_type == MempoolAcceptResult::ResultType::INVALID || - it->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); - } const auto res = WITH_LOCK(::cs_main, return AcceptToMemoryPool(chainstate, txs.back(), GetTime(), bypass_limits, /*test_accept=*/!single_submit)); const bool accepted = res.m_result_type == MempoolAcceptResult::ResultType::VALID; @@ -281,6 +285,12 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) Assert(added.size() == 1); Assert(txs.back() == *added.begin()); } + } else if (result_package.m_state.GetResult() != PackageValidationResult::PCKG_POLICY) { + // We don't know anything about the validity since transactions were randomly generated, so + // just use result_package.m_state here. This makes the expect_valid check meaningless, but + // we can still verify that the contents of m_tx_results are consistent with m_state. + const bool expect_valid{result_package.m_state.IsValid()}; + Assert(!CheckPackageMempoolAcceptResult(txs, result_package, expect_valid, nullptr)); } else { // This is empty if it fails early checks, or "full" if transactions are looked at deeper Assert(result_package.m_tx_results.size() == txs.size() || result_package.m_tx_results.empty()); diff --git a/src/test/fuzz/partially_downloaded_block.cpp b/src/test/fuzz/partially_downloaded_block.cpp index ae7a68762e..4a4b46da60 100644 --- a/src/test/fuzz/partially_downloaded_block.cpp +++ b/src/test/fuzz/partially_downloaded_block.cpp @@ -44,7 +44,7 @@ FUZZ_TARGET(partially_downloaded_block, .init = initialize_pdb) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - auto block{ConsumeDeserializable<CBlock>(fuzzed_data_provider)}; + auto block{ConsumeDeserializable<CBlock>(fuzzed_data_provider, TX_WITH_WITNESS)}; if (!block || block->vtx.size() == 0 || block->vtx.size() >= std::numeric_limits<uint16_t>::max()) { return; diff --git a/src/test/fuzz/policy_estimator.cpp b/src/test/fuzz/policy_estimator.cpp index 227ee9d2c4..a4e1947b9f 100644 --- a/src/test/fuzz/policy_estimator.cpp +++ b/src/test/fuzz/policy_estimator.cpp @@ -6,16 +6,15 @@ #include <policy/fees.h> #include <policy/fees_args.h> #include <primitives/transaction.h> +#include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <test/fuzz/util/mempool.h> #include <test/util/setup_common.h> -#include <txmempool.h> -#include <cstdint> +#include <memory> #include <optional> -#include <string> #include <vector> namespace { @@ -31,40 +30,55 @@ void initialize_policy_estimator() FUZZ_TARGET(policy_estimator, .init = initialize_policy_estimator) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + bool good_data{true}; + CBlockPolicyEstimator block_policy_estimator{FeeestPath(*g_setup->m_node.args), DEFAULT_ACCEPT_STALE_FEE_ESTIMATES}; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 10'000) + { CallOneOf( fuzzed_data_provider, [&] { - const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!mtx) { + good_data = false; return; } const CTransaction tx{*mtx}; - block_policy_estimator.processTransaction(ConsumeTxMemPoolEntry(fuzzed_data_provider, tx), fuzzed_data_provider.ConsumeBool()); + const CTxMemPoolEntry& entry = ConsumeTxMemPoolEntry(fuzzed_data_provider, tx); + const auto tx_submitted_in_package = fuzzed_data_provider.ConsumeBool(); + const auto tx_has_mempool_parents = fuzzed_data_provider.ConsumeBool(); + const auto tx_info = NewMempoolTransactionInfo(entry.GetSharedTx(), entry.GetFee(), + entry.GetTxSize(), entry.GetHeight(), + /*mempool_limit_bypassed=*/false, + tx_submitted_in_package, + /*chainstate_is_current=*/true, + tx_has_mempool_parents); + block_policy_estimator.processTransaction(tx_info); if (fuzzed_data_provider.ConsumeBool()) { - (void)block_policy_estimator.removeTx(tx.GetHash(), /*inBlock=*/fuzzed_data_provider.ConsumeBool()); + (void)block_policy_estimator.removeTx(tx.GetHash()); } }, [&] { - std::vector<CTxMemPoolEntry> mempool_entries; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { - const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + std::list<CTxMemPoolEntry> mempool_entries; + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) + { + const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!mtx) { + good_data = false; break; } const CTransaction tx{*mtx}; - mempool_entries.push_back(ConsumeTxMemPoolEntry(fuzzed_data_provider, tx)); + mempool_entries.emplace_back(CTxMemPoolEntry::ExplicitCopy, ConsumeTxMemPoolEntry(fuzzed_data_provider, tx)); } - std::vector<const CTxMemPoolEntry*> ptrs; - ptrs.reserve(mempool_entries.size()); + std::vector<RemovedMempoolTransactionInfo> txs; + txs.reserve(mempool_entries.size()); for (const CTxMemPoolEntry& mempool_entry : mempool_entries) { - ptrs.push_back(&mempool_entry); + txs.emplace_back(mempool_entry); } - block_policy_estimator.processBlock(fuzzed_data_provider.ConsumeIntegral<unsigned int>(), ptrs); + block_policy_estimator.processBlock(txs, fuzzed_data_provider.ConsumeIntegral<unsigned int>()); }, [&] { - (void)block_policy_estimator.removeTx(ConsumeUInt256(fuzzed_data_provider), /*inBlock=*/fuzzed_data_provider.ConsumeBool()); + (void)block_policy_estimator.removeTx(ConsumeUInt256(fuzzed_data_provider)); }, [&] { block_policy_estimator.FlushUnconfirmed(); @@ -77,8 +91,8 @@ FUZZ_TARGET(policy_estimator, .init = initialize_policy_estimator) (void)block_policy_estimator.HighestTargetTracked(fuzzed_data_provider.PickValueInArray(ALL_FEE_ESTIMATE_HORIZONS)); } { - FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); - AutoFile fuzzed_auto_file{fuzzed_auto_file_provider.open()}; + FuzzedFileProvider fuzzed_file_provider{fuzzed_data_provider}; + AutoFile fuzzed_auto_file{fuzzed_file_provider.open()}; block_policy_estimator.Write(fuzzed_auto_file); block_policy_estimator.Read(fuzzed_auto_file); } diff --git a/src/test/fuzz/policy_estimator_io.cpp b/src/test/fuzz/policy_estimator_io.cpp index c04ef8f5b0..3e7d093343 100644 --- a/src/test/fuzz/policy_estimator_io.cpp +++ b/src/test/fuzz/policy_estimator_io.cpp @@ -4,13 +4,13 @@ #include <policy/fees.h> #include <policy/fees_args.h> +#include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <test/util/setup_common.h> -#include <cstdint> -#include <vector> +#include <memory> namespace { const BasicTestingSetup* g_setup; @@ -25,8 +25,8 @@ void initialize_policy_estimator_io() FUZZ_TARGET(policy_estimator_io, .init = initialize_policy_estimator_io) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - FuzzedAutoFileProvider fuzzed_auto_file_provider = ConsumeAutoFile(fuzzed_data_provider); - AutoFile fuzzed_auto_file{fuzzed_auto_file_provider.open()}; + FuzzedFileProvider fuzzed_file_provider{fuzzed_data_provider}; + AutoFile fuzzed_auto_file{fuzzed_file_provider.open()}; // Re-using block_policy_estimator across runs to avoid costly creation of CBlockPolicyEstimator object. static CBlockPolicyEstimator block_policy_estimator{FeeestPath(*g_setup->m_node.args), DEFAULT_ACCEPT_STALE_FEE_ESTIMATES}; if (block_policy_estimator.Read(fuzzed_auto_file)) { diff --git a/src/test/fuzz/primitives_transaction.cpp b/src/test/fuzz/primitives_transaction.cpp index 48815c8910..64f1c8cf8d 100644 --- a/src/test/fuzz/primitives_transaction.cpp +++ b/src/test/fuzz/primitives_transaction.cpp @@ -24,8 +24,8 @@ FUZZ_TARGET(primitives_transaction) const CTxOut tx_out_1{ConsumeMoney(fuzzed_data_provider), script}; const CTxOut tx_out_2{ConsumeMoney(fuzzed_data_provider), ConsumeScript(fuzzed_data_provider)}; assert((tx_out_1 == tx_out_2) != (tx_out_1 != tx_out_2)); - const std::optional<CMutableTransaction> mutable_tx_1 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); - const std::optional<CMutableTransaction> mutable_tx_2 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mutable_tx_1 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); + const std::optional<CMutableTransaction> mutable_tx_2 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (mutable_tx_1 && mutable_tx_2) { const CTransaction tx_1{*mutable_tx_1}; const CTransaction tx_2{*mutable_tx_2}; diff --git a/src/test/fuzz/process_message.cpp b/src/test/fuzz/process_message.cpp index d38d1bb40e..56b391ed5c 100644 --- a/src/test/fuzz/process_message.cpp +++ b/src/test/fuzz/process_message.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022 The Bitcoin Core developers +// Copyright (c) 2020-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -8,9 +8,6 @@ #include <primitives/transaction.h> #include <protocol.h> #include <script/script.h> -#include <serialize.h> -#include <span.h> -#include <streams.h> #include <sync.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> @@ -20,15 +17,10 @@ #include <test/util/net.h> #include <test/util/setup_common.h> #include <test/util/validation.h> -#include <util/chaintype.h> #include <util/check.h> #include <util/time.h> -#include <validation.h> #include <validationinterface.h> -#include <version.h> - -#include <atomic> #include <cstdlib> #include <iostream> #include <memory> @@ -81,14 +73,22 @@ FUZZ_TARGET(process_message, .init = initialize_process_message) const auto mock_time = ConsumeTime(fuzzed_data_provider); SetMockTime(mock_time); - // fuzzed_data_provider is fully consumed after this call, don't use it - CDataStream random_bytes_data_stream{fuzzed_data_provider.ConsumeRemainingBytes<unsigned char>(), SER_NETWORK, PROTOCOL_VERSION}; - try { - g_setup->m_node.peerman->ProcessMessage(p2p_node, random_message_type, random_bytes_data_stream, - GetTime<std::chrono::microseconds>(), std::atomic<bool>{false}); - } catch (const std::ios_base::failure&) { + CSerializedNetMsg net_msg; + net_msg.m_type = random_message_type; + net_msg.data = ConsumeRandomLengthByteVector(fuzzed_data_provider, MAX_PROTOCOL_MESSAGE_LENGTH); + + connman.FlushSendBuffer(p2p_node); + (void)connman.ReceiveMsgFrom(p2p_node, std::move(net_msg)); + + bool more_work{true}; + while (more_work) { + p2p_node.fPauseSend = false; + try { + more_work = connman.ProcessMessagesOnce(p2p_node); + } catch (const std::ios_base::failure&) { + } + g_setup->m_node.peerman->SendMessages(&p2p_node); } - g_setup->m_node.peerman->SendMessages(&p2p_node); SyncWithValidationInterfaceQueue(); g_setup->m_node.connman->StopNodes(); } diff --git a/src/test/fuzz/process_messages.cpp b/src/test/fuzz/process_messages.cpp index 4cb388c20b..6b264907b5 100644 --- a/src/test/fuzz/process_messages.cpp +++ b/src/test/fuzz/process_messages.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2020-2022 The Bitcoin Core developers +// Copyright (c) 2020-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -6,6 +6,8 @@ #include <net.h> #include <net_processing.h> #include <protocol.h> +#include <script/script.h> +#include <sync.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> @@ -14,9 +16,14 @@ #include <test/util/net.h> #include <test/util/setup_common.h> #include <test/util/validation.h> -#include <validation.h> +#include <util/time.h> #include <validationinterface.h> +#include <ios> +#include <string> +#include <utility> +#include <vector> + namespace { const TestingSetup* g_setup; } // namespace @@ -55,7 +62,8 @@ FUZZ_TARGET(process_messages, .init = initialize_process_messages) connman.AddTestNode(p2p_node); } - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 30) + { const std::string random_message_type{fuzzed_data_provider.ConsumeBytesAsString(CMessageHeader::COMMAND_SIZE).c_str()}; const auto mock_time = ConsumeTime(fuzzed_data_provider); @@ -63,19 +71,23 @@ FUZZ_TARGET(process_messages, .init = initialize_process_messages) CSerializedNetMsg net_msg; net_msg.m_type = random_message_type; - net_msg.data = ConsumeRandomLengthByteVector(fuzzed_data_provider); + net_msg.data = ConsumeRandomLengthByteVector(fuzzed_data_provider, MAX_PROTOCOL_MESSAGE_LENGTH); CNode& random_node = *PickValue(fuzzed_data_provider, peers); connman.FlushSendBuffer(random_node); (void)connman.ReceiveMsgFrom(random_node, std::move(net_msg)); - random_node.fPauseSend = false; - try { - connman.ProcessMessagesOnce(random_node); - } catch (const std::ios_base::failure&) { + bool more_work{true}; + while (more_work) { // Ensure that every message is eventually processed in some way or another + random_node.fPauseSend = false; + + try { + more_work = connman.ProcessMessagesOnce(random_node); + } catch (const std::ios_base::failure&) { + } + g_setup->m_node.peerman->SendMessages(&random_node); } - g_setup->m_node.peerman->SendMessages(&random_node); } SyncWithValidationInterfaceQueue(); g_setup->m_node.connman->StopNodes(); diff --git a/src/test/fuzz/psbt.cpp b/src/test/fuzz/psbt.cpp index 825ed67ec1..e0692600bb 100644 --- a/src/test/fuzz/psbt.cpp +++ b/src/test/fuzz/psbt.cpp @@ -11,7 +11,6 @@ #include <script/script.h> #include <streams.h> #include <util/check.h> -#include <version.h> #include <cstdint> #include <optional> diff --git a/src/test/fuzz/rbf.cpp b/src/test/fuzz/rbf.cpp index dbe99029c3..aa6385d12d 100644 --- a/src/test/fuzz/rbf.cpp +++ b/src/test/fuzz/rbf.cpp @@ -33,7 +33,7 @@ FUZZ_TARGET(rbf, .init = initialize_rbf) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); SetMockTime(ConsumeTime(fuzzed_data_provider)); - std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!mtx) { return; } @@ -42,7 +42,7 @@ FUZZ_TARGET(rbf, .init = initialize_rbf) LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { - const std::optional<CMutableTransaction> another_mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> another_mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!another_mtx) { break; } diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 270cab58e2..2325bf0941 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -3,18 +3,14 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <base58.h> -#include <core_io.h> #include <key.h> #include <key_io.h> -#include <node/context.h> #include <primitives/block.h> #include <primitives/transaction.h> #include <psbt.h> -#include <rpc/blockchain.h> #include <rpc/client.h> #include <rpc/request.h> #include <rpc/server.h> -#include <rpc/util.h> #include <span.h> #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> @@ -22,19 +18,23 @@ #include <test/fuzz/util.h> #include <test/util/setup_common.h> #include <tinyformat.h> +#include <uint256.h> #include <univalue.h> -#include <util/chaintype.h> #include <util/strencodings.h> #include <util/string.h> #include <util/time.h> +#include <algorithm> +#include <cassert> #include <cstdint> +#include <cstdlib> +#include <exception> #include <iostream> #include <memory> #include <optional> #include <stdexcept> -#include <string> #include <vector> +enum class ChainType; namespace { struct RPCFuzzTestingSetup : public TestingSetup { @@ -184,7 +184,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ "waitfornewblock", }; -std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider, bool& good_data) { const size_t max_string_length = 4096; const size_t max_base58_bytes_length{64}; @@ -249,18 +249,20 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) }, [&] { // hex encoded block - std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider); + std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider, TX_WITH_WITNESS); if (!opt_block) { + good_data = false; return; } - CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION}; - data_stream << *opt_block; + DataStream data_stream{}; + data_stream << TX_WITH_WITNESS(*opt_block); r = HexStr(data_stream); }, [&] { // hex encoded block header std::optional<CBlockHeader> opt_block_header = ConsumeDeserializable<CBlockHeader>(fuzzed_data_provider); if (!opt_block_header) { + good_data = false; return; } DataStream data_stream{}; @@ -269,21 +271,24 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) }, [&] { // hex encoded tx - std::optional<CMutableTransaction> opt_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + std::optional<CMutableTransaction> opt_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!opt_tx) { + good_data = false; return; } - CDataStream data_stream{SER_NETWORK, fuzzed_data_provider.ConsumeBool() ? PROTOCOL_VERSION : (PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS)}; - data_stream << *opt_tx; + DataStream data_stream; + auto allow_witness = (fuzzed_data_provider.ConsumeBool() ? TX_WITH_WITNESS : TX_NO_WITNESS); + data_stream << allow_witness(*opt_tx); r = HexStr(data_stream); }, [&] { // base64 encoded psbt std::optional<PartiallySignedTransaction> opt_psbt = ConsumeDeserializable<PartiallySignedTransaction>(fuzzed_data_provider); if (!opt_psbt) { + good_data = false; return; } - CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION}; + DataStream data_stream{}; data_stream << *opt_psbt; r = EncodeBase64(data_stream); }, @@ -291,6 +296,7 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) // base58 encoded key CKey key = ConsumePrivateKey(fuzzed_data_provider); if (!key.IsValid()) { + good_data = false; return; } r = EncodeSecret(key); @@ -299,6 +305,7 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) // hex encoded pubkey CKey key = ConsumePrivateKey(fuzzed_data_provider); if (!key.IsValid()) { + good_data = false; return; } r = HexStr(key.GetPubKey()); @@ -306,18 +313,19 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) return r; } -std::string ConsumeArrayRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +std::string ConsumeArrayRPCArgument(FuzzedDataProvider& fuzzed_data_provider, bool& good_data) { std::vector<std::string> scalar_arguments; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) { - scalar_arguments.push_back(ConsumeScalarRPCArgument(fuzzed_data_provider)); + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 100) + { + scalar_arguments.push_back(ConsumeScalarRPCArgument(fuzzed_data_provider, good_data)); } return "[\"" + Join(scalar_arguments, "\",\"") + "\"]"; } -std::string ConsumeRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +std::string ConsumeRPCArgument(FuzzedDataProvider& fuzzed_data_provider, bool& good_data) { - return fuzzed_data_provider.ConsumeBool() ? ConsumeScalarRPCArgument(fuzzed_data_provider) : ConsumeArrayRPCArgument(fuzzed_data_provider); + return fuzzed_data_provider.ConsumeBool() ? ConsumeScalarRPCArgument(fuzzed_data_provider, good_data) : ConsumeArrayRPCArgument(fuzzed_data_provider, good_data); } RPCFuzzTestingSetup* InitializeRPCFuzzTestingSetup() @@ -353,6 +361,7 @@ void initialize_rpc() FUZZ_TARGET(rpc, .init = initialize_rpc) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + bool good_data{true}; SetMockTime(ConsumeTime(fuzzed_data_provider)); const std::string rpc_command = fuzzed_data_provider.ConsumeRandomLengthString(64); if (!g_limit_to_rpc_command.empty() && rpc_command != g_limit_to_rpc_command) { @@ -363,16 +372,15 @@ FUZZ_TARGET(rpc, .init = initialize_rpc) return; } std::vector<std::string> arguments; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) { - arguments.push_back(ConsumeRPCArgument(fuzzed_data_provider)); + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 100) + { + arguments.push_back(ConsumeRPCArgument(fuzzed_data_provider, good_data)); } try { rpc_testing_setup->CallRPC(rpc_command, arguments); } catch (const UniValue& json_rpc_error) { const std::string error_msg{json_rpc_error.find_value("message").get_str()}; - // Once c++20 is allowed, starts_with can be used. - // if (error_msg.starts_with("Internal bug detected")) { - if (0 == error_msg.rfind("Internal bug detected", 0)) { + if (error_msg.starts_with("Internal bug detected")) { // Only allow the intentional internal bug assert(error_msg.find("trigger_internal_bug") != std::string::npos); } diff --git a/src/test/fuzz/script_assets_test_minimizer.cpp b/src/test/fuzz/script_assets_test_minimizer.cpp index 66c862a6f9..511b581f60 100644 --- a/src/test/fuzz/script_assets_test_minimizer.cpp +++ b/src/test/fuzz/script_assets_test_minimizer.cpp @@ -54,7 +54,7 @@ CMutableTransaction TxFromHex(const std::string& str) { CMutableTransaction tx; try { - SpanReader{SERIALIZE_TRANSACTION_NO_WITNESS, CheckedParseHex(str)} >> tx; + SpanReader{CheckedParseHex(str)} >> TX_NO_WITNESS(tx); } catch (const std::ios_base::failure&) { throw std::runtime_error("Tx deserialization failure"); } @@ -68,7 +68,7 @@ std::vector<CTxOut> TxOutsFromJSON(const UniValue& univalue) for (size_t i = 0; i < univalue.size(); ++i) { CTxOut txout; try { - SpanReader{0, CheckedParseHex(univalue[i].get_str())} >> txout; + SpanReader{CheckedParseHex(univalue[i].get_str())} >> txout; } catch (const std::ios_base::failure&) { throw std::runtime_error("Prevout invalid format"); } diff --git a/src/test/fuzz/script_flags.cpp b/src/test/fuzz/script_flags.cpp index f8594fc233..accb32f1cc 100644 --- a/src/test/fuzz/script_flags.cpp +++ b/src/test/fuzz/script_flags.cpp @@ -3,27 +3,24 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <consensus/amount.h> -#include <pubkey.h> +#include <primitives/transaction.h> #include <script/interpreter.h> +#include <serialize.h> #include <streams.h> +#include <test/fuzz/fuzz.h> #include <test/util/script.h> -#include <version.h> -#include <test/fuzz/fuzz.h> +#include <cassert> +#include <ios> +#include <utility> +#include <vector> FUZZ_TARGET(script_flags) { - CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); - try { - int nVersion; - ds >> nVersion; - ds.SetVersion(nVersion); - } catch (const std::ios_base::failure&) { - return; - } - + if (buffer.size() > 100'000) return; + DataStream ds{buffer}; try { - const CTransaction tx(deserialize, ds); + const CTransaction tx(deserialize, TX_WITH_WITNESS, ds); unsigned int verify_flags; ds >> verify_flags; diff --git a/src/test/fuzz/script_interpreter.cpp b/src/test/fuzz/script_interpreter.cpp index 5d59771682..5e76443abe 100644 --- a/src/test/fuzz/script_interpreter.cpp +++ b/src/test/fuzz/script_interpreter.cpp @@ -20,13 +20,13 @@ FUZZ_TARGET(script_interpreter) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); { const CScript script_code = ConsumeScript(fuzzed_data_provider); - const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (mtx) { const CTransaction tx_to{*mtx}; const unsigned int in = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); if (in < tx_to.vin.size()) { (void)SignatureHash(script_code, tx_to, in, fuzzed_data_provider.ConsumeIntegral<int>(), ConsumeMoney(fuzzed_data_provider), fuzzed_data_provider.PickValueInArray({SigVersion::BASE, SigVersion::WITNESS_V0}), nullptr); - const std::optional<CMutableTransaction> mtx_precomputed = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mtx_precomputed = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (mtx_precomputed) { const CTransaction tx_precomputed{*mtx_precomputed}; const PrecomputedTransactionData precomputed_transaction_data{tx_precomputed}; diff --git a/src/test/fuzz/script_sigcache.cpp b/src/test/fuzz/script_sigcache.cpp index 486b1e5197..5fdbc9e106 100644 --- a/src/test/fuzz/script_sigcache.cpp +++ b/src/test/fuzz/script_sigcache.cpp @@ -30,7 +30,7 @@ FUZZ_TARGET(script_sigcache, .init = initialize_script_sigcache) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - const std::optional<CMutableTransaction> mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); const CTransaction tx{mutable_transaction ? *mutable_transaction : CMutableTransaction{}}; const unsigned int n_in = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); const CAmount amount = ConsumeMoney(fuzzed_data_provider); diff --git a/src/test/fuzz/script_sign.cpp b/src/test/fuzz/script_sign.cpp index 179715b6ea..9ae150e553 100644 --- a/src/test/fuzz/script_sign.cpp +++ b/src/test/fuzz/script_sign.cpp @@ -36,14 +36,13 @@ FUZZ_TARGET(script_sign, .init = initialize_script_sign) const std::vector<uint8_t> key = ConsumeRandomLengthByteVector(fuzzed_data_provider, 128); { - DataStream stream{ConsumeDataStream(fuzzed_data_provider)}; - CDataStream random_data_stream{stream, SER_NETWORK, INIT_PROTO_VERSION}; // temporary copy, to be removed along with the version flag SERIALIZE_TRANSACTION_NO_WITNESS + DataStream random_data_stream{ConsumeDataStream(fuzzed_data_provider)}; std::map<CPubKey, KeyOriginInfo> hd_keypaths; try { DeserializeHDKeypaths(random_data_stream, key, hd_keypaths); } catch (const std::ios_base::failure&) { } - CDataStream serialized{SER_NETWORK, PROTOCOL_VERSION}; + DataStream serialized{}; SerializeHDKeypaths(serialized, hd_keypaths, CompactSizeWriter(fuzzed_data_provider.ConsumeIntegral<uint8_t>())); } @@ -60,7 +59,7 @@ FUZZ_TARGET(script_sign, .init = initialize_script_sign) } hd_keypaths[*pub_key] = *key_origin_info; } - CDataStream serialized{SER_NETWORK, PROTOCOL_VERSION}; + DataStream serialized{}; try { SerializeHDKeypaths(serialized, hd_keypaths, CompactSizeWriter(fuzzed_data_provider.ConsumeIntegral<uint8_t>())); } catch (const std::ios_base::failure&) { @@ -86,7 +85,7 @@ FUZZ_TARGET(script_sign, .init = initialize_script_sign) } { - const std::optional<CMutableTransaction> mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); const std::optional<CTxOut> tx_out = ConsumeDeserializable<CTxOut>(fuzzed_data_provider); const unsigned int n_in = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); if (mutable_transaction && tx_out && mutable_transaction->vin.size() > n_in) { @@ -100,7 +99,7 @@ FUZZ_TARGET(script_sign, .init = initialize_script_sign) if (mutable_transaction) { CTransaction tx_from{*mutable_transaction}; CMutableTransaction tx_to; - const std::optional<CMutableTransaction> opt_tx_to = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> opt_tx_to = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (opt_tx_to) { tx_to = *opt_tx_to; } @@ -125,18 +124,7 @@ FUZZ_TARGET(script_sign, .init = initialize_script_sign) } (void)signature_creator.CreateSig(provider, vch_sig, address, ConsumeScript(fuzzed_data_provider), fuzzed_data_provider.PickValueInArray({SigVersion::BASE, SigVersion::WITNESS_V0})); } - std::map<COutPoint, Coin> coins; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { - const std::optional<COutPoint> outpoint = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); - if (!outpoint) { - break; - } - const std::optional<Coin> coin = ConsumeDeserializable<Coin>(fuzzed_data_provider); - if (!coin) { - break; - } - coins[*outpoint] = *coin; - } + std::map<COutPoint, Coin> coins{ConsumeCoins(fuzzed_data_provider)}; std::map<int, bilingual_str> input_errors; (void)SignTransaction(sign_transaction_tx_to, &provider, coins, fuzzed_data_provider.ConsumeIntegral<int>(), input_errors); } diff --git a/src/test/fuzz/signet.cpp b/src/test/fuzz/signet.cpp index 3ccf5eb6f0..6f1996572e 100644 --- a/src/test/fuzz/signet.cpp +++ b/src/test/fuzz/signet.cpp @@ -25,7 +25,7 @@ void initialize_signet() FUZZ_TARGET(signet, .init = initialize_signet) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - const std::optional<CBlock> block = ConsumeDeserializable<CBlock>(fuzzed_data_provider); + const std::optional<CBlock> block = ConsumeDeserializable<CBlock>(fuzzed_data_provider, TX_WITH_WITNESS); if (!block) { return; } diff --git a/src/test/fuzz/socks5.cpp b/src/test/fuzz/socks5.cpp index 05b8312ab2..af81fcb593 100644 --- a/src/test/fuzz/socks5.cpp +++ b/src/test/fuzz/socks5.cpp @@ -32,7 +32,9 @@ FUZZ_TARGET(socks5, .init = initialize_socks5) ProxyCredentials proxy_credentials; proxy_credentials.username = fuzzed_data_provider.ConsumeRandomLengthString(512); proxy_credentials.password = fuzzed_data_provider.ConsumeRandomLengthString(512); - InterruptSocks5(fuzzed_data_provider.ConsumeBool()); + if (fuzzed_data_provider.ConsumeBool()) { + g_socks5_interrupt(); + } // Set FUZZED_SOCKET_FAKE_LATENCY=1 to exercise recv timeout code paths. This // will slow down fuzzing. g_socks5_recv_timeout = (fuzzed_data_provider.ConsumeBool() && std::getenv("FUZZED_SOCKET_FAKE_LATENCY") != nullptr) ? 1ms : default_socks5_recv_timeout; diff --git a/src/test/fuzz/transaction.cpp b/src/test/fuzz/transaction.cpp index 88c2a334c7..2a043f7458 100644 --- a/src/test/fuzz/transaction.cpp +++ b/src/test/fuzz/transaction.cpp @@ -18,7 +18,6 @@ #include <util/chaintype.h> #include <util/rbf.h> #include <validation.h> -#include <version.h> #include <cassert> @@ -29,31 +28,21 @@ void initialize_transaction() FUZZ_TARGET(transaction, .init = initialize_transaction) { - CDataStream ds(buffer, SER_NETWORK, INIT_PROTO_VERSION); - try { - int nVersion; - ds >> nVersion; - ds.SetVersion(nVersion); - } catch (const std::ios_base::failure&) { - return; - } + DataStream ds{buffer}; bool valid_tx = true; const CTransaction tx = [&] { try { - return CTransaction(deserialize, ds); + return CTransaction(deserialize, TX_WITH_WITNESS, ds); } catch (const std::ios_base::failure&) { valid_tx = false; return CTransaction{CMutableTransaction{}}; } }(); bool valid_mutable_tx = true; - CDataStream ds_mtx(buffer, SER_NETWORK, INIT_PROTO_VERSION); + DataStream ds_mtx{buffer}; CMutableTransaction mutable_tx; try { - int nVersion; - ds_mtx >> nVersion; - ds_mtx.SetVersion(nVersion); - ds_mtx >> mutable_tx; + ds_mtx >> TX_WITH_WITNESS(mutable_tx); } catch (const std::ios_base::failure&) { valid_mutable_tx = false; } diff --git a/src/test/fuzz/tx_in.cpp b/src/test/fuzz/tx_in.cpp index fc16f80cde..d0ef467db4 100644 --- a/src/test/fuzz/tx_in.cpp +++ b/src/test/fuzz/tx_in.cpp @@ -8,7 +8,6 @@ #include <primitives/transaction.h> #include <streams.h> #include <test/fuzz/fuzz.h> -#include <version.h> #include <cassert> diff --git a/src/test/fuzz/tx_out.cpp b/src/test/fuzz/tx_out.cpp index 806216fbf5..37f86822d9 100644 --- a/src/test/fuzz/tx_out.cpp +++ b/src/test/fuzz/tx_out.cpp @@ -9,7 +9,6 @@ #include <primitives/transaction.h> #include <streams.h> #include <test/fuzz/fuzz.h> -#include <version.h> FUZZ_TARGET(tx_out) { diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index ee73f67f66..4ad0956201 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -59,9 +59,9 @@ struct TransactionsDelta final : public CValidationInterface { explicit TransactionsDelta(std::set<CTransactionRef>& r, std::set<CTransactionRef>& a) : m_removed{r}, m_added{a} {} - void TransactionAddedToMempool(const CTransactionRef& tx, uint64_t /* mempool_sequence */) override + void TransactionAddedToMempool(const NewMempoolTransactionInfo& tx, uint64_t /* mempool_sequence */) override { - Assert(m_added.insert(tx).second); + Assert(m_added.insert(tx.info.m_tx).second); } void TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason, uint64_t /* mempool_sequence */) override @@ -123,7 +123,6 @@ CTxMemPool MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeConte CTxMemPool::Options mempool_opts{MemPoolOptionsForTest(node)}; // ...override specific options for this specific fuzz suite - mempool_opts.estimator = nullptr; mempool_opts.check_ratio = 1; mempool_opts.require_standard = fuzzed_data_provider.ConsumeBool(); @@ -131,6 +130,56 @@ CTxMemPool MakeMempool(FuzzedDataProvider& fuzzed_data_provider, const NodeConte return CTxMemPool{mempool_opts}; } +void CheckATMPInvariants(const MempoolAcceptResult& res, bool txid_in_mempool, bool wtxid_in_mempool) +{ + + switch (res.m_result_type) { + case MempoolAcceptResult::ResultType::VALID: + { + Assert(txid_in_mempool); + Assert(wtxid_in_mempool); + Assert(res.m_state.IsValid()); + Assert(!res.m_state.IsInvalid()); + Assert(res.m_replaced_transactions); + Assert(res.m_vsize); + Assert(res.m_base_fees); + Assert(res.m_effective_feerate); + Assert(res.m_wtxids_fee_calculations); + Assert(!res.m_other_wtxid); + break; + } + case MempoolAcceptResult::ResultType::INVALID: + { + // It may be already in the mempool since in ATMP cases we don't set MEMPOOL_ENTRY or DIFFERENT_WITNESS + Assert(!res.m_state.IsValid()); + Assert(res.m_state.IsInvalid()); + + const bool is_reconsiderable{res.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE}; + Assert(!res.m_replaced_transactions); + Assert(!res.m_vsize); + Assert(!res.m_base_fees); + // Fee information is provided if the failure is TX_RECONSIDERABLE. + // In other cases, validation may be unable or unwilling to calculate the fees. + Assert(res.m_effective_feerate.has_value() == is_reconsiderable); + Assert(res.m_wtxids_fee_calculations.has_value() == is_reconsiderable); + Assert(!res.m_other_wtxid); + break; + } + case MempoolAcceptResult::ResultType::MEMPOOL_ENTRY: + { + // ATMP never sets this; only set in package settings + Assert(false); + break; + } + case MempoolAcceptResult::ResultType::DIFFERENT_WITNESS: + { + // ATMP never sets this; only set in package settings + Assert(false); + break; + } + } +} + FUZZ_TARGET(tx_pool_standard, .init = initialize_tx_pool) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); @@ -230,7 +279,7 @@ FUZZ_TARGET(tx_pool_standard, .init = initialize_tx_pool) tx->GetHash() : PickValue(fuzzed_data_provider, outpoints_rbf).hash; const auto delta = fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(-50 * COIN, +50 * COIN); - tx_pool.PrioritiseTransaction(txid, delta); + tx_pool.PrioritiseTransaction(txid.ToUint256(), delta); } // Remember all removed and added transactions @@ -258,9 +307,11 @@ FUZZ_TARGET(tx_pool_standard, .init = initialize_tx_pool) SyncWithValidationInterfaceQueue(); UnregisterSharedValidationInterface(txr); + bool txid_in_mempool = tx_pool.exists(GenTxid::Txid(tx->GetHash())); + bool wtxid_in_mempool = tx_pool.exists(GenTxid::Wtxid(tx->GetWitnessHash())); + CheckATMPInvariants(res, txid_in_mempool, wtxid_in_mempool); + Assert(accepted != added.empty()); - Assert(accepted == res.m_state.IsValid()); - Assert(accepted != res.m_state.IsInvalid()); if (accepted) { Assert(added.size() == 1); // For now, no package acceptance Assert(tx == *added.begin()); @@ -315,7 +366,7 @@ FUZZ_TARGET(tx_pool, .init = initialize_tx_pool) MockTime(fuzzed_data_provider, chainstate); - std::vector<uint256> txids; + std::vector<Txid> txids; txids.reserve(g_outpoints_coinbase_init_mature.size()); for (const auto& outpoint : g_outpoints_coinbase_init_mature) { txids.push_back(outpoint.hash); @@ -323,7 +374,7 @@ FUZZ_TARGET(tx_pool, .init = initialize_tx_pool) for (int i{0}; i <= 3; ++i) { // Add some immature and non-existent outpoints txids.push_back(g_outpoints_coinbase_init_immature.at(i).hash); - txids.push_back(ConsumeUInt256(fuzzed_data_provider)); + txids.push_back(Txid::FromUint256(ConsumeUInt256(fuzzed_data_provider))); } SetMempoolConstraints(*node.args, fuzzed_data_provider); @@ -343,11 +394,11 @@ FUZZ_TARGET(tx_pool, .init = initialize_tx_pool) tx_pool.RollingFeeUpdate(); } if (fuzzed_data_provider.ConsumeBool()) { - const auto& txid = fuzzed_data_provider.ConsumeBool() ? + const auto txid = fuzzed_data_provider.ConsumeBool() ? mut_tx.GetHash() : PickValue(fuzzed_data_provider, txids); const auto delta = fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(-50 * COIN, +50 * COIN); - tx_pool.PrioritiseTransaction(txid, delta); + tx_pool.PrioritiseTransaction(txid.ToUint256(), delta); } const auto tx = MakeTransactionRef(mut_tx); diff --git a/src/test/fuzz/txorphan.cpp b/src/test/fuzz/txorphan.cpp index a84dc951fc..5423ba8920 100644 --- a/src/test/fuzz/txorphan.cpp +++ b/src/test/fuzz/txorphan.cpp @@ -33,13 +33,14 @@ void initialize_orphanage() FUZZ_TARGET(txorphan, .init = initialize_orphanage) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + FastRandomContext limit_orphans_rng{/*fDeterministic=*/true}; SetMockTime(ConsumeTime(fuzzed_data_provider)); TxOrphanage orphanage; std::vector<COutPoint> outpoints; // initial outpoints used to construct transactions later for (uint8_t i = 0; i < 4; i++) { - outpoints.emplace_back(uint256{i}, 0); + outpoints.emplace_back(Txid::FromUint256(uint256{i}), 0); } // if true, allow duplicate input when constructing tx const bool duplicate_input = fuzzed_data_provider.ConsumeBool(); @@ -91,13 +92,13 @@ FUZZ_TARGET(txorphan, .init = initialize_orphanage) { CTransactionRef ref = orphanage.GetTxToReconsider(peer_id); if (ref) { - bool have_tx = orphanage.HaveTx(GenTxid::Txid(ref->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(ref->GetHash())); + bool have_tx = orphanage.HaveTx(GenTxid::Txid(ref->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(ref->GetWitnessHash())); Assert(have_tx); } } }, [&] { - bool have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); + bool have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetWitnessHash())); // AddTx should return false if tx is too big or already have it // tx weight is unknown, we only check when tx is already in orphanage { @@ -105,7 +106,7 @@ FUZZ_TARGET(txorphan, .init = initialize_orphanage) // have_tx == true -> add_tx == false Assert(!have_tx || !add_tx); } - have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); + have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetWitnessHash())); { bool add_tx = orphanage.AddTx(tx, peer_id); // if have_tx is still false, it must be too big @@ -114,12 +115,12 @@ FUZZ_TARGET(txorphan, .init = initialize_orphanage) } }, [&] { - bool have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); + bool have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetWitnessHash())); // EraseTx should return 0 if m_orphans doesn't have the tx { Assert(have_tx == orphanage.EraseTx(tx->GetHash())); } - have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetHash())); + have_tx = orphanage.HaveTx(GenTxid::Txid(tx->GetHash())) || orphanage.HaveTx(GenTxid::Wtxid(tx->GetWitnessHash())); // have_tx should be false and EraseTx should fail { Assert(!have_tx && !orphanage.EraseTx(tx->GetHash())); @@ -132,7 +133,7 @@ FUZZ_TARGET(txorphan, .init = initialize_orphanage) // test mocktime and expiry SetMockTime(ConsumeTime(fuzzed_data_provider)); auto limit = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); - orphanage.LimitOrphans(limit); + orphanage.LimitOrphans(limit, limit_orphans_rng); Assert(orphanage.Size() <= limit); }); } diff --git a/src/test/fuzz/util.cpp b/src/test/fuzz/util.cpp index 87ca2f6aed..23b0761355 100644 --- a/src/test/fuzz/util.cpp +++ b/src/test/fuzz/util.cpp @@ -10,7 +10,6 @@ #include <util/overflow.h> #include <util/rbf.h> #include <util/time.h> -#include <version.h> #include <memory> @@ -40,7 +39,7 @@ int64_t ConsumeTime(FuzzedDataProvider& fuzzed_data_provider, const std::optiona return fuzzed_data_provider.ConsumeIntegralInRange<int64_t>(min.value_or(time_min), max.value_or(time_max)); } -CMutableTransaction ConsumeTransaction(FuzzedDataProvider& fuzzed_data_provider, const std::optional<std::vector<uint256>>& prevout_txids, const int max_num_in, const int max_num_out) noexcept +CMutableTransaction ConsumeTransaction(FuzzedDataProvider& fuzzed_data_provider, const std::optional<std::vector<Txid>>& prevout_txids, const int max_num_in, const int max_num_out) noexcept { CMutableTransaction tx_mut; const auto p2wsh_op_true = fuzzed_data_provider.ConsumeBool(); @@ -53,7 +52,7 @@ CMutableTransaction ConsumeTransaction(FuzzedDataProvider& fuzzed_data_provider, for (int i = 0; i < num_in; ++i) { const auto& txid_prev = prevout_txids ? PickValue(fuzzed_data_provider, *prevout_txids) : - ConsumeUInt256(fuzzed_data_provider); + Txid::FromUint256(ConsumeUInt256(fuzzed_data_provider)); const auto index_out = fuzzed_data_provider.ConsumeIntegralInRange<uint32_t>(0, max_num_out); const auto sequence = ConsumeSequence(fuzzed_data_provider); const auto script_sig = p2wsh_op_true ? CScript{} : ConsumeScript(fuzzed_data_provider); @@ -164,6 +163,24 @@ uint32_t ConsumeSequence(FuzzedDataProvider& fuzzed_data_provider) noexcept fuzzed_data_provider.ConsumeIntegral<uint32_t>(); } +std::map<COutPoint, Coin> ConsumeCoins(FuzzedDataProvider& fuzzed_data_provider) noexcept +{ + std::map<COutPoint, Coin> coins; + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { + const std::optional<COutPoint> outpoint{ConsumeDeserializable<COutPoint>(fuzzed_data_provider)}; + if (!outpoint) { + break; + } + const std::optional<Coin> coin{ConsumeDeserializable<Coin>(fuzzed_data_provider)}; + if (!coin) { + break; + } + coins[*outpoint] = *coin; + } + + return coins; +} + CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) noexcept { CTxDestination tx_destination; diff --git a/src/test/fuzz/util.h b/src/test/fuzz/util.h index 8263cd4c08..de09745730 100644 --- a/src/test/fuzz/util.h +++ b/src/test/fuzz/util.h @@ -20,7 +20,6 @@ #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <uint256.h> -#include <version.h> #include <algorithm> #include <array> @@ -107,7 +106,7 @@ template <typename T, typename P> DataStream ds{buffer}; T obj; try { - ds >> WithParams(params, obj); + ds >> params(obj); } catch (const std::ios_base::failure&) { return std::nullopt; } @@ -118,7 +117,7 @@ template <typename T> [[nodiscard]] inline std::optional<T> ConsumeDeserializable(FuzzedDataProvider& fuzzed_data_provider, const std::optional<size_t>& max_length = std::nullopt) noexcept { const std::vector<uint8_t> buffer = ConsumeRandomLengthByteVector(fuzzed_data_provider, max_length); - CDataStream ds{buffer, SER_NETWORK, INIT_PROTO_VERSION}; + DataStream ds{buffer}; T obj; try { ds >> obj; @@ -145,7 +144,7 @@ template <typename WeakEnumType, size_t size> [[nodiscard]] int64_t ConsumeTime(FuzzedDataProvider& fuzzed_data_provider, const std::optional<int64_t>& min = std::nullopt, const std::optional<int64_t>& max = std::nullopt) noexcept; -[[nodiscard]] CMutableTransaction ConsumeTransaction(FuzzedDataProvider& fuzzed_data_provider, const std::optional<std::vector<uint256>>& prevout_txids, const int max_num_in = 10, const int max_num_out = 10) noexcept; +[[nodiscard]] CMutableTransaction ConsumeTransaction(FuzzedDataProvider& fuzzed_data_provider, const std::optional<std::vector<Txid>>& prevout_txids, const int max_num_in = 10, const int max_num_out = 10) noexcept; [[nodiscard]] CScriptWitness ConsumeScriptWitness(FuzzedDataProvider& fuzzed_data_provider, const size_t max_stack_elem_size = 32) noexcept; @@ -181,6 +180,8 @@ template <typename WeakEnumType, size_t size> return UintToArith256(ConsumeUInt256(fuzzed_data_provider)); } +[[nodiscard]] std::map<COutPoint, Coin> ConsumeCoins(FuzzedDataProvider& fuzzed_data_provider) noexcept; + [[nodiscard]] CTxDestination ConsumeTxDestination(FuzzedDataProvider& fuzzed_data_provider) noexcept; [[nodiscard]] CKey ConsumePrivateKey(FuzzedDataProvider& fuzzed_data_provider, std::optional<bool> compressed = std::nullopt) noexcept; @@ -263,31 +264,6 @@ public: static int close(void* cookie); }; -[[nodiscard]] inline FuzzedFileProvider ConsumeFile(FuzzedDataProvider& fuzzed_data_provider) noexcept -{ - return {fuzzed_data_provider}; -} - -class FuzzedAutoFileProvider -{ - FuzzedFileProvider m_fuzzed_file_provider; - -public: - FuzzedAutoFileProvider(FuzzedDataProvider& fuzzed_data_provider) : m_fuzzed_file_provider{fuzzed_data_provider} - { - } - - AutoFile open() - { - return AutoFile{m_fuzzed_file_provider.open()}; - } -}; - -[[nodiscard]] inline FuzzedAutoFileProvider ConsumeAutoFile(FuzzedDataProvider& fuzzed_data_provider) noexcept -{ - return {fuzzed_data_provider}; -} - #define WRITE_TO_STREAM_CASE(type, consume) \ [&] { \ type o = consume; \ diff --git a/src/test/fuzz/util/descriptor.cpp b/src/test/fuzz/util/descriptor.cpp new file mode 100644 index 0000000000..df78bdf314 --- /dev/null +++ b/src/test/fuzz/util/descriptor.cpp @@ -0,0 +1,86 @@ +// Copyright (c) 2023-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <test/fuzz/util/descriptor.h> + +void MockedDescriptorConverter::Init() { + // The data to use as a private key or a seed for an xprv. + std::array<std::byte, 32> key_data{std::byte{1}}; + // Generate keys of all kinds and store them in the keys array. + for (size_t i{0}; i < TOTAL_KEYS_GENERATED; i++) { + key_data[31] = std::byte(i); + + // If this is a "raw" key, generate a normal privkey. Otherwise generate + // an extended one. + if (IdIsCompPubKey(i) || IdIsUnCompPubKey(i) || IdIsXOnlyPubKey(i) || IdIsConstPrivKey(i)) { + CKey privkey; + privkey.Set(UCharCast(key_data.begin()), UCharCast(key_data.end()), !IdIsUnCompPubKey(i)); + if (IdIsCompPubKey(i) || IdIsUnCompPubKey(i)) { + CPubKey pubkey{privkey.GetPubKey()}; + keys_str[i] = HexStr(pubkey); + } else if (IdIsXOnlyPubKey(i)) { + const XOnlyPubKey pubkey{privkey.GetPubKey()}; + keys_str[i] = HexStr(pubkey); + } else { + keys_str[i] = EncodeSecret(privkey); + } + } else { + CExtKey ext_privkey; + ext_privkey.SetSeed(key_data); + if (IdIsXprv(i)) { + keys_str[i] = EncodeExtKey(ext_privkey); + } else { + const CExtPubKey ext_pubkey{ext_privkey.Neuter()}; + keys_str[i] = EncodeExtPubKey(ext_pubkey); + } + } + } +} + +std::optional<uint8_t> MockedDescriptorConverter::IdxFromHex(std::string_view hex_characters) const { + if (hex_characters.size() != 2) return {}; + auto idx = ParseHex(hex_characters); + if (idx.size() != 1) return {}; + return idx[0]; +} + +std::optional<std::string> MockedDescriptorConverter::GetDescriptor(std::string_view mocked_desc) const { + // The smallest fragment would be "pk(%00)" + if (mocked_desc.size() < 7) return {}; + + // The actual descriptor string to be returned. + std::string desc; + desc.reserve(mocked_desc.size()); + + // Replace all occurrences of '%' followed by two hex characters with the corresponding key. + for (size_t i = 0; i < mocked_desc.size();) { + if (mocked_desc[i] == '%') { + if (i + 3 >= mocked_desc.size()) return {}; + if (const auto idx = IdxFromHex(mocked_desc.substr(i + 1, 2))) { + desc += keys_str[*idx]; + i += 3; + } else { + return {}; + } + } else { + desc += mocked_desc[i++]; + } + } + + return desc; +} + +bool HasDeepDerivPath(const FuzzBufferType& buff, const int max_depth) +{ + auto depth{0}; + for (const auto& ch: buff) { + if (ch == ',') { + // A comma is always present between two key expressions, so we use that as a delimiter. + depth = 0; + } else if (ch == '/') { + if (++depth > max_depth) return true; + } + } + return false; +} diff --git a/src/test/fuzz/util/descriptor.h b/src/test/fuzz/util/descriptor.h new file mode 100644 index 0000000000..cd41dbafa3 --- /dev/null +++ b/src/test/fuzz/util/descriptor.h @@ -0,0 +1,58 @@ +// Copyright (c) 2023-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#ifndef BITCOIN_TEST_FUZZ_UTIL_DESCRIPTOR_H +#define BITCOIN_TEST_FUZZ_UTIL_DESCRIPTOR_H + +#include <key_io.h> +#include <util/strencodings.h> +#include <script/descriptor.h> +#include <test/fuzz/fuzz.h> + +#include <functional> + +/** + * Converts a mocked descriptor string to a valid one. Every key in a mocked descriptor key is + * represented by 2 hex characters preceded by the '%' character. We parse the two hex characters + * as an index in a list of pre-generated keys. This list contains keys of the various types + * accepted in descriptor keys expressions. + */ +class MockedDescriptorConverter { +private: + //! Types are raw (un)compressed pubkeys, raw xonly pubkeys, raw privkeys (WIF), xpubs, xprvs. + static constexpr uint8_t KEY_TYPES_COUNT{6}; + //! How many keys we'll generate in total. + static constexpr size_t TOTAL_KEYS_GENERATED{std::numeric_limits<uint8_t>::max() + 1}; + //! 256 keys of various types. + std::array<std::string, TOTAL_KEYS_GENERATED> keys_str; + +public: + // We derive the type of key to generate from the 1-byte id parsed from hex. + bool IdIsCompPubKey(uint8_t idx) const { return idx % KEY_TYPES_COUNT == 0; } + bool IdIsUnCompPubKey(uint8_t idx) const { return idx % KEY_TYPES_COUNT == 1; } + bool IdIsXOnlyPubKey(uint8_t idx) const { return idx % KEY_TYPES_COUNT == 2; } + bool IdIsConstPrivKey(uint8_t idx) const { return idx % KEY_TYPES_COUNT == 3; } + bool IdIsXpub(uint8_t idx) const { return idx % KEY_TYPES_COUNT == 4; } + bool IdIsXprv(uint8_t idx) const { return idx % KEY_TYPES_COUNT == 5; } + + //! When initializing the target, populate the list of keys. + void Init(); + + //! Parse an id in the keys vectors from a 2-characters hex string. + std::optional<uint8_t> IdxFromHex(std::string_view hex_characters) const; + + //! Get an actual descriptor string from a descriptor string whose keys were mocked. + std::optional<std::string> GetDescriptor(std::string_view mocked_desc) const; +}; + +//! Default maximum number of derivation indexes in a single derivation path when limiting its depth. +constexpr int MAX_DEPTH{2}; + +/** + * Whether the buffer, if it represents a valid descriptor, contains a derivation path deeper than + * a given maximum depth. Note this may also be hit for deriv paths in origins. + */ +bool HasDeepDerivPath(const FuzzBufferType& buff, const int max_depth = MAX_DEPTH); + +#endif // BITCOIN_TEST_FUZZ_UTIL_DESCRIPTOR_H diff --git a/src/test/fuzz/util/net.cpp b/src/test/fuzz/util/net.cpp index 5a286c05d2..eb0f14ede0 100644 --- a/src/test/fuzz/util/net.cpp +++ b/src/test/fuzz/util/net.cpp @@ -6,13 +6,13 @@ #include <compat/compat.h> #include <netaddress.h> +#include <node/protocol_version.h> #include <protocol.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/util.h> #include <test/util/net.h> #include <util/sock.h> #include <util/time.h> -#include <version.h> #include <array> #include <cassert> diff --git a/src/test/fuzz/utxo_total_supply.cpp b/src/test/fuzz/utxo_total_supply.cpp index 318797faf2..48ed266abe 100644 --- a/src/test/fuzz/utxo_total_supply.cpp +++ b/src/test/fuzz/utxo_total_supply.cpp @@ -16,7 +16,6 @@ #include <test/util/setup_common.h> #include <util/chaintype.h> #include <validation.h> -#include <version.h> FUZZ_TARGET(utxo_total_supply) { @@ -94,8 +93,8 @@ FUZZ_TARGET(utxo_total_supply) assert(ActiveHeight() == 0); // Get at which height we duplicate the coinbase // Assuming that the fuzzer will mine relatively short chains (less than 200 blocks), we want the duplicate coinbase to be not too high. - // Up to 2000 seems reasonable. - int64_t duplicate_coinbase_height = fuzzed_data_provider.ConsumeIntegralInRange(0, 20 * COINBASE_MATURITY); + // Up to 300 seems reasonable. + int64_t duplicate_coinbase_height = fuzzed_data_provider.ConsumeIntegralInRange(0, 300); // Always pad with OP_0 at the end to avoid bad-cb-length error const CScript duplicate_coinbase_script = CScript() << duplicate_coinbase_height << OP_0; // Mine the first block with this duplicate @@ -121,7 +120,7 @@ FUZZ_TARGET(utxo_total_supply) // Limit to avoid timeout, but enough to cover duplicate_coinbase_height // and CVE-2018-17144. - LIMITED_WHILE(fuzzed_data_provider.remaining_bytes(), 2'000) + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes(), 2'00) { CallOneOf( fuzzed_data_provider, diff --git a/src/test/fuzz/validation_load_mempool.cpp b/src/test/fuzz/validation_load_mempool.cpp index 5d020b4d59..00678742c9 100644 --- a/src/test/fuzz/validation_load_mempool.cpp +++ b/src/test/fuzz/validation_load_mempool.cpp @@ -38,7 +38,7 @@ FUZZ_TARGET(validation_load_mempool, .init = initialize_validation_load_mempool) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; SetMockTime(ConsumeTime(fuzzed_data_provider)); - FuzzedFileProvider fuzzed_file_provider = ConsumeFile(fuzzed_data_provider); + FuzzedFileProvider fuzzed_file_provider{fuzzed_data_provider}; CTxMemPool pool{MemPoolOptionsForTest(g_setup->m_node)}; diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index 54afcef989..f0d2b9ed72 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -122,12 +122,12 @@ BOOST_AUTO_TEST_CASE(siphash) (uint64_t(x+4)<<32)|(uint64_t(x+5)<<40)|(uint64_t(x+6)<<48)|(uint64_t(x+7)<<56)); } - CHashWriter ss{CLIENT_VERSION}; + HashWriter ss{}; CMutableTransaction tx; // Note these tests were originally written with tx.nVersion=1 // and the test would be affected by default tx version bumps if not fixed. tx.nVersion = 1; - ss << tx; + ss << TX_WITH_WITNESS(tx); BOOST_CHECK_EQUAL(SipHashUint256(1, 2, ss.GetHash()), 0x79751e980c2a0a35ULL); // Check consistency between CSipHasher and SipHashUint256[Extra]. diff --git a/src/test/i2p_tests.cpp b/src/test/i2p_tests.cpp index 5b8b0e9215..f80f07d190 100644 --- a/src/test/i2p_tests.cpp +++ b/src/test/i2p_tests.cpp @@ -9,6 +9,7 @@ #include <test/util/logging.h> #include <test/util/net.h> #include <test/util/setup_common.h> +#include <util/readwritefile.h> #include <util/threadinterrupt.h> #include <boost/test/unit_test.hpp> @@ -125,4 +126,47 @@ BOOST_AUTO_TEST_CASE(listen_ok_accept_fail) } } +BOOST_AUTO_TEST_CASE(damaged_private_key) +{ + const auto CreateSockOrig = CreateSock; + + CreateSock = [](const CService&) { + return std::make_unique<StaticContentsSock>("HELLO REPLY RESULT=OK VERSION=3.1\n" + "SESSION STATUS RESULT=OK DESTINATION=\n"); + }; + + const auto i2p_private_key_file = m_args.GetDataDirNet() / "test_i2p_private_key_damaged"; + + for (const auto& [file_contents, expected_error] : std::vector<std::tuple<std::string, std::string>>{ + {"", "The private key is too short (0 < 387)"}, + + {"abcd", "The private key is too short (4 < 387)"}, + + {std::string(386, '\0'), "The private key is too short (386 < 387)"}, + + {std::string(385, '\0') + '\0' + '\1', + "Certificate length (1) designates that the private key should be 388 bytes, but it is only " + "387 bytes"}, + + {std::string(385, '\0') + '\0' + '\5' + "abcd", + "Certificate length (5) designates that the private key should be 392 bytes, but it is only " + "391 bytes"}}) { + BOOST_REQUIRE(WriteBinaryFile(i2p_private_key_file, file_contents)); + + CThreadInterrupt interrupt; + i2p::sam::Session session(i2p_private_key_file, CService{}, &interrupt); + + { + ASSERT_DEBUG_LOG("Creating persistent SAM session"); + ASSERT_DEBUG_LOG(expected_error); + + i2p::Connection conn; + bool proxy_error; + BOOST_CHECK(!session.Connect(CService{}, conn, proxy_error)); + } + } + + CreateSock = CreateSockOrig; +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index db58a0baec..217e4a6d22 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -191,7 +191,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) CheckSort<descendant_score>(pool, sortedOrder); CTxMemPool::setEntries setAncestors; - setAncestors.insert(pool.mapTx.find(tx6.GetHash())); + setAncestors.insert(pool.GetIter(tx6.GetHash()).value()); CMutableTransaction tx7 = CMutableTransaction(); tx7.vin.resize(1); tx7.vin[0].prevout = COutPoint(tx6.GetHash(), 0); @@ -223,7 +223,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) tx8.vout.resize(1); tx8.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx8.vout[0].nValue = 10 * COIN; - setAncestors.insert(pool.mapTx.find(tx7.GetHash())); + setAncestors.insert(pool.GetIter(tx7.GetHash()).value()); pool.addUnchecked(entry.Fee(0LL).Time(NodeSeconds{2s}).FromTx(tx8), setAncestors); // Now tx8 should be sorted low, but tx6/tx both high @@ -247,8 +247,8 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) std::vector<std::string> snapshotOrder = sortedOrder; - setAncestors.insert(pool.mapTx.find(tx8.GetHash())); - setAncestors.insert(pool.mapTx.find(tx9.GetHash())); + setAncestors.insert(pool.GetIter(tx8.GetHash()).value()); + setAncestors.insert(pool.GetIter(tx9.GetHash()).value()); /* tx10 depends on tx8 and tx9 and has a high fee*/ CMutableTransaction tx10 = CMutableTransaction(); tx10.vin.resize(2); @@ -291,11 +291,11 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) BOOST_CHECK_EQUAL(pool.size(), 10U); // Now try removing tx10 and verify the sort order returns to normal - pool.removeRecursive(pool.mapTx.find(tx10.GetHash())->GetTx(), REMOVAL_REASON_DUMMY); + pool.removeRecursive(*Assert(pool.get(tx10.GetHash())), REMOVAL_REASON_DUMMY); CheckSort<descendant_score>(pool, snapshotOrder); - pool.removeRecursive(pool.mapTx.find(tx9.GetHash())->GetTx(), REMOVAL_REASON_DUMMY); - pool.removeRecursive(pool.mapTx.find(tx8.GetHash())->GetTx(), REMOVAL_REASON_DUMMY); + pool.removeRecursive(*Assert(pool.get(tx9.GetHash())), REMOVAL_REASON_DUMMY); + pool.removeRecursive(*Assert(pool.get(tx8.GetHash())), REMOVAL_REASON_DUMMY); } BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) diff --git a/src/test/merkleblock_tests.cpp b/src/test/merkleblock_tests.cpp index c7ec5fec15..9c6008bdca 100644 --- a/src/test/merkleblock_tests.cpp +++ b/src/test/merkleblock_tests.cpp @@ -21,13 +21,13 @@ BOOST_AUTO_TEST_CASE(merkleblock_construct_from_txids_found) { CBlock block = getBlock13b8a(); - std::set<uint256> txids; + std::set<Txid> txids; // Last txn in block. - uint256 txhash1 = uint256S("0x74d681e0e03bafa802c8aa084379aa98d9fcd632ddc2ed9782b586ec87451f20"); + Txid txhash1{TxidFromString("0x74d681e0e03bafa802c8aa084379aa98d9fcd632ddc2ed9782b586ec87451f20")}; // Second txn in block. - uint256 txhash2 = uint256S("0xf9fc751cb7dc372406a9f8d738d5e6f8f63bab71986a39cf36ee70ee17036d07"); + Txid txhash2{TxidFromString("0xf9fc751cb7dc372406a9f8d738d5e6f8f63bab71986a39cf36ee70ee17036d07")}; txids.insert(txhash1); txids.insert(txhash2); @@ -62,8 +62,8 @@ BOOST_AUTO_TEST_CASE(merkleblock_construct_from_txids_not_found) { CBlock block = getBlock13b8a(); - std::set<uint256> txids2; - txids2.insert(uint256S("0xc0ffee00003bafa802c8aa084379aa98d9fcd632ddc2ed9782b586ec87451f20")); + std::set<Txid> txids2; + txids2.insert(TxidFromString("0xc0ffee00003bafa802c8aa084379aa98d9fcd632ddc2ed9782b586ec87451f20")); CMerkleBlock merkleBlock(block, txids2); BOOST_CHECK_EQUAL(merkleBlock.header.GetHash().GetHex(), block.GetHash().GetHex()); diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index b4c7cac223..342d2514ed 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -118,19 +118,19 @@ void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const tx.vout.resize(1); tx.vout[0].nValue = 5000000000LL - 1000; // This tx has a low fee: 1000 satoshis - uint256 hashParentTx = tx.GetHash(); // save this txid for later use + Txid hashParentTx = tx.GetHash(); // save this txid for later use tx_mempool.addUnchecked(entry.Fee(1000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); // This tx has a medium fee: 10000 satoshis tx.vin[0].prevout.hash = txFirst[1]->GetHash(); tx.vout[0].nValue = 5000000000LL - 10000; - uint256 hashMediumFeeTx = tx.GetHash(); + Txid hashMediumFeeTx = tx.GetHash(); tx_mempool.addUnchecked(entry.Fee(10000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); // This tx has a high fee, but depends on the first transaction tx.vin[0].prevout.hash = hashParentTx; tx.vout[0].nValue = 5000000000LL - 1000 - 50000; // 50k satoshi fee - uint256 hashHighFeeTx = tx.GetHash(); + Txid hashHighFeeTx = tx.GetHash(); tx_mempool.addUnchecked(entry.Fee(50000).Time(Now<NodeSeconds>()).SpendsCoinbase(false).FromTx(tx)); std::unique_ptr<CBlockTemplate> pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); @@ -142,9 +142,9 @@ void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const // Test that a package below the block min tx fee doesn't get included tx.vin[0].prevout.hash = hashHighFeeTx; tx.vout[0].nValue = 5000000000LL - 1000 - 50000; // 0 fee - uint256 hashFreeTx = tx.GetHash(); + Txid hashFreeTx = tx.GetHash(); tx_mempool.addUnchecked(entry.Fee(0).FromTx(tx)); - size_t freeTxSize = ::GetSerializeSize(tx, PROTOCOL_VERSION); + size_t freeTxSize = ::GetSerializeSize(TX_WITH_WITNESS(tx)); // Calculate a fee on child transaction that will put the package just // below the block min tx fee (assuming 1 child tx of the same size). @@ -152,7 +152,7 @@ void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const tx.vin[0].prevout.hash = hashFreeTx; tx.vout[0].nValue = 5000000000LL - 1000 - 50000 - feeToUse; - uint256 hashLowFeeTx = tx.GetHash(); + Txid hashLowFeeTx = tx.GetHash(); tx_mempool.addUnchecked(entry.Fee(feeToUse).FromTx(tx)); pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); // Verify that the free tx and the low fee tx didn't get selected @@ -180,7 +180,7 @@ void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const tx.vout.resize(2); tx.vout[0].nValue = 5000000000LL - 100000000; tx.vout[1].nValue = 100000000; // 1BTC output - uint256 hashFreeTx2 = tx.GetHash(); + Txid hashFreeTx2 = tx.GetHash(); tx_mempool.addUnchecked(entry.Fee(0).SpendsCoinbase(true).FromTx(tx)); // This tx can't be mined by itself @@ -188,7 +188,7 @@ void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const tx.vout.resize(1); feeToUse = blockMinFeeRate.GetFee(freeTxSize); tx.vout[0].nValue = 5000000000LL - 100000000 - feeToUse; - uint256 hashLowFeeTx2 = tx.GetHash(); + Txid hashLowFeeTx2 = tx.GetHash(); tx_mempool.addUnchecked(entry.Fee(feeToUse).SpendsCoinbase(false).FromTx(tx)); pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); @@ -210,7 +210,7 @@ void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const void MinerTestingSetup::TestBasicMining(const CScript& scriptPubKey, const std::vector<CTransactionRef>& txFirst, int baseheight) { - uint256 hash; + Txid hash; CMutableTransaction tx; TestMemPoolEntryHelper entry; entry.nFee = 11; @@ -545,20 +545,20 @@ void MinerTestingSetup::TestPrioritisedMining(const CScript& scriptPubKey, const tx.vin[0].prevout.n = 0; tx.vout[0].nValue = 5000000000LL - 1000; // This tx has a low fee: 1000 satoshis - uint256 hashParentTx = tx.GetHash(); // save this txid for later use + Txid hashParentTx = tx.GetHash(); // save this txid for later use tx_mempool.addUnchecked(entry.Fee(1000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); // This tx has a medium fee: 10000 satoshis tx.vin[0].prevout.hash = txFirst[2]->GetHash(); tx.vout[0].nValue = 5000000000LL - 10000; - uint256 hashMediumFeeTx = tx.GetHash(); + Txid hashMediumFeeTx = tx.GetHash(); tx_mempool.addUnchecked(entry.Fee(10000).Time(Now<NodeSeconds>()).SpendsCoinbase(true).FromTx(tx)); tx_mempool.PrioritiseTransaction(hashMediumFeeTx, -5 * COIN); // This tx also has a low fee, but is prioritised tx.vin[0].prevout.hash = hashParentTx; tx.vout[0].nValue = 5000000000LL - 1000 - 1000; // 1000 satoshi fee - uint256 hashPrioritsedChild = tx.GetHash(); + Txid hashPrioritsedChild = tx.GetHash(); tx_mempool.addUnchecked(entry.Fee(1000).Time(Now<NodeSeconds>()).SpendsCoinbase(false).FromTx(tx)); tx_mempool.PrioritiseTransaction(hashPrioritsedChild, 2 * COIN); @@ -570,19 +570,19 @@ void MinerTestingSetup::TestPrioritisedMining(const CScript& scriptPubKey, const // When FreeChild is included, FreeChild's prioritisation should also not be included. tx.vin[0].prevout.hash = txFirst[3]->GetHash(); tx.vout[0].nValue = 5000000000LL; // 0 fee - uint256 hashFreeParent = tx.GetHash(); + Txid hashFreeParent = tx.GetHash(); tx_mempool.addUnchecked(entry.Fee(0).SpendsCoinbase(true).FromTx(tx)); tx_mempool.PrioritiseTransaction(hashFreeParent, 10 * COIN); tx.vin[0].prevout.hash = hashFreeParent; tx.vout[0].nValue = 5000000000LL; // 0 fee - uint256 hashFreeChild = tx.GetHash(); + Txid hashFreeChild = tx.GetHash(); tx_mempool.addUnchecked(entry.Fee(0).SpendsCoinbase(false).FromTx(tx)); tx_mempool.PrioritiseTransaction(hashFreeChild, 1 * COIN); tx.vin[0].prevout.hash = hashFreeChild; tx.vout[0].nValue = 5000000000LL; // 0 fee - uint256 hashFreeGrandchild = tx.GetHash(); + Txid hashFreeGrandchild = tx.GetHash(); tx_mempool.addUnchecked(entry.Fee(0).SpendsCoinbase(false).FromTx(tx)); auto pblocktemplate = AssemblerForTest(tx_mempool).CreateNewBlock(scriptPubKey); diff --git a/src/test/miniminer_tests.cpp b/src/test/miniminer_tests.cpp index f65356936b..5b007997b9 100644 --- a/src/test/miniminer_tests.cpp +++ b/src/test/miniminer_tests.cpp @@ -15,6 +15,11 @@ BOOST_FIXTURE_TEST_SUITE(miniminer_tests, TestingSetup) +const CAmount low_fee{CENT/2000}; // 500 ṩ +const CAmount med_fee{CENT/200}; // 5000 ṩ +const CAmount high_fee{CENT/10}; // 100_000 ṩ + + static inline CTransactionRef make_tx(const std::vector<COutPoint>& inputs, size_t num_outputs) { CMutableTransaction tx = CMutableTransaction(); @@ -67,21 +72,51 @@ Value Find(const std::map<Key, Value>& map, const Key& key) return it->second; } -BOOST_FIXTURE_TEST_CASE(miniminer_1p1c, TestChain100Setup) +BOOST_FIXTURE_TEST_CASE(miniminer_negative, TestChain100Setup) { CTxMemPool& pool = *Assert(m_node.mempool); LOCK2(::cs_main, pool.cs); TestMemPoolEntryHelper entry; - const CAmount low_fee{CENT/2000}; - const CAmount normal_fee{CENT/200}; - const CAmount high_fee{CENT/10}; + // Create a transaction that will be prioritised to have a negative modified fee. + const CAmount positive_base_fee{1000}; + const CAmount negative_fee_delta{-50000}; + const CAmount negative_modified_fees{positive_base_fee + negative_fee_delta}; + BOOST_CHECK(negative_modified_fees < 0); + const auto tx_mod_negative = make_tx({COutPoint{m_coinbase_txns[4]->GetHash(), 0}}, /*num_outputs=*/1); + pool.addUnchecked(entry.Fee(positive_base_fee).FromTx(tx_mod_negative)); + pool.PrioritiseTransaction(tx_mod_negative->GetHash(), negative_fee_delta); + const COutPoint only_outpoint{tx_mod_negative->GetHash(), 0}; + + // When target feerate is 0, transactions with negative fees are not selected. + node::MiniMiner mini_miner_target0(pool, {only_outpoint}); + BOOST_CHECK(mini_miner_target0.IsReadyToCalculate()); + const CFeeRate feerate_zero(0); + mini_miner_target0.BuildMockTemplate(feerate_zero); + // Check the quit condition: + BOOST_CHECK(negative_modified_fees < feerate_zero.GetFee(Assert(pool.GetEntry(tx_mod_negative->GetHash()))->GetTxSize())); + BOOST_CHECK(mini_miner_target0.GetMockTemplateTxids().empty()); + + // With no target feerate, the template includes all transactions, even negative feerate ones. + node::MiniMiner mini_miner_no_target(pool, {only_outpoint}); + BOOST_CHECK(mini_miner_no_target.IsReadyToCalculate()); + mini_miner_no_target.BuildMockTemplate(std::nullopt); + const auto template_txids{mini_miner_no_target.GetMockTemplateTxids()}; + BOOST_CHECK_EQUAL(template_txids.size(), 1); + BOOST_CHECK(template_txids.count(tx_mod_negative->GetHash().ToUint256()) > 0); +} + +BOOST_FIXTURE_TEST_CASE(miniminer_1p1c, TestChain100Setup) +{ + CTxMemPool& pool = *Assert(m_node.mempool); + LOCK2(::cs_main, pool.cs); + TestMemPoolEntryHelper entry; // Create a parent tx0 and child tx1 with normal fees: const auto tx0 = make_tx({COutPoint{m_coinbase_txns[0]->GetHash(), 0}}, /*num_outputs=*/2); - pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx0)); + pool.addUnchecked(entry.Fee(med_fee).FromTx(tx0)); const auto tx1 = make_tx({COutPoint{tx0->GetHash(), 0}}, /*num_outputs=*/1); - pool.addUnchecked(entry.Fee(normal_fee).FromTx(tx1)); + pool.addUnchecked(entry.Fee(med_fee).FromTx(tx1)); // Create a low-feerate parent tx2 and high-feerate child tx3 (cpfp) const auto tx2 = make_tx({COutPoint{m_coinbase_txns[1]->GetHash(), 0}}, /*num_outputs=*/2); @@ -94,9 +129,11 @@ BOOST_FIXTURE_TEST_CASE(miniminer_1p1c, TestChain100Setup) pool.addUnchecked(entry.Fee(low_fee).FromTx(tx4)); const auto tx5 = make_tx({COutPoint{tx4->GetHash(), 0}}, /*num_outputs=*/1); pool.addUnchecked(entry.Fee(low_fee).FromTx(tx5)); + const CAmount tx5_delta{CENT/100}; // Make tx5's modified fee much higher than its base fee. This should cause it to pass // the fee-related checks despite being low-feerate. - pool.PrioritiseTransaction(tx5->GetHash(), CENT/100); + pool.PrioritiseTransaction(tx5->GetHash(), tx5_delta); + const CAmount tx5_mod_fee{low_fee + tx5_delta}; // Create a high-feerate parent tx6, low-feerate child tx7 const auto tx6 = make_tx({COutPoint{m_coinbase_txns[3]->GetHash(), 0}}, /*num_outputs=*/2); @@ -142,9 +179,9 @@ BOOST_FIXTURE_TEST_CASE(miniminer_1p1c, TestChain100Setup) }; std::map<uint256, TxDimensions> tx_dims; for (const auto& tx : all_transactions) { - const auto it = pool.GetIter(tx->GetHash()).value(); - tx_dims.emplace(tx->GetHash(), TxDimensions{it->GetTxSize(), it->GetModifiedFee(), - CFeeRate(it->GetModifiedFee(), it->GetTxSize())}); + const auto& entry{*Assert(pool.GetEntry(tx->GetHash()))}; + tx_dims.emplace(tx->GetHash(), TxDimensions{entry.GetTxSize(), entry.GetModifiedFee(), + CFeeRate(entry.GetModifiedFee(), entry.GetTxSize())}); } const std::vector<CFeeRate> various_normal_feerates({CFeeRate(0), CFeeRate(500), CFeeRate(999), @@ -153,7 +190,7 @@ BOOST_FIXTURE_TEST_CASE(miniminer_1p1c, TestChain100Setup) CFeeRate(23330), CFeeRate(50000), CFeeRate(5*CENT)}); // All nonexistent entries have a bumpfee of zero, regardless of feerate - std::vector<COutPoint> nonexistent_outpoints({ COutPoint{GetRandHash(), 0}, COutPoint{GetRandHash(), 3} }); + std::vector<COutPoint> nonexistent_outpoints({ COutPoint{Txid::FromUint256(GetRandHash()), 0}, COutPoint{Txid::FromUint256(GetRandHash()), 3} }); for (const auto& outpoint : nonexistent_outpoints) BOOST_CHECK(!pool.isSpent(outpoint)); for (const auto& feerate : various_normal_feerates) { node::MiniMiner mini_miner(pool, nonexistent_outpoints); @@ -273,6 +310,64 @@ BOOST_FIXTURE_TEST_CASE(miniminer_1p1c, TestChain100Setup) } } } + + // Check m_inclusion_order for equivalent mempool- and manually-constructed MiniMiners. + // (We cannot check bump fees in manually-constructed MiniMiners because it doesn't know what + // outpoints are requested). + std::vector<node::MiniMinerMempoolEntry> miniminer_info; + { + const int32_t tx0_vsize{tx_dims.at(tx0->GetHash()).vsize}; + const int32_t tx1_vsize{tx_dims.at(tx1->GetHash()).vsize}; + const int32_t tx2_vsize{tx_dims.at(tx2->GetHash()).vsize}; + const int32_t tx3_vsize{tx_dims.at(tx3->GetHash()).vsize}; + const int32_t tx4_vsize{tx_dims.at(tx4->GetHash()).vsize}; + const int32_t tx5_vsize{tx_dims.at(tx5->GetHash()).vsize}; + const int32_t tx6_vsize{tx_dims.at(tx6->GetHash()).vsize}; + const int32_t tx7_vsize{tx_dims.at(tx7->GetHash()).vsize}; + + miniminer_info.emplace_back(tx0,/*vsize_self=*/tx0_vsize,/*vsize_ancestor=*/tx0_vsize,/*fee_self=*/med_fee,/*fee_ancestor=*/med_fee); + miniminer_info.emplace_back(tx1, tx1_vsize, tx0_vsize + tx1_vsize, med_fee, 2*med_fee); + miniminer_info.emplace_back(tx2, tx2_vsize, tx2_vsize, low_fee, low_fee); + miniminer_info.emplace_back(tx3, tx3_vsize, tx2_vsize + tx3_vsize, high_fee, low_fee + high_fee); + miniminer_info.emplace_back(tx4, tx4_vsize, tx4_vsize, low_fee, low_fee); + miniminer_info.emplace_back(tx5, tx5_vsize, tx4_vsize + tx5_vsize, tx5_mod_fee, low_fee + tx5_mod_fee); + miniminer_info.emplace_back(tx6, tx6_vsize, tx6_vsize, high_fee, high_fee); + miniminer_info.emplace_back(tx7, tx7_vsize, tx6_vsize + tx7_vsize, low_fee, high_fee + low_fee); + } + std::map<Txid, std::set<Txid>> descendant_caches; + descendant_caches.emplace(tx0->GetHash(), std::set<Txid>{tx0->GetHash(), tx1->GetHash()}); + descendant_caches.emplace(tx1->GetHash(), std::set<Txid>{tx1->GetHash()}); + descendant_caches.emplace(tx2->GetHash(), std::set<Txid>{tx2->GetHash(), tx3->GetHash()}); + descendant_caches.emplace(tx3->GetHash(), std::set<Txid>{tx3->GetHash()}); + descendant_caches.emplace(tx4->GetHash(), std::set<Txid>{tx4->GetHash(), tx5->GetHash()}); + descendant_caches.emplace(tx5->GetHash(), std::set<Txid>{tx5->GetHash()}); + descendant_caches.emplace(tx6->GetHash(), std::set<Txid>{tx6->GetHash(), tx7->GetHash()}); + descendant_caches.emplace(tx7->GetHash(), std::set<Txid>{tx7->GetHash()}); + + node::MiniMiner miniminer_manual(miniminer_info, descendant_caches); + // Use unspent outpoints to avoid entries being omitted. + node::MiniMiner miniminer_pool(pool, all_unspent_outpoints); + BOOST_CHECK(miniminer_manual.IsReadyToCalculate()); + BOOST_CHECK(miniminer_pool.IsReadyToCalculate()); + for (const auto& sequences : {miniminer_manual.Linearize(), miniminer_pool.Linearize()}) { + // tx6 is selected first: high feerate with no parents to bump + BOOST_CHECK_EQUAL(Find(sequences, tx6->GetHash()), 0); + + // tx2 + tx3 CPFP are selected next + BOOST_CHECK_EQUAL(Find(sequences, tx2->GetHash()), 1); + BOOST_CHECK_EQUAL(Find(sequences, tx3->GetHash()), 1); + + // tx4 + prioritised tx5 CPFP + BOOST_CHECK_EQUAL(Find(sequences, tx4->GetHash()), 2); + BOOST_CHECK_EQUAL(Find(sequences, tx5->GetHash()), 2); + + BOOST_CHECK_EQUAL(Find(sequences, tx0->GetHash()), 3); + BOOST_CHECK_EQUAL(Find(sequences, tx1->GetHash()), 3); + + + // tx7 is selected last: low feerate with no children + BOOST_CHECK_EQUAL(Find(sequences, tx7->GetHash()), 4); + } } BOOST_FIXTURE_TEST_CASE(miniminer_overlap, TestChain100Setup) @@ -308,10 +403,6 @@ BOOST_FIXTURE_TEST_CASE(miniminer_overlap, TestChain100Setup) LOCK2(::cs_main, pool.cs); TestMemPoolEntryHelper entry; - const CAmount low_fee{CENT/2000}; // 500 ṩ - const CAmount med_fee{CENT/200}; // 5000 ṩ - const CAmount high_fee{CENT/10}; // 100_000 ṩ - // Create 3 parents of different feerates, and 1 child spending outputs from all 3 parents. const auto tx0 = make_tx({COutPoint{m_coinbase_txns[0]->GetHash(), 0}}, /*num_outputs=*/2); pool.addUnchecked(entry.Fee(low_fee).FromTx(tx0)); @@ -356,15 +447,15 @@ BOOST_FIXTURE_TEST_CASE(miniminer_overlap, TestChain100Setup) // tx3's feerate is lower than tx2's. same fee, different weight. BOOST_CHECK(tx2_feerate > tx3_feerate); const auto tx3_anc_feerate = CFeeRate(low_fee + med_fee + high_fee + high_fee, tx_vsizes[0] + tx_vsizes[1] + tx_vsizes[2] + tx_vsizes[3]); - const auto tx3_iter = pool.GetIter(tx3->GetHash()); - BOOST_CHECK(tx3_anc_feerate == CFeeRate(tx3_iter.value()->GetModFeesWithAncestors(), tx3_iter.value()->GetSizeWithAncestors())); + const auto& tx3_entry{*Assert(pool.GetEntry(tx3->GetHash()))}; + BOOST_CHECK(tx3_anc_feerate == CFeeRate(tx3_entry.GetModFeesWithAncestors(), tx3_entry.GetSizeWithAncestors())); const auto tx4_feerate = CFeeRate(high_fee, tx_vsizes[4]); const auto tx6_anc_feerate = CFeeRate(high_fee + low_fee + med_fee, tx_vsizes[4] + tx_vsizes[5] + tx_vsizes[6]); - const auto tx6_iter = pool.GetIter(tx6->GetHash()); - BOOST_CHECK(tx6_anc_feerate == CFeeRate(tx6_iter.value()->GetModFeesWithAncestors(), tx6_iter.value()->GetSizeWithAncestors())); + const auto& tx6_entry{*Assert(pool.GetEntry(tx6->GetHash()))}; + BOOST_CHECK(tx6_anc_feerate == CFeeRate(tx6_entry.GetModFeesWithAncestors(), tx6_entry.GetSizeWithAncestors())); const auto tx7_anc_feerate = CFeeRate(high_fee + low_fee + high_fee, tx_vsizes[4] + tx_vsizes[5] + tx_vsizes[7]); - const auto tx7_iter = pool.GetIter(tx7->GetHash()); - BOOST_CHECK(tx7_anc_feerate == CFeeRate(tx7_iter.value()->GetModFeesWithAncestors(), tx7_iter.value()->GetSizeWithAncestors())); + const auto& tx7_entry{*Assert(pool.GetEntry(tx7->GetHash()))}; + BOOST_CHECK(tx7_anc_feerate == CFeeRate(tx7_entry.GetModFeesWithAncestors(), tx7_entry.GetSizeWithAncestors())); BOOST_CHECK(tx4_feerate > tx6_anc_feerate); BOOST_CHECK(tx4_feerate > tx7_anc_feerate); @@ -450,15 +541,66 @@ BOOST_FIXTURE_TEST_CASE(miniminer_overlap, TestChain100Setup) BOOST_CHECK(tx7_bumpfee != bump_fees.end()); BOOST_CHECK_EQUAL(tx7_bumpfee->second, 0); } + // Check linearization order + std::vector<node::MiniMinerMempoolEntry> miniminer_info; + miniminer_info.emplace_back(tx0,/*vsize_self=*/tx_vsizes[0], /*vsize_ancestor=*/tx_vsizes[0], /*fee_self=*/low_fee, /*fee_ancestor=*/low_fee); + miniminer_info.emplace_back(tx1, tx_vsizes[1], tx_vsizes[1], med_fee, med_fee); + miniminer_info.emplace_back(tx2, tx_vsizes[2], tx_vsizes[2], high_fee, high_fee); + miniminer_info.emplace_back(tx3, tx_vsizes[3], tx_vsizes[0]+tx_vsizes[1]+tx_vsizes[2]+tx_vsizes[3], high_fee, low_fee+med_fee+2*high_fee); + miniminer_info.emplace_back(tx4, tx_vsizes[4], tx_vsizes[4], high_fee, high_fee); + miniminer_info.emplace_back(tx5, tx_vsizes[5], tx_vsizes[4]+tx_vsizes[5], low_fee, low_fee + high_fee); + miniminer_info.emplace_back(tx6, tx_vsizes[6], tx_vsizes[4]+tx_vsizes[5]+tx_vsizes[6], med_fee, high_fee+low_fee+med_fee); + miniminer_info.emplace_back(tx7, tx_vsizes[7], tx_vsizes[4]+tx_vsizes[5]+tx_vsizes[7], high_fee, high_fee+low_fee+high_fee); + + std::map<Txid, std::set<Txid>> descendant_caches; + descendant_caches.emplace(tx0->GetHash(), std::set<Txid>{tx0->GetHash(), tx3->GetHash()}); + descendant_caches.emplace(tx1->GetHash(), std::set<Txid>{tx1->GetHash(), tx3->GetHash()}); + descendant_caches.emplace(tx2->GetHash(), std::set<Txid>{tx2->GetHash(), tx3->GetHash()}); + descendant_caches.emplace(tx3->GetHash(), std::set<Txid>{tx3->GetHash()}); + descendant_caches.emplace(tx4->GetHash(), std::set<Txid>{tx4->GetHash(), tx5->GetHash(), tx6->GetHash(), tx7->GetHash()}); + descendant_caches.emplace(tx5->GetHash(), std::set<Txid>{tx5->GetHash(), tx6->GetHash(), tx7->GetHash()}); + descendant_caches.emplace(tx6->GetHash(), std::set<Txid>{tx6->GetHash()}); + descendant_caches.emplace(tx7->GetHash(), std::set<Txid>{tx7->GetHash()}); + + node::MiniMiner miniminer_manual(miniminer_info, descendant_caches); + // Use unspent outpoints to avoid entries being omitted. + node::MiniMiner miniminer_pool(pool, all_unspent_outpoints); + BOOST_CHECK(miniminer_manual.IsReadyToCalculate()); + BOOST_CHECK(miniminer_pool.IsReadyToCalculate()); + for (const auto& sequences : {miniminer_manual.Linearize(), miniminer_pool.Linearize()}) { + // tx2 and tx4 selected first: high feerate with nothing to bump + BOOST_CHECK_EQUAL(Find(sequences, tx4->GetHash()), 0); + BOOST_CHECK_EQUAL(Find(sequences, tx2->GetHash()), 1); + + // tx5 + tx7 CPFP + BOOST_CHECK_EQUAL(Find(sequences, tx5->GetHash()), 2); + BOOST_CHECK_EQUAL(Find(sequences, tx7->GetHash()), 2); + + // tx0 and tx1 CPFP'd by tx3 + BOOST_CHECK_EQUAL(Find(sequences, tx0->GetHash()), 3); + BOOST_CHECK_EQUAL(Find(sequences, tx1->GetHash()), 3); + BOOST_CHECK_EQUAL(Find(sequences, tx3->GetHash()), 3); + + // tx6 at medium feerate + BOOST_CHECK_EQUAL(Find(sequences, tx6->GetHash()), 4); + } } BOOST_FIXTURE_TEST_CASE(calculate_cluster, TestChain100Setup) { CTxMemPool& pool = *Assert(m_node.mempool); LOCK2(cs_main, pool.cs); + // TODO this can be removed once the mempool interface uses Txid, Wtxid + auto convert_to_uint256_vec = [](const std::vector<Txid>& vec) -> std::vector<uint256> { + std::vector<uint256> out; + std::transform(vec.begin(), vec.end(), std::back_inserter(out), + [](const Txid& txid) { return txid.ToUint256(); }); + return out; + }; + // Add chain of size 500 TestMemPoolEntryHelper entry; - std::vector<uint256> chain_txids; + std::vector<Txid> chain_txids; auto& lasttx = m_coinbase_txns[0]; for (auto i{0}; i < 500; ++i) { const auto tx = make_tx({COutPoint{lasttx->GetHash(), 0}}, /*num_outputs=*/1); @@ -469,7 +611,7 @@ BOOST_FIXTURE_TEST_CASE(calculate_cluster, TestChain100Setup) const auto cluster_500tx = pool.GatherClusters({lasttx->GetHash()}); CTxMemPool::setEntries cluster_500tx_set{cluster_500tx.begin(), cluster_500tx.end()}; BOOST_CHECK_EQUAL(cluster_500tx.size(), cluster_500tx_set.size()); - const auto vec_iters_500 = pool.GetIterVec(chain_txids); + const auto vec_iters_500 = pool.GetIterVec(convert_to_uint256_vec(chain_txids)); for (const auto& iter : vec_iters_500) BOOST_CHECK(cluster_500tx_set.count(iter)); // GatherClusters stops at 500 transactions. @@ -484,9 +626,9 @@ BOOST_FIXTURE_TEST_CASE(calculate_cluster, TestChain100Setup) * txc0 txc1 txc2 ... txc48 * Note that each transaction's ancestor size is 1 or 3, and each descendant size is 1, 2 or 3. * However, all of these transactions are in the same cluster. */ - std::vector<uint256> zigzag_txids; + std::vector<Txid> zigzag_txids; for (auto p{0}; p < 50; ++p) { - const auto txp = make_tx({COutPoint{GetRandHash(), 0}}, /*num_outputs=*/2); + const auto txp = make_tx({COutPoint{Txid::FromUint256(GetRandHash()), 0}}, /*num_outputs=*/2); pool.addUnchecked(entry.Fee(CENT).FromTx(txp)); zigzag_txids.push_back(txp->GetHash()); } @@ -495,7 +637,7 @@ BOOST_FIXTURE_TEST_CASE(calculate_cluster, TestChain100Setup) pool.addUnchecked(entry.Fee(CENT).FromTx(txc)); zigzag_txids.push_back(txc->GetHash()); } - const auto vec_iters_zigzag = pool.GetIterVec(zigzag_txids); + const auto vec_iters_zigzag = pool.GetIterVec(convert_to_uint256_vec(zigzag_txids)); // It doesn't matter which tx we calculate cluster for, everybody is in it. const std::vector<size_t> indices{0, 22, 72, zigzag_txids.size() - 1}; for (const auto index : indices) { @@ -507,4 +649,64 @@ BOOST_FIXTURE_TEST_CASE(calculate_cluster, TestChain100Setup) } } +BOOST_FIXTURE_TEST_CASE(manual_ctor, TestChain100Setup) +{ + CTxMemPool& pool = *Assert(m_node.mempool); + LOCK2(cs_main, pool.cs); + { + // 3 pairs of grandparent + fee-bumping parent, plus 1 low-feerate child. + // 0 fee + high fee + auto grandparent_zero_fee = make_tx({{m_coinbase_txns.at(0)->GetHash(), 0}}, 1); + auto parent_high_feerate = make_tx({{grandparent_zero_fee->GetHash(), 0}}, 1); + // double low fee + med fee + auto grandparent_double_low_feerate = make_tx({{m_coinbase_txns.at(2)->GetHash(), 0}}, 1); + auto parent_med_feerate = make_tx({{grandparent_double_low_feerate->GetHash(), 0}}, 1); + // low fee + double low fee + auto grandparent_low_feerate = make_tx({{m_coinbase_txns.at(1)->GetHash(), 0}}, 1); + auto parent_double_low_feerate = make_tx({{grandparent_low_feerate->GetHash(), 0}}, 1); + // child is below the cpfp package feerates because it is larger than everything else + auto child = make_tx({{parent_high_feerate->GetHash(), 0}, {parent_double_low_feerate->GetHash(), 0}, {parent_med_feerate->GetHash(), 0}}, 1); + + // We artificially record each transaction (except the child) with a uniform vsize of 100vB. + const int64_t tx_vsize{100}; + const int64_t child_vsize{1000}; + + std::vector<node::MiniMinerMempoolEntry> miniminer_info; + miniminer_info.emplace_back(grandparent_zero_fee, /*vsize_self=*/tx_vsize,/*vsize_ancestor=*/tx_vsize, /*fee_self=*/0,/*fee_ancestor=*/0); + miniminer_info.emplace_back(parent_high_feerate, tx_vsize, 2*tx_vsize, high_fee, high_fee); + miniminer_info.emplace_back(grandparent_double_low_feerate, tx_vsize, tx_vsize, 2*low_fee, 2*low_fee); + miniminer_info.emplace_back(parent_med_feerate, tx_vsize, 2*tx_vsize, med_fee, 2*low_fee+med_fee); + miniminer_info.emplace_back(grandparent_low_feerate, tx_vsize, tx_vsize, low_fee, low_fee); + miniminer_info.emplace_back(parent_double_low_feerate, tx_vsize, 2*tx_vsize, 2*low_fee, 3*low_fee); + miniminer_info.emplace_back(child, child_vsize, 6*tx_vsize+child_vsize, low_fee, high_fee+med_fee+6*low_fee); + std::map<Txid, std::set<Txid>> descendant_caches; + descendant_caches.emplace(grandparent_zero_fee->GetHash(), std::set<Txid>{grandparent_zero_fee->GetHash(), parent_high_feerate->GetHash(), child->GetHash()}); + descendant_caches.emplace(grandparent_low_feerate->GetHash(), std::set<Txid>{grandparent_low_feerate->GetHash(), parent_double_low_feerate->GetHash(), child->GetHash()}); + descendant_caches.emplace(grandparent_double_low_feerate->GetHash(), std::set<Txid>{grandparent_double_low_feerate->GetHash(), parent_med_feerate->GetHash(), child->GetHash()}); + descendant_caches.emplace(parent_high_feerate->GetHash(), std::set<Txid>{parent_high_feerate->GetHash(), child->GetHash()}); + descendant_caches.emplace(parent_med_feerate->GetHash(), std::set<Txid>{parent_med_feerate->GetHash(), child->GetHash()}); + descendant_caches.emplace(parent_double_low_feerate->GetHash(), std::set<Txid>{parent_double_low_feerate->GetHash(), child->GetHash()}); + descendant_caches.emplace(child->GetHash(), std::set<Txid>{child->GetHash()}); + + node::MiniMiner miniminer_manual(miniminer_info, descendant_caches); + BOOST_CHECK(miniminer_manual.IsReadyToCalculate()); + const auto sequences{miniminer_manual.Linearize()}; + + // CPFP zero + high + BOOST_CHECK_EQUAL(sequences.at(grandparent_zero_fee->GetHash()), 0); + BOOST_CHECK_EQUAL(sequences.at(parent_high_feerate->GetHash()), 0); + + // CPFP double low + med + BOOST_CHECK_EQUAL(sequences.at(grandparent_double_low_feerate->GetHash()), 1); + BOOST_CHECK_EQUAL(sequences.at(parent_med_feerate->GetHash()), 1); + + // CPFP low + double low + BOOST_CHECK_EQUAL(sequences.at(grandparent_low_feerate->GetHash()), 2); + BOOST_CHECK_EQUAL(sequences.at(parent_double_low_feerate->GetHash()), 2); + + // Child at the end + BOOST_CHECK_EQUAL(sequences.at(child->GetHash()), 3); + } +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/miniscript_tests.cpp b/src/test/miniscript_tests.cpp index 996c379962..1f28e61bc9 100644 --- a/src/test/miniscript_tests.cpp +++ b/src/test/miniscript_tests.cpp @@ -369,7 +369,7 @@ void TestSatisfy(const KeyConverter& converter, const std::string& testcase, con CScriptWitness witness_nonmal; const bool nonmal_success = node->Satisfy(satisfier, witness_nonmal.stack, true) == miniscript::Availability::YES; // Compute witness size (excluding script push, control block, and witness count encoding). - const size_t wit_size = GetSerializeSize(witness_nonmal.stack, PROTOCOL_VERSION) - GetSizeOfCompactSize(witness_nonmal.stack.size()); + const size_t wit_size = GetSerializeSize(witness_nonmal.stack) - GetSizeOfCompactSize(witness_nonmal.stack.size()); SatisfactionToWitness(converter.MsContext(), witness_nonmal, script, builder); if (nonmal_success) { diff --git a/src/test/net_peer_connection_tests.cpp b/src/test/net_peer_connection_tests.cpp new file mode 100644 index 0000000000..58cbe9eb72 --- /dev/null +++ b/src/test/net_peer_connection_tests.cpp @@ -0,0 +1,157 @@ +// Copyright (c) 2023-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <compat/compat.h> +#include <net.h> +#include <net_processing.h> +#include <netaddress.h> +#include <netbase.h> +#include <netgroup.h> +#include <node/connection_types.h> +#include <node/protocol_version.h> +#include <protocol.h> +#include <random.h> +#include <test/util/logging.h> +#include <test/util/net.h> +#include <test/util/random.h> +#include <test/util/setup_common.h> +#include <tinyformat.h> +#include <util/chaintype.h> + +#include <algorithm> +#include <cstdint> +#include <memory> +#include <optional> +#include <string> +#include <vector> + +#include <boost/test/unit_test.hpp> + +struct LogIPsTestingSetup : public TestingSetup { + LogIPsTestingSetup() + : TestingSetup{ChainType::MAIN, /*extra_args=*/{"-logips"}} {} +}; + +BOOST_FIXTURE_TEST_SUITE(net_peer_connection_tests, LogIPsTestingSetup) + +static CService ip(uint32_t i) +{ + struct in_addr s; + s.s_addr = i; + return CService{CNetAddr{s}, Params().GetDefaultPort()}; +} + +/** Create a peer and connect to it. If the optional `address` (IP/CJDNS only) isn't passed, a random address is created. */ +static void AddPeer(NodeId& id, std::vector<CNode*>& nodes, PeerManager& peerman, ConnmanTestMsg& connman, ConnectionType conn_type, bool onion_peer = false, std::optional<std::string> address = std::nullopt) +{ + CAddress addr{}; + + if (address.has_value()) { + addr = CAddress{MaybeFlipIPv6toCJDNS(LookupNumeric(address.value(), Params().GetDefaultPort())), NODE_NONE}; + } else if (onion_peer) { + auto tor_addr{g_insecure_rand_ctx.randbytes(ADDR_TORV3_SIZE)}; + BOOST_REQUIRE(addr.SetSpecial(OnionToString(tor_addr))); + } + + while (!addr.IsLocal() && !addr.IsRoutable()) { + addr = CAddress{ip(g_insecure_rand_ctx.randbits(32)), NODE_NONE}; + } + + BOOST_REQUIRE(addr.IsValid()); + + const bool inbound_onion{onion_peer && conn_type == ConnectionType::INBOUND}; + + nodes.emplace_back(new CNode{++id, + /*sock=*/nullptr, + addr, + /*nKeyedNetGroupIn=*/0, + /*nLocalHostNonceIn=*/0, + CAddress{}, + /*addrNameIn=*/"", + conn_type, + /*inbound_onion=*/inbound_onion}); + CNode& node = *nodes.back(); + node.SetCommonVersion(PROTOCOL_VERSION); + + peerman.InitializeNode(node, ServiceFlags(NODE_NETWORK | NODE_WITNESS)); + node.fSuccessfullyConnected = true; + + connman.AddTestNode(node); +} + +BOOST_AUTO_TEST_CASE(test_addnode_getaddednodeinfo_and_connection_detection) +{ + auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman, Params()); + auto peerman = PeerManager::make(*connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, {}); + NodeId id{0}; + std::vector<CNode*> nodes; + + // Connect a localhost peer. + { + ASSERT_DEBUG_LOG("Added connection to 127.0.0.1:8333 peer=1"); + AddPeer(id, nodes, *peerman, *connman, ConnectionType::MANUAL, /*onion_peer=*/false, /*address=*/"127.0.0.1"); + BOOST_REQUIRE(nodes.back() != nullptr); + } + + // Call ConnectNode(), which is also called by RPC addnode onetry, for a localhost + // address that resolves to multiple IPs, including that of the connected peer. + // The connection attempt should consistently fail due to the check in ConnectNode(). + for (int i = 0; i < 10; ++i) { + ASSERT_DEBUG_LOG("Not opening a connection to localhost, already connected to 127.0.0.1:8333"); + BOOST_CHECK(!connman->ConnectNodePublic(*peerman, "localhost", ConnectionType::MANUAL)); + } + + // Add 3 more peer connections. + AddPeer(id, nodes, *peerman, *connman, ConnectionType::OUTBOUND_FULL_RELAY); + AddPeer(id, nodes, *peerman, *connman, ConnectionType::BLOCK_RELAY, /*onion_peer=*/true); + AddPeer(id, nodes, *peerman, *connman, ConnectionType::INBOUND); + + BOOST_TEST_MESSAGE("Call AddNode() for all the peers"); + for (auto node : connman->TestNodes()) { + BOOST_CHECK(connman->AddNode({/*m_added_node=*/node->addr.ToStringAddrPort(), /*m_use_v2transport=*/true})); + BOOST_TEST_MESSAGE(strprintf("peer id=%s addr=%s", node->GetId(), node->addr.ToStringAddrPort())); + } + + BOOST_TEST_MESSAGE("\nCall AddNode() with 2 addrs resolving to existing localhost addnode entry; neither should be added"); + BOOST_CHECK(!connman->AddNode({/*m_added_node=*/"127.0.0.1", /*m_use_v2transport=*/true})); + // OpenBSD doesn't support the IPv4 shorthand notation with omitted zero-bytes. +#if !defined(__OpenBSD__) + BOOST_CHECK(!connman->AddNode({/*m_added_node=*/"127.1", /*m_use_v2transport=*/true})); +#endif + + BOOST_TEST_MESSAGE("\nExpect GetAddedNodeInfo to return expected number of peers with `include_connected` true/false"); + BOOST_CHECK_EQUAL(connman->GetAddedNodeInfo(/*include_connected=*/true).size(), nodes.size()); + BOOST_CHECK(connman->GetAddedNodeInfo(/*include_connected=*/false).empty()); + + // Test AddedNodesContain() + for (auto node : connman->TestNodes()) { + BOOST_CHECK(connman->AddedNodesContain(node->addr)); + } + AddPeer(id, nodes, *peerman, *connman, ConnectionType::OUTBOUND_FULL_RELAY); + BOOST_CHECK(!connman->AddedNodesContain(nodes.back()->addr)); + + BOOST_TEST_MESSAGE("\nPrint GetAddedNodeInfo contents:"); + for (const auto& info : connman->GetAddedNodeInfo(/*include_connected=*/true)) { + BOOST_TEST_MESSAGE(strprintf("\nadded node: %s", info.m_params.m_added_node)); + BOOST_TEST_MESSAGE(strprintf("connected: %s", info.fConnected)); + if (info.fConnected) { + BOOST_TEST_MESSAGE(strprintf("IP address: %s", info.resolvedAddress.ToStringAddrPort())); + BOOST_TEST_MESSAGE(strprintf("direction: %s", info.fInbound ? "inbound" : "outbound")); + } + } + + BOOST_TEST_MESSAGE("\nCheck that all connected peers are correctly detected as connected"); + for (auto node : connman->TestNodes()) { + BOOST_CHECK(connman->AlreadyConnectedPublic(node->addr)); + } + + // Clean up + for (auto node : connman->TestNodes()) { + peerman->FinalizeNode(*node); + } + connman->ClearTestNodes(); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/net_tests.cpp b/src/test/net_tests.cpp index 7c98c382e4..70a8279f04 100644 --- a/src/test/net_tests.cpp +++ b/src/test/net_tests.cpp @@ -12,6 +12,7 @@ #include <netaddress.h> #include <netbase.h> #include <netmessagemaker.h> +#include <node/protocol_version.h> #include <serialize.h> #include <span.h> #include <streams.h> @@ -22,7 +23,6 @@ #include <util/strencodings.h> #include <util/string.h> #include <validation.h> -#include <version.h> #include <boost/test/unit_test.hpp> @@ -331,17 +331,17 @@ BOOST_AUTO_TEST_CASE(cnetaddr_serialize_v1) DataStream s{}; const auto ser_params{CAddress::V1_NETWORK}; - s << WithParams(ser_params, addr); + s << ser_params(addr); BOOST_CHECK_EQUAL(HexStr(s), "00000000000000000000000000000000"); s.clear(); addr = LookupHost("1.2.3.4", false).value(); - s << WithParams(ser_params, addr); + s << ser_params(addr); BOOST_CHECK_EQUAL(HexStr(s), "00000000000000000000ffff01020304"); s.clear(); addr = LookupHost("1a1b:2a2b:3a3b:4a4b:5a5b:6a6b:7a7b:8a8b", false).value(); - s << WithParams(ser_params, addr); + s << ser_params(addr); BOOST_CHECK_EQUAL(HexStr(s), "1a1b2a2b3a3b4a4b5a5b6a6b7a7b8a8b"); s.clear(); @@ -349,12 +349,12 @@ BOOST_AUTO_TEST_CASE(cnetaddr_serialize_v1) BOOST_CHECK(!addr.SetSpecial("6hzph5hv6337r6p2.onion")); BOOST_REQUIRE(addr.SetSpecial("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion")); - s << WithParams(ser_params, addr); + s << ser_params(addr); BOOST_CHECK_EQUAL(HexStr(s), "00000000000000000000000000000000"); s.clear(); addr.SetInternal("a"); - s << WithParams(ser_params, addr); + s << ser_params(addr); BOOST_CHECK_EQUAL(HexStr(s), "fd6b88c08724ca978112ca1bbdcafac2"); s.clear(); } @@ -365,17 +365,17 @@ BOOST_AUTO_TEST_CASE(cnetaddr_serialize_v2) DataStream s{}; const auto ser_params{CAddress::V2_NETWORK}; - s << WithParams(ser_params, addr); + s << ser_params(addr); BOOST_CHECK_EQUAL(HexStr(s), "021000000000000000000000000000000000"); s.clear(); addr = LookupHost("1.2.3.4", false).value(); - s << WithParams(ser_params, addr); + s << ser_params(addr); BOOST_CHECK_EQUAL(HexStr(s), "010401020304"); s.clear(); addr = LookupHost("1a1b:2a2b:3a3b:4a4b:5a5b:6a6b:7a7b:8a8b", false).value(); - s << WithParams(ser_params, addr); + s << ser_params(addr); BOOST_CHECK_EQUAL(HexStr(s), "02101a1b2a2b3a3b4a4b5a5b6a6b7a7b8a8b"); s.clear(); @@ -383,12 +383,12 @@ BOOST_AUTO_TEST_CASE(cnetaddr_serialize_v2) BOOST_CHECK(!addr.SetSpecial("6hzph5hv6337r6p2.onion")); BOOST_REQUIRE(addr.SetSpecial("kpgvmscirrdqpekbqjsvw5teanhatztpp2gl6eee4zkowvwfxwenqaid.onion")); - s << WithParams(ser_params, addr); + s << ser_params(addr); BOOST_CHECK_EQUAL(HexStr(s), "042053cd5648488c4707914182655b7664034e09e66f7e8cbf1084e654eb56c5bd88"); s.clear(); BOOST_REQUIRE(addr.SetInternal("a")); - s << WithParams(ser_params, addr); + s << ser_params(addr); BOOST_CHECK_EQUAL(HexStr(s), "0210fd6b88c08724ca978112ca1bbdcafac2"); s.clear(); } @@ -403,7 +403,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s << Span{ParseHex("01" // network type (IPv4) "04" // address length "01020304")}; // address - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(addr.IsValid()); BOOST_CHECK(addr.IsIPv4()); BOOST_CHECK(addr.IsAddrV1Compatible()); @@ -414,7 +414,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s << Span{ParseHex("01" // network type (IPv4) "04" // address length "0102")}; // address - BOOST_CHECK_EXCEPTION(s >> WithParams(ser_params, addr), std::ios_base::failure, HasReason("end of data")); + BOOST_CHECK_EXCEPTION(s >> ser_params(addr), std::ios_base::failure, HasReason("end of data")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); @@ -422,7 +422,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s << Span{ParseHex("01" // network type (IPv4) "05" // address length "01020304")}; // address - BOOST_CHECK_EXCEPTION(s >> WithParams(ser_params, addr), std::ios_base::failure, + BOOST_CHECK_EXCEPTION(s >> ser_params(addr), std::ios_base::failure, HasReason("BIP155 IPv4 address with length 5 (should be 4)")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); @@ -431,7 +431,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s << Span{ParseHex("01" // network type (IPv4) "fd0102" // address length (513 as CompactSize) "01020304")}; // address - BOOST_CHECK_EXCEPTION(s >> WithParams(ser_params, addr), std::ios_base::failure, + BOOST_CHECK_EXCEPTION(s >> ser_params(addr), std::ios_base::failure, HasReason("Address too long: 513 > 512")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); @@ -440,7 +440,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s << Span{ParseHex("02" // network type (IPv6) "10" // address length "0102030405060708090a0b0c0d0e0f10")}; // address - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(addr.IsValid()); BOOST_CHECK(addr.IsIPv6()); BOOST_CHECK(addr.IsAddrV1Compatible()); @@ -453,7 +453,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "10" // address length "fd6b88c08724ca978112ca1bbdcafac2")}; // address: 0xfd + sha256("bitcoin")[0:5] + // sha256(name)[0:10] - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(addr.IsInternal()); BOOST_CHECK(addr.IsAddrV1Compatible()); BOOST_CHECK_EQUAL(addr.ToStringAddr(), "zklycewkdo64v6wc.internal"); @@ -463,7 +463,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s << Span{ParseHex("02" // network type (IPv6) "04" // address length "00")}; // address - BOOST_CHECK_EXCEPTION(s >> WithParams(ser_params, addr), std::ios_base::failure, + BOOST_CHECK_EXCEPTION(s >> ser_params(addr), std::ios_base::failure, HasReason("BIP155 IPv6 address with length 4 (should be 16)")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); @@ -472,7 +472,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s << Span{ParseHex("02" // network type (IPv6) "10" // address length "00000000000000000000ffff01020304")}; // address - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(!addr.IsValid()); BOOST_REQUIRE(s.empty()); @@ -480,7 +480,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s << Span{ParseHex("02" // network type (IPv6) "10" // address length "fd87d87eeb430102030405060708090a")}; // address - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(!addr.IsValid()); BOOST_REQUIRE(s.empty()); @@ -488,7 +488,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) s << Span{ParseHex("03" // network type (TORv2) "0a" // address length "f1f2f3f4f5f6f7f8f9fa")}; // address - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(!addr.IsValid()); BOOST_REQUIRE(s.empty()); @@ -498,7 +498,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "79bcc625184b05194975c28b66b66b04" // address "69f7f6556fb1ac3189a79b40dda32f1f" )}; - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(addr.IsValid()); BOOST_CHECK(addr.IsTor()); BOOST_CHECK(!addr.IsAddrV1Compatible()); @@ -511,7 +511,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "00" // address length "00" // address )}; - BOOST_CHECK_EXCEPTION(s >> WithParams(ser_params, addr), std::ios_base::failure, + BOOST_CHECK_EXCEPTION(s >> ser_params(addr), std::ios_base::failure, HasReason("BIP155 TORv3 address with length 0 (should be 32)")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); @@ -521,7 +521,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "20" // address length "a2894dabaec08c0051a481a6dac88b64" // address "f98232ae42d4b6fd2fa81952dfe36a87")}; - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(addr.IsValid()); BOOST_CHECK(addr.IsI2P()); BOOST_CHECK(!addr.IsAddrV1Compatible()); @@ -534,7 +534,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "03" // address length "00" // address )}; - BOOST_CHECK_EXCEPTION(s >> WithParams(ser_params, addr), std::ios_base::failure, + BOOST_CHECK_EXCEPTION(s >> ser_params(addr), std::ios_base::failure, HasReason("BIP155 I2P address with length 3 (should be 32)")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); @@ -544,7 +544,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "10" // address length "fc000001000200030004000500060007" // address )}; - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(addr.IsValid()); BOOST_CHECK(addr.IsCJDNS()); BOOST_CHECK(!addr.IsAddrV1Compatible()); @@ -556,7 +556,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "10" // address length "aa000001000200030004000500060007" // address )}; - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(addr.IsCJDNS()); BOOST_CHECK(!addr.IsValid()); BOOST_REQUIRE(s.empty()); @@ -566,7 +566,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "01" // address length "00" // address )}; - BOOST_CHECK_EXCEPTION(s >> WithParams(ser_params, addr), std::ios_base::failure, + BOOST_CHECK_EXCEPTION(s >> ser_params(addr), std::ios_base::failure, HasReason("BIP155 CJDNS address with length 1 (should be 16)")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); @@ -576,7 +576,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "fe00000002" // address length (CompactSize's MAX_SIZE) "01020304050607" // address )}; - BOOST_CHECK_EXCEPTION(s >> WithParams(ser_params, addr), std::ios_base::failure, + BOOST_CHECK_EXCEPTION(s >> ser_params(addr), std::ios_base::failure, HasReason("Address too long: 33554432 > 512")); BOOST_REQUIRE(!s.empty()); // The stream is not consumed on invalid input. s.clear(); @@ -586,7 +586,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "04" // address length "01020304" // address )}; - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(!addr.IsValid()); BOOST_REQUIRE(s.empty()); @@ -595,7 +595,7 @@ BOOST_AUTO_TEST_CASE(cnetaddr_unserialize_v2) "00" // address length "" // address )}; - s >> WithParams(ser_params, addr); + s >> ser_params(addr); BOOST_CHECK(!addr.IsValid()); BOOST_REQUIRE(s.empty()); } @@ -845,7 +845,6 @@ BOOST_AUTO_TEST_CASE(initial_advertise_from_version_message) const uint64_t services{NODE_NETWORK | NODE_WITNESS}; const int64_t time{0}; - const CNetMsgMaker msg_maker{PROTOCOL_VERSION}; // Force ChainstateManager::IsInitialBlockDownload() to return false. // Otherwise PushAddress() isn't called by PeerManager::ProcessMessage(). @@ -858,14 +857,14 @@ BOOST_AUTO_TEST_CASE(initial_advertise_from_version_message) std::chrono::microseconds time_received_dummy{0}; const auto msg_version = - msg_maker.Make(NetMsgType::VERSION, PROTOCOL_VERSION, services, time, services, CAddress::V1_NETWORK(peer_us)); - CDataStream msg_version_stream{msg_version.data, SER_NETWORK, PROTOCOL_VERSION}; + NetMsg::Make(NetMsgType::VERSION, PROTOCOL_VERSION, services, time, services, CAddress::V1_NETWORK(peer_us)); + DataStream msg_version_stream{msg_version.data}; m_node.peerman->ProcessMessage( peer, NetMsgType::VERSION, msg_version_stream, time_received_dummy, interrupt_dummy); - const auto msg_verack = msg_maker.Make(NetMsgType::VERACK); - CDataStream msg_verack_stream{msg_verack.data, SER_NETWORK, PROTOCOL_VERSION}; + const auto msg_verack = NetMsg::Make(NetMsgType::VERACK); + DataStream msg_verack_stream{msg_verack.data}; // Will set peer.fSuccessfullyConnected to true (necessary in SendMessages()). m_node.peerman->ProcessMessage( @@ -1047,10 +1046,10 @@ class V2TransportTester public: /** Construct a tester object. test_initiator: whether the tested transport is initiator. */ - V2TransportTester(bool test_initiator) : - m_transport(0, test_initiator, SER_NETWORK, INIT_PROTO_VERSION), - m_cipher{GenerateRandomTestKey(), MakeByteSpan(InsecureRand256())}, - m_test_initiator(test_initiator) {} + explicit V2TransportTester(bool test_initiator) + : m_transport{0, test_initiator}, + m_cipher{GenerateRandomTestKey(), MakeByteSpan(InsecureRand256())}, + m_test_initiator(test_initiator) {} /** Data type returned by Interact: * @@ -1127,7 +1126,7 @@ public: void SendV1Version(const MessageStartChars& magic) { CMessageHeader hdr(magic, "version", 126 + InsecureRandRange(11)); - CDataStream ser(SER_NETWORK, CLIENT_VERSION); + DataStream ser{}; ser << hdr; m_to_send.insert(m_to_send.end(), UCharCast(ser.data()), UCharCast(ser.data() + ser.size())); } diff --git a/src/test/netbase_tests.cpp b/src/test/netbase_tests.cpp index 74ff531cd9..fa70f62eb4 100644 --- a/src/test/netbase_tests.cpp +++ b/src/test/netbase_tests.cpp @@ -12,7 +12,6 @@ #include <test/util/setup_common.h> #include <util/strencodings.h> #include <util/translation.h> -#include <version.h> #include <string> @@ -27,13 +26,6 @@ static CNetAddr ResolveIP(const std::string& ip) return LookupHost(ip, false).value_or(CNetAddr{}); } -static CSubNet ResolveSubNet(const std::string& subnet) -{ - CSubNet ret; - LookupSubNet(subnet, ret); - return ret; -} - static CNetAddr CreateInternal(const std::string& host) { CNetAddr addr; @@ -159,49 +151,49 @@ BOOST_AUTO_TEST_CASE(embedded_test) BOOST_AUTO_TEST_CASE(subnet_test) { - BOOST_CHECK(ResolveSubNet("1.2.3.0/24") == ResolveSubNet("1.2.3.0/255.255.255.0")); - BOOST_CHECK(ResolveSubNet("1.2.3.0/24") != ResolveSubNet("1.2.4.0/255.255.255.0")); - BOOST_CHECK(ResolveSubNet("1.2.3.0/24").Match(ResolveIP("1.2.3.4"))); - BOOST_CHECK(!ResolveSubNet("1.2.2.0/24").Match(ResolveIP("1.2.3.4"))); - BOOST_CHECK(ResolveSubNet("1.2.3.4").Match(ResolveIP("1.2.3.4"))); - BOOST_CHECK(ResolveSubNet("1.2.3.4/32").Match(ResolveIP("1.2.3.4"))); - BOOST_CHECK(!ResolveSubNet("1.2.3.4").Match(ResolveIP("5.6.7.8"))); - BOOST_CHECK(!ResolveSubNet("1.2.3.4/32").Match(ResolveIP("5.6.7.8"))); - BOOST_CHECK(ResolveSubNet("::ffff:127.0.0.1").Match(ResolveIP("127.0.0.1"))); - BOOST_CHECK(ResolveSubNet("1:2:3:4:5:6:7:8").Match(ResolveIP("1:2:3:4:5:6:7:8"))); - BOOST_CHECK(!ResolveSubNet("1:2:3:4:5:6:7:8").Match(ResolveIP("1:2:3:4:5:6:7:9"))); - BOOST_CHECK(ResolveSubNet("1:2:3:4:5:6:7:0/112").Match(ResolveIP("1:2:3:4:5:6:7:1234"))); - BOOST_CHECK(ResolveSubNet("192.168.0.1/24").Match(ResolveIP("192.168.0.2"))); - BOOST_CHECK(ResolveSubNet("192.168.0.20/29").Match(ResolveIP("192.168.0.18"))); - BOOST_CHECK(ResolveSubNet("1.2.2.1/24").Match(ResolveIP("1.2.2.4"))); - BOOST_CHECK(ResolveSubNet("1.2.2.110/31").Match(ResolveIP("1.2.2.111"))); - BOOST_CHECK(ResolveSubNet("1.2.2.20/26").Match(ResolveIP("1.2.2.63"))); + BOOST_CHECK(LookupSubNet("1.2.3.0/24") == LookupSubNet("1.2.3.0/255.255.255.0")); + BOOST_CHECK(LookupSubNet("1.2.3.0/24") != LookupSubNet("1.2.4.0/255.255.255.0")); + BOOST_CHECK(LookupSubNet("1.2.3.0/24").Match(ResolveIP("1.2.3.4"))); + BOOST_CHECK(!LookupSubNet("1.2.2.0/24").Match(ResolveIP("1.2.3.4"))); + BOOST_CHECK(LookupSubNet("1.2.3.4").Match(ResolveIP("1.2.3.4"))); + BOOST_CHECK(LookupSubNet("1.2.3.4/32").Match(ResolveIP("1.2.3.4"))); + BOOST_CHECK(!LookupSubNet("1.2.3.4").Match(ResolveIP("5.6.7.8"))); + BOOST_CHECK(!LookupSubNet("1.2.3.4/32").Match(ResolveIP("5.6.7.8"))); + BOOST_CHECK(LookupSubNet("::ffff:127.0.0.1").Match(ResolveIP("127.0.0.1"))); + BOOST_CHECK(LookupSubNet("1:2:3:4:5:6:7:8").Match(ResolveIP("1:2:3:4:5:6:7:8"))); + BOOST_CHECK(!LookupSubNet("1:2:3:4:5:6:7:8").Match(ResolveIP("1:2:3:4:5:6:7:9"))); + BOOST_CHECK(LookupSubNet("1:2:3:4:5:6:7:0/112").Match(ResolveIP("1:2:3:4:5:6:7:1234"))); + BOOST_CHECK(LookupSubNet("192.168.0.1/24").Match(ResolveIP("192.168.0.2"))); + BOOST_CHECK(LookupSubNet("192.168.0.20/29").Match(ResolveIP("192.168.0.18"))); + BOOST_CHECK(LookupSubNet("1.2.2.1/24").Match(ResolveIP("1.2.2.4"))); + BOOST_CHECK(LookupSubNet("1.2.2.110/31").Match(ResolveIP("1.2.2.111"))); + BOOST_CHECK(LookupSubNet("1.2.2.20/26").Match(ResolveIP("1.2.2.63"))); // All-Matching IPv6 Matches arbitrary IPv6 - BOOST_CHECK(ResolveSubNet("::/0").Match(ResolveIP("1:2:3:4:5:6:7:1234"))); + BOOST_CHECK(LookupSubNet("::/0").Match(ResolveIP("1:2:3:4:5:6:7:1234"))); // But not `::` or `0.0.0.0` because they are considered invalid addresses - BOOST_CHECK(!ResolveSubNet("::/0").Match(ResolveIP("::"))); - BOOST_CHECK(!ResolveSubNet("::/0").Match(ResolveIP("0.0.0.0"))); + BOOST_CHECK(!LookupSubNet("::/0").Match(ResolveIP("::"))); + BOOST_CHECK(!LookupSubNet("::/0").Match(ResolveIP("0.0.0.0"))); // Addresses from one network (IPv4) don't belong to subnets of another network (IPv6) - BOOST_CHECK(!ResolveSubNet("::/0").Match(ResolveIP("1.2.3.4"))); + BOOST_CHECK(!LookupSubNet("::/0").Match(ResolveIP("1.2.3.4"))); // All-Matching IPv4 does not Match IPv6 - BOOST_CHECK(!ResolveSubNet("0.0.0.0/0").Match(ResolveIP("1:2:3:4:5:6:7:1234"))); + BOOST_CHECK(!LookupSubNet("0.0.0.0/0").Match(ResolveIP("1:2:3:4:5:6:7:1234"))); // Invalid subnets Match nothing (not even invalid addresses) BOOST_CHECK(!CSubNet().Match(ResolveIP("1.2.3.4"))); - BOOST_CHECK(!ResolveSubNet("").Match(ResolveIP("4.5.6.7"))); - BOOST_CHECK(!ResolveSubNet("bloop").Match(ResolveIP("0.0.0.0"))); - BOOST_CHECK(!ResolveSubNet("bloop").Match(ResolveIP("hab"))); + BOOST_CHECK(!LookupSubNet("").Match(ResolveIP("4.5.6.7"))); + BOOST_CHECK(!LookupSubNet("bloop").Match(ResolveIP("0.0.0.0"))); + BOOST_CHECK(!LookupSubNet("bloop").Match(ResolveIP("hab"))); // Check valid/invalid - BOOST_CHECK(ResolveSubNet("1.2.3.0/0").IsValid()); - BOOST_CHECK(!ResolveSubNet("1.2.3.0/-1").IsValid()); - BOOST_CHECK(ResolveSubNet("1.2.3.0/32").IsValid()); - BOOST_CHECK(!ResolveSubNet("1.2.3.0/33").IsValid()); - BOOST_CHECK(!ResolveSubNet("1.2.3.0/300").IsValid()); - BOOST_CHECK(ResolveSubNet("1:2:3:4:5:6:7:8/0").IsValid()); - BOOST_CHECK(ResolveSubNet("1:2:3:4:5:6:7:8/33").IsValid()); - BOOST_CHECK(!ResolveSubNet("1:2:3:4:5:6:7:8/-1").IsValid()); - BOOST_CHECK(ResolveSubNet("1:2:3:4:5:6:7:8/128").IsValid()); - BOOST_CHECK(!ResolveSubNet("1:2:3:4:5:6:7:8/129").IsValid()); - BOOST_CHECK(!ResolveSubNet("fuzzy").IsValid()); + BOOST_CHECK(LookupSubNet("1.2.3.0/0").IsValid()); + BOOST_CHECK(!LookupSubNet("1.2.3.0/-1").IsValid()); + BOOST_CHECK(LookupSubNet("1.2.3.0/32").IsValid()); + BOOST_CHECK(!LookupSubNet("1.2.3.0/33").IsValid()); + BOOST_CHECK(!LookupSubNet("1.2.3.0/300").IsValid()); + BOOST_CHECK(LookupSubNet("1:2:3:4:5:6:7:8/0").IsValid()); + BOOST_CHECK(LookupSubNet("1:2:3:4:5:6:7:8/33").IsValid()); + BOOST_CHECK(!LookupSubNet("1:2:3:4:5:6:7:8/-1").IsValid()); + BOOST_CHECK(LookupSubNet("1:2:3:4:5:6:7:8/128").IsValid()); + BOOST_CHECK(!LookupSubNet("1:2:3:4:5:6:7:8/129").IsValid()); + BOOST_CHECK(!LookupSubNet("fuzzy").IsValid()); //CNetAddr constructor test BOOST_CHECK(CSubNet(ResolveIP("127.0.0.1")).IsValid()); @@ -247,85 +239,85 @@ BOOST_AUTO_TEST_CASE(subnet_test) BOOST_CHECK(!CSubNet(tor_addr, 200).IsValid()); BOOST_CHECK(!CSubNet(tor_addr, ResolveIP("255.0.0.0")).IsValid()); - subnet = ResolveSubNet("1.2.3.4/255.255.255.255"); + subnet = LookupSubNet("1.2.3.4/255.255.255.255"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.4/32"); - subnet = ResolveSubNet("1.2.3.4/255.255.255.254"); + subnet = LookupSubNet("1.2.3.4/255.255.255.254"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.4/31"); - subnet = ResolveSubNet("1.2.3.4/255.255.255.252"); + subnet = LookupSubNet("1.2.3.4/255.255.255.252"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.4/30"); - subnet = ResolveSubNet("1.2.3.4/255.255.255.248"); + subnet = LookupSubNet("1.2.3.4/255.255.255.248"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.0/29"); - subnet = ResolveSubNet("1.2.3.4/255.255.255.240"); + subnet = LookupSubNet("1.2.3.4/255.255.255.240"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.0/28"); - subnet = ResolveSubNet("1.2.3.4/255.255.255.224"); + subnet = LookupSubNet("1.2.3.4/255.255.255.224"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.0/27"); - subnet = ResolveSubNet("1.2.3.4/255.255.255.192"); + subnet = LookupSubNet("1.2.3.4/255.255.255.192"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.0/26"); - subnet = ResolveSubNet("1.2.3.4/255.255.255.128"); + subnet = LookupSubNet("1.2.3.4/255.255.255.128"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.0/25"); - subnet = ResolveSubNet("1.2.3.4/255.255.255.0"); + subnet = LookupSubNet("1.2.3.4/255.255.255.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.3.0/24"); - subnet = ResolveSubNet("1.2.3.4/255.255.254.0"); + subnet = LookupSubNet("1.2.3.4/255.255.254.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.2.0/23"); - subnet = ResolveSubNet("1.2.3.4/255.255.252.0"); + subnet = LookupSubNet("1.2.3.4/255.255.252.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.0.0/22"); - subnet = ResolveSubNet("1.2.3.4/255.255.248.0"); + subnet = LookupSubNet("1.2.3.4/255.255.248.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.0.0/21"); - subnet = ResolveSubNet("1.2.3.4/255.255.240.0"); + subnet = LookupSubNet("1.2.3.4/255.255.240.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.0.0/20"); - subnet = ResolveSubNet("1.2.3.4/255.255.224.0"); + subnet = LookupSubNet("1.2.3.4/255.255.224.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.0.0/19"); - subnet = ResolveSubNet("1.2.3.4/255.255.192.0"); + subnet = LookupSubNet("1.2.3.4/255.255.192.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.0.0/18"); - subnet = ResolveSubNet("1.2.3.4/255.255.128.0"); + subnet = LookupSubNet("1.2.3.4/255.255.128.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.0.0/17"); - subnet = ResolveSubNet("1.2.3.4/255.255.0.0"); + subnet = LookupSubNet("1.2.3.4/255.255.0.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.0.0/16"); - subnet = ResolveSubNet("1.2.3.4/255.254.0.0"); + subnet = LookupSubNet("1.2.3.4/255.254.0.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.2.0.0/15"); - subnet = ResolveSubNet("1.2.3.4/255.252.0.0"); + subnet = LookupSubNet("1.2.3.4/255.252.0.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.0.0.0/14"); - subnet = ResolveSubNet("1.2.3.4/255.248.0.0"); + subnet = LookupSubNet("1.2.3.4/255.248.0.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.0.0.0/13"); - subnet = ResolveSubNet("1.2.3.4/255.240.0.0"); + subnet = LookupSubNet("1.2.3.4/255.240.0.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.0.0.0/12"); - subnet = ResolveSubNet("1.2.3.4/255.224.0.0"); + subnet = LookupSubNet("1.2.3.4/255.224.0.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.0.0.0/11"); - subnet = ResolveSubNet("1.2.3.4/255.192.0.0"); + subnet = LookupSubNet("1.2.3.4/255.192.0.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.0.0.0/10"); - subnet = ResolveSubNet("1.2.3.4/255.128.0.0"); + subnet = LookupSubNet("1.2.3.4/255.128.0.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.0.0.0/9"); - subnet = ResolveSubNet("1.2.3.4/255.0.0.0"); + subnet = LookupSubNet("1.2.3.4/255.0.0.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "1.0.0.0/8"); - subnet = ResolveSubNet("1.2.3.4/254.0.0.0"); + subnet = LookupSubNet("1.2.3.4/254.0.0.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "0.0.0.0/7"); - subnet = ResolveSubNet("1.2.3.4/252.0.0.0"); + subnet = LookupSubNet("1.2.3.4/252.0.0.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "0.0.0.0/6"); - subnet = ResolveSubNet("1.2.3.4/248.0.0.0"); + subnet = LookupSubNet("1.2.3.4/248.0.0.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "0.0.0.0/5"); - subnet = ResolveSubNet("1.2.3.4/240.0.0.0"); + subnet = LookupSubNet("1.2.3.4/240.0.0.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "0.0.0.0/4"); - subnet = ResolveSubNet("1.2.3.4/224.0.0.0"); + subnet = LookupSubNet("1.2.3.4/224.0.0.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "0.0.0.0/3"); - subnet = ResolveSubNet("1.2.3.4/192.0.0.0"); + subnet = LookupSubNet("1.2.3.4/192.0.0.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "0.0.0.0/2"); - subnet = ResolveSubNet("1.2.3.4/128.0.0.0"); + subnet = LookupSubNet("1.2.3.4/128.0.0.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "0.0.0.0/1"); - subnet = ResolveSubNet("1.2.3.4/0.0.0.0"); + subnet = LookupSubNet("1.2.3.4/0.0.0.0"); BOOST_CHECK_EQUAL(subnet.ToString(), "0.0.0.0/0"); - subnet = ResolveSubNet("1:2:3:4:5:6:7:8/ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"); + subnet = LookupSubNet("1:2:3:4:5:6:7:8/ffff:ffff:ffff:ffff:ffff:ffff:ffff:ffff"); BOOST_CHECK_EQUAL(subnet.ToString(), "1:2:3:4:5:6:7:8/128"); - subnet = ResolveSubNet("1:2:3:4:5:6:7:8/ffff:0000:0000:0000:0000:0000:0000:0000"); + subnet = LookupSubNet("1:2:3:4:5:6:7:8/ffff:0000:0000:0000:0000:0000:0000:0000"); BOOST_CHECK_EQUAL(subnet.ToString(), "1::/16"); - subnet = ResolveSubNet("1:2:3:4:5:6:7:8/0000:0000:0000:0000:0000:0000:0000:0000"); + subnet = LookupSubNet("1:2:3:4:5:6:7:8/0000:0000:0000:0000:0000:0000:0000:0000"); BOOST_CHECK_EQUAL(subnet.ToString(), "::/0"); // Invalid netmasks (with 1-bits after 0-bits) - subnet = ResolveSubNet("1.2.3.4/255.255.232.0"); + subnet = LookupSubNet("1.2.3.4/255.255.232.0"); BOOST_CHECK(!subnet.IsValid()); - subnet = ResolveSubNet("1.2.3.4/255.0.255.255"); + subnet = LookupSubNet("1.2.3.4/255.0.255.255"); BOOST_CHECK(!subnet.IsValid()); - subnet = ResolveSubNet("1:2:3:4:5:6:7:8/ffff:ffff:ffff:fffe:ffff:ffff:ffff:ff0f"); + subnet = LookupSubNet("1:2:3:4:5:6:7:8/ffff:ffff:ffff:fffe:ffff:ffff:ffff:ff0f"); BOOST_CHECK(!subnet.IsValid()); } @@ -479,15 +471,15 @@ BOOST_AUTO_TEST_CASE(netbase_dont_resolve_strings_with_embedded_nul_characters) BOOST_CHECK(!LookupHost("127.0.0.1\0"s, false).has_value()); BOOST_CHECK(!LookupHost("127.0.0.1\0example.com"s, false).has_value()); BOOST_CHECK(!LookupHost("127.0.0.1\0example.com\0"s, false).has_value()); - CSubNet ret; - BOOST_CHECK(LookupSubNet("1.2.3.0/24"s, ret)); - BOOST_CHECK(!LookupSubNet("1.2.3.0/24\0"s, ret)); - BOOST_CHECK(!LookupSubNet("1.2.3.0/24\0example.com"s, ret)); - BOOST_CHECK(!LookupSubNet("1.2.3.0/24\0example.com\0"s, ret)); - BOOST_CHECK(LookupSubNet("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion"s, ret)); - BOOST_CHECK(!LookupSubNet("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion\0"s, ret)); - BOOST_CHECK(!LookupSubNet("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion\0example.com"s, ret)); - BOOST_CHECK(!LookupSubNet("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion\0example.com\0"s, ret)); + + BOOST_CHECK(LookupSubNet("1.2.3.0/24"s).IsValid()); + BOOST_CHECK(!LookupSubNet("1.2.3.0/24\0"s).IsValid()); + BOOST_CHECK(!LookupSubNet("1.2.3.0/24\0example.com"s).IsValid()); + BOOST_CHECK(!LookupSubNet("1.2.3.0/24\0example.com\0"s).IsValid()); + BOOST_CHECK(LookupSubNet("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion"s).IsValid()); + BOOST_CHECK(!LookupSubNet("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion\0"s).IsValid()); + BOOST_CHECK(!LookupSubNet("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion\0example.com"s).IsValid()); + BOOST_CHECK(!LookupSubNet("pg6mmjiyjmcrsslvykfwnntlaru7p5svn6y2ymmju6nubxndf4pscryd.onion\0example.com\0"s).IsValid()); } // Since CNetAddr (un)ser is tested separately in net_tests.cpp here we only diff --git a/src/test/orphanage_tests.cpp b/src/test/orphanage_tests.cpp index af53737fec..4231fcc909 100644 --- a/src/test/orphanage_tests.cpp +++ b/src/test/orphanage_tests.cpp @@ -3,6 +3,7 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <arith_uint256.h> +#include <primitives/transaction.h> #include <pubkey.h> #include <script/sign.h> #include <script/signingprovider.h> @@ -29,8 +30,8 @@ public: CTransactionRef RandomOrphan() EXCLUSIVE_LOCKS_REQUIRED(!m_mutex) { LOCK(m_mutex); - std::map<uint256, OrphanTx>::iterator it; - it = m_orphans.lower_bound(InsecureRand256()); + std::map<Txid, OrphanTx>::iterator it; + it = m_orphans.lower_bound(Txid::FromUint256(InsecureRand256())); if (it == m_orphans.end()) it = m_orphans.begin(); return it->second.tx; @@ -67,7 +68,7 @@ BOOST_AUTO_TEST_CASE(DoS_mapOrphans) CMutableTransaction tx; tx.vin.resize(1); tx.vin[0].prevout.n = 0; - tx.vin[0].prevout.hash = InsecureRand256(); + tx.vin[0].prevout.hash = Txid::FromUint256(InsecureRand256()); tx.vin[0].scriptSig << OP_1; tx.vout.resize(1); tx.vout[0].nValue = 1*CENT; @@ -111,7 +112,7 @@ BOOST_AUTO_TEST_CASE(DoS_mapOrphans) } SignatureData empty; BOOST_CHECK(SignSignature(keystore, *txPrev, tx, 0, SIGHASH_ALL, empty)); - // Re-use same signature for other inputs + // Reuse same signature for other inputs // (they don't have to be valid for this test) for (unsigned int j = 1; j < tx.vin.size(); j++) tx.vin[j].scriptSig = tx.vin[0].scriptSig; @@ -128,11 +129,12 @@ BOOST_AUTO_TEST_CASE(DoS_mapOrphans) } // Test LimitOrphanTxSize() function: - orphanage.LimitOrphans(40); + FastRandomContext rng{/*fDeterministic=*/true}; + orphanage.LimitOrphans(40, rng); BOOST_CHECK(orphanage.CountOrphans() <= 40); - orphanage.LimitOrphans(10); + orphanage.LimitOrphans(10, rng); BOOST_CHECK(orphanage.CountOrphans() <= 10); - orphanage.LimitOrphans(0); + orphanage.LimitOrphans(0, rng); BOOST_CHECK(orphanage.CountOrphans() == 0); } diff --git a/src/test/pmt_tests.cpp b/src/test/pmt_tests.cpp index a1e672d174..edecf70c6f 100644 --- a/src/test/pmt_tests.cpp +++ b/src/test/pmt_tests.cpp @@ -9,7 +9,6 @@ #include <test/util/random.h> #include <test/util/setup_common.h> #include <uint256.h> -#include <version.h> #include <vector> diff --git a/src/test/policyestimator_tests.cpp b/src/test/policyestimator_tests.cpp index bc9c672560..ede73c6895 100644 --- a/src/test/policyestimator_tests.cpp +++ b/src/test/policyestimator_tests.cpp @@ -8,6 +8,7 @@ #include <txmempool.h> #include <uint256.h> #include <util/time.h> +#include <validationinterface.h> #include <test/util/setup_common.h> @@ -19,7 +20,7 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) { CBlockPolicyEstimator& feeEst = *Assert(m_node.fee_estimator); CTxMemPool& mpool = *Assert(m_node.mempool); - LOCK2(cs_main, mpool.cs); + RegisterValidationInterface(&feeEst); TestMemPoolEntryHelper entry; CAmount basefee(2000); CAmount deltaFee(100); @@ -59,8 +60,23 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) for (int j = 0; j < 10; j++) { // For each fee for (int k = 0; k < 4; k++) { // add 4 fee txs tx.vin[0].prevout.n = 10000*blocknum+100*j+k; // make transaction unique + { + LOCK2(cs_main, mpool.cs); + mpool.addUnchecked(entry.Fee(feeV[j]).Time(Now<NodeSeconds>()).Height(blocknum).FromTx(tx)); + // Since TransactionAddedToMempool callbacks are generated in ATMP, + // not addUnchecked, we cheat and create one manually here + const int64_t virtual_size = GetVirtualTransactionSize(*MakeTransactionRef(tx)); + const NewMempoolTransactionInfo tx_info{NewMempoolTransactionInfo(MakeTransactionRef(tx), + feeV[j], + virtual_size, + entry.nHeight, + /*mempool_limit_bypassed=*/false, + /*submitted_in_package=*/false, + /*chainstate_is_current=*/true, + /*has_no_mempool_parents=*/true)}; + GetMainSignals().TransactionAddedToMempool(tx_info, mpool.GetAndIncrementSequence()); + } uint256 hash = tx.GetHash(); - mpool.addUnchecked(entry.Fee(feeV[j]).Time(Now<NodeSeconds>()).Height(blocknum).FromTx(tx)); txHashes[j].push_back(hash); } } @@ -76,10 +92,17 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) txHashes[9-h].pop_back(); } } - mpool.removeForBlock(block, ++blocknum); + + { + LOCK(mpool.cs); + mpool.removeForBlock(block, ++blocknum); + } + block.clear(); // Check after just a few txs that combining buckets works as expected if (blocknum == 3) { + // Wait for fee estimator to catch up + SyncWithValidationInterfaceQueue(); // At this point we should need to combine 3 buckets to get enough data points // So estimateFee(1) should fail and estimateFee(2) should return somewhere around // 9*baserate. estimateFee(2) %'s are 100,100,90 = average 97% @@ -89,6 +112,9 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) } } + // Wait for fee estimator to catch up + SyncWithValidationInterfaceQueue(); + std::vector<CAmount> origFeeEst; // Highest feerate is 10*baseRate and gets in all blocks, // second highest feerate is 9*baseRate and gets in 9/10 blocks = 90%, @@ -114,8 +140,13 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) // Mine 50 more blocks with no transactions happening, estimates shouldn't change // We haven't decayed the moving average enough so we still have enough data points in every bucket - while (blocknum < 250) + while (blocknum < 250) { + LOCK(mpool.cs); mpool.removeForBlock(block, ++blocknum); + } + + // Wait for fee estimator to catch up + SyncWithValidationInterfaceQueue(); BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); for (int i = 2; i < 10;i++) { @@ -130,14 +161,35 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) for (int j = 0; j < 10; j++) { // For each fee multiple for (int k = 0; k < 4; k++) { // add 4 fee txs tx.vin[0].prevout.n = 10000*blocknum+100*j+k; + { + LOCK2(cs_main, mpool.cs); + mpool.addUnchecked(entry.Fee(feeV[j]).Time(Now<NodeSeconds>()).Height(blocknum).FromTx(tx)); + // Since TransactionAddedToMempool callbacks are generated in ATMP, + // not addUnchecked, we cheat and create one manually here + const int64_t virtual_size = GetVirtualTransactionSize(*MakeTransactionRef(tx)); + const NewMempoolTransactionInfo tx_info{NewMempoolTransactionInfo(MakeTransactionRef(tx), + feeV[j], + virtual_size, + entry.nHeight, + /*mempool_limit_bypassed=*/false, + /*submitted_in_package=*/false, + /*chainstate_is_current=*/true, + /*has_no_mempool_parents=*/true)}; + GetMainSignals().TransactionAddedToMempool(tx_info, mpool.GetAndIncrementSequence()); + } uint256 hash = tx.GetHash(); - mpool.addUnchecked(entry.Fee(feeV[j]).Time(Now<NodeSeconds>()).Height(blocknum).FromTx(tx)); txHashes[j].push_back(hash); } } - mpool.removeForBlock(block, ++blocknum); + { + LOCK(mpool.cs); + mpool.removeForBlock(block, ++blocknum); + } } + // Wait for fee estimator to catch up + SyncWithValidationInterfaceQueue(); + for (int i = 1; i < 10;i++) { BOOST_CHECK(feeEst.estimateFee(i) == CFeeRate(0) || feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); } @@ -152,8 +204,16 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) txHashes[j].pop_back(); } } - mpool.removeForBlock(block, 266); + + { + LOCK(mpool.cs); + mpool.removeForBlock(block, 266); + } block.clear(); + + // Wait for fee estimator to catch up + SyncWithValidationInterfaceQueue(); + BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); for (int i = 2; i < 10;i++) { BOOST_CHECK(feeEst.estimateFee(i) == CFeeRate(0) || feeEst.estimateFee(i).GetFeePerK() > origFeeEst[i-1] - deltaFee); @@ -165,17 +225,39 @@ BOOST_AUTO_TEST_CASE(BlockPolicyEstimates) for (int j = 0; j < 10; j++) { // For each fee multiple for (int k = 0; k < 4; k++) { // add 4 fee txs tx.vin[0].prevout.n = 10000*blocknum+100*j+k; + { + LOCK2(cs_main, mpool.cs); + mpool.addUnchecked(entry.Fee(feeV[j]).Time(Now<NodeSeconds>()).Height(blocknum).FromTx(tx)); + // Since TransactionAddedToMempool callbacks are generated in ATMP, + // not addUnchecked, we cheat and create one manually here + const int64_t virtual_size = GetVirtualTransactionSize(*MakeTransactionRef(tx)); + const NewMempoolTransactionInfo tx_info{NewMempoolTransactionInfo(MakeTransactionRef(tx), + feeV[j], + virtual_size, + entry.nHeight, + /*mempool_limit_bypassed=*/false, + /*submitted_in_package=*/false, + /*chainstate_is_current=*/true, + /*has_no_mempool_parents=*/true)}; + GetMainSignals().TransactionAddedToMempool(tx_info, mpool.GetAndIncrementSequence()); + } uint256 hash = tx.GetHash(); - mpool.addUnchecked(entry.Fee(feeV[j]).Time(Now<NodeSeconds>()).Height(blocknum).FromTx(tx)); CTransactionRef ptx = mpool.get(hash); if (ptx) block.push_back(ptx); } } - mpool.removeForBlock(block, ++blocknum); + + { + LOCK(mpool.cs); + mpool.removeForBlock(block, ++blocknum); + } + block.clear(); } + // Wait for fee estimator to catch up + SyncWithValidationInterfaceQueue(); BOOST_CHECK(feeEst.estimateFee(1) == CFeeRate(0)); for (int i = 2; i < 9; i++) { // At 9, the original estimate was already at the bottom (b/c scale = 2) BOOST_CHECK(feeEst.estimateFee(i).GetFeePerK() < origFeeEst[i-1] - deltaFee); diff --git a/src/test/pool_tests.cpp b/src/test/pool_tests.cpp index 8a07e09a44..5ad4afa3a1 100644 --- a/src/test/pool_tests.cpp +++ b/src/test/pool_tests.cpp @@ -156,21 +156,20 @@ BOOST_AUTO_TEST_CASE(random_allocations) BOOST_AUTO_TEST_CASE(memusage_test) { - auto std_map = std::unordered_map<int, int>{}; - - using Map = std::unordered_map<int, - int, - std::hash<int>, - std::equal_to<int>, - PoolAllocator<std::pair<const int, int>, - sizeof(std::pair<const int, int>) + sizeof(void*) * 4, - alignof(void*)>>; + auto std_map = std::unordered_map<int64_t, int64_t>{}; + + using Map = std::unordered_map<int64_t, + int64_t, + std::hash<int64_t>, + std::equal_to<int64_t>, + PoolAllocator<std::pair<const int64_t, int64_t>, + sizeof(std::pair<const int64_t, int64_t>) + sizeof(void*) * 4>>; auto resource = Map::allocator_type::ResourceType(1024); PoolResourceTester::CheckAllDataAccountedFor(resource); { - auto resource_map = Map{0, std::hash<int>{}, std::equal_to<int>{}, &resource}; + auto resource_map = Map{0, std::hash<int64_t>{}, std::equal_to<int64_t>{}, &resource}; // can't have the same resource usage BOOST_TEST(memusage::DynamicUsage(std_map) != memusage::DynamicUsage(resource_map)); @@ -182,6 +181,11 @@ BOOST_AUTO_TEST_CASE(memusage_test) // Eventually the resource_map should have a much lower memory usage because it has less malloc overhead BOOST_TEST(memusage::DynamicUsage(resource_map) <= memusage::DynamicUsage(std_map) * 90 / 100); + + // Make sure the pool is actually used by the nodes + auto max_nodes_per_chunk = resource.ChunkSizeBytes() / sizeof(Map::value_type); + auto min_num_allocated_chunks = resource_map.size() / max_nodes_per_chunk + 1; + BOOST_TEST(resource.NumAllocatedChunks() >= min_num_allocated_chunks); } PoolResourceTester::CheckAllDataAccountedFor(resource); diff --git a/src/test/pow_tests.cpp b/src/test/pow_tests.cpp index 5bd14f22c6..3a44d1da49 100644 --- a/src/test/pow_tests.cpp +++ b/src/test/pow_tests.cpp @@ -177,7 +177,7 @@ void sanity_check_chainparams(const ArgsManager& args, ChainType chain_type) // check max target * 4*nPowTargetTimespan doesn't overflow -- see pow.cpp:CalculateNextWorkRequired() if (!consensus.fPowNoRetargeting) { - arith_uint256 targ_max("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"); + arith_uint256 targ_max{UintToArith256(uint256S("0xFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"))}; targ_max /= consensus.nPowTargetTimespan*4; BOOST_CHECK(UintToArith256(consensus.powLimit) < targ_max); } diff --git a/src/test/script_p2sh_tests.cpp b/src/test/script_p2sh_tests.cpp index 739ab75de3..54dcc218b9 100644 --- a/src/test/script_p2sh_tests.cpp +++ b/src/test/script_p2sh_tests.cpp @@ -18,9 +18,15 @@ #include <boost/test/unit_test.hpp> // Helpers: +static bool IsStandardTx(const CTransaction& tx, bool permit_bare_multisig, std::string& reason) +{ + return IsStandardTx(tx, std::nullopt, permit_bare_multisig, CFeeRate{DUST_RELAY_TX_FEE}, reason); +} + static bool IsStandardTx(const CTransaction& tx, std::string& reason) { - return IsStandardTx(tx, std::nullopt, DEFAULT_PERMIT_BAREMULTISIG, CFeeRate{DUST_RELAY_TX_FEE}, reason); + return IsStandardTx(tx, std::nullopt, /*permit_bare_multisig=*/true, CFeeRate{DUST_RELAY_TX_FEE}, reason) && + IsStandardTx(tx, std::nullopt, /*permit_bare_multisig=*/false, CFeeRate{DUST_RELAY_TX_FEE}, reason); } static std::vector<unsigned char> Serialize(const CScript& s) @@ -201,7 +207,9 @@ BOOST_AUTO_TEST_CASE(set) { SignatureData empty; BOOST_CHECK_MESSAGE(SignSignature(keystore, CTransaction(txFrom), txTo[i], 0, SIGHASH_ALL, empty), strprintf("SignSignature %d", i)); - BOOST_CHECK_MESSAGE(IsStandardTx(CTransaction(txTo[i]), reason), strprintf("txTo[%d].IsStandard", i)); + BOOST_CHECK_MESSAGE(IsStandardTx(CTransaction(txTo[i]), /*permit_bare_multisig=*/true, reason), strprintf("txTo[%d].IsStandard", i)); + bool no_pbms_is_std = IsStandardTx(CTransaction(txTo[i]), /*permit_bare_multisig=*/false, reason); + BOOST_CHECK_MESSAGE((i == 0 ? no_pbms_is_std : !no_pbms_is_std), strprintf("txTo[%d].IsStandard(permbaremulti=false)", i)); } } diff --git a/src/test/script_standard_tests.cpp b/src/test/script_standard_tests.cpp index 58bdb37b7c..29e2d4a569 100644 --- a/src/test/script_standard_tests.cpp +++ b/src/test/script_standard_tests.cpp @@ -136,10 +136,8 @@ BOOST_AUTO_TEST_CASE(script_standard_Solver_success) BOOST_AUTO_TEST_CASE(script_standard_Solver_failure) { - CKey key; - CPubKey pubkey; - key.MakeNewKey(true); - pubkey = key.GetPubKey(); + CKey key = GenerateRandomKey(); + CPubKey pubkey = key.GetPubKey(); CScript s; std::vector<std::vector<unsigned char> > solutions; @@ -192,10 +190,8 @@ BOOST_AUTO_TEST_CASE(script_standard_Solver_failure) BOOST_AUTO_TEST_CASE(script_standard_ExtractDestination) { - CKey key; - CPubKey pubkey; - key.MakeNewKey(true); - pubkey = key.GetPubKey(); + CKey key = GenerateRandomKey(); + CPubKey pubkey = key.GetPubKey(); CScript s; CTxDestination address; diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 624d0b2c12..1f674408b2 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -141,8 +141,8 @@ void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScript } #if defined(HAVE_CONSENSUS_LIB) - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << tx2; + DataStream stream; + stream << TX_WITH_WITNESS(tx2); uint32_t libconsensus_flags{flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL}; if (libconsensus_flags == flags) { int expectedSuccessCode = expect ? 1 : 0; @@ -1051,10 +1051,9 @@ sign_multisig(const CScript& scriptPubKey, const CKey& key, const CTransaction& BOOST_AUTO_TEST_CASE(script_CHECKMULTISIG12) { ScriptError err; - CKey key1, key2, key3; - key1.MakeNewKey(true); - key2.MakeNewKey(false); - key3.MakeNewKey(true); + CKey key1 = GenerateRandomKey(); + CKey key2 = GenerateRandomKey(/*compressed=*/false); + CKey key3 = GenerateRandomKey(); CScript scriptPubKey12; scriptPubKey12 << OP_1 << ToByteVector(key1.GetPubKey()) << ToByteVector(key2.GetPubKey()) << OP_2 << OP_CHECKMULTISIG; @@ -1081,11 +1080,10 @@ BOOST_AUTO_TEST_CASE(script_CHECKMULTISIG12) BOOST_AUTO_TEST_CASE(script_CHECKMULTISIG23) { ScriptError err; - CKey key1, key2, key3, key4; - key1.MakeNewKey(true); - key2.MakeNewKey(false); - key3.MakeNewKey(true); - key4.MakeNewKey(false); + CKey key1 = GenerateRandomKey(); + CKey key2 = GenerateRandomKey(/*compressed=*/false); + CKey key3 = GenerateRandomKey(); + CKey key4 = GenerateRandomKey(/*compressed=*/false); CScript scriptPubKey23; scriptPubKey23 << OP_2 << ToByteVector(key1.GetPubKey()) << ToByteVector(key2.GetPubKey()) << ToByteVector(key3.GetPubKey()) << OP_3 << OP_CHECKMULTISIG; @@ -1112,7 +1110,7 @@ BOOST_AUTO_TEST_CASE(script_CHECKMULTISIG23) BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_OK, ScriptErrorString(err)); keys.clear(); - keys.push_back(key2); keys.push_back(key2); // Can't re-use sig + keys.push_back(key2); keys.push_back(key2); // Can't reuse sig CScript badsig1 = sign_multisig(scriptPubKey23, keys, CTransaction(txTo23)); BOOST_CHECK(!VerifyScript(badsig1, scriptPubKey23, nullptr, gFlags, MutableTransactionSignatureChecker(&txTo23, 0, txFrom23.vout[0].nValue, MissingDataBehavior::ASSERT_FAIL), &err)); BOOST_CHECK_MESSAGE(err == SCRIPT_ERR_EVAL_FALSE, ScriptErrorString(err)); @@ -1165,8 +1163,7 @@ BOOST_AUTO_TEST_CASE(script_combineSigs) std::vector<CPubKey> pubkeys; for (int i = 0; i < 3; i++) { - CKey key; - key.MakeNewKey(i%2 == 1); + CKey key = GenerateRandomKey(/*compressed=*/i%2 == 1); keys.push_back(key); pubkeys.push_back(key.GetPubKey()); BOOST_CHECK(keystore.AddKey(key)); @@ -1470,7 +1467,7 @@ BOOST_AUTO_TEST_CASE(script_HasValidOps) static CMutableTransaction TxFromHex(const std::string& str) { CMutableTransaction tx; - SpanReader{SERIALIZE_TRANSACTION_NO_WITNESS, ParseHex(str)} >> tx; + SpanReader{ParseHex(str)} >> TX_NO_WITNESS(tx); return tx; } @@ -1480,7 +1477,7 @@ static std::vector<CTxOut> TxOutsFromJSON(const UniValue& univalue) std::vector<CTxOut> prevouts; for (size_t i = 0; i < univalue.size(); ++i) { CTxOut txout; - SpanReader{0, ParseHex(univalue[i].get_str())} >> txout; + SpanReader{ParseHex(univalue[i].get_str())} >> txout; prevouts.push_back(std::move(txout)); } return prevouts; @@ -1513,8 +1510,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_returns_true) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << spendTx; + DataStream stream; + stream << TX_WITH_WITNESS(spendTx); bitcoinconsensus_error err; int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); @@ -1536,8 +1533,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_index_err) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << spendTx; + DataStream stream; + stream << TX_WITH_WITNESS(spendTx); bitcoinconsensus_error err; int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); @@ -1559,8 +1556,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_size) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << spendTx; + DataStream stream; + stream << TX_WITH_WITNESS(spendTx); bitcoinconsensus_error err; int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size() * 2, nIn, libconsensus_flags, &err); @@ -1582,7 +1579,7 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_serialization) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + DataStream stream; stream << 0xffffffff; bitcoinconsensus_error err; @@ -1605,8 +1602,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_amount_required_err) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << spendTx; + DataStream stream; + stream << TX_WITH_WITNESS(spendTx); bitcoinconsensus_error err; int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); @@ -1628,8 +1625,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_invalid_flags) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << spendTx; + DataStream stream; + stream << TX_WITH_WITNESS(spendTx); bitcoinconsensus_error err; int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); @@ -1651,8 +1648,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_spent_outputs_required_err) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << spendTx; + DataStream stream; + stream << TX_WITH_WITNESS(spendTx); bitcoinconsensus_error err; int result{bitcoinconsensus_verify_script_with_spent_outputs(scriptPubKey.data(), scriptPubKey.size(), creditTx.vout[0].nValue, UCharCast(stream.data()), stream.size(), nullptr, 0, nIn, libconsensus_flags, &err)}; @@ -1718,8 +1715,8 @@ static void AssetTest(const UniValue& test) CachingTransactionSignatureChecker txcheck(&tx, idx, prevouts[idx].nValue, true, txdata); #if defined(HAVE_CONSENSUS_LIB) - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << tx; + DataStream stream; + stream << TX_WITH_WITNESS(tx); std::vector<UTXO> utxos; utxos.resize(prevouts.size()); for (size_t i = 0; i < prevouts.size(); i++) { @@ -1752,8 +1749,8 @@ static void AssetTest(const UniValue& test) CachingTransactionSignatureChecker txcheck(&tx, idx, prevouts[idx].nValue, true, txdata); #if defined(HAVE_CONSENSUS_LIB) - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << tx; + DataStream stream; + stream << TX_WITH_WITNESS(tx); std::vector<UTXO> utxos; utxos.resize(prevouts.size()); for (size_t i = 0; i < prevouts.size(); i++) { @@ -1816,7 +1813,7 @@ BOOST_AUTO_TEST_CASE(bip341_keypath_test_vectors) for (const auto& vec : vectors.getValues()) { auto txhex = ParseHex(vec["given"]["rawUnsignedTx"].get_str()); CMutableTransaction tx; - SpanReader{PROTOCOL_VERSION, txhex} >> tx; + SpanReader{txhex} >> TX_WITH_WITNESS(tx); std::vector<CTxOut> utxos; for (const auto& utxo_spent : vec["given"]["utxosSpent"].getValues()) { auto script_bytes = ParseHex(utxo_spent["scriptPubKey"].get_str()); diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index d18d2623b1..d75eb499b4 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -36,7 +36,7 @@ public: READWRITE(obj.boolval); READWRITE(obj.stringval); READWRITE(obj.charstrval); - READWRITE(obj.txval); + READWRITE(TX_WITH_WITNESS(obj.txval)); } bool operator==(const CSerializeMethodsTestSingle& rhs) const @@ -56,35 +56,37 @@ public: SERIALIZE_METHODS(CSerializeMethodsTestMany, obj) { - READWRITE(obj.intval, obj.boolval, obj.stringval, obj.charstrval, obj.txval); + READWRITE(obj.intval, obj.boolval, obj.stringval, obj.charstrval, TX_WITH_WITNESS(obj.txval)); } }; BOOST_AUTO_TEST_CASE(sizes) { - BOOST_CHECK_EQUAL(sizeof(unsigned char), GetSerializeSize((unsigned char)0, 0)); - BOOST_CHECK_EQUAL(sizeof(int8_t), GetSerializeSize(int8_t(0), 0)); - BOOST_CHECK_EQUAL(sizeof(uint8_t), GetSerializeSize(uint8_t(0), 0)); - BOOST_CHECK_EQUAL(sizeof(int16_t), GetSerializeSize(int16_t(0), 0)); - BOOST_CHECK_EQUAL(sizeof(uint16_t), GetSerializeSize(uint16_t(0), 0)); - BOOST_CHECK_EQUAL(sizeof(int32_t), GetSerializeSize(int32_t(0), 0)); - BOOST_CHECK_EQUAL(sizeof(uint32_t), GetSerializeSize(uint32_t(0), 0)); - BOOST_CHECK_EQUAL(sizeof(int64_t), GetSerializeSize(int64_t(0), 0)); - BOOST_CHECK_EQUAL(sizeof(uint64_t), GetSerializeSize(uint64_t(0), 0)); + BOOST_CHECK_EQUAL(sizeof(unsigned char), GetSerializeSize((unsigned char)0)); + BOOST_CHECK_EQUAL(sizeof(int8_t), GetSerializeSize(int8_t(0))); + BOOST_CHECK_EQUAL(sizeof(uint8_t), GetSerializeSize(uint8_t(0))); + BOOST_CHECK_EQUAL(sizeof(int16_t), GetSerializeSize(int16_t(0))); + BOOST_CHECK_EQUAL(sizeof(uint16_t), GetSerializeSize(uint16_t(0))); + BOOST_CHECK_EQUAL(sizeof(int32_t), GetSerializeSize(int32_t(0))); + BOOST_CHECK_EQUAL(sizeof(uint32_t), GetSerializeSize(uint32_t(0))); + BOOST_CHECK_EQUAL(sizeof(int64_t), GetSerializeSize(int64_t(0))); + BOOST_CHECK_EQUAL(sizeof(uint64_t), GetSerializeSize(uint64_t(0))); // Bool is serialized as uint8_t - BOOST_CHECK_EQUAL(sizeof(uint8_t), GetSerializeSize(bool(0), 0)); + BOOST_CHECK_EQUAL(sizeof(uint8_t), GetSerializeSize(bool(0))); // Sanity-check GetSerializeSize and c++ type matching - BOOST_CHECK_EQUAL(GetSerializeSize((unsigned char)0, 0), 1U); - BOOST_CHECK_EQUAL(GetSerializeSize(int8_t(0), 0), 1U); - BOOST_CHECK_EQUAL(GetSerializeSize(uint8_t(0), 0), 1U); - BOOST_CHECK_EQUAL(GetSerializeSize(int16_t(0), 0), 2U); - BOOST_CHECK_EQUAL(GetSerializeSize(uint16_t(0), 0), 2U); - BOOST_CHECK_EQUAL(GetSerializeSize(int32_t(0), 0), 4U); - BOOST_CHECK_EQUAL(GetSerializeSize(uint32_t(0), 0), 4U); - BOOST_CHECK_EQUAL(GetSerializeSize(int64_t(0), 0), 8U); - BOOST_CHECK_EQUAL(GetSerializeSize(uint64_t(0), 0), 8U); - BOOST_CHECK_EQUAL(GetSerializeSize(bool(0), 0), 1U); + BOOST_CHECK_EQUAL(GetSerializeSize((unsigned char)0), 1U); + BOOST_CHECK_EQUAL(GetSerializeSize(int8_t(0)), 1U); + BOOST_CHECK_EQUAL(GetSerializeSize(uint8_t(0)), 1U); + BOOST_CHECK_EQUAL(GetSerializeSize(int16_t(0)), 2U); + BOOST_CHECK_EQUAL(GetSerializeSize(uint16_t(0)), 2U); + BOOST_CHECK_EQUAL(GetSerializeSize(int32_t(0)), 4U); + BOOST_CHECK_EQUAL(GetSerializeSize(uint32_t(0)), 4U); + BOOST_CHECK_EQUAL(GetSerializeSize(int64_t(0)), 8U); + BOOST_CHECK_EQUAL(GetSerializeSize(uint64_t(0)), 8U); + BOOST_CHECK_EQUAL(GetSerializeSize(bool(0)), 1U); + BOOST_CHECK_EQUAL(GetSerializeSize(std::array<uint8_t, 1>{0}), 1U); + BOOST_CHECK_EQUAL(GetSerializeSize(std::array<uint8_t, 2>{0, 0}), 2U); } BOOST_AUTO_TEST_CASE(varints) @@ -95,13 +97,13 @@ BOOST_AUTO_TEST_CASE(varints) DataStream::size_type size = 0; for (int i = 0; i < 100000; i++) { ss << VARINT_MODE(i, VarIntMode::NONNEGATIVE_SIGNED); - size += ::GetSerializeSize(VARINT_MODE(i, VarIntMode::NONNEGATIVE_SIGNED), 0); + size += ::GetSerializeSize(VARINT_MODE(i, VarIntMode::NONNEGATIVE_SIGNED)); BOOST_CHECK(size == ss.size()); } for (uint64_t i = 0; i < 100000000000ULL; i += 999999937) { ss << VARINT(i); - size += ::GetSerializeSize(VARINT(i), 0); + size += ::GetSerializeSize(VARINT(i)); BOOST_CHECK(size == ss.size()); } @@ -179,6 +181,16 @@ BOOST_AUTO_TEST_CASE(vector_bool) BOOST_CHECK((HashWriter{} << vec1).GetHash() == (HashWriter{} << vec2).GetHash()); } +BOOST_AUTO_TEST_CASE(array) +{ + std::array<uint8_t, 32> array1{1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1}; + DataStream ds; + ds << array1; + std::array<uint8_t, 32> array2; + ds >> array2; + BOOST_CHECK(array1 == array2); +} + BOOST_AUTO_TEST_CASE(noncanonical) { // Write some non-canonical CompactSize encodings, and @@ -228,7 +240,7 @@ BOOST_AUTO_TEST_CASE(class_methods) CSerializeMethodsTestMany methodtest2(intval, boolval, stringval, charstrval, tx_ref); CSerializeMethodsTestSingle methodtest3; CSerializeMethodsTestMany methodtest4; - CDataStream ss(SER_DISK, PROTOCOL_VERSION); + DataStream ss; BOOST_CHECK(methodtest1 == methodtest2); ss << methodtest1; ss >> methodtest4; @@ -238,8 +250,8 @@ BOOST_AUTO_TEST_CASE(class_methods) BOOST_CHECK(methodtest2 == methodtest3); BOOST_CHECK(methodtest3 == methodtest4); - CDataStream ss2{SER_DISK, PROTOCOL_VERSION}; - ss2 << intval << boolval << stringval << charstrval << txval; + DataStream ss2; + ss2 << intval << boolval << stringval << charstrval << TX_WITH_WITNESS(txval); ss2 >> methodtest3; BOOST_CHECK(methodtest3 == methodtest4); { @@ -255,10 +267,15 @@ BOOST_AUTO_TEST_CASE(class_methods) } } -enum class BaseFormat { - RAW, - HEX, +struct BaseFormat { + const enum { + RAW, + HEX, + } m_base_format; + SER_PARAMS_OPFUNC }; +constexpr BaseFormat RAW{BaseFormat::RAW}; +constexpr BaseFormat HEX{BaseFormat::HEX}; /// (Un)serialize a number as raw byte or 2 hexadecimal chars. class Base @@ -272,7 +289,7 @@ public: template <typename Stream> void Serialize(Stream& s) const { - if (s.GetParams() == BaseFormat::RAW) { + if (s.GetParams().m_base_format == BaseFormat::RAW) { s << m_base_data; } else { s << Span{HexStr(Span{&m_base_data, 1})}; @@ -282,7 +299,7 @@ public: template <typename Stream> void Unserialize(Stream& s) { - if (s.GetParams() == BaseFormat::RAW) { + if (s.GetParams().m_base_format == BaseFormat::RAW) { s >> m_base_data; } else { std::string hex{"aa"}; @@ -301,6 +318,8 @@ public: LOWER, UPPER, } m_derived_format; + + SER_PARAMS_OPFUNC }; class Derived : public Base @@ -310,7 +329,7 @@ public: SERIALIZE_METHODS_PARAMS(Derived, obj, DerivedAndBaseFormat, fmt) { - READWRITE(WithParams(fmt.m_base_format, AsBase<Base>(obj))); + READWRITE(fmt.m_base_format(AsBase<Base>(obj))); if (ser_action.ForRead()) { std::string str; @@ -330,20 +349,20 @@ BOOST_AUTO_TEST_CASE(with_params_base) DataStream stream; - stream << WithParams(BaseFormat::RAW, b); + stream << RAW(b); BOOST_CHECK_EQUAL(stream.str(), "\x0F"); b.m_base_data = 0; - stream >> WithParams(BaseFormat::RAW, b); + stream >> RAW(b); BOOST_CHECK_EQUAL(b.m_base_data, 0x0F); stream.clear(); - stream << WithParams(BaseFormat::HEX, b); + stream << HEX(b); BOOST_CHECK_EQUAL(stream.str(), "0f"); b.m_base_data = 0; - stream >> WithParams(BaseFormat::HEX, b); + stream >> HEX(b); BOOST_CHECK_EQUAL(b.m_base_data, 0x0F); } @@ -353,45 +372,42 @@ BOOST_AUTO_TEST_CASE(with_params_vector_of_base) DataStream stream; - stream << WithParams(BaseFormat::RAW, v); + stream << RAW(v); BOOST_CHECK_EQUAL(stream.str(), "\x02\x0F\xFF"); v[0].m_base_data = 0; v[1].m_base_data = 0; - stream >> WithParams(BaseFormat::RAW, v); + stream >> RAW(v); BOOST_CHECK_EQUAL(v[0].m_base_data, 0x0F); BOOST_CHECK_EQUAL(v[1].m_base_data, 0xFF); stream.clear(); - stream << WithParams(BaseFormat::HEX, v); + stream << HEX(v); BOOST_CHECK_EQUAL(stream.str(), "\x02" "0fff"); v[0].m_base_data = 0; v[1].m_base_data = 0; - stream >> WithParams(BaseFormat::HEX, v); + stream >> HEX(v); BOOST_CHECK_EQUAL(v[0].m_base_data, 0x0F); BOOST_CHECK_EQUAL(v[1].m_base_data, 0xFF); } +constexpr DerivedAndBaseFormat RAW_LOWER{{BaseFormat::RAW}, DerivedAndBaseFormat::DerivedFormat::LOWER}; +constexpr DerivedAndBaseFormat HEX_UPPER{{BaseFormat::HEX}, DerivedAndBaseFormat::DerivedFormat::UPPER}; + BOOST_AUTO_TEST_CASE(with_params_derived) { Derived d; d.m_base_data = 0x0F; d.m_derived_data = "xY"; - DerivedAndBaseFormat fmt; - DataStream stream; - fmt.m_base_format = BaseFormat::RAW; - fmt.m_derived_format = DerivedAndBaseFormat::DerivedFormat::LOWER; - stream << WithParams(fmt, d); + stream << RAW_LOWER(d); - fmt.m_base_format = BaseFormat::HEX; - fmt.m_derived_format = DerivedAndBaseFormat::DerivedFormat::UPPER; - stream << WithParams(fmt, d); + stream << HEX_UPPER(d); BOOST_CHECK_EQUAL(stream.str(), "\x0F\x02xy" "0f\x02XY"); diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 9653edd84b..d08d5519a1 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -15,7 +15,6 @@ #include <test/util/random.h> #include <test/util/setup_common.h> #include <util/strencodings.h> -#include <version.h> #include <iostream> @@ -78,8 +77,8 @@ uint256 static SignatureHashOld(CScript scriptCode, const CTransaction& txTo, un } // Serialize and hash - CHashWriter ss{SERIALIZE_TRANSACTION_NO_WITNESS}; - ss << txTmp << nHashType; + HashWriter ss{}; + ss << TX_NO_WITNESS(txTmp) << nHashType; return ss.GetHash(); } @@ -102,7 +101,7 @@ void static RandomTransaction(CMutableTransaction& tx, bool fSingle) for (int in = 0; in < ins; in++) { tx.vin.emplace_back(); CTxIn &txin = tx.vin.back(); - txin.prevout.hash = InsecureRand256(); + txin.prevout.hash = Txid::FromUint256(InsecureRand256()); txin.prevout.n = InsecureRandBits(2); RandomScript(txin.scriptSig); txin.nSequence = (InsecureRandBool()) ? InsecureRand32() : std::numeric_limits<uint32_t>::max(); @@ -138,8 +137,8 @@ BOOST_AUTO_TEST_CASE(sighash_test) sho = SignatureHashOld(scriptCode, CTransaction(txTo), nIn, nHashType); sh = SignatureHash(scriptCode, txTo, nIn, nHashType, 0, SigVersion::BASE); #if defined(PRINT_SIGHASH_JSON) - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); - ss << txTo; + DataStream ss; + ss << TX_WITH_WITNESS(txTo); std::cout << "\t[\"" ; std::cout << HexStr(ss) << "\", \""; @@ -188,8 +187,8 @@ BOOST_AUTO_TEST_CASE(sighash_from_data) nHashType = test[3].getInt<int>(); sigHashHex = test[4].get_str(); - CDataStream stream(ParseHex(raw_tx), SER_NETWORK, PROTOCOL_VERSION); - stream >> tx; + DataStream stream(ParseHex(raw_tx)); + stream >> TX_WITH_WITNESS(tx); TxValidationState state; BOOST_CHECK_MESSAGE(CheckTransaction(*tx, state), strTest); diff --git a/src/test/sigopcount_tests.cpp b/src/test/sigopcount_tests.cpp index c0bed50e1d..2081acdf4d 100644 --- a/src/test/sigopcount_tests.cpp +++ b/src/test/sigopcount_tests.cpp @@ -50,8 +50,7 @@ BOOST_AUTO_TEST_CASE(GetSigOpCount) std::vector<CPubKey> keys; for (int i = 0; i < 3; i++) { - CKey k; - k.MakeNewKey(true); + CKey k = GenerateRandomKey(); keys.push_back(k.GetPubKey()); } CScript s2 = GetScriptForMultisig(1, keys); @@ -120,8 +119,7 @@ BOOST_AUTO_TEST_CASE(GetTxSigOpCost) CCoinsView coinsDummy; CCoinsViewCache coins(&coinsDummy); // Create key - CKey key; - key.MakeNewKey(true); + CKey key = GenerateRandomKey(); CPubKey pubkey = key.GetPubKey(); // Default flags const uint32_t flags{SCRIPT_VERIFY_WITNESS | SCRIPT_VERIFY_P2SH}; diff --git a/src/test/span_tests.cpp b/src/test/span_tests.cpp new file mode 100644 index 0000000000..f6cac10b09 --- /dev/null +++ b/src/test/span_tests.cpp @@ -0,0 +1,73 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <span.h> + +#include <boost/test/unit_test.hpp> +#include <array> +#include <set> +#include <vector> + +namespace { +struct Ignore +{ + template<typename T> Ignore(T&&) {} +}; +template<typename T> +bool Spannable(T&& value, decltype(Span{value})* enable = nullptr) +{ + return true; +} +bool Spannable(Ignore) +{ + return false; +} + +#if defined(__clang__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wunneeded-member-function" +# pragma clang diagnostic ignored "-Wunused-member-function" +#endif +struct SpannableYes +{ + int* data(); + size_t size(); +}; +struct SpannableNo +{ + void* data(); + size_t size(); +}; +#if defined(__clang__) +# pragma clang diagnostic pop +#endif +} // namespace + +BOOST_AUTO_TEST_SUITE(span_tests) + +// Make sure template Span template deduction guides accurately enable calls to +// Span constructor overloads that work, and disable calls to constructor overloads that +// don't work. This makes it is possible to use the Span constructor in a SFINAE +// contexts like in the Spannable function above to detect whether types are or +// aren't compatible with Spans at compile time. +// +// Previously there was a bug where writing a SFINAE check for vector<bool> was +// not possible, because in libstdc++ vector<bool> has a data() memeber +// returning void*, and the Span template guide ignored the data() return value, +// so the template substitution would succeed, but the constructor would fail, +// resulting in a fatal compile error, rather than a SFINAE error that could be +// handled. +BOOST_AUTO_TEST_CASE(span_constructor_sfinae) +{ + BOOST_CHECK(Spannable(std::vector<int>{})); + BOOST_CHECK(!Spannable(std::set<int>{})); + BOOST_CHECK(!Spannable(std::vector<bool>{})); + BOOST_CHECK(Spannable(std::array<int, 3>{})); + BOOST_CHECK(Spannable(Span<int>{})); + BOOST_CHECK(Spannable("char array")); + BOOST_CHECK(Spannable(SpannableYes{})); + BOOST_CHECK(!Spannable(SpannableNo{})); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/streams_tests.cpp b/src/test/streams_tests.cpp index f03f7c1da2..7d1ac5a19a 100644 --- a/src/test/streams_tests.cpp +++ b/src/test/streams_tests.cpp @@ -74,49 +74,49 @@ BOOST_AUTO_TEST_CASE(streams_vector_writer) // point should yield the same results, even if the first test grew the // vector. - CVectorWriter{INIT_PROTO_VERSION, vch, 0, a, b}; + VectorWriter{vch, 0, a, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{1, 2}})); - CVectorWriter{INIT_PROTO_VERSION, vch, 0, a, b}; + VectorWriter{vch, 0, a, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{1, 2}})); vch.clear(); - CVectorWriter{INIT_PROTO_VERSION, vch, 2, a, b}; + VectorWriter{vch, 2, a, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2}})); - CVectorWriter{INIT_PROTO_VERSION, vch, 2, a, b}; + VectorWriter{vch, 2, a, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2}})); vch.clear(); vch.resize(5, 0); - CVectorWriter{INIT_PROTO_VERSION, vch, 2, a, b}; + VectorWriter{vch, 2, a, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2, 0}})); - CVectorWriter{INIT_PROTO_VERSION, vch, 2, a, b}; + VectorWriter{vch, 2, a, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 1, 2, 0}})); vch.clear(); vch.resize(4, 0); - CVectorWriter{INIT_PROTO_VERSION, vch, 3, a, b}; + VectorWriter{vch, 3, a, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 1, 2}})); - CVectorWriter{INIT_PROTO_VERSION, vch, 3, a, b}; + VectorWriter{vch, 3, a, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 1, 2}})); vch.clear(); vch.resize(4, 0); - CVectorWriter{INIT_PROTO_VERSION, vch, 4, a, b}; + VectorWriter{vch, 4, a, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 0, 1, 2}})); - CVectorWriter{INIT_PROTO_VERSION, vch, 4, a, b}; + VectorWriter{vch, 4, a, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{0, 0, 0, 0, 1, 2}})); vch.clear(); - CVectorWriter{INIT_PROTO_VERSION, vch, 0, bytes}; + VectorWriter{vch, 0, bytes}; BOOST_CHECK((vch == std::vector<unsigned char>{{3, 4, 5, 6}})); - CVectorWriter{INIT_PROTO_VERSION, vch, 0, bytes}; + VectorWriter{vch, 0, bytes}; BOOST_CHECK((vch == std::vector<unsigned char>{{3, 4, 5, 6}})); vch.clear(); vch.resize(4, 8); - CVectorWriter{INIT_PROTO_VERSION, vch, 2, a, bytes, b}; + VectorWriter{vch, 2, a, bytes, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{8, 8, 1, 3, 4, 5, 6, 2}})); - CVectorWriter{INIT_PROTO_VERSION, vch, 2, a, bytes, b}; + VectorWriter{vch, 2, a, bytes, b}; BOOST_CHECK((vch == std::vector<unsigned char>{{8, 8, 1, 3, 4, 5, 6, 2}})); vch.clear(); } @@ -125,7 +125,7 @@ BOOST_AUTO_TEST_CASE(streams_vector_reader) { std::vector<unsigned char> vch = {1, 255, 3, 4, 5, 6}; - SpanReader reader{INIT_PROTO_VERSION, vch}; + SpanReader reader{vch}; BOOST_CHECK_EQUAL(reader.size(), 6U); BOOST_CHECK(!reader.empty()); @@ -155,7 +155,7 @@ BOOST_AUTO_TEST_CASE(streams_vector_reader) BOOST_CHECK_THROW(reader >> d, std::ios_base::failure); // Read a 4 bytes as a signed int from the beginning of the buffer. - SpanReader new_reader{INIT_PROTO_VERSION, vch}; + SpanReader new_reader{vch}; new_reader >> d; BOOST_CHECK_EQUAL(d, 67370753); // 1,255,3,4 in little-endian base-256 BOOST_CHECK_EQUAL(new_reader.size(), 2U); @@ -169,7 +169,7 @@ BOOST_AUTO_TEST_CASE(streams_vector_reader) BOOST_AUTO_TEST_CASE(streams_vector_reader_rvalue) { std::vector<uint8_t> data{0x82, 0xa7, 0x31}; - SpanReader reader{INIT_PROTO_VERSION, data}; + SpanReader reader{data}; uint32_t varint = 0; // Deserialize into r-value reader >> VARINT(varint); @@ -249,7 +249,7 @@ BOOST_AUTO_TEST_CASE(streams_serializedata_xor) BOOST_AUTO_TEST_CASE(streams_buffered_file) { fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp"; - CAutoFile file{fsbridge::fopen(streams_test_filename, "w+b"), 333}; + AutoFile file{fsbridge::fopen(streams_test_filename, "w+b")}; // The value at each offset is the offset. for (uint8_t j = 0; j < 40; ++j) { @@ -271,9 +271,6 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) BufferedFile bf{file, 25, 10}; BOOST_CHECK(!bf.eof()); - // This member has no functional effect. - BOOST_CHECK_EQUAL(bf.GetVersion(), 333); - uint8_t i; bf >> i; BOOST_CHECK_EQUAL(i, 0); @@ -383,7 +380,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file) BOOST_AUTO_TEST_CASE(streams_buffered_file_skip) { fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp"; - CAutoFile file{fsbridge::fopen(streams_test_filename, "w+b"), 333}; + AutoFile file{fsbridge::fopen(streams_test_filename, "w+b")}; // The value at each offset is the byte offset (e.g. byte 1 in the file has the value 0x01). for (uint8_t j = 0; j < 40; ++j) { file << j; @@ -436,7 +433,7 @@ BOOST_AUTO_TEST_CASE(streams_buffered_file_rand) fs::path streams_test_filename = m_args.GetDataDirBase() / "streams_test_tmp"; for (int rep = 0; rep < 50; ++rep) { - CAutoFile file{fsbridge::fopen(streams_test_filename, "w+b"), 333}; + AutoFile file{fsbridge::fopen(streams_test_filename, "w+b")}; size_t fileSize = InsecureRandRange(256); for (uint8_t i = 0; i < fileSize; ++i) { file << i; diff --git a/src/test/system_tests.cpp b/src/test/system_tests.cpp index 740f461548..6a96b60db0 100644 --- a/src/test/system_tests.cpp +++ b/src/test/system_tests.cpp @@ -90,7 +90,13 @@ BOOST_AUTO_TEST_CASE(run_command) }); } { - BOOST_REQUIRE_THROW(RunCommandParseJSON("echo \"{\""), std::runtime_error); // Unable to parse JSON + // Unable to parse JSON +#ifdef WIN32 + const std::string command{"cmd.exe /c echo {"}; +#else + const std::string command{"echo {"}; +#endif + BOOST_CHECK_EXCEPTION(RunCommandParseJSON(command), std::runtime_error, HasReason("Unable to parse JSON: {")); } // Test std::in, except for Windows #ifndef WIN32 diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index a4c0db8aea..d1cb2531aa 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -27,6 +27,7 @@ #include <test/util/transaction_utils.h> #include <util/strencodings.h> #include <util/string.h> +#include <util/transaction_identifier.h> #include <validation.h> #include <functional> @@ -219,7 +220,7 @@ BOOST_AUTO_TEST_CASE(tx_valid) fValid = false; break; } - COutPoint outpoint{uint256S(vinput[0].get_str()), uint32_t(vinput[1].getInt<int>())}; + COutPoint outpoint{TxidFromString(vinput[0].get_str()), uint32_t(vinput[1].getInt<int>())}; mapprevOutScriptPubKeys[outpoint] = ParseScript(vinput[2].get_str()); if (vinput.size() >= 4) { @@ -233,8 +234,8 @@ BOOST_AUTO_TEST_CASE(tx_valid) } std::string transaction = test[1].get_str(); - CDataStream stream(ParseHex(transaction), SER_NETWORK, PROTOCOL_VERSION); - CTransaction tx(deserialize, stream); + DataStream stream(ParseHex(transaction)); + CTransaction tx(deserialize, TX_WITH_WITNESS, stream); TxValidationState state; BOOST_CHECK_MESSAGE(CheckTransaction(tx, state), strTest); @@ -307,7 +308,7 @@ BOOST_AUTO_TEST_CASE(tx_invalid) fValid = false; break; } - COutPoint outpoint{uint256S(vinput[0].get_str()), uint32_t(vinput[1].getInt<int>())}; + COutPoint outpoint{TxidFromString(vinput[0].get_str()), uint32_t(vinput[1].getInt<int>())}; mapprevOutScriptPubKeys[outpoint] = ParseScript(vinput[2].get_str()); if (vinput.size() >= 4) { @@ -321,8 +322,8 @@ BOOST_AUTO_TEST_CASE(tx_invalid) } std::string transaction = test[1].get_str(); - CDataStream stream(ParseHex(transaction), SER_NETWORK, PROTOCOL_VERSION ); - CTransaction tx(deserialize, stream); + DataStream stream(ParseHex(transaction)); + CTransaction tx(deserialize, TX_WITH_WITNESS, stream); TxValidationState state; if (!CheckTransaction(tx, state) || state.IsInvalid()) { @@ -371,9 +372,9 @@ BOOST_AUTO_TEST_CASE(basic_transaction_tests) // Random real transaction (e2769b09e784f32f62ef849763d4f45b98e07ba658647343b915ff832b110436) unsigned char ch[] = {0x01, 0x00, 0x00, 0x00, 0x01, 0x6b, 0xff, 0x7f, 0xcd, 0x4f, 0x85, 0x65, 0xef, 0x40, 0x6d, 0xd5, 0xd6, 0x3d, 0x4f, 0xf9, 0x4f, 0x31, 0x8f, 0xe8, 0x20, 0x27, 0xfd, 0x4d, 0xc4, 0x51, 0xb0, 0x44, 0x74, 0x01, 0x9f, 0x74, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x49, 0x30, 0x46, 0x02, 0x21, 0x00, 0xda, 0x0d, 0xc6, 0xae, 0xce, 0xfe, 0x1e, 0x06, 0xef, 0xdf, 0x05, 0x77, 0x37, 0x57, 0xde, 0xb1, 0x68, 0x82, 0x09, 0x30, 0xe3, 0xb0, 0xd0, 0x3f, 0x46, 0xf5, 0xfc, 0xf1, 0x50, 0xbf, 0x99, 0x0c, 0x02, 0x21, 0x00, 0xd2, 0x5b, 0x5c, 0x87, 0x04, 0x00, 0x76, 0xe4, 0xf2, 0x53, 0xf8, 0x26, 0x2e, 0x76, 0x3e, 0x2d, 0xd5, 0x1e, 0x7f, 0xf0, 0xbe, 0x15, 0x77, 0x27, 0xc4, 0xbc, 0x42, 0x80, 0x7f, 0x17, 0xbd, 0x39, 0x01, 0x41, 0x04, 0xe6, 0xc2, 0x6e, 0xf6, 0x7d, 0xc6, 0x10, 0xd2, 0xcd, 0x19, 0x24, 0x84, 0x78, 0x9a, 0x6c, 0xf9, 0xae, 0xa9, 0x93, 0x0b, 0x94, 0x4b, 0x7e, 0x2d, 0xb5, 0x34, 0x2b, 0x9d, 0x9e, 0x5b, 0x9f, 0xf7, 0x9a, 0xff, 0x9a, 0x2e, 0xe1, 0x97, 0x8d, 0xd7, 0xfd, 0x01, 0xdf, 0xc5, 0x22, 0xee, 0x02, 0x28, 0x3d, 0x3b, 0x06, 0xa9, 0xd0, 0x3a, 0xcf, 0x80, 0x96, 0x96, 0x8d, 0x7d, 0xbb, 0x0f, 0x91, 0x78, 0xff, 0xff, 0xff, 0xff, 0x02, 0x8b, 0xa7, 0x94, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, 0xa9, 0x14, 0xba, 0xde, 0xec, 0xfd, 0xef, 0x05, 0x07, 0x24, 0x7f, 0xc8, 0xf7, 0x42, 0x41, 0xd7, 0x3b, 0xc0, 0x39, 0x97, 0x2d, 0x7b, 0x88, 0xac, 0x40, 0x94, 0xa8, 0x02, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, 0xa9, 0x14, 0xc1, 0x09, 0x32, 0x48, 0x3f, 0xec, 0x93, 0xed, 0x51, 0xf5, 0xfe, 0x95, 0xe7, 0x25, 0x59, 0xf2, 0xcc, 0x70, 0x43, 0xf9, 0x88, 0xac, 0x00, 0x00, 0x00, 0x00, 0x00}; std::vector<unsigned char> vch(ch, ch + sizeof(ch) -1); - CDataStream stream(vch, SER_DISK, CLIENT_VERSION); + DataStream stream(vch); CMutableTransaction tx; - stream >> tx; + stream >> TX_WITH_WITNESS(tx); TxValidationState state; BOOST_CHECK_MESSAGE(CheckTransaction(CTransaction(tx), state) && state.IsValid(), "Simple deserialized transaction should be valid."); @@ -418,9 +419,9 @@ static void CreateCreditAndSpend(const FillableSigningProvider& keystore, const outputm.vout.resize(1); outputm.vout[0].nValue = 1; outputm.vout[0].scriptPubKey = outscript; - CDataStream ssout(SER_NETWORK, PROTOCOL_VERSION); - ssout << outputm; - ssout >> output; + DataStream ssout; + ssout << TX_WITH_WITNESS(outputm); + ssout >> TX_WITH_WITNESS(output); assert(output->vin.size() == 1); assert(output->vin[0] == outputm.vin[0]); assert(output->vout.size() == 1); @@ -437,9 +438,9 @@ static void CreateCreditAndSpend(const FillableSigningProvider& keystore, const SignatureData empty; bool ret = SignSignature(keystore, *output, inputm, 0, SIGHASH_ALL, empty); assert(ret == success); - CDataStream ssin(SER_NETWORK, PROTOCOL_VERSION); - ssin << inputm; - ssin >> input; + DataStream ssin; + ssin << TX_WITH_WITNESS(inputm); + ssin >> TX_WITH_WITNESS(input); assert(input.vin.size() == 1); assert(input.vin[0] == inputm.vin[0]); assert(input.vout.size() == 1); @@ -486,8 +487,7 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) CMutableTransaction mtx; mtx.nVersion = 1; - CKey key; - key.MakeNewKey(true); // Need to use compressed keys in segwit or the signing will fail + CKey key = GenerateRandomKey(); // Need to use compressed keys in segwit or the signing will fail FillableSigningProvider keystore; BOOST_CHECK(keystore.AddKeyPubKey(key, key.GetPubKey())); CKeyID hash = key.GetPubKey().GetID(); @@ -504,9 +504,7 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) // create a big transaction of 4500 inputs signed by the same key for(uint32_t ij = 0; ij < 4500; ij++) { uint32_t i = mtx.vin.size(); - uint256 prevId; - prevId.SetHex("0000000000000000000000000000000000000000000000000000000000000100"); - COutPoint outpoint(prevId, i); + COutPoint outpoint(TxidFromString("0000000000000000000000000000000000000000000000000000000000000100"), i); mtx.vin.resize(mtx.vin.size() + 1); mtx.vin[i].prevout = outpoint; @@ -524,17 +522,15 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) assert(hashSigned); } - CDataStream ssout(SER_NETWORK, PROTOCOL_VERSION); - ssout << mtx; - CTransaction tx(deserialize, ssout); + DataStream ssout; + ssout << TX_WITH_WITNESS(mtx); + CTransaction tx(deserialize, TX_WITH_WITNESS, ssout); // check all inputs concurrently, with the cache PrecomputedTransactionData txdata(tx); - CCheckQueue<CScriptCheck> scriptcheckqueue(128); + CCheckQueue<CScriptCheck> scriptcheckqueue(/*batch_size=*/128, /*worker_threads_num=*/20); CCheckQueueControl<CScriptCheck> control(&scriptcheckqueue); - scriptcheckqueue.StartWorkerThreads(20); - std::vector<Coin> coins; for(uint32_t i = 0; i < mtx.vin.size(); i++) { Coin coin; @@ -553,7 +549,6 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) bool controlCheck = control.Wait(); assert(controlCheck); - scriptcheckqueue.StopWorkerThreads(); } SignatureData CombineSignatures(const CMutableTransaction& input1, const CMutableTransaction& input2, const CTransactionRef tx) @@ -568,18 +563,16 @@ SignatureData CombineSignatures(const CMutableTransaction& input1, const CMutabl BOOST_AUTO_TEST_CASE(test_witness) { FillableSigningProvider keystore, keystore2; - CKey key1, key2, key3, key1L, key2L; - CPubKey pubkey1, pubkey2, pubkey3, pubkey1L, pubkey2L; - key1.MakeNewKey(true); - key2.MakeNewKey(true); - key3.MakeNewKey(true); - key1L.MakeNewKey(false); - key2L.MakeNewKey(false); - pubkey1 = key1.GetPubKey(); - pubkey2 = key2.GetPubKey(); - pubkey3 = key3.GetPubKey(); - pubkey1L = key1L.GetPubKey(); - pubkey2L = key2L.GetPubKey(); + CKey key1 = GenerateRandomKey(); + CKey key2 = GenerateRandomKey(); + CKey key3 = GenerateRandomKey(); + CKey key1L = GenerateRandomKey(/*compressed=*/false); + CKey key2L = GenerateRandomKey(/*compressed=*/false); + CPubKey pubkey1 = key1.GetPubKey(); + CPubKey pubkey2 = key2.GetPubKey(); + CPubKey pubkey3 = key3.GetPubKey(); + CPubKey pubkey1L = key1L.GetPubKey(); + CPubKey pubkey2L = key2L.GetPubKey(); BOOST_CHECK(keystore.AddKeyPubKey(key1, pubkey1)); BOOST_CHECK(keystore.AddKeyPubKey(key2, pubkey2)); BOOST_CHECK(keystore.AddKeyPubKey(key1L, pubkey1L)); @@ -760,8 +753,7 @@ BOOST_AUTO_TEST_CASE(test_IsStandard) t.vin[0].scriptSig << std::vector<unsigned char>(65, 0); t.vout.resize(1); t.vout[0].nValue = 90*CENT; - CKey key; - key.MakeNewKey(true); + CKey key = GenerateRandomKey(); t.vout[0].scriptPubKey = GetScriptForDestination(PKHash(key.GetPubKey())); constexpr auto CheckIsStandard = [](const auto& t) { diff --git a/src/test/txindex_tests.cpp b/src/test/txindex_tests.cpp index 9fa59bab57..e2432a4718 100644 --- a/src/test/txindex_tests.cpp +++ b/src/test/txindex_tests.cpp @@ -33,7 +33,7 @@ BOOST_FIXTURE_TEST_CASE(txindex_initial_sync, TestChain100Setup) BOOST_REQUIRE(txindex.StartBackgroundSync()); // Allow tx index to catch up with the block index. - IndexWaitSynced(txindex); + IndexWaitSynced(txindex, *Assert(m_node.shutdown)); // Check that txindex excludes genesis block transactions. const CBlock& genesis_block = Params().GenesisBlock(); diff --git a/src/test/txpackage_tests.cpp b/src/test/txpackage_tests.cpp index 4d9a5ef7f3..f6456526bb 100644 --- a/src/test/txpackage_tests.cpp +++ b/src/test/txpackage_tests.cpp @@ -9,7 +9,9 @@ #include <primitives/transaction.h> #include <script/script.h> #include <test/util/random.h> +#include <test/util/script.h> #include <test/util/setup_common.h> +#include <test/util/txmempool.h> #include <validation.h> #include <boost/test/unit_test.hpp> @@ -27,7 +29,7 @@ inline CTransactionRef create_placeholder_tx(size_t num_inputs, size_t num_outpu mtx.vout.resize(num_outputs); auto random_script = CScript() << ToByteVector(InsecureRand256()) << ToByteVector(InsecureRand256()); for (size_t i{0}; i < num_inputs; ++i) { - mtx.vin[i].prevout.hash = InsecureRand256(); + mtx.vin[i].prevout.hash = Txid::FromUint256(InsecureRand256()); mtx.vin[i].prevout.n = 0; mtx.vin[i].scriptSig = random_script; } @@ -47,7 +49,7 @@ BOOST_FIXTURE_TEST_CASE(package_sanitization_tests, TestChain100Setup) package_too_many.emplace_back(create_placeholder_tx(1, 1)); } PackageValidationState state_too_many; - BOOST_CHECK(!CheckPackage(package_too_many, state_too_many)); + BOOST_CHECK(!IsWellFormedPackage(package_too_many, state_too_many, /*require_sorted=*/true)); BOOST_CHECK_EQUAL(state_too_many.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(state_too_many.GetRejectReason(), "package-too-many-transactions"); @@ -62,7 +64,7 @@ BOOST_FIXTURE_TEST_CASE(package_sanitization_tests, TestChain100Setup) } BOOST_CHECK(package_too_large.size() <= MAX_PACKAGE_COUNT); PackageValidationState state_too_large; - BOOST_CHECK(!CheckPackage(package_too_large, state_too_large)); + BOOST_CHECK(!IsWellFormedPackage(package_too_large, state_too_large, /*require_sorted=*/true)); BOOST_CHECK_EQUAL(state_too_large.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(state_too_large.GetRejectReason(), "package-too-large"); @@ -73,9 +75,39 @@ BOOST_FIXTURE_TEST_CASE(package_sanitization_tests, TestChain100Setup) package_duplicate_txids_empty.emplace_back(MakeTransactionRef(empty_tx)); } PackageValidationState state_duplicates; - BOOST_CHECK(!CheckPackage(package_duplicate_txids_empty, state_duplicates)); + BOOST_CHECK(!IsWellFormedPackage(package_duplicate_txids_empty, state_duplicates, /*require_sorted=*/true)); BOOST_CHECK_EQUAL(state_duplicates.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(state_duplicates.GetRejectReason(), "package-contains-duplicates"); + BOOST_CHECK(!IsConsistentPackage(package_duplicate_txids_empty)); + + // Packages can't have transactions spending the same prevout + CMutableTransaction tx_zero_1; + CMutableTransaction tx_zero_2; + COutPoint same_prevout{Txid::FromUint256(InsecureRand256()), 0}; + tx_zero_1.vin.emplace_back(same_prevout); + tx_zero_2.vin.emplace_back(same_prevout); + // Different vouts (not the same tx) + tx_zero_1.vout.emplace_back(CENT, P2WSH_OP_TRUE); + tx_zero_2.vout.emplace_back(2 * CENT, P2WSH_OP_TRUE); + Package package_conflicts{MakeTransactionRef(tx_zero_1), MakeTransactionRef(tx_zero_2)}; + BOOST_CHECK(!IsConsistentPackage(package_conflicts)); + // Transactions are considered sorted when they have no dependencies. + BOOST_CHECK(IsTopoSortedPackage(package_conflicts)); + PackageValidationState state_conflicts; + BOOST_CHECK(!IsWellFormedPackage(package_conflicts, state_conflicts, /*require_sorted=*/true)); + BOOST_CHECK_EQUAL(state_conflicts.GetResult(), PackageValidationResult::PCKG_POLICY); + BOOST_CHECK_EQUAL(state_conflicts.GetRejectReason(), "conflict-in-package"); + + // IsConsistentPackage only cares about conflicts between transactions, not about a transaction + // conflicting with itself (i.e. duplicate prevouts in vin). + CMutableTransaction dup_tx; + const COutPoint rand_prevout{Txid::FromUint256(InsecureRand256()), 0}; + dup_tx.vin.emplace_back(rand_prevout); + dup_tx.vin.emplace_back(rand_prevout); + Package package_with_dup_tx{MakeTransactionRef(dup_tx)}; + BOOST_CHECK(IsConsistentPackage(package_with_dup_tx)); + package_with_dup_tx.emplace_back(create_placeholder_tx(1, 1)); + BOOST_CHECK(IsConsistentPackage(package_with_dup_tx)); } BOOST_FIXTURE_TEST_CASE(package_validation_tests, TestChain100Setup) @@ -84,8 +116,7 @@ BOOST_FIXTURE_TEST_CASE(package_validation_tests, TestChain100Setup) unsigned int initialPoolSize = m_node.mempool->size(); // Parent and Child Package - CKey parent_key; - parent_key.MakeNewKey(true); + CKey parent_key = GenerateRandomKey(); CScript parent_locking_script = GetScriptForDestination(PKHash(parent_key.GetPubKey())); auto mtx_parent = CreateValidMempoolTransaction(/*input_transaction=*/m_coinbase_txns[0], /*input_vout=*/0, /*input_height=*/0, /*input_signing_key=*/coinbaseKey, @@ -93,44 +124,42 @@ BOOST_FIXTURE_TEST_CASE(package_validation_tests, TestChain100Setup) /*output_amount=*/CAmount(49 * COIN), /*submit=*/false); CTransactionRef tx_parent = MakeTransactionRef(mtx_parent); - CKey child_key; - child_key.MakeNewKey(true); + CKey child_key = GenerateRandomKey(); CScript child_locking_script = GetScriptForDestination(PKHash(child_key.GetPubKey())); auto mtx_child = CreateValidMempoolTransaction(/*input_transaction=*/tx_parent, /*input_vout=*/0, /*input_height=*/101, /*input_signing_key=*/parent_key, /*output_destination=*/child_locking_script, /*output_amount=*/CAmount(48 * COIN), /*submit=*/false); CTransactionRef tx_child = MakeTransactionRef(mtx_child); - const auto result_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, {tx_parent, tx_child}, /*test_accept=*/true); - BOOST_CHECK_MESSAGE(result_parent_child.m_state.IsValid(), - "Package validation unexpectedly failed: " << result_parent_child.m_state.GetRejectReason()); - BOOST_CHECK(result_parent_child.m_tx_results.size() == 2); - auto it_parent = result_parent_child.m_tx_results.find(tx_parent->GetWitnessHash()); - auto it_child = result_parent_child.m_tx_results.find(tx_child->GetWitnessHash()); - BOOST_CHECK(it_parent != result_parent_child.m_tx_results.end()); - BOOST_CHECK_MESSAGE(it_parent->second.m_state.IsValid(), - "Package validation unexpectedly failed: " << it_parent->second.m_state.GetRejectReason()); - BOOST_CHECK(it_parent->second.m_effective_feerate.value().GetFee(GetVirtualTransactionSize(*tx_parent)) == COIN); - BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().size(), 1); - BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().front(), tx_parent->GetWitnessHash()); - BOOST_CHECK(it_child != result_parent_child.m_tx_results.end()); - BOOST_CHECK_MESSAGE(it_child->second.m_state.IsValid(), - "Package validation unexpectedly failed: " << it_child->second.m_state.GetRejectReason()); - BOOST_CHECK(it_child->second.m_effective_feerate.value().GetFee(GetVirtualTransactionSize(*tx_child)) == COIN); - BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().size(), 1); - BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().front(), tx_child->GetWitnessHash()); + Package package_parent_child{tx_parent, tx_child}; + const auto result_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_parent_child, /*test_accept=*/true); + if (auto err_parent_child{CheckPackageMempoolAcceptResult(package_parent_child, result_parent_child, /*expect_valid=*/true, nullptr)}) { + BOOST_ERROR(err_parent_child.value()); + } else { + auto it_parent = result_parent_child.m_tx_results.find(tx_parent->GetWitnessHash()); + auto it_child = result_parent_child.m_tx_results.find(tx_child->GetWitnessHash()); + + BOOST_CHECK(it_parent->second.m_effective_feerate.value().GetFee(GetVirtualTransactionSize(*tx_parent)) == COIN); + BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().size(), 1); + BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().front(), tx_parent->GetWitnessHash()); + BOOST_CHECK(it_child->second.m_effective_feerate.value().GetFee(GetVirtualTransactionSize(*tx_child)) == COIN); + BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().size(), 1); + BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().front(), tx_child->GetWitnessHash()); + } // A single, giant transaction submitted through ProcessNewPackage fails on single tx policy. CTransactionRef giant_ptx = create_placeholder_tx(999, 999); BOOST_CHECK(GetVirtualTransactionSize(*giant_ptx) > DEFAULT_ANCESTOR_SIZE_LIMIT_KVB * 1000); - auto result_single_large = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, {giant_ptx}, /*test_accept=*/true); - BOOST_CHECK(result_single_large.m_state.IsInvalid()); - BOOST_CHECK_EQUAL(result_single_large.m_state.GetResult(), PackageValidationResult::PCKG_TX); - BOOST_CHECK_EQUAL(result_single_large.m_state.GetRejectReason(), "transaction failed"); - BOOST_CHECK(result_single_large.m_tx_results.size() == 1); - auto it_giant_tx = result_single_large.m_tx_results.find(giant_ptx->GetWitnessHash()); - BOOST_CHECK(it_giant_tx != result_single_large.m_tx_results.end()); - BOOST_CHECK_EQUAL(it_giant_tx->second.m_state.GetRejectReason(), "tx-size"); + Package package_single_giant{giant_ptx}; + auto result_single_large = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_single_giant, /*test_accept=*/true); + if (auto err_single_large{CheckPackageMempoolAcceptResult(package_single_giant, result_single_large, /*expect_valid=*/false, nullptr)}) { + BOOST_ERROR(err_single_large.value()); + } else { + BOOST_CHECK_EQUAL(result_single_large.m_state.GetResult(), PackageValidationResult::PCKG_TX); + BOOST_CHECK_EQUAL(result_single_large.m_state.GetRejectReason(), "transaction failed"); + auto it_giant_tx = result_single_large.m_tx_results.find(giant_ptx->GetWitnessHash()); + BOOST_CHECK_EQUAL(it_giant_tx->second.m_state.GetRejectReason(), "tx-size"); + } // Check that mempool size hasn't changed. BOOST_CHECK_EQUAL(m_node.mempool->size(), initialPoolSize); @@ -139,11 +168,9 @@ BOOST_FIXTURE_TEST_CASE(package_validation_tests, TestChain100Setup) BOOST_FIXTURE_TEST_CASE(noncontextual_package_tests, TestChain100Setup) { // The signatures won't be verified so we can just use a placeholder - CKey placeholder_key; - placeholder_key.MakeNewKey(true); + CKey placeholder_key = GenerateRandomKey(); CScript spk = GetScriptForDestination(PKHash(placeholder_key.GetPubKey())); - CKey placeholder_key_2; - placeholder_key_2.MakeNewKey(true); + CKey placeholder_key_2 = GenerateRandomKey(); CScript spk2 = GetScriptForDestination(PKHash(placeholder_key_2.GetPubKey())); // Parent and Child Package @@ -157,8 +184,8 @@ BOOST_FIXTURE_TEST_CASE(noncontextual_package_tests, TestChain100Setup) CTransactionRef tx_child = MakeTransactionRef(mtx_child); PackageValidationState state; - BOOST_CHECK(CheckPackage({tx_parent, tx_child}, state)); - BOOST_CHECK(!CheckPackage({tx_child, tx_parent}, state)); + BOOST_CHECK(IsWellFormedPackage({tx_parent, tx_child}, state, /*require_sorted=*/true)); + BOOST_CHECK(!IsWellFormedPackage({tx_child, tx_parent}, state, /*require_sorted=*/true)); BOOST_CHECK_EQUAL(state.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(state.GetRejectReason(), "package-not-sorted"); BOOST_CHECK(IsChildWithParents({tx_parent, tx_child})); @@ -186,7 +213,7 @@ BOOST_FIXTURE_TEST_CASE(noncontextual_package_tests, TestChain100Setup) package.push_back(MakeTransactionRef(child)); PackageValidationState state; - BOOST_CHECK(CheckPackage(package, state)); + BOOST_CHECK(IsWellFormedPackage(package, state, /*require_sorted=*/true)); BOOST_CHECK(IsChildWithParents(package)); BOOST_CHECK(IsChildWithParentsTree(package)); @@ -224,8 +251,8 @@ BOOST_FIXTURE_TEST_CASE(noncontextual_package_tests, TestChain100Setup) BOOST_CHECK(!IsChildWithParentsTree({tx_parent, tx_parent_also_child, tx_child})); // IsChildWithParents does not detect unsorted parents. BOOST_CHECK(IsChildWithParents({tx_parent_also_child, tx_parent, tx_child})); - BOOST_CHECK(CheckPackage({tx_parent, tx_parent_also_child, tx_child}, state)); - BOOST_CHECK(!CheckPackage({tx_parent_also_child, tx_parent, tx_child}, state)); + BOOST_CHECK(IsWellFormedPackage({tx_parent, tx_parent_also_child, tx_child}, state, /*require_sorted=*/true)); + BOOST_CHECK(!IsWellFormedPackage({tx_parent_also_child, tx_parent, tx_child}, state, /*require_sorted=*/true)); BOOST_CHECK_EQUAL(state.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(state.GetRejectReason(), "package-not-sorted"); } @@ -235,8 +262,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) { LOCK(cs_main); unsigned int expected_pool_size = m_node.mempool->size(); - CKey parent_key; - parent_key.MakeNewKey(true); + CKey parent_key = GenerateRandomKey(); CScript parent_locking_script = GetScriptForDestination(PKHash(parent_key.GetPubKey())); // Unrelated transactions are not allowed in package submission. @@ -250,6 +276,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) } auto result_unrelated_submit = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_unrelated, /*test_accept=*/false); + // We don't expect m_tx_results for each transaction when basic sanity checks haven't passed. BOOST_CHECK(result_unrelated_submit.m_state.IsInvalid()); BOOST_CHECK_EQUAL(result_unrelated_submit.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(result_unrelated_submit.m_state.GetRejectReason(), "package-not-child-with-parents"); @@ -266,8 +293,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) package_parent_child.push_back(tx_parent); package_3gen.push_back(tx_parent); - CKey child_key; - child_key.MakeNewKey(true); + CKey child_key = GenerateRandomKey(); CScript child_locking_script = GetScriptForDestination(PKHash(child_key.GetPubKey())); auto mtx_child = CreateValidMempoolTransaction(/*input_transaction=*/tx_parent, /*input_vout=*/0, /*input_height=*/101, /*input_signing_key=*/parent_key, @@ -277,8 +303,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) package_parent_child.push_back(tx_child); package_3gen.push_back(tx_child); - CKey grandchild_key; - grandchild_key.MakeNewKey(true); + CKey grandchild_key = GenerateRandomKey(); CScript grandchild_locking_script = GetScriptForDestination(PKHash(grandchild_key.GetPubKey())); auto mtx_grandchild = CreateValidMempoolTransaction(/*input_transaction=*/tx_child, /*input_vout=*/0, /*input_height=*/101, /*input_signing_key=*/child_key, @@ -305,20 +330,20 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) CMutableTransaction mtx_parent_invalid{mtx_parent}; mtx_parent_invalid.vin[0].scriptWitness = bad_witness; CTransactionRef tx_parent_invalid = MakeTransactionRef(mtx_parent_invalid); + Package package_invalid_parent{tx_parent_invalid, tx_child}; auto result_quit_early = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {tx_parent_invalid, tx_child}, /*test_accept=*/ false); - BOOST_CHECK(result_quit_early.m_state.IsInvalid()); + package_invalid_parent, /*test_accept=*/ false); + if (auto err_parent_invalid{CheckPackageMempoolAcceptResult(package_invalid_parent, result_quit_early, /*expect_valid=*/false, m_node.mempool.get())}) { + BOOST_ERROR(err_parent_invalid.value()); + } else { + auto it_parent = result_quit_early.m_tx_results.find(tx_parent_invalid->GetWitnessHash()); + auto it_child = result_quit_early.m_tx_results.find(tx_child->GetWitnessHash()); + BOOST_CHECK_EQUAL(it_parent->second.m_state.GetResult(), TxValidationResult::TX_WITNESS_MUTATED); + BOOST_CHECK_EQUAL(it_parent->second.m_state.GetRejectReason(), "bad-witness-nonstandard"); + BOOST_CHECK_EQUAL(it_child->second.m_state.GetResult(), TxValidationResult::TX_MISSING_INPUTS); + BOOST_CHECK_EQUAL(it_child->second.m_state.GetRejectReason(), "bad-txns-inputs-missingorspent"); + } BOOST_CHECK_EQUAL(result_quit_early.m_state.GetResult(), PackageValidationResult::PCKG_TX); - BOOST_CHECK(!result_quit_early.m_tx_results.empty()); - BOOST_CHECK_EQUAL(result_quit_early.m_tx_results.size(), 2); - auto it_parent = result_quit_early.m_tx_results.find(tx_parent_invalid->GetWitnessHash()); - auto it_child = result_quit_early.m_tx_results.find(tx_child->GetWitnessHash()); - BOOST_CHECK(it_parent != result_quit_early.m_tx_results.end()); - BOOST_CHECK(it_child != result_quit_early.m_tx_results.end()); - BOOST_CHECK_EQUAL(it_parent->second.m_state.GetResult(), TxValidationResult::TX_WITNESS_MUTATED); - BOOST_CHECK_EQUAL(it_parent->second.m_state.GetRejectReason(), "bad-witness-nonstandard"); - BOOST_CHECK_EQUAL(it_child->second.m_state.GetResult(), TxValidationResult::TX_MISSING_INPUTS); - BOOST_CHECK_EQUAL(it_child->second.m_state.GetRejectReason(), "bad-txns-inputs-missingorspent"); } // Child with missing parent. @@ -350,36 +375,27 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) BOOST_CHECK(it_parent->second.m_effective_feerate == CFeeRate(1 * COIN, GetVirtualTransactionSize(*tx_parent))); BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().size(), 1); BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().front(), tx_parent->GetWitnessHash()); - BOOST_CHECK(it_child != submit_parent_child.m_tx_results.end()); - BOOST_CHECK(it_child->second.m_state.IsValid()); BOOST_CHECK(it_child->second.m_effective_feerate == CFeeRate(1 * COIN, GetVirtualTransactionSize(*tx_child))); BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().size(), 1); BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().front(), tx_child->GetWitnessHash()); BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_parent->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_child->GetHash()))); } // Already-in-mempool transactions should be detected and de-duplicated. { const auto submit_deduped = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_parent_child, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_deduped.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_deduped.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_deduped.m_tx_results.size(), package_parent_child.size()); - auto it_parent_deduped = submit_deduped.m_tx_results.find(tx_parent->GetWitnessHash()); - auto it_child_deduped = submit_deduped.m_tx_results.find(tx_child->GetWitnessHash()); - BOOST_CHECK(it_parent_deduped != submit_deduped.m_tx_results.end()); - BOOST_CHECK(it_parent_deduped->second.m_state.IsValid()); - BOOST_CHECK(it_parent_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); - BOOST_CHECK(it_child_deduped != submit_deduped.m_tx_results.end()); - BOOST_CHECK(it_child_deduped->second.m_state.IsValid()); - BOOST_CHECK(it_child_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + if (auto err_deduped{CheckPackageMempoolAcceptResult(package_parent_child, submit_deduped, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_deduped.value()); + } else { + auto it_parent_deduped = submit_deduped.m_tx_results.find(tx_parent->GetWitnessHash()); + auto it_child_deduped = submit_deduped.m_tx_results.find(tx_child->GetWitnessHash()); + BOOST_CHECK(it_parent_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK(it_child_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + } BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_parent->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_child->GetHash()))); } } @@ -411,8 +427,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) witness2.stack.emplace_back(2); witness2.stack.emplace_back(witnessScript.begin(), witnessScript.end()); - CKey child_key; - child_key.MakeNewKey(true); + CKey child_key = GenerateRandomKey(); CScript child_locking_script = GetScriptForDestination(WitnessV0KeyHash(child_key.GetPubKey())); CMutableTransaction mtx_child1; mtx_child1.nVersion = 1; @@ -439,51 +454,39 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // Try submitting Package1{parent, child1} and Package2{parent, child2} where the children are // same-txid-different-witness. { + Package package_parent_child1{ptx_parent, ptx_child1}; const auto submit_witness1 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_parent, ptx_child1}, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_witness1.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_witness1.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_witness1.m_tx_results.size(), 2); - auto it_parent1 = submit_witness1.m_tx_results.find(ptx_parent->GetWitnessHash()); - auto it_child1 = submit_witness1.m_tx_results.find(ptx_child1->GetWitnessHash()); - BOOST_CHECK(it_parent1 != submit_witness1.m_tx_results.end()); - BOOST_CHECK_MESSAGE(it_parent1->second.m_state.IsValid(), - "Transaction unexpectedly failed: " << it_parent1->second.m_state.GetRejectReason()); - BOOST_CHECK(it_child1 != submit_witness1.m_tx_results.end()); - BOOST_CHECK_MESSAGE(it_child1->second.m_state.IsValid(), - "Transaction unexpectedly failed: " << it_child1->second.m_state.GetRejectReason()); - - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_child1->GetHash()))); + package_parent_child1, /*test_accept=*/false); + if (auto err_witness1{CheckPackageMempoolAcceptResult(package_parent_child1, submit_witness1, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_witness1.value()); + } // Child2 would have been validated individually. + Package package_parent_child2{ptx_parent, ptx_child2}; const auto submit_witness2 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_parent, ptx_child2}, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_witness2.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_witness2.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_witness2.m_tx_results.size(), 2); - auto it_parent2_deduped = submit_witness2.m_tx_results.find(ptx_parent->GetWitnessHash()); - auto it_child2 = submit_witness2.m_tx_results.find(ptx_child2->GetWitnessHash()); - BOOST_CHECK(it_parent2_deduped != submit_witness2.m_tx_results.end()); - BOOST_CHECK(it_parent2_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); - BOOST_CHECK(it_child2 != submit_witness2.m_tx_results.end()); - BOOST_CHECK(it_child2->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); - BOOST_CHECK_EQUAL(ptx_child1->GetWitnessHash(), it_child2->second.m_other_wtxid.value()); - - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_child2->GetHash()))); - BOOST_CHECK(!m_node.mempool->exists(GenTxid::Wtxid(ptx_child2->GetWitnessHash()))); + package_parent_child2, /*test_accept=*/false); + if (auto err_witness2{CheckPackageMempoolAcceptResult(package_parent_child2, submit_witness2, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_witness2.value()); + } else { + auto it_parent2_deduped = submit_witness2.m_tx_results.find(ptx_parent->GetWitnessHash()); + auto it_child2 = submit_witness2.m_tx_results.find(ptx_child2->GetWitnessHash()); + BOOST_CHECK(it_parent2_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK(it_child2->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); + BOOST_CHECK_EQUAL(ptx_child1->GetWitnessHash(), it_child2->second.m_other_wtxid.value()); + } // Deduplication should work when wtxid != txid. Submit package with the already-in-mempool // transactions again, which should not fail. const auto submit_segwit_dedup = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_parent, ptx_child1}, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_segwit_dedup.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_segwit_dedup.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_segwit_dedup.m_tx_results.size(), 2); - auto it_parent_dup = submit_segwit_dedup.m_tx_results.find(ptx_parent->GetWitnessHash()); - auto it_child_dup = submit_segwit_dedup.m_tx_results.find(ptx_child1->GetWitnessHash()); - BOOST_CHECK(it_parent_dup->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); - BOOST_CHECK(it_child_dup->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + package_parent_child1, /*test_accept=*/false); + if (auto err_segwit_dedup{CheckPackageMempoolAcceptResult(package_parent_child1, submit_segwit_dedup, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_segwit_dedup.value()); + } else { + auto it_parent_dup = submit_segwit_dedup.m_tx_results.find(ptx_parent->GetWitnessHash()); + auto it_child_dup = submit_segwit_dedup.m_tx_results.find(ptx_child1->GetWitnessHash()); + BOOST_CHECK(it_parent_dup->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK(it_child_dup->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + } } // Try submitting Package1{child2, grandchild} where child2 is same-txid-different-witness as @@ -493,8 +496,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // This tests a potential censorship vector in which an attacker broadcasts a competing package // where a parent's witness is mutated. The honest package should be accepted despite the fact // that we don't allow witness replacement. - CKey grandchild_key; - grandchild_key.MakeNewKey(true); + CKey grandchild_key = GenerateRandomKey(); CScript grandchild_locking_script = GetScriptForDestination(WitnessV0KeyHash(grandchild_key.GetPubKey())); auto mtx_grandchild = CreateValidMempoolTransaction(/*input_transaction=*/ptx_child2, /*input_vout=*/0, /*input_height=*/0, /*input_signing_key=*/child_key, @@ -504,21 +506,17 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // We already submitted child1 above. { + Package package_child2_grandchild{ptx_child2, ptx_grandchild}; const auto submit_spend_ignored = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_child2, ptx_grandchild}, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_spend_ignored.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_spend_ignored.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_spend_ignored.m_tx_results.size(), 2); - auto it_child2_ignored = submit_spend_ignored.m_tx_results.find(ptx_child2->GetWitnessHash()); - auto it_grandchild = submit_spend_ignored.m_tx_results.find(ptx_grandchild->GetWitnessHash()); - BOOST_CHECK(it_child2_ignored != submit_spend_ignored.m_tx_results.end()); - BOOST_CHECK(it_child2_ignored->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); - BOOST_CHECK(it_grandchild != submit_spend_ignored.m_tx_results.end()); - BOOST_CHECK(it_grandchild->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_child2->GetHash()))); - BOOST_CHECK(!m_node.mempool->exists(GenTxid::Wtxid(ptx_child2->GetWitnessHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Wtxid(ptx_grandchild->GetWitnessHash()))); + package_child2_grandchild, /*test_accept=*/false); + if (auto err_spend_ignored{CheckPackageMempoolAcceptResult(package_child2_grandchild, submit_spend_ignored, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_spend_ignored.value()); + } else { + auto it_child2_ignored = submit_spend_ignored.m_tx_results.find(ptx_child2->GetWitnessHash()); + auto it_grandchild = submit_spend_ignored.m_tx_results.find(ptx_grandchild->GetWitnessHash()); + BOOST_CHECK(it_child2_ignored->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); + BOOST_CHECK(it_grandchild->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + } } // A package Package{parent1, parent2, parent3, child} where the parents are a mixture of @@ -588,8 +586,7 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) BOOST_CHECK(m_node.mempool->m_min_relay_feerate.GetFee(GetVirtualTransactionSize(*ptx_parent3)) <= low_fee_amt); // child spends parent1, parent2, and parent3 - CKey mixed_grandchild_key; - mixed_grandchild_key.MakeNewKey(true); + CKey mixed_grandchild_key = GenerateRandomKey(); CScript mixed_child_spk = GetScriptForDestination(WitnessV0KeyHash(mixed_grandchild_key.GetPubKey())); CMutableTransaction mtx_mixed_child; @@ -610,36 +607,28 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // child should be accepted { const auto mixed_result = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_mixed, false); - BOOST_CHECK_MESSAGE(mixed_result.m_state.IsValid(), mixed_result.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(mixed_result.m_tx_results.size(), package_mixed.size()); - auto it_parent1 = mixed_result.m_tx_results.find(ptx_parent1->GetWitnessHash()); - auto it_parent2 = mixed_result.m_tx_results.find(ptx_parent2_v1->GetWitnessHash()); - auto it_parent3 = mixed_result.m_tx_results.find(ptx_parent3->GetWitnessHash()); - auto it_child = mixed_result.m_tx_results.find(ptx_mixed_child->GetWitnessHash()); - BOOST_CHECK(it_parent1 != mixed_result.m_tx_results.end()); - BOOST_CHECK(it_parent2 != mixed_result.m_tx_results.end()); - BOOST_CHECK(it_parent3 != mixed_result.m_tx_results.end()); - BOOST_CHECK(it_child != mixed_result.m_tx_results.end()); - - BOOST_CHECK(it_parent1->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); - BOOST_CHECK(it_parent2->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); - BOOST_CHECK(it_parent3->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK_EQUAL(ptx_parent2_v2->GetWitnessHash(), it_parent2->second.m_other_wtxid.value()); - - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent1->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent2_v1->GetHash()))); - BOOST_CHECK(!m_node.mempool->exists(GenTxid::Wtxid(ptx_parent2_v1->GetWitnessHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent3->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_mixed_child->GetHash()))); - - // package feerate should include parent3 and child. It should not include parent1 or parent2_v1. - const CFeeRate expected_feerate(1 * COIN, GetVirtualTransactionSize(*ptx_parent3) + GetVirtualTransactionSize(*ptx_mixed_child)); - BOOST_CHECK(it_parent3->second.m_effective_feerate.value() == expected_feerate); - BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); - std::vector<uint256> expected_wtxids({ptx_parent3->GetWitnessHash(), ptx_mixed_child->GetWitnessHash()}); - BOOST_CHECK(it_parent3->second.m_wtxids_fee_calculations.value() == expected_wtxids); - BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); + if (auto err_mixed{CheckPackageMempoolAcceptResult(package_mixed, mixed_result, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_mixed.value()); + } else { + auto it_parent1 = mixed_result.m_tx_results.find(ptx_parent1->GetWitnessHash()); + auto it_parent2 = mixed_result.m_tx_results.find(ptx_parent2_v1->GetWitnessHash()); + auto it_parent3 = mixed_result.m_tx_results.find(ptx_parent3->GetWitnessHash()); + auto it_child = mixed_result.m_tx_results.find(ptx_mixed_child->GetWitnessHash()); + + BOOST_CHECK(it_parent1->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK(it_parent2->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); + BOOST_CHECK(it_parent3->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK_EQUAL(ptx_parent2_v2->GetWitnessHash(), it_parent2->second.m_other_wtxid.value()); + + // package feerate should include parent3 and child. It should not include parent1 or parent2_v1. + const CFeeRate expected_feerate(1 * COIN, GetVirtualTransactionSize(*ptx_parent3) + GetVirtualTransactionSize(*ptx_mixed_child)); + BOOST_CHECK(it_parent3->second.m_effective_feerate.value() == expected_feerate); + BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); + std::vector<Wtxid> expected_wtxids({ptx_parent3->GetWitnessHash(), ptx_mixed_child->GetWitnessHash()}); + BOOST_CHECK(it_parent3->second.m_wtxids_fee_calculations.value() == expected_wtxids); + BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); + } } } @@ -649,11 +638,9 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) MockMempoolMinFee(CFeeRate(5000)); LOCK(::cs_main); size_t expected_pool_size = m_node.mempool->size(); - CKey child_key; - child_key.MakeNewKey(true); + CKey child_key = GenerateRandomKey(); CScript parent_spk = GetScriptForDestination(WitnessV0KeyHash(child_key.GetPubKey())); - CKey grandchild_key; - grandchild_key.MakeNewKey(true); + CKey grandchild_key = GenerateRandomKey(); CScript child_spk = GetScriptForDestination(WitnessV0KeyHash(grandchild_key.GetPubKey())); // low-fee parent and high-fee child package @@ -684,15 +671,17 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_cpfp_deprio = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_cpfp, /*test_accept=*/ false); - BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_state.GetResult(), PackageValidationResult::PCKG_TX); - BOOST_CHECK(submit_cpfp_deprio.m_state.IsInvalid()); - BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_tx_results.find(tx_parent->GetWitnessHash())->second.m_state.GetResult(), - TxValidationResult::TX_MEMPOOL_POLICY); - BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_tx_results.find(tx_child->GetWitnessHash())->second.m_state.GetResult(), - TxValidationResult::TX_MISSING_INPUTS); - BOOST_CHECK(submit_cpfp_deprio.m_tx_results.find(tx_parent->GetWitnessHash())->second.m_state.GetRejectReason() == "min relay fee not met"); - BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - const CFeeRate expected_feerate(0, GetVirtualTransactionSize(*tx_parent) + GetVirtualTransactionSize(*tx_child)); + if (auto err_cpfp_deprio{CheckPackageMempoolAcceptResult(package_cpfp, submit_cpfp_deprio, /*expect_valid=*/false, m_node.mempool.get())}) { + BOOST_ERROR(err_cpfp_deprio.value()); + } else { + BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_state.GetResult(), PackageValidationResult::PCKG_TX); + BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_tx_results.find(tx_parent->GetWitnessHash())->second.m_state.GetResult(), + TxValidationResult::TX_MEMPOOL_POLICY); + BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_tx_results.find(tx_child->GetWitnessHash())->second.m_state.GetResult(), + TxValidationResult::TX_MISSING_INPUTS); + BOOST_CHECK(submit_cpfp_deprio.m_tx_results.find(tx_parent->GetWitnessHash())->second.m_state.GetRejectReason() == "min relay fee not met"); + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + } } // Clear the prioritisation of the parent transaction. @@ -704,31 +693,27 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_cpfp = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_cpfp, /*test_accept=*/ false); + if (auto err_cpfp{CheckPackageMempoolAcceptResult(package_cpfp, submit_cpfp, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_cpfp.value()); + } else { + auto it_parent = submit_cpfp.m_tx_results.find(tx_parent->GetWitnessHash()); + auto it_child = submit_cpfp.m_tx_results.find(tx_child->GetWitnessHash()); + BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_parent->second.m_base_fees.value() == coinbase_value - parent_value); + BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_child->second.m_base_fees.value() == COIN); + + const CFeeRate expected_feerate(coinbase_value - child_value, + GetVirtualTransactionSize(*tx_parent) + GetVirtualTransactionSize(*tx_child)); + BOOST_CHECK(it_parent->second.m_effective_feerate.value() == expected_feerate); + BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); + std::vector<Wtxid> expected_wtxids({tx_parent->GetWitnessHash(), tx_child->GetWitnessHash()}); + BOOST_CHECK(it_parent->second.m_wtxids_fee_calculations.value() == expected_wtxids); + BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); + BOOST_CHECK(expected_feerate.GetFeePerK() > 1000); + } expected_pool_size += 2; - BOOST_CHECK_MESSAGE(submit_cpfp.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_cpfp.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_cpfp.m_tx_results.size(), package_cpfp.size()); - auto it_parent = submit_cpfp.m_tx_results.find(tx_parent->GetWitnessHash()); - auto it_child = submit_cpfp.m_tx_results.find(tx_child->GetWitnessHash()); - BOOST_CHECK(it_parent != submit_cpfp.m_tx_results.end()); - BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_parent->second.m_base_fees.value() == coinbase_value - parent_value); - BOOST_CHECK(it_child != submit_cpfp.m_tx_results.end()); - BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_child->second.m_base_fees.value() == COIN); - BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_parent->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_child->GetHash()))); - - const CFeeRate expected_feerate(coinbase_value - child_value, - GetVirtualTransactionSize(*tx_parent) + GetVirtualTransactionSize(*tx_child)); - BOOST_CHECK(it_parent->second.m_effective_feerate.value() == expected_feerate); - BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); - std::vector<uint256> expected_wtxids({tx_parent->GetWitnessHash(), tx_child->GetWitnessHash()}); - BOOST_CHECK(it_parent->second.m_wtxids_fee_calculations.value() == expected_wtxids); - BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); - BOOST_CHECK(expected_feerate.GetFeePerK() > 1000); } // Just because we allow low-fee parents doesn't mean we allow low-feerate packages. @@ -754,15 +739,28 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) package_still_too_low.push_back(tx_child_cheap); BOOST_CHECK(m_node.mempool->GetMinFee().GetFee(GetVirtualTransactionSize(*tx_child_cheap)) <= child_fee); BOOST_CHECK(m_node.mempool->GetMinFee().GetFee(GetVirtualTransactionSize(*tx_parent_cheap) + GetVirtualTransactionSize(*tx_child_cheap)) > parent_fee + child_fee); + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - // Cheap package should fail with package-fee-too-low. + // Cheap package should fail for being too low fee. { - BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_package_too_low = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_still_too_low, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_package_too_low.m_state.IsInvalid(), "Package validation unexpectedly succeeded"); - BOOST_CHECK_EQUAL(submit_package_too_low.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); - BOOST_CHECK_EQUAL(submit_package_too_low.m_state.GetRejectReason(), "package-fee-too-low"); + if (auto err_package_too_low{CheckPackageMempoolAcceptResult(package_still_too_low, submit_package_too_low, /*expect_valid=*/false, m_node.mempool.get())}) { + BOOST_ERROR(err_package_too_low.value()); + } else { + // Individual feerate of parent is too low. + BOOST_CHECK_EQUAL(submit_package_too_low.m_tx_results.at(tx_parent_cheap->GetWitnessHash()).m_state.GetResult(), + TxValidationResult::TX_RECONSIDERABLE); + BOOST_CHECK(submit_package_too_low.m_tx_results.at(tx_parent_cheap->GetWitnessHash()).m_effective_feerate.value() == + CFeeRate(parent_fee, GetVirtualTransactionSize(*tx_parent_cheap))); + // Package feerate of parent + child is too low. + BOOST_CHECK_EQUAL(submit_package_too_low.m_tx_results.at(tx_child_cheap->GetWitnessHash()).m_state.GetResult(), + TxValidationResult::TX_RECONSIDERABLE); + BOOST_CHECK(submit_package_too_low.m_tx_results.at(tx_child_cheap->GetWitnessHash()).m_effective_feerate.value() == + CFeeRate(parent_fee + child_fee, GetVirtualTransactionSize(*tx_parent_cheap) + GetVirtualTransactionSize(*tx_child_cheap))); + } + BOOST_CHECK_EQUAL(submit_package_too_low.m_state.GetResult(), PackageValidationResult::PCKG_TX); + BOOST_CHECK_EQUAL(submit_package_too_low.m_state.GetRejectReason(), "transaction failed"); BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); } @@ -773,25 +771,26 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) { const auto submit_prioritised_package = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_still_too_low, /*test_accept=*/false); + if (auto err_prioritised{CheckPackageMempoolAcceptResult(package_still_too_low, submit_prioritised_package, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_prioritised.value()); + } else { + const CFeeRate expected_feerate(1 * COIN + parent_fee + child_fee, + GetVirtualTransactionSize(*tx_parent_cheap) + GetVirtualTransactionSize(*tx_child_cheap)); + BOOST_CHECK_EQUAL(submit_prioritised_package.m_tx_results.size(), package_still_too_low.size()); + auto it_parent = submit_prioritised_package.m_tx_results.find(tx_parent_cheap->GetWitnessHash()); + auto it_child = submit_prioritised_package.m_tx_results.find(tx_child_cheap->GetWitnessHash()); + BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_parent->second.m_base_fees.value() == parent_fee); + BOOST_CHECK(it_parent->second.m_effective_feerate.value() == expected_feerate); + BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_child->second.m_base_fees.value() == child_fee); + BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); + std::vector<Wtxid> expected_wtxids({tx_parent_cheap->GetWitnessHash(), tx_child_cheap->GetWitnessHash()}); + BOOST_CHECK(it_parent->second.m_wtxids_fee_calculations.value() == expected_wtxids); + BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); + } expected_pool_size += 2; - BOOST_CHECK_MESSAGE(submit_prioritised_package.m_state.IsValid(), - "Package validation unexpectedly failed" << submit_prioritised_package.m_state.GetRejectReason()); - const CFeeRate expected_feerate(1 * COIN + parent_fee + child_fee, - GetVirtualTransactionSize(*tx_parent_cheap) + GetVirtualTransactionSize(*tx_child_cheap)); - BOOST_CHECK_EQUAL(submit_prioritised_package.m_tx_results.size(), package_still_too_low.size()); - auto it_parent = submit_prioritised_package.m_tx_results.find(tx_parent_cheap->GetWitnessHash()); - auto it_child = submit_prioritised_package.m_tx_results.find(tx_child_cheap->GetWitnessHash()); - BOOST_CHECK(it_parent != submit_prioritised_package.m_tx_results.end()); - BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_parent->second.m_base_fees.value() == parent_fee); - BOOST_CHECK(it_parent->second.m_effective_feerate.value() == expected_feerate); - BOOST_CHECK(it_child != submit_prioritised_package.m_tx_results.end()); - BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_child->second.m_base_fees.value() == child_fee); - BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); - std::vector<uint256> expected_wtxids({tx_parent_cheap->GetWitnessHash(), tx_child_cheap->GetWitnessHash()}); - BOOST_CHECK(it_parent->second.m_wtxids_fee_calculations.value() == expected_wtxids); - BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); } // Package feerate is calculated without topology in mind; it's just aggregating fees and sizes. @@ -820,31 +819,27 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_rich_parent = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_rich_parent, /*test_accept=*/false); + if (auto err_rich_parent{CheckPackageMempoolAcceptResult(package_rich_parent, submit_rich_parent, /*expect_valid=*/false, m_node.mempool.get())}) { + BOOST_ERROR(err_rich_parent.value()); + } else { + // The child would have been validated on its own and failed. + BOOST_CHECK_EQUAL(submit_rich_parent.m_state.GetResult(), PackageValidationResult::PCKG_TX); + BOOST_CHECK_EQUAL(submit_rich_parent.m_state.GetRejectReason(), "transaction failed"); + + auto it_parent = submit_rich_parent.m_tx_results.find(tx_parent_rich->GetWitnessHash()); + auto it_child = submit_rich_parent.m_tx_results.find(tx_child_poor->GetWitnessHash()); + BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::INVALID); + BOOST_CHECK(it_parent->second.m_state.GetRejectReason() == ""); + BOOST_CHECK_MESSAGE(it_parent->second.m_base_fees.value() == high_parent_fee, + strprintf("rich parent: expected fee %s, got %s", high_parent_fee, it_parent->second.m_base_fees.value())); + BOOST_CHECK(it_parent->second.m_effective_feerate == CFeeRate(high_parent_fee, GetVirtualTransactionSize(*tx_parent_rich))); + BOOST_CHECK_EQUAL(it_child->second.m_result_type, MempoolAcceptResult::ResultType::INVALID); + BOOST_CHECK_EQUAL(it_child->second.m_state.GetResult(), TxValidationResult::TX_MEMPOOL_POLICY); + BOOST_CHECK(it_child->second.m_state.GetRejectReason() == "min relay fee not met"); + } expected_pool_size += 1; - BOOST_CHECK_MESSAGE(submit_rich_parent.m_state.IsInvalid(), "Package validation unexpectedly succeeded"); - - // The child would have been validated on its own and failed. - BOOST_CHECK_EQUAL(submit_rich_parent.m_state.GetResult(), PackageValidationResult::PCKG_TX); - BOOST_CHECK_EQUAL(submit_rich_parent.m_state.GetRejectReason(), "transaction failed"); - - auto it_parent = submit_rich_parent.m_tx_results.find(tx_parent_rich->GetWitnessHash()); - auto it_child = submit_rich_parent.m_tx_results.find(tx_child_poor->GetWitnessHash()); - BOOST_CHECK(it_parent != submit_rich_parent.m_tx_results.end()); - BOOST_CHECK(it_child != submit_rich_parent.m_tx_results.end()); - BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::INVALID); - BOOST_CHECK(it_parent->second.m_state.GetRejectReason() == ""); - BOOST_CHECK_MESSAGE(it_parent->second.m_base_fees.value() == high_parent_fee, - strprintf("rich parent: expected fee %s, got %s", high_parent_fee, it_parent->second.m_base_fees.value())); - BOOST_CHECK(it_parent->second.m_effective_feerate == CFeeRate(high_parent_fee, GetVirtualTransactionSize(*tx_parent_rich))); - BOOST_CHECK(it_child != submit_rich_parent.m_tx_results.end()); - BOOST_CHECK_EQUAL(it_child->second.m_result_type, MempoolAcceptResult::ResultType::INVALID); - BOOST_CHECK_EQUAL(it_child->second.m_state.GetResult(), TxValidationResult::TX_MEMPOOL_POLICY); - BOOST_CHECK(it_child->second.m_state.GetRejectReason() == "min relay fee not met"); - BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_parent_rich->GetHash()))); - BOOST_CHECK(!m_node.mempool->exists(GenTxid::Txid(tx_child_poor->GetHash()))); } } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/uint256_tests.cpp b/src/test/uint256_tests.cpp index 9caefe43e2..5746961550 100644 --- a/src/test/uint256_tests.cpp +++ b/src/test/uint256_tests.cpp @@ -6,7 +6,6 @@ #include <streams.h> #include <test/util/setup_common.h> #include <uint256.h> -#include <version.h> #include <boost/test/unit_test.hpp> @@ -184,8 +183,8 @@ BOOST_AUTO_TEST_CASE( methods ) // GetHex SetHex begin() end() size() GetLow64 G BOOST_CHECK(OneL.begin() + 32 == OneL.end()); BOOST_CHECK(MaxL.begin() + 32 == MaxL.end()); BOOST_CHECK(TmpL.begin() + 32 == TmpL.end()); - BOOST_CHECK(GetSerializeSize(R1L, PROTOCOL_VERSION) == 32); - BOOST_CHECK(GetSerializeSize(ZeroL, PROTOCOL_VERSION) == 32); + BOOST_CHECK(GetSerializeSize(R1L) == 32); + BOOST_CHECK(GetSerializeSize(ZeroL) == 32); DataStream ss{}; ss << R1L; @@ -230,8 +229,8 @@ BOOST_AUTO_TEST_CASE( methods ) // GetHex SetHex begin() end() size() GetLow64 G BOOST_CHECK(OneS.begin() + 20 == OneS.end()); BOOST_CHECK(MaxS.begin() + 20 == MaxS.end()); BOOST_CHECK(TmpS.begin() + 20 == TmpS.end()); - BOOST_CHECK(GetSerializeSize(R1S, PROTOCOL_VERSION) == 20); - BOOST_CHECK(GetSerializeSize(ZeroS, PROTOCOL_VERSION) == 20); + BOOST_CHECK(GetSerializeSize(R1S) == 20); + BOOST_CHECK(GetSerializeSize(ZeroS) == 20); ss << R1S; BOOST_CHECK(ss.str() == std::string(R1Array,R1Array+20)); @@ -260,8 +259,8 @@ BOOST_AUTO_TEST_CASE( conversion ) BOOST_CHECK(UintToArith256(OneL) == 1); BOOST_CHECK(ArithToUint256(0) == ZeroL); BOOST_CHECK(ArithToUint256(1) == OneL); - BOOST_CHECK(arith_uint256(R1L.GetHex()) == UintToArith256(R1L)); - BOOST_CHECK(arith_uint256(R2L.GetHex()) == UintToArith256(R2L)); + BOOST_CHECK(arith_uint256(UintToArith256(uint256S(R1L.GetHex()))) == UintToArith256(R1L)); + BOOST_CHECK(arith_uint256(UintToArith256(uint256S(R2L.GetHex()))) == UintToArith256(R2L)); BOOST_CHECK(R1L.GetHex() == UintToArith256(R1L).GetHex()); BOOST_CHECK(R2L.GetHex() == UintToArith256(R2L).GetHex()); } @@ -279,6 +278,34 @@ BOOST_AUTO_TEST_CASE( operator_with_self ) BOOST_CHECK(v == UintToArith256(uint256S("0"))); } +BOOST_AUTO_TEST_CASE(parse) +{ + { + std::string s_12{"0000000000000000000000000000000000000000000000000000000000000012"}; + BOOST_CHECK_EQUAL(uint256S("12\0").GetHex(), s_12); + BOOST_CHECK_EQUAL(uint256S(std::string{"12\0", 3}).GetHex(), s_12); + BOOST_CHECK_EQUAL(uint256S("0x12").GetHex(), s_12); + BOOST_CHECK_EQUAL(uint256S(" 0x12").GetHex(), s_12); + BOOST_CHECK_EQUAL(uint256S(" 12").GetHex(), s_12); + } + { + std::string s_1{uint256::ONE.GetHex()}; + BOOST_CHECK_EQUAL(uint256S("1\0").GetHex(), s_1); + BOOST_CHECK_EQUAL(uint256S(std::string{"1\0", 2}).GetHex(), s_1); + BOOST_CHECK_EQUAL(uint256S("0x1").GetHex(), s_1); + BOOST_CHECK_EQUAL(uint256S(" 0x1").GetHex(), s_1); + BOOST_CHECK_EQUAL(uint256S(" 1").GetHex(), s_1); + } + { + std::string s_0{uint256::ZERO.GetHex()}; + BOOST_CHECK_EQUAL(uint256S("\0").GetHex(), s_0); + BOOST_CHECK_EQUAL(uint256S(std::string{"\0", 1}).GetHex(), s_0); + BOOST_CHECK_EQUAL(uint256S("0x").GetHex(), s_0); + BOOST_CHECK_EQUAL(uint256S(" 0x").GetHex(), s_0); + BOOST_CHECK_EQUAL(uint256S(" ").GetHex(), s_0); + } +} + BOOST_AUTO_TEST_CASE( check_ONE ) { uint256 one = uint256S("0000000000000000000000000000000000000000000000000000000000000001"); diff --git a/src/test/util/coins.cpp b/src/test/util/coins.cpp index 9b6c5535c5..742dbc04d1 100644 --- a/src/test/util/coins.cpp +++ b/src/test/util/coins.cpp @@ -16,8 +16,7 @@ COutPoint AddTestCoin(CCoinsViewCache& coins_view) { Coin new_coin; - const uint256 txid{InsecureRand256()}; - COutPoint outpoint{txid, /*nIn=*/0}; + COutPoint outpoint{Txid::FromUint256(InsecureRand256()), /*nIn=*/0}; new_coin.nHeight = 1; new_coin.out.nValue = InsecureRandMoneyAmount(); new_coin.out.scriptPubKey.assign(uint32_t{56}, 1); diff --git a/src/test/util/index.cpp b/src/test/util/index.cpp index e653d5dbf0..cfeba35756 100644 --- a/src/test/util/index.cpp +++ b/src/test/util/index.cpp @@ -5,16 +5,16 @@ #include <test/util/index.h> #include <index/base.h> -#include <shutdown.h> #include <util/check.h> +#include <util/signalinterrupt.h> #include <util/time.h> -void IndexWaitSynced(const BaseIndex& index) +void IndexWaitSynced(const BaseIndex& index, const util::SignalInterrupt& interrupt) { while (!index.BlockUntilSyncedToCurrentChain()) { // Assert shutdown was not requested to abort the test, instead of looping forever, in case // there was an unexpected error in the index that caused it to stop syncing and request a shutdown. - Assert(!ShutdownRequested()); + Assert(!interrupt); UninterruptibleSleep(100ms); } diff --git a/src/test/util/index.h b/src/test/util/index.h index 95309f6273..a3bd1dddc3 100644 --- a/src/test/util/index.h +++ b/src/test/util/index.h @@ -6,8 +6,11 @@ #define BITCOIN_TEST_UTIL_INDEX_H class BaseIndex; +namespace util { +class SignalInterrupt; +} // namespace util /** Block until the index is synced to the current chain */ -void IndexWaitSynced(const BaseIndex& index); +void IndexWaitSynced(const BaseIndex& index, const util::SignalInterrupt& interrupt); #endif // BITCOIN_TEST_UTIL_INDEX_H diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index bf5a653090..9257a4964a 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -4,11 +4,15 @@ #include <test/util/net.h> -#include <chainparams.h> -#include <node/eviction.h> #include <net.h> #include <net_processing.h> +#include <netaddress.h> #include <netmessagemaker.h> +#include <node/connection_types.h> +#include <node/eviction.h> +#include <protocol.h> +#include <random.h> +#include <serialize.h> #include <span.h> #include <vector> @@ -22,13 +26,12 @@ void ConnmanTestMsg::Handshake(CNode& node, { auto& peerman{static_cast<PeerManager&>(*m_msgproc)}; auto& connman{*this}; - const CNetMsgMaker mm{0}; peerman.InitializeNode(node, local_services); FlushSendBuffer(node); // Drop the version message added by InitializeNode. CSerializedNetMsg msg_version{ - mm.Make(NetMsgType::VERSION, + NetMsg::Make(NetMsgType::VERSION, version, // Using<CustomUintFormatter<8>>(remote_services), // int64_t{}, // dummy time @@ -55,7 +58,7 @@ void ConnmanTestMsg::Handshake(CNode& node, assert(statestats.m_relay_txs == (relay_txs && !node.IsBlockOnlyConn())); assert(statestats.their_services == remote_services); if (successfully_connected) { - CSerializedNetMsg msg_verack{mm.Make(NetMsgType::VERACK)}; + CSerializedNetMsg msg_verack{NetMsg::Make(NetMsgType::VERACK)}; (void)connman.ReceiveMsgFrom(node, std::move(msg_verack)); node.fPauseSend = false; connman.ProcessMessagesOnce(node); @@ -98,6 +101,17 @@ bool ConnmanTestMsg::ReceiveMsgFrom(CNode& node, CSerializedNetMsg&& ser_msg) co return complete; } +CNode* ConnmanTestMsg::ConnectNodePublic(PeerManager& peerman, const char* pszDest, ConnectionType conn_type) +{ + CNode* node = ConnectNode(CAddress{}, pszDest, /*fCountFailure=*/false, conn_type, /*use_v2transport=*/true); + if (!node) return nullptr; + node->SetCommonVersion(PROTOCOL_VERSION); + peerman.InitializeNode(*node, ServiceFlags(NODE_NETWORK | NODE_WITNESS)); + node->fSuccessfullyConnected = true; + AddTestNode(*node); + return node; +} + std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(int n_candidates, FastRandomContext& random_context) { std::vector<NodeEvictionCandidate> candidates; diff --git a/src/test/util/net.h b/src/test/util/net.h index 497292542b..d91d801132 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -6,16 +6,30 @@ #define BITCOIN_TEST_UTIL_NET_H #include <compat/compat.h> -#include <node/eviction.h> -#include <netaddress.h> #include <net.h> +#include <net_permissions.h> +#include <net_processing.h> +#include <netaddress.h> +#include <node/connection_types.h> +#include <node/eviction.h> +#include <sync.h> #include <util/sock.h> +#include <algorithm> #include <array> #include <cassert> +#include <chrono> +#include <cstdint> #include <cstring> #include <memory> #include <string> +#include <unordered_map> +#include <vector> + +class FastRandomContext; + +template <typename C> +class Span; struct ConnmanTestMsg : public CConnman { using CConnman::CConnman; @@ -25,6 +39,12 @@ struct ConnmanTestMsg : public CConnman { m_peer_connect_timeout = timeout; } + std::vector<CNode*> TestNodes() + { + LOCK(m_nodes_mutex); + return m_nodes; + } + void AddTestNode(CNode& node) { LOCK(m_nodes_mutex); @@ -50,12 +70,20 @@ struct ConnmanTestMsg : public CConnman { bool relay_txs) EXCLUSIVE_LOCKS_REQUIRED(NetEventsInterface::g_msgproc_mutex); - void ProcessMessagesOnce(CNode& node) EXCLUSIVE_LOCKS_REQUIRED(NetEventsInterface::g_msgproc_mutex) { m_msgproc->ProcessMessages(&node, flagInterruptMsgProc); } + bool ProcessMessagesOnce(CNode& node) EXCLUSIVE_LOCKS_REQUIRED(NetEventsInterface::g_msgproc_mutex) + { + return m_msgproc->ProcessMessages(&node, flagInterruptMsgProc); + } void NodeReceiveMsgBytes(CNode& node, Span<const uint8_t> msg_bytes, bool& complete) const; bool ReceiveMsgFrom(CNode& node, CSerializedNetMsg&& ser_msg) const; void FlushSendBuffer(CNode& node) const; + + bool AlreadyConnectedPublic(const CAddress& addr) { return AlreadyConnectedToAddress(addr); }; + + CNode* ConnectNodePublic(PeerManager& peerman, const char* pszDest, ConnectionType conn_type) + EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex); }; constexpr ServiceFlags ALL_SERVICE_FLAGS[]{ diff --git a/src/test/util/script.h b/src/test/util/script.h index 428b3e10b3..96c4d55e5a 100644 --- a/src/test/util/script.h +++ b/src/test/util/script.h @@ -18,6 +18,18 @@ static const CScript P2WSH_OP_TRUE{ return hash; }())}; +static const std::vector<uint8_t> EMPTY{}; +static const CScript P2WSH_EMPTY{ + CScript{} + << OP_0 + << ToByteVector([] { + uint256 hash; + CSHA256().Write(EMPTY.data(), EMPTY.size()).Finalize(hash.begin()); + return hash; + }())}; +static const std::vector<std::vector<uint8_t>> P2WSH_EMPTY_TRUE_STACK{{static_cast<uint8_t>(OP_TRUE)}, {}}; +static const std::vector<std::vector<uint8_t>> P2WSH_EMPTY_TWO_STACK{{static_cast<uint8_t>(OP_2)}, {}}; + /** Flags that are not forbidden by an assert in script validation */ bool IsValidFlagCombination(unsigned flags); diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index e70c105c8a..8789e86196 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -40,7 +40,6 @@ #include <rpc/server.h> #include <scheduler.h> #include <script/sigcache.h> -#include <shutdown.h> #include <streams.h> #include <test/util/net.h> #include <test/util/random.h> @@ -49,6 +48,8 @@ #include <txdb.h> #include <txmempool.h> #include <util/chaintype.h> +#include <util/check.h> +#include <util/rbf.h> #include <util/strencodings.h> #include <util/string.h> #include <util/thread.h> @@ -87,10 +88,20 @@ std::ostream& operator<<(std::ostream& os, const uint256& num) return os; } +struct NetworkSetup +{ + NetworkSetup() + { + Assert(SetupNetworking()); + } +}; +static NetworkSetup g_networksetup_instance; + BasicTestingSetup::BasicTestingSetup(const ChainType chainType, const std::vector<const char*>& extra_args) : m_path_root{fs::temp_directory_path() / "test_common_" PACKAGE_NAME / g_insecure_rand_ctx_temp_path.rand256().ToString()}, m_args{} { + m_node.shutdown = &m_interrupt; m_node.args = &gArgs; std::vector<const char*> arguments = Cat( { @@ -129,7 +140,6 @@ BasicTestingSetup::BasicTestingSetup(const ChainType chainType, const std::vecto LogInstance().StartLogging(); m_node.kernel = std::make_unique<kernel::Context>(); SetupEnvironment(); - SetupNetworking(); ValidationCacheSizes validation_cache_sizes{}; ApplyArgsManOptions(*m_node.args, validation_cache_sizes); @@ -146,6 +156,7 @@ BasicTestingSetup::BasicTestingSetup(const ChainType chainType, const std::vecto BasicTestingSetup::~BasicTestingSetup() { + m_node.kernel.reset(); SetMockTime(0s); // Reset mocktime for following tests LogInstance().DisconnectTestLogger(); fs::remove_all(m_path_root); @@ -168,7 +179,7 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, const std::vecto m_cache_sizes = CalculateCacheSizes(m_args); - m_node.notifications = std::make_unique<KernelNotifications>(m_node.exit_status); + m_node.notifications = std::make_unique<KernelNotifications>(*Assert(m_node.shutdown), m_node.exit_status); const ChainstateManager::Options chainman_opts{ .chainparams = chainparams, @@ -176,26 +187,23 @@ ChainTestingSetup::ChainTestingSetup(const ChainType chainType, const std::vecto .adjusted_time_callback = GetAdjustedTime, .check_block_index = true, .notifications = *m_node.notifications, + .worker_threads_num = 2, }; const BlockManager::Options blockman_opts{ .chainparams = chainman_opts.chainparams, .blocks_dir = m_args.GetBlocksDirPath(), .notifications = chainman_opts.notifications, }; - m_node.chainman = std::make_unique<ChainstateManager>(m_node.kernel->interrupt, chainman_opts, blockman_opts); + m_node.chainman = std::make_unique<ChainstateManager>(*Assert(m_node.shutdown), chainman_opts, blockman_opts); m_node.chainman->m_blockman.m_block_tree_db = std::make_unique<BlockTreeDB>(DBParams{ .path = m_args.GetDataDirNet() / "blocks" / "index", .cache_bytes = static_cast<size_t>(m_cache_sizes.block_tree_db), .memory_only = true}); - - constexpr int script_check_threads = 2; - StartScriptCheckWorkerThreads(script_check_threads); } ChainTestingSetup::~ChainTestingSetup() { if (m_node.scheduler) m_node.scheduler->stop(); - StopScriptCheckWorkerThreads(); GetMainSignals().FlushBackgroundCallbacks(); GetMainSignals().UnregisterBackgroundSignalScheduler(); m_node.connman.reset(); @@ -204,8 +212,9 @@ ChainTestingSetup::~ChainTestingSetup() m_node.netgroupman.reset(); m_node.args = nullptr; m_node.mempool.reset(); - m_node.scheduler.reset(); + m_node.fee_estimator.reset(); m_node.chainman.reset(); + m_node.scheduler.reset(); } void ChainTestingSetup::LoadVerifyActivateChainstate() @@ -336,56 +345,106 @@ CBlock TestChain100Setup::CreateAndProcessBlock( return block; } - -CMutableTransaction TestChain100Setup::CreateValidMempoolTransaction(CTransactionRef input_transaction, - int input_vout, - int input_height, - CKey input_signing_key, - CScript output_destination, - CAmount output_amount, - bool submit) +std::pair<CMutableTransaction, CAmount> TestChain100Setup::CreateValidTransaction(const std::vector<CTransactionRef>& input_transactions, + const std::vector<COutPoint>& inputs, + int input_height, + const std::vector<CKey>& input_signing_keys, + const std::vector<CTxOut>& outputs, + const std::optional<CFeeRate>& feerate, + const std::optional<uint32_t>& fee_output) { - // Transaction we will submit to the mempool CMutableTransaction mempool_txn; + mempool_txn.vin.reserve(inputs.size()); + mempool_txn.vout.reserve(outputs.size()); - // Create an input - COutPoint outpoint_to_spend(input_transaction->GetHash(), input_vout); - CTxIn input(outpoint_to_spend); - mempool_txn.vin.push_back(input); - - // Create an output - CTxOut output(output_amount, output_destination); - mempool_txn.vout.push_back(output); + for (const auto& outpoint : inputs) { + mempool_txn.vin.emplace_back(outpoint, CScript(), MAX_BIP125_RBF_SEQUENCE); + } + mempool_txn.vout = outputs; - // Sign the transaction // - Add the signing key to a keystore FillableSigningProvider keystore; - keystore.AddKey(input_signing_key); + for (const auto& input_signing_key : input_signing_keys) { + keystore.AddKey(input_signing_key); + } // - Populate a CoinsViewCache with the unspent output CCoinsView coins_view; CCoinsViewCache coins_cache(&coins_view); - AddCoins(coins_cache, *input_transaction.get(), input_height); - // - Use GetCoin to properly populate utxo_to_spend, - Coin utxo_to_spend; - assert(coins_cache.GetCoin(outpoint_to_spend, utxo_to_spend)); - // - Then add it to a map to pass in to SignTransaction + for (const auto& input_transaction : input_transactions) { + AddCoins(coins_cache, *input_transaction.get(), input_height); + } + // Build Outpoint to Coin map for SignTransaction std::map<COutPoint, Coin> input_coins; - input_coins.insert({outpoint_to_spend, utxo_to_spend}); + CAmount inputs_amount{0}; + for (const auto& outpoint_to_spend : inputs) { + // - Use GetCoin to properly populate utxo_to_spend, + Coin utxo_to_spend; + assert(coins_cache.GetCoin(outpoint_to_spend, utxo_to_spend)); + input_coins.insert({outpoint_to_spend, utxo_to_spend}); + inputs_amount += utxo_to_spend.out.nValue; + } // - Default signature hashing type int nHashType = SIGHASH_ALL; std::map<int, bilingual_str> input_errors; assert(SignTransaction(mempool_txn, &keystore, input_coins, nHashType, input_errors)); + CAmount current_fee = inputs_amount - std::accumulate(outputs.begin(), outputs.end(), CAmount(0), + [](const CAmount& acc, const CTxOut& out) { + return acc + out.nValue; + }); + // Deduct fees from fee_output to meet feerate if set + if (feerate.has_value()) { + assert(fee_output.has_value()); + assert(fee_output.value() < mempool_txn.vout.size()); + CAmount target_fee = feerate.value().GetFee(GetVirtualTransactionSize(CTransaction{mempool_txn})); + CAmount deduction = target_fee - current_fee; + if (deduction > 0) { + // Only deduct fee if there's anything to deduct. If the caller has put more fees than + // the target feerate, don't change the fee. + mempool_txn.vout[fee_output.value()].nValue -= deduction; + // Re-sign since an output has changed + input_errors.clear(); + assert(SignTransaction(mempool_txn, &keystore, input_coins, nHashType, input_errors)); + current_fee = target_fee; + } + } + return {mempool_txn, current_fee}; +} +CMutableTransaction TestChain100Setup::CreateValidMempoolTransaction(const std::vector<CTransactionRef>& input_transactions, + const std::vector<COutPoint>& inputs, + int input_height, + const std::vector<CKey>& input_signing_keys, + const std::vector<CTxOut>& outputs, + bool submit) +{ + CMutableTransaction mempool_txn = CreateValidTransaction(input_transactions, inputs, input_height, input_signing_keys, outputs, std::nullopt, std::nullopt).first; // If submit=true, add transaction to the mempool. if (submit) { LOCK(cs_main); const MempoolAcceptResult result = m_node.chainman->ProcessTransaction(MakeTransactionRef(mempool_txn)); assert(result.m_result_type == MempoolAcceptResult::ResultType::VALID); } - return mempool_txn; } +CMutableTransaction TestChain100Setup::CreateValidMempoolTransaction(CTransactionRef input_transaction, + uint32_t input_vout, + int input_height, + CKey input_signing_key, + CScript output_destination, + CAmount output_amount, + bool submit) +{ + COutPoint input{input_transaction->GetHash(), input_vout}; + CTxOut output{output_amount, output_destination}; + return CreateValidMempoolTransaction(/*input_transactions=*/{input_transaction}, + /*inputs=*/{input}, + /*input_height=*/input_height, + /*input_signing_keys=*/{input_signing_key}, + /*outputs=*/{output}, + /*submit=*/submit); +} + std::vector<CTransactionRef> TestChain100Setup::PopulateMempool(FastRandomContext& det_rand, size_t num_transactions, bool submit) { std::vector<CTransactionRef> mempool_transactions; @@ -448,7 +507,7 @@ void TestChain100Setup::MockMempoolMinFee(const CFeeRate& target_feerate) // Manually create an invalid transaction. Manually set the fee in the CTxMemPoolEntry to // achieve the exact target feerate. CMutableTransaction mtx = CMutableTransaction(); - mtx.vin.emplace_back(COutPoint{g_insecure_rand_ctx.rand256(), 0}); + mtx.vin.emplace_back(COutPoint{Txid::FromUint256(g_insecure_rand_ctx.rand256()), 0}); mtx.vout.emplace_back(1 * COIN, GetScriptForDestination(WitnessV0ScriptHash(CScript() << OP_TRUE))); const auto tx{MakeTransactionRef(mtx)}; LockPoints lp; @@ -468,7 +527,9 @@ void TestChain100Setup::MockMempoolMinFee(const CFeeRate& target_feerate) CBlock getBlock13b8a() { CBlock block; - CDataStream stream(ParseHex("0100000090f0a9f110702f808219ebea1173056042a714bad51b916cb6800000000000005275289558f51c9966699404ae2294730c3c9f9bda53523ce50e9b95e558da2fdb261b4d4c86041b1ab1bf930901000000010000000000000000000000000000000000000000000000000000000000000000ffffffff07044c86041b0146ffffffff0100f2052a01000000434104e18f7afbe4721580e81e8414fc8c24d7cfacf254bb5c7b949450c3e997c2dc1242487a8169507b631eb3771f2b425483fb13102c4eb5d858eef260fe70fbfae0ac00000000010000000196608ccbafa16abada902780da4dc35dafd7af05fa0da08cf833575f8cf9e836000000004a493046022100dab24889213caf43ae6adc41cf1c9396c08240c199f5225acf45416330fd7dbd022100fe37900e0644bf574493a07fc5edba06dbc07c311b947520c2d514bc5725dcb401ffffffff0100f2052a010000001976a914f15d1921f52e4007b146dfa60f369ed2fc393ce288ac000000000100000001fb766c1288458c2bafcfec81e48b24d98ec706de6b8af7c4e3c29419bfacb56d000000008c493046022100f268ba165ce0ad2e6d93f089cfcd3785de5c963bb5ea6b8c1b23f1ce3e517b9f022100da7c0f21adc6c401887f2bfd1922f11d76159cbc597fbd756a23dcbb00f4d7290141042b4e8625a96127826915a5b109852636ad0da753c9e1d5606a50480cd0c40f1f8b8d898235e571fe9357d9ec842bc4bba1827daaf4de06d71844d0057707966affffffff0280969800000000001976a9146963907531db72d0ed1a0cfb471ccb63923446f388ac80d6e34c000000001976a914f0688ba1c0d1ce182c7af6741e02658c7d4dfcd388ac000000000100000002c40297f730dd7b5a99567eb8d27b78758f607507c52292d02d4031895b52f2ff010000008b483045022100f7edfd4b0aac404e5bab4fd3889e0c6c41aa8d0e6fa122316f68eddd0a65013902205b09cc8b2d56e1cd1f7f2fafd60a129ed94504c4ac7bdc67b56fe67512658b3e014104732012cb962afa90d31b25d8fb0e32c94e513ab7a17805c14ca4c3423e18b4fb5d0e676841733cb83abaf975845c9f6f2a8097b7d04f4908b18368d6fc2d68ecffffffffca5065ff9617cbcba45eb23726df6498a9b9cafed4f54cbab9d227b0035ddefb000000008a473044022068010362a13c7f9919fa832b2dee4e788f61f6f5d344a7c2a0da6ae740605658022006d1af525b9a14a35c003b78b72bd59738cd676f845d1ff3fc25049e01003614014104732012cb962afa90d31b25d8fb0e32c94e513ab7a17805c14ca4c3423e18b4fb5d0e676841733cb83abaf975845c9f6f2a8097b7d04f4908b18368d6fc2d68ecffffffff01001ec4110200000043410469ab4181eceb28985b9b4e895c13fa5e68d85761b7eee311db5addef76fa8621865134a221bd01f28ec9999ee3e021e60766e9d1f3458c115fb28650605f11c9ac000000000100000001cdaf2f758e91c514655e2dc50633d1e4c84989f8aa90a0dbc883f0d23ed5c2fa010000008b48304502207ab51be6f12a1962ba0aaaf24a20e0b69b27a94fac5adf45aa7d2d18ffd9236102210086ae728b370e5329eead9accd880d0cb070aea0c96255fae6c4f1ddcce1fd56e014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff02404b4c00000000001976a9142b6ba7c9d796b75eef7942fc9288edd37c32f5c388ac002d3101000000001976a9141befba0cdc1ad56529371864d9f6cb042faa06b588ac000000000100000001b4a47603e71b61bc3326efd90111bf02d2f549b067f4c4a8fa183b57a0f800cb010000008a4730440220177c37f9a505c3f1a1f0ce2da777c339bd8339ffa02c7cb41f0a5804f473c9230220585b25a2ee80eb59292e52b987dad92acb0c64eced92ed9ee105ad153cdb12d001410443bd44f683467e549dae7d20d1d79cbdb6df985c6e9c029c8d0c6cb46cc1a4d3cf7923c5021b27f7a0b562ada113bc85d5fda5a1b41e87fe6e8802817cf69996ffffffff0280651406000000001976a9145505614859643ab7b547cd7f1f5e7e2a12322d3788ac00aa0271000000001976a914ea4720a7a52fc166c55ff2298e07baf70ae67e1b88ac00000000010000000586c62cd602d219bb60edb14a3e204de0705176f9022fe49a538054fb14abb49e010000008c493046022100f2bc2aba2534becbdf062eb993853a42bbbc282083d0daf9b4b585bd401aa8c9022100b1d7fd7ee0b95600db8535bbf331b19eed8d961f7a8e54159c53675d5f69df8c014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff03ad0e58ccdac3df9dc28a218bcf6f1997b0a93306faaa4b3a28ae83447b2179010000008b483045022100be12b2937179da88599e27bb31c3525097a07cdb52422d165b3ca2f2020ffcf702200971b51f853a53d644ebae9ec8f3512e442b1bcb6c315a5b491d119d10624c83014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff2acfcab629bbc8685792603762c921580030ba144af553d271716a95089e107b010000008b483045022100fa579a840ac258871365dd48cd7552f96c8eea69bd00d84f05b283a0dab311e102207e3c0ee9234814cfbb1b659b83671618f45abc1326b9edcc77d552a4f2a805c0014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffffdcdc6023bbc9944a658ddc588e61eacb737ddf0a3cd24f113b5a8634c517fcd2000000008b4830450221008d6df731df5d32267954bd7d2dda2302b74c6c2a6aa5c0ca64ecbabc1af03c75022010e55c571d65da7701ae2da1956c442df81bbf076cdbac25133f99d98a9ed34c014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffffe15557cd5ce258f479dfd6dc6514edf6d7ed5b21fcfa4a038fd69f06b83ac76e010000008b483045022023b3e0ab071eb11de2eb1cc3a67261b866f86bf6867d4558165f7c8c8aca2d86022100dc6e1f53a91de3efe8f63512850811f26284b62f850c70ca73ed5de8771fb451014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff01404b4c00000000001976a9142b6ba7c9d796b75eef7942fc9288edd37c32f5c388ac00000000010000000166d7577163c932b4f9690ca6a80b6e4eb001f0a2fa9023df5595602aae96ed8d000000008a4730440220262b42546302dfb654a229cefc86432b89628ff259dc87edd1154535b16a67e102207b4634c020a97c3e7bbd0d4d19da6aa2269ad9dded4026e896b213d73ca4b63f014104979b82d02226b3a4597523845754d44f13639e3bf2df5e82c6aab2bdc79687368b01b1ab8b19875ae3c90d661a3d0a33161dab29934edeb36aa01976be3baf8affffffff02404b4c00000000001976a9144854e695a02af0aeacb823ccbc272134561e0a1688ac40420f00000000001976a914abee93376d6b37b5c2940655a6fcaf1c8e74237988ac0000000001000000014e3f8ef2e91349a9059cb4f01e54ab2597c1387161d3da89919f7ea6acdbb371010000008c49304602210081f3183471a5ca22307c0800226f3ef9c353069e0773ac76bb580654d56aa523022100d4c56465bdc069060846f4fbf2f6b20520b2a80b08b168b31e66ddb9c694e240014104976c79848e18251612f8940875b2b08d06e6dc73b9840e8860c066b7e87432c477e9a59a453e71e6d76d5fe34058b800a098fc1740ce3012e8fc8a00c96af966ffffffff02c0e1e400000000001976a9144134e75a6fcb6042034aab5e18570cf1f844f54788ac404b4c00000000001976a9142b6ba7c9d796b75eef7942fc9288edd37c32f5c388ac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + DataStream stream{ + ParseHex("0100000090f0a9f110702f808219ebea1173056042a714bad51b916cb6800000000000005275289558f51c9966699404ae2294730c3c9f9bda53523ce50e9b95e558da2fdb261b4d4c86041b1ab1bf930901000000010000000000000000000000000000000000000000000000000000000000000000ffffffff07044c86041b0146ffffffff0100f2052a01000000434104e18f7afbe4721580e81e8414fc8c24d7cfacf254bb5c7b949450c3e997c2dc1242487a8169507b631eb3771f2b425483fb13102c4eb5d858eef260fe70fbfae0ac00000000010000000196608ccbafa16abada902780da4dc35dafd7af05fa0da08cf833575f8cf9e836000000004a493046022100dab24889213caf43ae6adc41cf1c9396c08240c199f5225acf45416330fd7dbd022100fe37900e0644bf574493a07fc5edba06dbc07c311b947520c2d514bc5725dcb401ffffffff0100f2052a010000001976a914f15d1921f52e4007b146dfa60f369ed2fc393ce288ac000000000100000001fb766c1288458c2bafcfec81e48b24d98ec706de6b8af7c4e3c29419bfacb56d000000008c493046022100f268ba165ce0ad2e6d93f089cfcd3785de5c963bb5ea6b8c1b23f1ce3e517b9f022100da7c0f21adc6c401887f2bfd1922f11d76159cbc597fbd756a23dcbb00f4d7290141042b4e8625a96127826915a5b109852636ad0da753c9e1d5606a50480cd0c40f1f8b8d898235e571fe9357d9ec842bc4bba1827daaf4de06d71844d0057707966affffffff0280969800000000001976a9146963907531db72d0ed1a0cfb471ccb63923446f388ac80d6e34c000000001976a914f0688ba1c0d1ce182c7af6741e02658c7d4dfcd388ac000000000100000002c40297f730dd7b5a99567eb8d27b78758f607507c52292d02d4031895b52f2ff010000008b483045022100f7edfd4b0aac404e5bab4fd3889e0c6c41aa8d0e6fa122316f68eddd0a65013902205b09cc8b2d56e1cd1f7f2fafd60a129ed94504c4ac7bdc67b56fe67512658b3e014104732012cb962afa90d31b25d8fb0e32c94e513ab7a17805c14ca4c3423e18b4fb5d0e676841733cb83abaf975845c9f6f2a8097b7d04f4908b18368d6fc2d68ecffffffffca5065ff9617cbcba45eb23726df6498a9b9cafed4f54cbab9d227b0035ddefb000000008a473044022068010362a13c7f9919fa832b2dee4e788f61f6f5d344a7c2a0da6ae740605658022006d1af525b9a14a35c003b78b72bd59738cd676f845d1ff3fc25049e01003614014104732012cb962afa90d31b25d8fb0e32c94e513ab7a17805c14ca4c3423e18b4fb5d0e676841733cb83abaf975845c9f6f2a8097b7d04f4908b18368d6fc2d68ecffffffff01001ec4110200000043410469ab4181eceb28985b9b4e895c13fa5e68d85761b7eee311db5addef76fa8621865134a221bd01f28ec9999ee3e021e60766e9d1f3458c115fb28650605f11c9ac000000000100000001cdaf2f758e91c514655e2dc50633d1e4c84989f8aa90a0dbc883f0d23ed5c2fa010000008b48304502207ab51be6f12a1962ba0aaaf24a20e0b69b27a94fac5adf45aa7d2d18ffd9236102210086ae728b370e5329eead9accd880d0cb070aea0c96255fae6c4f1ddcce1fd56e014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff02404b4c00000000001976a9142b6ba7c9d796b75eef7942fc9288edd37c32f5c388ac002d3101000000001976a9141befba0cdc1ad56529371864d9f6cb042faa06b588ac000000000100000001b4a47603e71b61bc3326efd90111bf02d2f549b067f4c4a8fa183b57a0f800cb010000008a4730440220177c37f9a505c3f1a1f0ce2da777c339bd8339ffa02c7cb41f0a5804f473c9230220585b25a2ee80eb59292e52b987dad92acb0c64eced92ed9ee105ad153cdb12d001410443bd44f683467e549dae7d20d1d79cbdb6df985c6e9c029c8d0c6cb46cc1a4d3cf7923c5021b27f7a0b562ada113bc85d5fda5a1b41e87fe6e8802817cf69996ffffffff0280651406000000001976a9145505614859643ab7b547cd7f1f5e7e2a12322d3788ac00aa0271000000001976a914ea4720a7a52fc166c55ff2298e07baf70ae67e1b88ac00000000010000000586c62cd602d219bb60edb14a3e204de0705176f9022fe49a538054fb14abb49e010000008c493046022100f2bc2aba2534becbdf062eb993853a42bbbc282083d0daf9b4b585bd401aa8c9022100b1d7fd7ee0b95600db8535bbf331b19eed8d961f7a8e54159c53675d5f69df8c014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff03ad0e58ccdac3df9dc28a218bcf6f1997b0a93306faaa4b3a28ae83447b2179010000008b483045022100be12b2937179da88599e27bb31c3525097a07cdb52422d165b3ca2f2020ffcf702200971b51f853a53d644ebae9ec8f3512e442b1bcb6c315a5b491d119d10624c83014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff2acfcab629bbc8685792603762c921580030ba144af553d271716a95089e107b010000008b483045022100fa579a840ac258871365dd48cd7552f96c8eea69bd00d84f05b283a0dab311e102207e3c0ee9234814cfbb1b659b83671618f45abc1326b9edcc77d552a4f2a805c0014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffffdcdc6023bbc9944a658ddc588e61eacb737ddf0a3cd24f113b5a8634c517fcd2000000008b4830450221008d6df731df5d32267954bd7d2dda2302b74c6c2a6aa5c0ca64ecbabc1af03c75022010e55c571d65da7701ae2da1956c442df81bbf076cdbac25133f99d98a9ed34c014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffffe15557cd5ce258f479dfd6dc6514edf6d7ed5b21fcfa4a038fd69f06b83ac76e010000008b483045022023b3e0ab071eb11de2eb1cc3a67261b866f86bf6867d4558165f7c8c8aca2d86022100dc6e1f53a91de3efe8f63512850811f26284b62f850c70ca73ed5de8771fb451014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff01404b4c00000000001976a9142b6ba7c9d796b75eef7942fc9288edd37c32f5c388ac00000000010000000166d7577163c932b4f9690ca6a80b6e4eb001f0a2fa9023df5595602aae96ed8d000000008a4730440220262b42546302dfb654a229cefc86432b89628ff259dc87edd1154535b16a67e102207b4634c020a97c3e7bbd0d4d19da6aa2269ad9dded4026e896b213d73ca4b63f014104979b82d02226b3a4597523845754d44f13639e3bf2df5e82c6aab2bdc79687368b01b1ab8b19875ae3c90d661a3d0a33161dab29934edeb36aa01976be3baf8affffffff02404b4c00000000001976a9144854e695a02af0aeacb823ccbc272134561e0a1688ac40420f00000000001976a914abee93376d6b37b5c2940655a6fcaf1c8e74237988ac0000000001000000014e3f8ef2e91349a9059cb4f01e54ab2597c1387161d3da89919f7ea6acdbb371010000008c49304602210081f3183471a5ca22307c0800226f3ef9c353069e0773ac76bb580654d56aa523022100d4c56465bdc069060846f4fbf2f6b20520b2a80b08b168b31e66ddb9c694e240014104976c79848e18251612f8940875b2b08d06e6dc73b9840e8860c066b7e87432c477e9a59a453e71e6d76d5fe34058b800a098fc1740ce3012e8fc8a00c96af966ffffffff02c0e1e400000000001976a9144134e75a6fcb6042034aab5e18570cf1f844f54788ac404b4c00000000001976a9142b6ba7c9d796b75eef7942fc9288edd37c32f5c388ac00000000"), + }; + stream >> TX_WITH_WITNESS(block); return block; } diff --git a/src/test/util/setup_common.h b/src/test/util/setup_common.h index 7cd4fdb417..9ff4c372a5 100644 --- a/src/test/util/setup_common.h +++ b/src/test/util/setup_common.h @@ -5,14 +5,14 @@ #ifndef BITCOIN_TEST_UTIL_SETUP_COMMON_H #define BITCOIN_TEST_UTIL_SETUP_COMMON_H -#include <common/args.h> +#include <common/args.h> // IWYU pragma: export #include <key.h> #include <node/caches.h> #include <node/context.h> // IWYU pragma: export #include <primitives/transaction.h> #include <pubkey.h> #include <stdexcept> -#include <util/chaintype.h> +#include <util/chaintype.h> // IWYU pragma: export #include <util/check.h> #include <util/fs.h> #include <util/string.h> @@ -47,6 +47,7 @@ static constexpr CAmount CENT{1000000}; * This just configures logging, data dir and chain parameters. */ struct BasicTestingSetup { + util::SignalInterrupt m_interrupt; node::NodeContext m_node; // keep as first member to be destructed last explicit BasicTestingSetup(const ChainType chainType = ChainType::MAIN, const std::vector<const char*>& extra_args = {}); @@ -124,7 +125,44 @@ struct TestChain100Setup : public TestingSetup { void mineBlocks(int num_blocks); /** - * Create a transaction and submit to the mempool. + * Create a transaction, optionally setting the fee based on the feerate. + * Note: The feerate may not be met exactly depending on whether the signatures can have different sizes. + * + * @param input_transactions The transactions to spend + * @param inputs Outpoints with which to construct transaction vin. + * @param input_height The height of the block that included the input transactions. + * @param input_signing_keys The keys to spend the input transactions. + * @param outputs Transaction vout. + * @param feerate The feerate the transaction should pay. + * @param fee_output The index of the output to take the fee from. + * @return The transaction and the fee it pays + */ + std::pair<CMutableTransaction, CAmount> CreateValidTransaction(const std::vector<CTransactionRef>& input_transactions, + const std::vector<COutPoint>& inputs, + int input_height, + const std::vector<CKey>& input_signing_keys, + const std::vector<CTxOut>& outputs, + const std::optional<CFeeRate>& feerate, + const std::optional<uint32_t>& fee_output); + /** + * Create a transaction and, optionally, submit to the mempool. + * + * @param input_transactions The transactions to spend + * @param inputs Outpoints with which to construct transaction vin. + * @param input_height The height of the block that included the input transaction(s). + * @param input_signing_keys The keys to spend inputs. + * @param outputs Transaction vout. + * @param submit Whether or not to submit to mempool + */ + CMutableTransaction CreateValidMempoolTransaction(const std::vector<CTransactionRef>& input_transactions, + const std::vector<COutPoint>& inputs, + int input_height, + const std::vector<CKey>& input_signing_keys, + const std::vector<CTxOut>& outputs, + bool submit = true); + + /** + * Create a 1-in-1-out transaction and, optionally, submit to the mempool. * * @param input_transaction The transaction to spend * @param input_vout The vout to spend from the input_transaction @@ -135,7 +173,7 @@ struct TestChain100Setup : public TestingSetup { * @param submit Whether or not to submit to mempool */ CMutableTransaction CreateValidMempoolTransaction(CTransactionRef input_transaction, - int input_vout, + uint32_t input_vout, int input_height, CKey input_signing_key, CScript output_destination, diff --git a/src/test/util/txmempool.cpp b/src/test/util/txmempool.cpp index c945f35d79..379c3c9329 100644 --- a/src/test/util/txmempool.cpp +++ b/src/test/util/txmempool.cpp @@ -11,13 +11,13 @@ #include <util/check.h> #include <util/time.h> #include <util/translation.h> +#include <validation.h> using node::NodeContext; CTxMemPool::Options MemPoolOptionsForTest(const NodeContext& node) { CTxMemPool::Options mempool_opts{ - .estimator = node.fee_estimator.get(), // Default to always checking mempool regardless of // chainparams.DefaultConsistencyChecks for tests .check_ratio = 1, @@ -36,3 +36,83 @@ CTxMemPoolEntry TestMemPoolEntryHelper::FromTx(const CTransactionRef& tx) const { return CTxMemPoolEntry{tx, nFee, TicksSinceEpoch<std::chrono::seconds>(time), nHeight, m_sequence, spendsCoinbase, sigOpCost, lp}; } + +std::optional<std::string> CheckPackageMempoolAcceptResult(const Package& txns, + const PackageMempoolAcceptResult& result, + bool expect_valid, + const CTxMemPool* mempool) +{ + if (expect_valid) { + if (result.m_state.IsInvalid()) { + return strprintf("Package validation unexpectedly failed: %s", result.m_state.ToString()); + } + } else { + if (result.m_state.IsValid()) { + return strprintf("Package validation unexpectedly succeeded. %s", result.m_state.ToString()); + } + } + if (result.m_state.GetResult() != PackageValidationResult::PCKG_POLICY && txns.size() != result.m_tx_results.size()) { + return strprintf("txns size %u does not match tx results size %u", txns.size(), result.m_tx_results.size()); + } + for (const auto& tx : txns) { + const auto& wtxid = tx->GetWitnessHash(); + if (result.m_tx_results.count(wtxid) == 0) { + return strprintf("result not found for tx %s", wtxid.ToString()); + } + + const auto& atmp_result = result.m_tx_results.at(wtxid); + const bool valid{atmp_result.m_result_type == MempoolAcceptResult::ResultType::VALID}; + if (expect_valid && atmp_result.m_state.IsInvalid()) { + return strprintf("tx %s unexpectedly failed: %s", wtxid.ToString(), atmp_result.m_state.ToString()); + } + + //m_replaced_transactions should exist iff the result was VALID + if (atmp_result.m_replaced_transactions.has_value() != valid) { + return strprintf("tx %s result should %shave m_replaced_transactions", + wtxid.ToString(), valid ? "" : "not "); + } + + // m_vsize and m_base_fees should exist iff the result was VALID or MEMPOOL_ENTRY + const bool mempool_entry{atmp_result.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY}; + if (atmp_result.m_base_fees.has_value() != (valid || mempool_entry)) { + return strprintf("tx %s result should %shave m_base_fees", wtxid.ToString(), valid || mempool_entry ? "" : "not "); + } + if (atmp_result.m_vsize.has_value() != (valid || mempool_entry)) { + return strprintf("tx %s result should %shave m_vsize", wtxid.ToString(), valid || mempool_entry ? "" : "not "); + } + + // m_other_wtxid should exist iff the result was DIFFERENT_WITNESS + const bool diff_witness{atmp_result.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS}; + if (atmp_result.m_other_wtxid.has_value() != diff_witness) { + return strprintf("tx %s result should %shave m_other_wtxid", wtxid.ToString(), diff_witness ? "" : "not "); + } + + // m_effective_feerate and m_wtxids_fee_calculations should exist iff the result was valid + // or if the failure was TX_RECONSIDERABLE + const bool valid_or_reconsiderable{atmp_result.m_result_type == MempoolAcceptResult::ResultType::VALID || + atmp_result.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE}; + if (atmp_result.m_effective_feerate.has_value() != valid_or_reconsiderable) { + return strprintf("tx %s result should %shave m_effective_feerate", + wtxid.ToString(), valid ? "" : "not "); + } + if (atmp_result.m_wtxids_fee_calculations.has_value() != valid_or_reconsiderable) { + return strprintf("tx %s result should %shave m_effective_feerate", + wtxid.ToString(), valid ? "" : "not "); + } + + if (mempool) { + // The tx by txid should be in the mempool iff the result was not INVALID. + const bool txid_in_mempool{atmp_result.m_result_type != MempoolAcceptResult::ResultType::INVALID}; + if (mempool->exists(GenTxid::Txid(tx->GetHash())) != txid_in_mempool) { + return strprintf("tx %s should %sbe in mempool", wtxid.ToString(), txid_in_mempool ? "" : "not "); + } + // Additionally, if the result was DIFFERENT_WITNESS, we shouldn't be able to find the tx in mempool by wtxid. + if (tx->HasWitness() && atmp_result.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS) { + if (mempool->exists(GenTxid::Wtxid(wtxid))) { + return strprintf("wtxid %s should not be in mempool", wtxid.ToString()); + } + } + } + } + return std::nullopt; +} diff --git a/src/test/util/txmempool.h b/src/test/util/txmempool.h index 4b0daf0d42..a866d1ce74 100644 --- a/src/test/util/txmempool.h +++ b/src/test/util/txmempool.h @@ -5,12 +5,14 @@ #ifndef BITCOIN_TEST_UTIL_TXMEMPOOL_H #define BITCOIN_TEST_UTIL_TXMEMPOOL_H +#include <policy/packages.h> #include <txmempool.h> #include <util/time.h> namespace node { struct NodeContext; } +struct PackageMempoolAcceptResult; CTxMemPool::Options MemPoolOptionsForTest(const node::NodeContext& node); @@ -36,4 +38,12 @@ struct TestMemPoolEntryHelper { TestMemPoolEntryHelper& SigOpsCost(unsigned int _sigopsCost) { sigOpCost = _sigopsCost; return *this; } }; +/** Check expected properties for every PackageMempoolAcceptResult, regardless of value. Returns + * a string if an error occurs with error populated, nullopt otherwise. If mempool is provided, + * checks that the expected transactions are in mempool (this should be set to nullptr for a test_accept). +*/ +std::optional<std::string> CheckPackageMempoolAcceptResult(const Package& txns, + const PackageMempoolAcceptResult& result, + bool expect_valid, + const CTxMemPool* mempool); #endif // BITCOIN_TEST_UTIL_TXMEMPOOL_H diff --git a/src/test/util_tests.cpp b/src/test/util_tests.cpp index 7d6c96ab40..47808a2a58 100644 --- a/src/test/util_tests.cpp +++ b/src/test/util_tests.cpp @@ -12,7 +12,6 @@ #include <util/bitdeque.h> #include <util/fs.h> #include <util/fs_helpers.h> -#include <util/getuniquepath.h> #include <util/message.h> // For MessageSign(), MessageVerify(), MESSAGE_MAGIC #include <util/moneystr.h> #include <util/overflow.h> @@ -1106,15 +1105,16 @@ BOOST_AUTO_TEST_CASE(test_ParseFixedPoint) BOOST_CHECK(!ParseFixedPoint("31.999999999999999999999", 3, &amount)); } -static void TestOtherThread(fs::path dirname, fs::path lockname, bool *result) -{ - *result = LockDirectory(dirname, lockname); -} - #ifndef WIN32 // Cannot do this test on WIN32 due to lack of fork() static constexpr char LockCommand = 'L'; static constexpr char UnlockCommand = 'U'; static constexpr char ExitCommand = 'X'; +enum : char { + ResSuccess = 2, // Start with 2 to avoid accidental collision with common values 0 and 1 + ResErrorWrite, + ResErrorLock, + ResUnlockSuccess, +}; [[noreturn]] static void TestOtherProcess(fs::path dirname, fs::path lockname, int fd) { @@ -1122,15 +1122,22 @@ static constexpr char ExitCommand = 'X'; while (true) { int rv = read(fd, &ch, 1); // Wait for command assert(rv == 1); - switch(ch) { + switch (ch) { case LockCommand: - ch = LockDirectory(dirname, lockname); + ch = [&] { + switch (util::LockDirectory(dirname, lockname)) { + case util::LockResult::Success: return ResSuccess; + case util::LockResult::ErrorWrite: return ResErrorWrite; + case util::LockResult::ErrorLock: return ResErrorLock; + } // no default case, so the compiler can warn about missing cases + assert(false); + }(); rv = write(fd, &ch, 1); assert(rv == 1); break; case UnlockCommand: ReleaseDirectoryLocks(); - ch = true; // Always succeeds + ch = ResUnlockSuccess; // Always succeeds rv = write(fd, &ch, 1); assert(rv == 1); break; @@ -1165,63 +1172,71 @@ BOOST_AUTO_TEST_CASE(test_LockDirectory) TestOtherProcess(dirname, lockname, fd[0]); } BOOST_CHECK_EQUAL(close(fd[0]), 0); // Parent: close child end + + char ch; + // Lock on non-existent directory should fail + BOOST_CHECK_EQUAL(write(fd[1], &LockCommand, 1), 1); + BOOST_CHECK_EQUAL(read(fd[1], &ch, 1), 1); + BOOST_CHECK_EQUAL(ch, ResErrorWrite); #endif // Lock on non-existent directory should fail - BOOST_CHECK_EQUAL(LockDirectory(dirname, lockname), false); + BOOST_CHECK_EQUAL(util::LockDirectory(dirname, lockname), util::LockResult::ErrorWrite); fs::create_directories(dirname); // Probing lock on new directory should succeed - BOOST_CHECK_EQUAL(LockDirectory(dirname, lockname, true), true); + BOOST_CHECK_EQUAL(util::LockDirectory(dirname, lockname, true), util::LockResult::Success); // Persistent lock on new directory should succeed - BOOST_CHECK_EQUAL(LockDirectory(dirname, lockname), true); + BOOST_CHECK_EQUAL(util::LockDirectory(dirname, lockname), util::LockResult::Success); // Another lock on the directory from the same thread should succeed - BOOST_CHECK_EQUAL(LockDirectory(dirname, lockname), true); + BOOST_CHECK_EQUAL(util::LockDirectory(dirname, lockname), util::LockResult::Success); // Another lock on the directory from a different thread within the same process should succeed - bool threadresult; - std::thread thr(TestOtherThread, dirname, lockname, &threadresult); + util::LockResult threadresult; + std::thread thr([&] { threadresult = util::LockDirectory(dirname, lockname); }); thr.join(); - BOOST_CHECK_EQUAL(threadresult, true); + BOOST_CHECK_EQUAL(threadresult, util::LockResult::Success); #ifndef WIN32 // Try to acquire lock in child process while we're holding it, this should fail. - char ch; BOOST_CHECK_EQUAL(write(fd[1], &LockCommand, 1), 1); BOOST_CHECK_EQUAL(read(fd[1], &ch, 1), 1); - BOOST_CHECK_EQUAL((bool)ch, false); + BOOST_CHECK_EQUAL(ch, ResErrorLock); // Give up our lock ReleaseDirectoryLocks(); // Probing lock from our side now should succeed, but not hold on to the lock. - BOOST_CHECK_EQUAL(LockDirectory(dirname, lockname, true), true); + BOOST_CHECK_EQUAL(util::LockDirectory(dirname, lockname, true), util::LockResult::Success); // Try to acquire the lock in the child process, this should be successful. BOOST_CHECK_EQUAL(write(fd[1], &LockCommand, 1), 1); BOOST_CHECK_EQUAL(read(fd[1], &ch, 1), 1); - BOOST_CHECK_EQUAL((bool)ch, true); + BOOST_CHECK_EQUAL(ch, ResSuccess); // When we try to probe the lock now, it should fail. - BOOST_CHECK_EQUAL(LockDirectory(dirname, lockname, true), false); + BOOST_CHECK_EQUAL(util::LockDirectory(dirname, lockname, true), util::LockResult::ErrorLock); // Unlock the lock in the child process BOOST_CHECK_EQUAL(write(fd[1], &UnlockCommand, 1), 1); BOOST_CHECK_EQUAL(read(fd[1], &ch, 1), 1); - BOOST_CHECK_EQUAL((bool)ch, true); + BOOST_CHECK_EQUAL(ch, ResUnlockSuccess); // When we try to probe the lock now, it should succeed. - BOOST_CHECK_EQUAL(LockDirectory(dirname, lockname, true), true); + BOOST_CHECK_EQUAL(util::LockDirectory(dirname, lockname, true), util::LockResult::Success); // Re-lock the lock in the child process, then wait for it to exit, check // successful return. After that, we check that exiting the process // has released the lock as we would expect by probing it. int processstatus; BOOST_CHECK_EQUAL(write(fd[1], &LockCommand, 1), 1); + // The following line invokes the ~CNetCleanup dtor without + // a paired SetupNetworking call. This is acceptable as long as + // ~CNetCleanup is a no-op for non-Windows platforms. BOOST_CHECK_EQUAL(write(fd[1], &ExitCommand, 1), 1); BOOST_CHECK_EQUAL(waitpid(pid, &processstatus, 0), pid); BOOST_CHECK_EQUAL(processstatus, 0); - BOOST_CHECK_EQUAL(LockDirectory(dirname, lockname, true), true); + BOOST_CHECK_EQUAL(util::LockDirectory(dirname, lockname, true), util::LockResult::Success); // Restore SIGCHLD signal(SIGCHLD, old_handler); @@ -1232,22 +1247,6 @@ BOOST_AUTO_TEST_CASE(test_LockDirectory) fs::remove_all(dirname); } -BOOST_AUTO_TEST_CASE(test_DirIsWritable) -{ - // Should be able to write to the data dir. - fs::path tmpdirname = m_args.GetDataDirBase(); - BOOST_CHECK_EQUAL(DirIsWritable(tmpdirname), true); - - // Should not be able to write to a non-existent dir. - tmpdirname = GetUniquePath(tmpdirname); - BOOST_CHECK_EQUAL(DirIsWritable(tmpdirname), false); - - fs::create_directory(tmpdirname); - // Should be able to write to it now. - BOOST_CHECK_EQUAL(DirIsWritable(tmpdirname), true); - fs::remove(tmpdirname); -} - BOOST_AUTO_TEST_CASE(test_ToLower) { BOOST_CHECK_EQUAL(ToLower('@'), '@'); diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 64cb5522eb..35e5c6a037 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -283,8 +283,7 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) // Check that all txs are in the pool { - LOCK(m_node.mempool->cs); - BOOST_CHECK_EQUAL(m_node.mempool->mapTx.size(), txs.size()); + BOOST_CHECK_EQUAL(m_node.mempool->size(), txs.size()); } // Run a thread that simulates an RPC caller that is polling while @@ -295,7 +294,7 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) // not some intermediate amount. while (true) { LOCK(m_node.mempool->cs); - if (m_node.mempool->mapTx.size() == 0) { + if (m_node.mempool->size() == 0) { // We are done with the reorg break; } @@ -304,7 +303,7 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) // be atomic. So the caller assumes that the returned mempool // is consistent. That is, it has all txs that were there // before the reorg. - assert(m_node.mempool->mapTx.size() == txs.size()); + assert(m_node.mempool->size() == txs.size()); continue; } LOCK(cs_main); diff --git a/src/test/validation_chainstatemanager_tests.cpp b/src/test/validation_chainstatemanager_tests.cpp index 227d7d4633..368ba8bee4 100644 --- a/src/test/validation_chainstatemanager_tests.cpp +++ b/src/test/validation_chainstatemanager_tests.cpp @@ -379,7 +379,7 @@ struct SnapshotTestSetup : TestChain100Setup { LOCK(::cs_main); chainman.ResetChainstates(); BOOST_CHECK_EQUAL(chainman.GetAll().size(), 0); - m_node.notifications = std::make_unique<KernelNotifications>(m_node.exit_status); + m_node.notifications = std::make_unique<KernelNotifications>(*Assert(m_node.shutdown), m_node.exit_status); const ChainstateManager::Options chainman_opts{ .chainparams = ::Params(), .datadir = chainman.m_options.datadir, @@ -394,7 +394,7 @@ struct SnapshotTestSetup : TestChain100Setup { // For robustness, ensure the old manager is destroyed before creating a // new one. m_node.chainman.reset(); - m_node.chainman = std::make_unique<ChainstateManager>(m_node.kernel->interrupt, chainman_opts, blockman_opts); + m_node.chainman = std::make_unique<ChainstateManager>(*Assert(m_node.shutdown), chainman_opts, blockman_opts); } return *Assert(m_node.chainman); } @@ -579,7 +579,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_init, SnapshotTestSetup) // it will initialize instead of attempting to complete validation. // // Note that this is not a realistic use of DisconnectTip(). - DisconnectedBlockTransactions unused_pool{MAX_DISCONNECTED_TX_POOL_SIZE * 1000}; + DisconnectedBlockTransactions unused_pool{MAX_DISCONNECTED_TX_POOL_BYTES}; BlockValidationState unused_state; { LOCK2(::cs_main, bg_chainstate.MempoolMutex()); @@ -724,7 +724,7 @@ BOOST_FIXTURE_TEST_CASE(chainstatemanager_snapshot_completion_hash_mismatch, Sna badcoin.out.nValue = InsecureRand32(); badcoin.nHeight = 1; badcoin.out.scriptPubKey.assign(InsecureRandBits(6), 0); - uint256 txid = InsecureRand256(); + Txid txid = Txid::FromUint256(InsecureRand256()); ibd_coins.AddCoin(COutPoint(txid, 0), std::move(badcoin), false); fs::path snapshot_chainstate_dir = gArgs.GetDataDirNet() / "chainstate_snapshot"; diff --git a/src/test/versionbits_tests.cpp b/src/test/versionbits_tests.cpp index 9e69992752..f462895edb 100644 --- a/src/test/versionbits_tests.cpp +++ b/src/test/versionbits_tests.cpp @@ -424,7 +424,7 @@ BOOST_AUTO_TEST_CASE(versionbits_computeblockversion) uint32_t chain_all_vbits{0}; for (int i = 0; i < (int)Consensus::MAX_VERSION_BITS_DEPLOYMENTS; ++i) { const auto dep = static_cast<Consensus::DeploymentPos>(i); - // Check that no bits are re-used (within the same chain). This is + // Check that no bits are reused (within the same chain). This is // disallowed because the transition to FAILED (on timeout) does // not take precedence over STARTED/LOCKED_IN. So all softforks on // the same bit might overlap, even when non-overlapping start-end diff --git a/src/torcontrol.cpp b/src/torcontrol.cpp index 60cf31a964..11d13d050a 100644 --- a/src/torcontrol.cpp +++ b/src/torcontrol.cpp @@ -251,7 +251,7 @@ std::map<std::string,std::string> ParseTorReplyMapping(const std::string &s) /** * Unescape value. Per https://spec.torproject.org/control-spec section 2.1.1: * - * For future-proofing, controller implementors MAY use the following + * For future-proofing, controller implementers MAY use the following * rules to be compatible with buggy Tor implementations and with * future ones that implement the spec as intended: * diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 461662ad93..acee56fe78 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -12,9 +12,9 @@ #include <consensus/tx_verify.h> #include <consensus/validation.h> #include <logging.h> -#include <policy/fees.h> #include <policy/policy.h> #include <policy/settings.h> +#include <random.h> #include <reverse_iterator.h> #include <util/check.h> #include <util/moneystr.h> @@ -124,7 +124,7 @@ void CTxMemPool::UpdateTransactionsFromBlock(const std::vector<uint256>& vHashes if (it == mapTx.end()) { continue; } - auto iter = mapNextTx.lower_bound(COutPoint(hash, 0)); + auto iter = mapNextTx.lower_bound(COutPoint(Txid::FromUint256(hash), 0)); // First calculate the children, and update CTxMemPoolEntry::m_children to // include them, and update their CTxMemPoolEntry::m_parents to include this tx. // we cache the in-mempool children to avoid duplicate updates @@ -196,25 +196,20 @@ util::Result<CTxMemPool::setEntries> CTxMemPool::CalculateAncestorsAndCheckLimit return ancestors; } -bool CTxMemPool::CheckPackageLimits(const Package& package, - const int64_t total_vsize, - std::string &errString) const +util::Result<void> CTxMemPool::CheckPackageLimits(const Package& package, + const int64_t total_vsize) const { size_t pack_count = package.size(); // Package itself is busting mempool limits; should be rejected even if no staged_ancestors exist if (pack_count > static_cast<uint64_t>(m_limits.ancestor_count)) { - errString = strprintf("package count %u exceeds ancestor count limit [limit: %u]", pack_count, m_limits.ancestor_count); - return false; + return util::Error{Untranslated(strprintf("package count %u exceeds ancestor count limit [limit: %u]", pack_count, m_limits.ancestor_count))}; } else if (pack_count > static_cast<uint64_t>(m_limits.descendant_count)) { - errString = strprintf("package count %u exceeds descendant count limit [limit: %u]", pack_count, m_limits.descendant_count); - return false; + return util::Error{Untranslated(strprintf("package count %u exceeds descendant count limit [limit: %u]", pack_count, m_limits.descendant_count))}; } else if (total_vsize > m_limits.ancestor_size_vbytes) { - errString = strprintf("package size %u exceeds ancestor size limit [limit: %u]", total_vsize, m_limits.ancestor_size_vbytes); - return false; + return util::Error{Untranslated(strprintf("package size %u exceeds ancestor size limit [limit: %u]", total_vsize, m_limits.ancestor_size_vbytes))}; } else if (total_vsize > m_limits.descendant_size_vbytes) { - errString = strprintf("package size %u exceeds descendant size limit [limit: %u]", total_vsize, m_limits.descendant_size_vbytes); - return false; + return util::Error{Untranslated(strprintf("package size %u exceeds descendant size limit [limit: %u]", total_vsize, m_limits.descendant_size_vbytes))}; } CTxMemPoolEntry::Parents staged_ancestors; @@ -224,8 +219,7 @@ bool CTxMemPool::CheckPackageLimits(const Package& package, if (piter) { staged_ancestors.insert(**piter); if (staged_ancestors.size() + package.size() > static_cast<uint64_t>(m_limits.ancestor_count)) { - errString = strprintf("too many unconfirmed parents [limit: %u]", m_limits.ancestor_count); - return false; + return util::Error{Untranslated(strprintf("too many unconfirmed parents [limit: %u]", m_limits.ancestor_count))}; } } } @@ -236,8 +230,8 @@ bool CTxMemPool::CheckPackageLimits(const Package& package, const auto ancestors{CalculateAncestorsAndCheckLimits(total_vsize, package.size(), staged_ancestors, m_limits)}; // It's possible to overestimate the ancestor/descendant totals. - if (!ancestors.has_value()) errString = "possibly " + util::ErrorString(ancestors).original; - return ancestors.has_value(); + if (!ancestors.has_value()) return util::Error{Untranslated("possibly " + util::ErrorString(ancestors).original)}; + return {}; } util::Result<CTxMemPool::setEntries> CTxMemPool::CalculateMemPoolAncestors( @@ -402,7 +396,6 @@ void CTxMemPoolEntry::UpdateAncestorState(int32_t modifySize, CAmount modifyFee, CTxMemPool::CTxMemPool(const Options& opts) : m_check_ratio{opts.check_ratio}, - minerPolicyEstimator{opts.estimator}, m_max_size_bytes{opts.max_size_bytes}, m_expiry{opts.expiry}, m_incremental_relay_feerate{opts.incremental_relay_feerate}, @@ -412,6 +405,7 @@ CTxMemPool::CTxMemPool(const Options& opts) m_max_datacarrier_bytes{opts.max_datacarrier_bytes}, m_require_standard{opts.require_standard}, m_full_rbf{opts.full_rbf}, + m_persist_v1_dat{opts.persist_v1_dat}, m_limits{opts.limits} { } @@ -432,12 +426,12 @@ void CTxMemPool::AddTransactionsUpdated(unsigned int n) nTransactionsUpdated += n; } -void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry, setEntries &setAncestors, bool validFeeEstimate) +void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry, setEntries &setAncestors) { // Add to memory pool without checking anything. // Used by AcceptToMemoryPool(), which DOES do // all the appropriate checks. - indexed_transaction_set::iterator newit = mapTx.insert(entry).first; + indexed_transaction_set::iterator newit = mapTx.emplace(CTxMemPoolEntry::ExplicitCopy, entry).first; // Update transaction for any feeDelta created by PrioritiseTransaction CAmount delta{0}; @@ -476,12 +470,9 @@ void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry, setEntries &setAnces nTransactionsUpdated++; totalTxSize += entry.GetTxSize(); m_total_fee += entry.GetFee(); - if (minerPolicyEstimator) { - minerPolicyEstimator->processTransaction(entry, validFeeEstimate); - } - vTxHashes.emplace_back(tx.GetWitnessHash(), newit); - newit->vTxHashesIdx = vTxHashes.size() - 1; + txns_randomized.emplace_back(newit->GetSharedTx()); + newit->idx_randomized = txns_randomized.size() - 1; TRACE3(mempool, added, entry.GetTx().GetHash().data(), @@ -511,20 +502,21 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) std::chrono::duration_cast<std::chrono::duration<std::uint64_t>>(it->GetTime()).count() ); - const uint256 hash = it->GetTx().GetHash(); for (const CTxIn& txin : it->GetTx().vin) mapNextTx.erase(txin.prevout); - RemoveUnbroadcastTx(hash, true /* add logging because unchecked */ ); + RemoveUnbroadcastTx(it->GetTx().GetHash(), true /* add logging because unchecked */); - if (vTxHashes.size() > 1) { - vTxHashes[it->vTxHashesIdx] = std::move(vTxHashes.back()); - vTxHashes[it->vTxHashesIdx].second->vTxHashesIdx = it->vTxHashesIdx; - vTxHashes.pop_back(); - if (vTxHashes.size() * 2 < vTxHashes.capacity()) - vTxHashes.shrink_to_fit(); + if (txns_randomized.size() > 1) { + // Update idx_randomized of the to-be-moved entry. + Assert(GetEntry(txns_randomized.back()->GetHash()))->idx_randomized = it->idx_randomized; + // Remove entry from txns_randomized by replacing it with the back and deleting the back. + txns_randomized[it->idx_randomized] = std::move(txns_randomized.back()); + txns_randomized.pop_back(); + if (txns_randomized.size() * 2 < txns_randomized.capacity()) + txns_randomized.shrink_to_fit(); } else - vTxHashes.clear(); + txns_randomized.clear(); totalTxSize -= it->GetTxSize(); m_total_fee -= it->GetFee(); @@ -532,7 +524,6 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) cachedInnerUsage -= memusage::DynamicUsage(it->GetMemPoolParentsConst()) + memusage::DynamicUsage(it->GetMemPoolChildrenConst()); mapTx.erase(it); nTransactionsUpdated++; - if (minerPolicyEstimator) {minerPolicyEstimator->removeTx(hash, false);} } // Calculates descendants of entry that are not already in setDescendants, and adds to @@ -633,33 +624,26 @@ void CTxMemPool::removeConflicts(const CTransaction &tx) } /** - * Called when a block is connected. Removes from mempool and updates the miner fee estimator. + * Called when a block is connected. Removes from mempool. */ void CTxMemPool::removeForBlock(const std::vector<CTransactionRef>& vtx, unsigned int nBlockHeight) { AssertLockHeld(cs); - std::vector<const CTxMemPoolEntry*> entries; - for (const auto& tx : vtx) - { - uint256 hash = tx->GetHash(); - - indexed_transaction_set::iterator i = mapTx.find(hash); - if (i != mapTx.end()) - entries.push_back(&*i); - } - // Before the txs in the new block have been removed from the mempool, update policy estimates - if (minerPolicyEstimator) {minerPolicyEstimator->processBlock(nBlockHeight, entries);} + std::vector<RemovedMempoolTransactionInfo> txs_removed_for_block; + txs_removed_for_block.reserve(vtx.size()); for (const auto& tx : vtx) { txiter it = mapTx.find(tx->GetHash()); if (it != mapTx.end()) { setEntries stage; stage.insert(it); + txs_removed_for_block.emplace_back(*it); RemoveStaged(stage, true, MemPoolRemovalReason::BLOCK); } removeConflicts(*tx); ClearPrioritisation(tx->GetHash()); } + GetMainSignals().MempoolTransactionsRemovedForBlock(txs_removed_for_block, nBlockHeight); lastRollingFeeUpdate = GetTime(); blockSinceLastRollingFeeBump = true; } @@ -836,6 +820,18 @@ static TxMempoolInfo GetInfo(CTxMemPool::indexed_transaction_set::const_iterator return TxMempoolInfo{it->GetSharedTx(), it->GetTime(), it->GetFee(), it->GetTxSize(), it->GetModifiedFee() - it->GetFee()}; } +std::vector<CTxMemPoolEntryRef> CTxMemPool::entryAll() const +{ + AssertLockHeld(cs); + + std::vector<CTxMemPoolEntryRef> ret; + ret.reserve(mapTx.size()); + for (const auto& it : GetSortedDepthAndScore()) { + ret.emplace_back(*it); + } + return ret; +} + std::vector<TxMempoolInfo> CTxMemPool::infoAll() const { LOCK(cs); @@ -850,6 +846,13 @@ std::vector<TxMempoolInfo> CTxMemPool::infoAll() const return ret; } +const CTxMemPoolEntry* CTxMemPool::GetEntry(const Txid& txid) const +{ + AssertLockHeld(cs); + const auto i = mapTx.find(txid); + return i == mapTx.end() ? nullptr : &(*i); +} + CTransactionRef CTxMemPool::get(const uint256& hash) const { LOCK(cs); @@ -1033,7 +1036,7 @@ void CCoinsViewMemPool::Reset() size_t CTxMemPool::DynamicMemoryUsage() const { LOCK(cs); // Estimate the overhead of mapTx to be 15 pointers + an allocation, as no exact formula for boost::multi_index_contained is implemented. - return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 15 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(vTxHashes) + cachedInnerUsage; + return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 15 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(txns_randomized) + cachedInnerUsage; } void CTxMemPool::RemoveUnbroadcastTx(const uint256& txid, const bool unchecked) { @@ -1070,10 +1073,10 @@ int CTxMemPool::Expire(std::chrono::seconds time) return stage.size(); } -void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry, bool validFeeEstimate) +void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry) { auto ancestors{AssumeCalculateMemPoolAncestors(__func__, entry, Limits::NoLimits())}; - return addUnchecked(entry, ancestors, validFeeEstimate); + return addUnchecked(entry, ancestors); } void CTxMemPool::UpdateChild(txiter entry, txiter child, bool add) diff --git a/src/txmempool.h b/src/txmempool.h index cbeabb31fa..9da51756e6 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -202,8 +202,6 @@ struct entry_time {}; struct ancestor_score {}; struct index_by_wtxid {}; -class CBlockPolicyEstimator; - /** * Information about a mempool transaction. */ @@ -303,7 +301,6 @@ class CTxMemPool protected: const int m_check_ratio; //!< Value n means that 1 times in n we check. std::atomic<unsigned int> nTransactionsUpdated{0}; //!< Used by getblocktemplate to trigger CreateNewBlock() invocation - CBlockPolicyEstimator* const minerPolicyEstimator; uint64_t totalTxSize GUARDED_BY(cs){0}; //!< sum of all mempool tx's virtual sizes. Differs from serialized tx size since witness data is discounted. Defined in BIP 141. CAmount m_total_fee GUARDED_BY(cs){0}; //!< sum of all mempool tx's fees (NOT modified fee) @@ -392,7 +389,7 @@ public: indexed_transaction_set mapTx GUARDED_BY(cs); using txiter = indexed_transaction_set::nth_index<0>::type::const_iterator; - std::vector<std::pair<uint256, txiter>> vTxHashes GUARDED_BY(cs); //!< All tx witness hashes/entries in mapTx, in random order + std::vector<CTransactionRef> txns_randomized GUARDED_BY(cs); //!< All transactions in mapTx, in random order typedef std::set<txiter, CompareIteratorByHash> setEntries; @@ -446,6 +443,7 @@ public: const std::optional<unsigned> m_max_datacarrier_bytes; const bool m_require_standard; const bool m_full_rbf; + const bool m_persist_v1_dat; const Limits m_limits; @@ -471,13 +469,13 @@ public: // Note that addUnchecked is ONLY called from ATMP outside of tests // and any other callers may break wallet's in-mempool tracking (due to // lack of CValidationInterface::TransactionAddedToMempool callbacks). - void addUnchecked(const CTxMemPoolEntry& entry, bool validFeeEstimate = true) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main); - void addUnchecked(const CTxMemPoolEntry& entry, setEntries& setAncestors, bool validFeeEstimate = true) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main); + void addUnchecked(const CTxMemPoolEntry& entry) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main); + void addUnchecked(const CTxMemPoolEntry& entry, setEntries& setAncestors) EXCLUSIVE_LOCKS_REQUIRED(cs, cs_main); void removeRecursive(const CTransaction& tx, MemPoolRemovalReason reason) EXCLUSIVE_LOCKS_REQUIRED(cs); /** After reorg, filter the entries that would no longer be valid in the next block, and update * the entries' cached LockPoints if needed. The mempool does not have any knowledge of - * consensus rules. It just appplies the callable function and removes the ones for which it + * consensus rules. It just applies the callable function and removes the ones for which it * returns true. * @param[in] filter_final_and_mature Predicate that checks the relevant validation rules * and updates an entry's LockPoints. @@ -607,11 +605,10 @@ public: * to mempool. The transactions need not be direct * ancestors/descendants of each other. * @param[in] total_vsize Sum of virtual sizes for all transactions in package. - * @param[out] errString Populated with error reason if a limit is hit. + * @returns {} or the error reason if a limit is hit. */ - bool CheckPackageLimits(const Package& package, - int64_t total_vsize, - std::string &errString) const EXCLUSIVE_LOCKS_REQUIRED(cs); + util::Result<void> CheckPackageLimits(const Package& package, + int64_t total_vsize) const EXCLUSIVE_LOCKS_REQUIRED(cs); /** Populate setDescendants with all in-mempool descendants of hash. * Assumes that setDescendants includes all in-mempool descendants of anything @@ -684,6 +681,8 @@ public: return (mapTx.count(gtxid.GetHash()) != 0); } + const CTxMemPoolEntry* GetEntry(const Txid& txid) const LIFETIMEBOUND EXCLUSIVE_LOCKS_REQUIRED(cs); + CTransactionRef get(const uint256& hash) const; txiter get_iter_from_wtxid(const uint256& wtxid) const EXCLUSIVE_LOCKS_REQUIRED(cs) { @@ -695,6 +694,7 @@ public: /** Returns info for a transaction if its entry_sequence < last_sequence */ TxMempoolInfo info_for_relay(const GenTxid& gtxid, uint64_t last_sequence) const; + std::vector<CTxMemPoolEntryRef> entryAll() const EXCLUSIVE_LOCKS_REQUIRED(cs); std::vector<TxMempoolInfo> infoAll() const; size_t DynamicMemoryUsage() const; diff --git a/src/txorphanage.cpp b/src/txorphanage.cpp index 7455d914e8..e907fffd4f 100644 --- a/src/txorphanage.cpp +++ b/src/txorphanage.cpp @@ -7,6 +7,7 @@ #include <consensus/validation.h> #include <logging.h> #include <policy/policy.h> +#include <primitives/transaction.h> #include <cassert> @@ -20,8 +21,8 @@ bool TxOrphanage::AddTx(const CTransactionRef& tx, NodeId peer) { LOCK(m_mutex); - const uint256& hash = tx->GetHash(); - const uint256& wtxid = tx->GetWitnessHash(); + const Txid& hash = tx->GetHash(); + const Wtxid& wtxid = tx->GetWitnessHash(); if (m_orphans.count(hash)) return false; @@ -53,16 +54,16 @@ bool TxOrphanage::AddTx(const CTransactionRef& tx, NodeId peer) return true; } -int TxOrphanage::EraseTx(const uint256& txid) +int TxOrphanage::EraseTx(const Txid& txid) { LOCK(m_mutex); return EraseTxNoLock(txid); } -int TxOrphanage::EraseTxNoLock(const uint256& txid) +int TxOrphanage::EraseTxNoLock(const Txid& txid) { AssertLockHeld(m_mutex); - std::map<uint256, OrphanTx>::iterator it = m_orphans.find(txid); + std::map<Txid, OrphanTx>::iterator it = m_orphans.find(txid); if (it == m_orphans.end()) return 0; for (const CTxIn& txin : it->second.tx->vin) @@ -100,10 +101,10 @@ void TxOrphanage::EraseForPeer(NodeId peer) m_peer_work_set.erase(peer); int nErased = 0; - std::map<uint256, OrphanTx>::iterator iter = m_orphans.begin(); + std::map<Txid, OrphanTx>::iterator iter = m_orphans.begin(); while (iter != m_orphans.end()) { - std::map<uint256, OrphanTx>::iterator maybeErase = iter++; // increment to avoid iterator becoming invalid + std::map<Txid, OrphanTx>::iterator maybeErase = iter++; // increment to avoid iterator becoming invalid if (maybeErase->second.fromPeer == peer) { nErased += EraseTxNoLock(maybeErase->second.tx->GetHash()); @@ -112,7 +113,7 @@ void TxOrphanage::EraseForPeer(NodeId peer) if (nErased > 0) LogPrint(BCLog::TXPACKAGES, "Erased %d orphan tx from peer=%d\n", nErased, peer); } -void TxOrphanage::LimitOrphans(unsigned int max_orphans) +void TxOrphanage::LimitOrphans(unsigned int max_orphans, FastRandomContext& rng) { LOCK(m_mutex); @@ -123,10 +124,10 @@ void TxOrphanage::LimitOrphans(unsigned int max_orphans) // Sweep out expired orphan pool entries: int nErased = 0; int64_t nMinExpTime = nNow + ORPHAN_TX_EXPIRE_TIME - ORPHAN_TX_EXPIRE_INTERVAL; - std::map<uint256, OrphanTx>::iterator iter = m_orphans.begin(); + std::map<Txid, OrphanTx>::iterator iter = m_orphans.begin(); while (iter != m_orphans.end()) { - std::map<uint256, OrphanTx>::iterator maybeErase = iter++; + std::map<Txid, OrphanTx>::iterator maybeErase = iter++; if (maybeErase->second.nTimeExpire <= nNow) { nErased += EraseTxNoLock(maybeErase->second.tx->GetHash()); } else { @@ -137,7 +138,6 @@ void TxOrphanage::LimitOrphans(unsigned int max_orphans) nNextSweep = nMinExpTime + ORPHAN_TX_EXPIRE_INTERVAL; if (nErased > 0) LogPrint(BCLog::TXPACKAGES, "Erased %d orphan tx due to expiration\n", nErased); } - FastRandomContext rng; while (m_orphans.size() > max_orphans) { // Evict a random orphan: @@ -159,7 +159,7 @@ void TxOrphanage::AddChildrenToWorkSet(const CTransaction& tx) for (const auto& elem : it_by_prev->second) { // Get this source peer's work set, emplacing an empty set if it didn't exist // (note: if this peer wasn't still connected, we would have removed the orphan tx already) - std::set<uint256>& orphan_work_set = m_peer_work_set.try_emplace(elem->second.fromPeer).first->second; + std::set<Txid>& orphan_work_set = m_peer_work_set.try_emplace(elem->second.fromPeer).first->second; // Add this tx to the work set orphan_work_set.insert(elem->first); LogPrint(BCLog::TXPACKAGES, "added %s (wtxid=%s) to peer %d workset\n", @@ -173,9 +173,9 @@ bool TxOrphanage::HaveTx(const GenTxid& gtxid) const { LOCK(m_mutex); if (gtxid.IsWtxid()) { - return m_wtxid_to_orphan_it.count(gtxid.GetHash()); + return m_wtxid_to_orphan_it.count(Wtxid::FromUint256(gtxid.GetHash())); } else { - return m_orphans.count(gtxid.GetHash()); + return m_orphans.count(Txid::FromUint256(gtxid.GetHash())); } } @@ -187,7 +187,7 @@ CTransactionRef TxOrphanage::GetTxToReconsider(NodeId peer) if (work_set_it != m_peer_work_set.end()) { auto& work_set = work_set_it->second; while (!work_set.empty()) { - uint256 txid = *work_set.begin(); + Txid txid = *work_set.begin(); work_set.erase(work_set.begin()); const auto orphan_it = m_orphans.find(txid); @@ -215,7 +215,7 @@ void TxOrphanage::EraseForBlock(const CBlock& block) { LOCK(m_mutex); - std::vector<uint256> vOrphanErase; + std::vector<Txid> vOrphanErase; for (const CTransactionRef& ptx : block.vtx) { const CTransaction& tx = *ptx; @@ -226,7 +226,7 @@ void TxOrphanage::EraseForBlock(const CBlock& block) if (itByPrev == m_outpoint_to_orphan_it.end()) continue; for (auto mi = itByPrev->second.begin(); mi != itByPrev->second.end(); ++mi) { const CTransaction& orphanTx = *(*mi)->second.tx; - const uint256& orphanHash = orphanTx.GetHash(); + const auto& orphanHash = orphanTx.GetHash(); vOrphanErase.push_back(orphanHash); } } @@ -235,7 +235,7 @@ void TxOrphanage::EraseForBlock(const CBlock& block) // Erase orphan transactions included or precluded by this block if (vOrphanErase.size()) { int nErased = 0; - for (const uint256& orphanHash : vOrphanErase) { + for (const auto& orphanHash : vOrphanErase) { nErased += EraseTxNoLock(orphanHash); } LogPrint(BCLog::TXPACKAGES, "Erased %d orphan tx included or conflicted by block\n", nErased); diff --git a/src/txorphanage.h b/src/txorphanage.h index a4705bf382..2fd14e6fd2 100644 --- a/src/txorphanage.h +++ b/src/txorphanage.h @@ -34,7 +34,7 @@ public: CTransactionRef GetTxToReconsider(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); /** Erase an orphan by txid */ - int EraseTx(const uint256& txid) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); + int EraseTx(const Txid& txid) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); /** Erase all orphans announced by a peer (eg, after that peer disconnects) */ void EraseForPeer(NodeId peer) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); @@ -43,7 +43,7 @@ public: void EraseForBlock(const CBlock& block) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); /** Limit the orphanage to the given maximum */ - void LimitOrphans(unsigned int max_orphans) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); + void LimitOrphans(unsigned int max_orphans, FastRandomContext& rng) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex); /** Add any orphans that list a particular tx as a parent into the from peer's work set */ void AddChildrenToWorkSet(const CTransaction& tx) EXCLUSIVE_LOCKS_REQUIRED(!m_mutex);; @@ -71,10 +71,10 @@ protected: /** Map from txid to orphan transaction record. Limited by * -maxorphantx/DEFAULT_MAX_ORPHAN_TRANSACTIONS */ - std::map<uint256, OrphanTx> m_orphans GUARDED_BY(m_mutex); + std::map<Txid, OrphanTx> m_orphans GUARDED_BY(m_mutex); /** Which peer provided the orphans that need to be reconsidered */ - std::map<NodeId, std::set<uint256>> m_peer_work_set GUARDED_BY(m_mutex); + std::map<NodeId, std::set<Txid>> m_peer_work_set GUARDED_BY(m_mutex); using OrphanMap = decltype(m_orphans); @@ -96,10 +96,10 @@ protected: /** Index from wtxid into the m_orphans to lookup orphan * transactions using their witness ids. */ - std::map<uint256, OrphanMap::iterator> m_wtxid_to_orphan_it GUARDED_BY(m_mutex); + std::map<Wtxid, OrphanMap::iterator> m_wtxid_to_orphan_it GUARDED_BY(m_mutex); /** Erase an orphan by txid */ - int EraseTxNoLock(const uint256& txid) EXCLUSIVE_LOCKS_REQUIRED(m_mutex); + int EraseTxNoLock(const Txid& txid) EXCLUSIVE_LOCKS_REQUIRED(m_mutex); }; #endif // BITCOIN_TXORPHANAGE_H diff --git a/src/txrequest.cpp b/src/txrequest.cpp index 4c94d4562d..ce5fbd9a7f 100644 --- a/src/txrequest.cpp +++ b/src/txrequest.cpp @@ -72,16 +72,10 @@ struct Announcement { /** Whether this is a wtxid request. */ const bool m_is_wtxid : 1; - /** What state this announcement is in. - * This is a uint8_t instead of a State to silence a GCC warning in versions prior to 9.3. - * See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=61414 */ - uint8_t m_state : 3; - - /** Convert m_state to a State enum. */ - State GetState() const { return static_cast<State>(m_state); } - - /** Convert a State enum to a uint8_t and store it in m_state. */ - void SetState(State state) { m_state = static_cast<uint8_t>(state); } + /** What state this announcement is in. */ + State m_state : 3 {State::CANDIDATE_DELAYED}; + State GetState() const { return m_state; } + void SetState(State state) { m_state = state; } /** Whether this announcement is selected. There can be at most 1 selected peer per txhash. */ bool IsSelected() const @@ -103,9 +97,9 @@ struct Announcement { /** Construct a new announcement from scratch, initially in CANDIDATE_DELAYED state. */ Announcement(const GenTxid& gtxid, NodeId peer, bool preferred, std::chrono::microseconds reqtime, - SequenceNumber sequence) : - m_txhash(gtxid.GetHash()), m_time(reqtime), m_peer(peer), m_sequence(sequence), m_preferred(preferred), - m_is_wtxid(gtxid.IsWtxid()), m_state(static_cast<uint8_t>(State::CANDIDATE_DELAYED)) {} + SequenceNumber sequence) + : m_txhash(gtxid.GetHash()), m_time(reqtime), m_peer(peer), m_sequence(sequence), m_preferred(preferred), + m_is_wtxid{gtxid.IsWtxid()} {} }; //! Type alias for priorities. diff --git a/src/undo.h b/src/undo.h index a98f046735..1fb9ac0688 100644 --- a/src/undo.h +++ b/src/undo.h @@ -11,7 +11,6 @@ #include <consensus/consensus.h> #include <primitives/transaction.h> #include <serialize.h> -#include <version.h> /** Formatter for undo information for a CTxIn * diff --git a/src/util/check.h b/src/util/check.h index 7ddcebf506..a02a1de8dc 100644 --- a/src/util/check.h +++ b/src/util/check.h @@ -7,6 +7,7 @@ #include <attributes.h> +#include <cassert> // IWYU pragma: export #include <stdexcept> #include <string> #include <string_view> @@ -20,8 +21,6 @@ public: NonFatalCheckError(std::string_view msg, std::string_view file, int line, std::string_view func); }; -#define STR_INTERNAL_BUG(msg) StrFormatInternalBug((msg), __FILE__, __LINE__, __func__) - /** Helper for CHECK_NONFATAL() */ template <typename T> T&& inline_check_non_fatal(LIFETIMEBOUND T&& val, const char* file, int line, const char* func, const char* assertion) @@ -32,20 +31,6 @@ T&& inline_check_non_fatal(LIFETIMEBOUND T&& val, const char* file, int line, co return std::forward<T>(val); } -/** - * Identity function. Throw a NonFatalCheckError when the condition evaluates to false - * - * This should only be used - * - where the condition is assumed to be true, not for error handling or validating user input - * - where a failure to fulfill the condition is recoverable and does not abort the program - * - * For example in RPC code, where it is undesirable to crash the whole program, this can be generally used to replace - * asserts or recoverable logic errors. A NonFatalCheckError in RPC code is caught and passed as a string to the RPC - * caller, which can then report the issue to the developers. - */ -#define CHECK_NONFATAL(condition) \ - inline_check_non_fatal(condition, __FILE__, __LINE__, __func__, #condition) - #if defined(NDEBUG) #error "Cannot compile without assertions!" #endif @@ -69,6 +54,25 @@ T&& inline_assertion_check(LIFETIMEBOUND T&& val, [[maybe_unused]] const char* f return std::forward<T>(val); } +// All macros may use __func__ inside a lambda, so put them under nolint. +// NOLINTBEGIN(bugprone-lambda-function-name) + +#define STR_INTERNAL_BUG(msg) StrFormatInternalBug((msg), __FILE__, __LINE__, __func__) + +/** + * Identity function. Throw a NonFatalCheckError when the condition evaluates to false + * + * This should only be used + * - where the condition is assumed to be true, not for error handling or validating user input + * - where a failure to fulfill the condition is recoverable and does not abort the program + * + * For example in RPC code, where it is undesirable to crash the whole program, this can be generally used to replace + * asserts or recoverable logic errors. A NonFatalCheckError in RPC code is caught and passed as a string to the RPC + * caller, which can then report the issue to the developers. + */ +#define CHECK_NONFATAL(condition) \ + inline_check_non_fatal(condition, __FILE__, __LINE__, __func__, #condition) + /** Identity function. Abort if the value compares equal to zero */ #define Assert(val) inline_assertion_check<true>(val, __FILE__, __LINE__, __func__, #val) @@ -91,4 +95,6 @@ T&& inline_assertion_check(LIFETIMEBOUND T&& val, [[maybe_unused]] const char* f throw NonFatalCheckError( \ "Unreachable code reached (non-fatal)", __FILE__, __LINE__, __func__) +// NOLINTEND(bugprone-lambda-function-name) + #endif // BITCOIN_UTIL_CHECK_H diff --git a/src/util/fs.cpp b/src/util/fs.cpp index 14f7a44661..348c1b3383 100644 --- a/src/util/fs.cpp +++ b/src/util/fs.cpp @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022 The Bitcoin Core developers +// Copyright (c) 2017-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -18,6 +18,7 @@ #endif #include <cassert> +#include <cerrno> #include <string> namespace fsbridge { @@ -130,4 +131,4 @@ std::string get_filesystem_error_message(const fs::filesystem_error& e) #endif } -} // fsbridge +} // namespace fsbridge diff --git a/src/util/fs.h b/src/util/fs.h index 8f79f6cba6..f841e0d76c 100644 --- a/src/util/fs.h +++ b/src/util/fs.h @@ -1,4 +1,4 @@ -// Copyright (c) 2017-2022 The Bitcoin Core developers +// Copyright (c) 2017-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -14,6 +14,8 @@ #include <ios> #include <ostream> #include <string> +#include <system_error> +#include <type_traits> #include <utility> /** Filesystem operations and types */ @@ -35,7 +37,7 @@ public: // Allow path objects arguments for compatibility. path(std::filesystem::path path) : std::filesystem::path::path(std::move(path)) {} path& operator=(std::filesystem::path path) { std::filesystem::path::operator=(std::move(path)); return *this; } - path& operator/=(std::filesystem::path path) { std::filesystem::path::operator/=(path); return *this; } + path& operator/=(const std::filesystem::path& path) { std::filesystem::path::operator/=(path); return *this; } // Allow literal string arguments, which are safe as long as the literals are ASCII. path(const char* c) : std::filesystem::path(c) {} @@ -52,12 +54,15 @@ public: // Disallow std::string conversion method to avoid locale-dependent encoding on windows. std::string string() const = delete; - std::string u8string() const + /** + * Return a UTF-8 representation of the path as a std::string, for + * compatibility with code using std::string. For code using the newer + * std::u8string type, it is more efficient to call the inherited + * std::filesystem::path::u8string method instead. + */ + std::string utf8string() const { - const auto& utf8_str{std::filesystem::path::u8string()}; - // utf8_str might either be std::string (C++17) or std::u8string - // (C++20). Convert both to std::string. This method can be removed - // after switching to C++20. + const std::u8string& utf8_str{std::filesystem::path::u8string()}; return std::string{utf8_str.begin(), utf8_str.end()}; } @@ -69,11 +74,7 @@ public: static inline path u8path(const std::string& utf8_str) { -#if __cplusplus < 202002L - return std::filesystem::u8path(utf8_str); -#else return std::filesystem::path(std::u8string{utf8_str.begin(), utf8_str.end()}); -#endif } // Disallow implicit std::string conversion for absolute to avoid @@ -97,9 +98,9 @@ static inline auto quoted(const std::string& s) } // Allow safe path append operations. -static inline path operator/(path p1, path p2) +static inline path operator/(path p1, const path& p2) { - p1 /= std::move(p2); + p1 /= p2; return p1; } static inline path operator/(path p1, const char* p2) @@ -140,7 +141,7 @@ static inline bool copy_file(const path& from, const path& to, copy_options opti * Because \ref PathToString and \ref PathFromString functions don't specify an * encoding, they are meant to be used internally, not externally. They are not * appropriate to use in applications requiring UTF-8, where - * fs::path::u8string() and fs::u8path() methods should be used instead. Other + * fs::path::u8string() / fs::path::utf8string() and fs::u8path() methods should be used instead. Other * applications could require still different encodings. For example, JSON, XML, * or URI applications might prefer to use higher-level escapes (\uXXXX or * &XXXX; or %XX) instead of multibyte encoding. Rust, Python, Java applications @@ -154,13 +155,13 @@ static inline std::string PathToString(const path& path) // use here, because these methods encode the path using C++'s narrow // multibyte encoding, which on Windows corresponds to the current "code // page", which is unpredictable and typically not able to represent all - // valid paths. So fs::path::u8string() and + // valid paths. So fs::path::utf8string() and // fs::u8path() functions are used instead on Windows. On - // POSIX, u8string/u8path functions are not safe to use because paths are + // POSIX, u8string/utf8string/u8path functions are not safe to use because paths are // not always valid UTF-8, so plain string methods which do not transform // the path there are used. #ifdef WIN32 - return path.u8string(); + return path.utf8string(); #else static_assert(std::is_same<path::string_type, std::string>::value, "PathToString not implemented on this platform"); return path.std::filesystem::path::string(); @@ -184,6 +185,7 @@ static inline path PathFromString(const std::string& string) * already exists or is a symlink to an existing directory. * This is a temporary workaround for an issue in libstdc++ that has been fixed * upstream [PR101510]. + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101510 */ static inline bool create_directories(const std::filesystem::path& p) { diff --git a/src/util/fs_helpers.cpp b/src/util/fs_helpers.cpp index 2a9eb3502e..4de8833a3f 100644 --- a/src/util/fs_helpers.cpp +++ b/src/util/fs_helpers.cpp @@ -11,13 +11,10 @@ #include <logging.h> #include <sync.h> -#include <tinyformat.h> #include <util/fs.h> -#include <util/getuniquepath.h> #include <util/syserror.h> #include <cerrno> -#include <filesystem> #include <fstream> #include <map> #include <memory> @@ -53,31 +50,35 @@ static GlobalMutex cs_dir_locks; * is called. */ static std::map<std::string, std::unique_ptr<fsbridge::FileLock>> dir_locks GUARDED_BY(cs_dir_locks); - -bool LockDirectory(const fs::path& directory, const fs::path& lockfile_name, bool probe_only) +namespace util { +LockResult LockDirectory(const fs::path& directory, const fs::path& lockfile_name, bool probe_only) { LOCK(cs_dir_locks); fs::path pathLockFile = directory / lockfile_name; // If a lock for this directory already exists in the map, don't try to re-lock it if (dir_locks.count(fs::PathToString(pathLockFile))) { - return true; + return LockResult::Success; } // Create empty lock file if it doesn't exist. - FILE* file = fsbridge::fopen(pathLockFile, "a"); - if (file) fclose(file); + if (auto created{fsbridge::fopen(pathLockFile, "a")}) { + std::fclose(created); + } else { + return LockResult::ErrorWrite; + } auto lock = std::make_unique<fsbridge::FileLock>(pathLockFile); if (!lock->TryLock()) { - return error("Error while attempting to lock directory %s: %s", fs::PathToString(directory), lock->GetReason()); + error("Error while attempting to lock directory %s: %s", fs::PathToString(directory), lock->GetReason()); + return LockResult::ErrorLock; } if (!probe_only) { // Lock successful and we're not just probing, put it into the map dir_locks.emplace(fs::PathToString(pathLockFile), std::move(lock)); } - return true; + return LockResult::Success; } - +} // namespace util void UnlockDirectory(const fs::path& directory, const fs::path& lockfile_name) { LOCK(cs_dir_locks); @@ -90,19 +91,6 @@ void ReleaseDirectoryLocks() dir_locks.clear(); } -bool DirIsWritable(const fs::path& directory) -{ - fs::path tmpFile = GetUniquePath(directory); - - FILE* file = fsbridge::fopen(tmpFile, "a"); - if (!file) return false; - - fclose(file); - remove(tmpFile); - - return true; -} - bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes) { constexpr uint64_t min_disk_space = 52428800; // 50 MiB @@ -263,7 +251,7 @@ bool RenameOver(fs::path src, fs::path dest) { #ifdef __MINGW64__ // This is a workaround for a bug in libstdc++ which - // implements std::filesystem::rename with _wrename function. + // implements fs::rename with _wrename function. // This bug has been fixed in upstream: // - GCC 10.3: 8dd1c1085587c9f8a21bb5e588dfe1e8cdbba79e // - GCC 11.1: 1dfd95f0a0ca1d9e6cbc00e6cbfd1fa20a98f312 diff --git a/src/util/fs_helpers.h b/src/util/fs_helpers.h index e7db01a89b..ea3778eac3 100644 --- a/src/util/fs_helpers.h +++ b/src/util/fs_helpers.h @@ -35,9 +35,15 @@ void AllocateFileRange(FILE* file, unsigned int offset, unsigned int length); */ [[nodiscard]] bool RenameOver(fs::path src, fs::path dest); -bool LockDirectory(const fs::path& directory, const fs::path& lockfile_name, bool probe_only = false); +namespace util { +enum class LockResult { + Success, + ErrorWrite, + ErrorLock, +}; +[[nodiscard]] LockResult LockDirectory(const fs::path& directory, const fs::path& lockfile_name, bool probe_only = false); +} // namespace util void UnlockDirectory(const fs::path& directory, const fs::path& lockfile_name); -bool DirIsWritable(const fs::path& directory); bool CheckDiskSpace(const fs::path& dir, uint64_t additional_bytes = 0); /** Get the size of a file by scanning it. diff --git a/src/util/getuniquepath.cpp b/src/util/getuniquepath.cpp deleted file mode 100644 index 105b4d52d2..0000000000 --- a/src/util/getuniquepath.cpp +++ /dev/null @@ -1,14 +0,0 @@ -// Copyright (c) 2021-2022 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#include <random.h> -#include <util/fs.h> -#include <util/strencodings.h> - -fs::path GetUniquePath(const fs::path& base) -{ - FastRandomContext rnd; - fs::path tmpFile = base / fs::u8path(HexStr(rnd.randbytes(8))); - return tmpFile; -}
\ No newline at end of file diff --git a/src/util/getuniquepath.h b/src/util/getuniquepath.h deleted file mode 100644 index 1563652300..0000000000 --- a/src/util/getuniquepath.h +++ /dev/null @@ -1,19 +0,0 @@ -// Copyright (c) 2021 The Bitcoin Core developers -// Distributed under the MIT software license, see the accompanying -// file COPYING or http://www.opensource.org/licenses/mit-license.php. - -#ifndef BITCOIN_UTIL_GETUNIQUEPATH_H -#define BITCOIN_UTIL_GETUNIQUEPATH_H - -#include <util/fs.h> - -/** - * Helper function for getting a unique path - * - * @param[in] base Base path - * @returns base joined with a random 8-character long string. - * @post Returned path is unique with high probability. - */ -fs::path GetUniquePath(const fs::path& base); - -#endif // BITCOIN_UTIL_GETUNIQUEPATH_H
\ No newline at end of file diff --git a/src/util/hasher.h b/src/util/hasher.h index 506ae9415d..3ad6d5bb94 100644 --- a/src/util/hasher.h +++ b/src/util/hasher.h @@ -45,7 +45,7 @@ public: * a slight performance penalty (around 1.6%), but this is compensated by * memory savings of about 9% which allow for a larger dbcache setting. * - * @see https://gcc.gnu.org/onlinedocs/gcc-9.2.0/libstdc++/manual/manual/unordered_associative.html + * @see https://gcc.gnu.org/onlinedocs/gcc-13.2.0/libstdc++/manual/manual/unordered_associative.html */ size_t operator()(const COutPoint& id) const noexcept { return SipHashUint256Extra(k0, k1, id.hash, id.n); diff --git a/src/util/overloaded.h b/src/util/overloaded.h index 6be7453f81..5bb4b4e563 100644 --- a/src/util/overloaded.h +++ b/src/util/overloaded.h @@ -1,4 +1,4 @@ -// Copyright (c) 2021 The Bitcoin Core developers +// Copyright (c) 2021-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. @@ -15,7 +15,7 @@ namespace util { //! https://en.cppreference.com/w/cpp/utility/variant/visit#Example template<class... Ts> struct Overloaded : Ts... { using Ts::operator()...; }; -//! Explicit deduction guide (not needed as of C++20) +//! Explicit deduction guide (not needed after clang-17) template<class... Ts> Overloaded(Ts...) -> Overloaded<Ts...>; } // namespace util diff --git a/src/util/signalinterrupt.cpp b/src/util/signalinterrupt.cpp index c551ba8044..f36f3dbf9e 100644 --- a/src/util/signalinterrupt.cpp +++ b/src/util/signalinterrupt.cpp @@ -30,15 +30,16 @@ SignalInterrupt::operator bool() const return m_flag; } -void SignalInterrupt::reset() +bool SignalInterrupt::reset() { // Cancel existing interrupt by waiting for it, this will reset condition flags and remove // the token from the pipe. - if (*this) wait(); + if (*this && !wait()) return false; m_flag = false; + return true; } -void SignalInterrupt::operator()() +bool SignalInterrupt::operator()() { #ifdef WIN32 std::unique_lock<std::mutex> lk(m_mutex); @@ -52,13 +53,14 @@ void SignalInterrupt::operator()() // Write an arbitrary byte to the write end of the pipe. int res = m_pipe_w.TokenWrite('x'); if (res != 0) { - throw std::ios_base::failure("Could not write interrupt token"); + return false; } } #endif + return true; } -void SignalInterrupt::wait() +bool SignalInterrupt::wait() { #ifdef WIN32 std::unique_lock<std::mutex> lk(m_mutex); @@ -66,9 +68,10 @@ void SignalInterrupt::wait() #else int res = m_pipe_r.TokenRead(); if (res != 'x') { - throw std::ios_base::failure("Did not read expected interrupt token"); + return false; } #endif + return true; } } // namespace util diff --git a/src/util/signalinterrupt.h b/src/util/signalinterrupt.h index ca02feda91..605d124206 100644 --- a/src/util/signalinterrupt.h +++ b/src/util/signalinterrupt.h @@ -30,9 +30,9 @@ class SignalInterrupt public: SignalInterrupt(); explicit operator bool() const; - void operator()(); - void reset(); - void wait(); + [[nodiscard]] bool operator()(); + [[nodiscard]] bool reset(); + [[nodiscard]] bool wait(); private: std::atomic<bool> m_flag; diff --git a/src/util/sock.cpp b/src/util/sock.cpp index d16dc56aa3..e896b87160 100644 --- a/src/util/sock.cpp +++ b/src/util/sock.cpp @@ -242,7 +242,7 @@ bool Sock::WaitMany(std::chrono::milliseconds timeout, EventsPerSock& events_per #endif /* USE_POLL */ } -void Sock::SendComplete(const std::string& data, +void Sock::SendComplete(Span<const unsigned char> data, std::chrono::milliseconds timeout, CThreadInterrupt& interrupt) const { @@ -283,6 +283,13 @@ void Sock::SendComplete(const std::string& data, } } +void Sock::SendComplete(Span<const char> data, + std::chrono::milliseconds timeout, + CThreadInterrupt& interrupt) const +{ + SendComplete(MakeUCharSpan(data), timeout, interrupt); +} + std::string Sock::RecvUntilTerminator(uint8_t terminator, std::chrono::milliseconds timeout, CThreadInterrupt& interrupt, diff --git a/src/util/sock.h b/src/util/sock.h index d78e01929b..65e7ffc165 100644 --- a/src/util/sock.h +++ b/src/util/sock.h @@ -228,7 +228,14 @@ public: * @throws std::runtime_error if the operation cannot be completed. In this case only some of * the data will be written to the socket. */ - virtual void SendComplete(const std::string& data, + virtual void SendComplete(Span<const unsigned char> data, + std::chrono::milliseconds timeout, + CThreadInterrupt& interrupt) const; + + /** + * Convenience method, equivalent to `SendComplete(MakeUCharSpan(data), timeout, interrupt)`. + */ + virtual void SendComplete(Span<const char> data, std::chrono::milliseconds timeout, CThreadInterrupt& interrupt) const; diff --git a/src/util/strencodings.h b/src/util/strencodings.h index d792562735..439678c24a 100644 --- a/src/util/strencodings.h +++ b/src/util/strencodings.h @@ -260,7 +260,6 @@ bool TimingResistantEqual(const T& a, const T& b) } /** Parse number as fixed point according to JSON number syntax. - * See https://json.org/number.gif * @returns true on success, false on error. * @note The result must be in the range (-10^18,10^18), otherwise an overflow error will trigger. */ diff --git a/src/util/trace.h b/src/util/trace.h index 1fe743f043..b7c275f19b 100644 --- a/src/util/trace.h +++ b/src/util/trace.h @@ -13,28 +13,19 @@ #include <sys/sdt.h> -// Disabling this warning can be removed once we switch to C++20 -#if defined(__clang__) && __cplusplus < 202002L -#define BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH _Pragma("clang diagnostic push") _Pragma("clang diagnostic ignored \"-Wgnu-zero-variadic-macro-arguments\"") -#define BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP _Pragma("clang diagnostic pop") -#else -#define BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH -#define BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP -#endif - -#define TRACE(context, event) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE(context, event) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP -#define TRACE1(context, event, a) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE1(context, event, a) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP -#define TRACE2(context, event, a, b) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE2(context, event, a, b) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP -#define TRACE3(context, event, a, b, c) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE3(context, event, a, b, c) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP -#define TRACE4(context, event, a, b, c, d) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE4(context, event, a, b, c, d) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP -#define TRACE5(context, event, a, b, c, d, e) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE5(context, event, a, b, c, d, e) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP -#define TRACE6(context, event, a, b, c, d, e, f) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE6(context, event, a, b, c, d, e, f) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP -#define TRACE7(context, event, a, b, c, d, e, f, g) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE7(context, event, a, b, c, d, e, f, g) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP -#define TRACE8(context, event, a, b, c, d, e, f, g, h) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE8(context, event, a, b, c, d, e, f, g, h) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP -#define TRACE9(context, event, a, b, c, d, e, f, g, h, i) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE9(context, event, a, b, c, d, e, f, g, h, i) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP -#define TRACE10(context, event, a, b, c, d, e, f, g, h, i, j) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE10(context, event, a, b, c, d, e, f, g, h, i, j) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP -#define TRACE11(context, event, a, b, c, d, e, f, g, h, i, j, k) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE11(context, event, a, b, c, d, e, f, g, h, i, j, k) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP -#define TRACE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_PUSH DTRACE_PROBE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) BITCOIN_DISABLE_WARN_ZERO_VARIADIC_POP +#define TRACE(context, event) DTRACE_PROBE(context, event) +#define TRACE1(context, event, a) DTRACE_PROBE1(context, event, a) +#define TRACE2(context, event, a, b) DTRACE_PROBE2(context, event, a, b) +#define TRACE3(context, event, a, b, c) DTRACE_PROBE3(context, event, a, b, c) +#define TRACE4(context, event, a, b, c, d) DTRACE_PROBE4(context, event, a, b, c, d) +#define TRACE5(context, event, a, b, c, d, e) DTRACE_PROBE5(context, event, a, b, c, d, e) +#define TRACE6(context, event, a, b, c, d, e, f) DTRACE_PROBE6(context, event, a, b, c, d, e, f) +#define TRACE7(context, event, a, b, c, d, e, f, g) DTRACE_PROBE7(context, event, a, b, c, d, e, f, g) +#define TRACE8(context, event, a, b, c, d, e, f, g, h) DTRACE_PROBE8(context, event, a, b, c, d, e, f, g, h) +#define TRACE9(context, event, a, b, c, d, e, f, g, h, i) DTRACE_PROBE9(context, event, a, b, c, d, e, f, g, h, i) +#define TRACE10(context, event, a, b, c, d, e, f, g, h, i, j) DTRACE_PROBE10(context, event, a, b, c, d, e, f, g, h, i, j) +#define TRACE11(context, event, a, b, c, d, e, f, g, h, i, j, k) DTRACE_PROBE11(context, event, a, b, c, d, e, f, g, h, i, j, k) +#define TRACE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) DTRACE_PROBE12(context, event, a, b, c, d, e, f, g, h, i, j, k, l) #else diff --git a/src/util/transaction_identifier.h b/src/util/transaction_identifier.h new file mode 100644 index 0000000000..89e10dee01 --- /dev/null +++ b/src/util/transaction_identifier.h @@ -0,0 +1,73 @@ +#ifndef BITCOIN_UTIL_TRANSACTION_IDENTIFIER_H +#define BITCOIN_UTIL_TRANSACTION_IDENTIFIER_H + +#include <attributes.h> +#include <uint256.h> +#include <util/types.h> + +/** transaction_identifier represents the two canonical transaction identifier + * types (txid, wtxid).*/ +template <bool has_witness> +class transaction_identifier +{ + uint256 m_wrapped; + + // Note: Use FromUint256 externally instead. + transaction_identifier(const uint256& wrapped) : m_wrapped{wrapped} {} + + // TODO: Comparisons with uint256 should be disallowed once we have + // converted most of the code to using the new txid types. + constexpr int Compare(const uint256& other) const { return m_wrapped.Compare(other); } + constexpr int Compare(const transaction_identifier<has_witness>& other) const { return m_wrapped.Compare(other.m_wrapped); } + template <typename Other> + constexpr int Compare(const Other& other) const + { + static_assert(ALWAYS_FALSE<Other>, "Forbidden comparison type"); + return 0; + } + +public: + transaction_identifier() : m_wrapped{} {} + + template <typename Other> + bool operator==(const Other& other) const { return Compare(other) == 0; } + template <typename Other> + bool operator!=(const Other& other) const { return Compare(other) != 0; } + template <typename Other> + bool operator<(const Other& other) const { return Compare(other) < 0; } + + const uint256& ToUint256() const LIFETIMEBOUND { return m_wrapped; } + static transaction_identifier FromUint256(const uint256& id) { return {id}; } + + /** Wrapped `uint256` methods. */ + constexpr bool IsNull() const { return m_wrapped.IsNull(); } + constexpr void SetNull() { m_wrapped.SetNull(); } + std::string GetHex() const { return m_wrapped.GetHex(); } + std::string ToString() const { return m_wrapped.ToString(); } + constexpr const std::byte* data() const { return reinterpret_cast<const std::byte*>(m_wrapped.data()); } + constexpr const std::byte* begin() const { return reinterpret_cast<const std::byte*>(m_wrapped.begin()); } + constexpr const std::byte* end() const { return reinterpret_cast<const std::byte*>(m_wrapped.end()); } + template <typename Stream> void Serialize(Stream& s) const { m_wrapped.Serialize(s); } + template <typename Stream> void Unserialize(Stream& s) { m_wrapped.Unserialize(s); } + + /** Conversion function to `uint256`. + * + * Note: new code should use `ToUint256`. + * + * TODO: This should be removed once the majority of the code has switched + * to using the Txid and Wtxid types. Until then it makes for a smoother + * transition to allow this conversion. */ + operator const uint256&() const LIFETIMEBOUND { return m_wrapped; } +}; + +/** Txid commits to all transaction fields except the witness. */ +using Txid = transaction_identifier<false>; +/** Wtxid commits to all transaction fields including the witness. */ +using Wtxid = transaction_identifier<true>; + +inline Txid TxidFromString(std::string_view str) +{ + return Txid::FromUint256(uint256S(str.data())); +} + +#endif // BITCOIN_UTIL_TRANSACTION_IDENTIFIER_H diff --git a/src/util/types.h b/src/util/types.h index 0047b00026..e930216aea 100644 --- a/src/util/types.h +++ b/src/util/types.h @@ -5,6 +5,7 @@ #ifndef BITCOIN_UTIL_TYPES_H #define BITCOIN_UTIL_TYPES_H +// Not needed after C++23 (DR, https://cplusplus.github.io/CWG/issues/2518.html) template <class> inline constexpr bool ALWAYS_FALSE{false}; diff --git a/src/validation.cpp b/src/validation.cpp index a6cab6b095..0f3d5d1454 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -5,10 +5,6 @@ #include <validation.h> -#include <kernel/chain.h> -#include <kernel/coinstats.h> -#include <kernel/mempool_persist.h> - #include <arith_uint256.h> #include <chain.h> #include <checkqueue.h> @@ -22,7 +18,9 @@ #include <cuckoocache.h> #include <flatfile.h> #include <hash.h> +#include <kernel/chain.h> #include <kernel/chainparams.h> +#include <kernel/coinstats.h> #include <kernel/disconnected_transactions.h> #include <kernel/mempool_entry.h> #include <kernel/messagestartchars.h> @@ -47,12 +45,13 @@ #include <txmempool.h> #include <uint256.h> #include <undo.h> -#include <util/check.h> // For NDEBUG compile time check +#include <util/check.h> #include <util/fs.h> #include <util/fs_helpers.h> #include <util/hasher.h> #include <util/moneystr.h> #include <util/rbf.h> +#include <util/result.h> #include <util/signalinterrupt.h> #include <util/strencodings.h> #include <util/time.h> @@ -355,7 +354,7 @@ void Chainstate::MaybeUpdateMempoolForReorg( const std::optional<LockPoints> new_lock_points{CalculateLockPointsAtTip(m_chain.Tip(), view_mempool, tx)}; if (new_lock_points.has_value() && CheckSequenceLocksAtTip(m_chain.Tip(), *new_lock_points)) { // Now update the mempool entry lockpoints as well. - m_mempool->mapTx.modify(it, [&new_lock_points](CTxMemPoolEntry& e) { e.UpdateLockPoints(*new_lock_points); }); + it->UpdateLockPoints(*new_lock_points); } else { return true; } @@ -364,9 +363,7 @@ void Chainstate::MaybeUpdateMempoolForReorg( // If the transaction spends any coinbase outputs, it must be mature. if (it->GetSpendsCoinbase()) { for (const CTxIn& txin : tx.vin) { - auto it2 = m_mempool->mapTx.find(txin.prevout.hash); - if (it2 != m_mempool->mapTx.end()) - continue; + if (m_mempool->exists(GenTxid::Txid(txin.prevout.hash))) continue; const Coin& coin{CoinsTip().AccessCoin(txin.prevout)}; assert(!coin.IsSpent()); const auto mempool_spend_height{m_chain.Tip()->nHeight + 1}; @@ -548,6 +545,9 @@ public: } }; + /** Clean up all non-chainstate coins from m_view and m_viewmempool. */ + void CleanupTemporaryCoins() EXCLUSIVE_LOCKS_REQUIRED(cs_main, m_pool.cs); + // Single transaction acceptance MempoolAcceptResult AcceptSingleTransaction(const CTransactionRef& ptx, ATMPArgs& args) EXCLUSIVE_LOCKS_REQUIRED(cs_main); @@ -618,7 +618,7 @@ private: const CTransactionRef& m_ptx; /** Txid. */ - const uint256& m_hash; + const Txid& m_hash; TxValidationState m_state; /** A temporary cache containing serialized transaction data for signature verification. * Reused across PolicyScriptChecks and ConsensusScriptChecks. */ @@ -669,11 +669,11 @@ private: AssertLockHeld(m_pool.cs); CAmount mempoolRejectFee = m_pool.GetMinFee().GetFee(package_size); if (mempoolRejectFee > 0 && package_fee < mempoolRejectFee) { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool min fee not met", strprintf("%d < %d", package_fee, mempoolRejectFee)); + return state.Invalid(TxValidationResult::TX_RECONSIDERABLE, "mempool min fee not met", strprintf("%d < %d", package_fee, mempoolRejectFee)); } if (package_fee < m_pool.m_min_relay_feerate.GetFee(package_size)) { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "min relay fee not met", + return state.Invalid(TxValidationResult::TX_RECONSIDERABLE, "min relay fee not met", strprintf("%d < %d", package_fee, m_pool.m_min_relay_feerate.GetFee(package_size))); } return true; @@ -697,7 +697,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) AssertLockHeld(m_pool.cs); const CTransactionRef& ptx = ws.m_ptx; const CTransaction& tx = *ws.m_ptx; - const uint256& hash = ws.m_hash; + const Txid& hash = ws.m_hash; // Copy/alias what we need out of args const int64_t nAcceptTime = args.m_accept_time; @@ -723,7 +723,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) } // Transactions smaller than 65 non-witness bytes are not relayed to mitigate CVE-2017-12842. - if (::GetSerializeSize(tx, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) < MIN_STANDARD_TX_NONWITNESS_SIZE) + if (::GetSerializeSize(TX_NO_WITNESS(tx)) < MIN_STANDARD_TX_NONWITNESS_SIZE) return state.Invalid(TxValidationResult::TX_NOT_STANDARD, "tx-size-small"); // Only accept nLockTime-using transactions that can be mined in the next @@ -866,6 +866,8 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // method of ensuring the tx remains bumped. For example, the fee-bumping child could disappear // due to a replacement. if (!bypass_limits && ws.m_modified_fees < m_pool.m_min_relay_feerate.GetFee(ws.m_vsize)) { + // Even though this is a fee-related failure, this result is TX_MEMPOOL_POLICY, not + // TX_RECONSIDERABLE, because it cannot be bypassed using package validation. return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "min relay fee not met", strprintf("%d < %d", ws.m_modified_fees, m_pool.m_min_relay_feerate.GetFee(ws.m_vsize))); } @@ -980,6 +982,9 @@ bool MemPoolAccept::ReplacementChecks(Workspace& ws) // descendant transaction of a direct conflict to pay a higher feerate than the transaction that // might replace them, under these rules. if (const auto err_string{PaysMoreThanConflicts(ws.m_iters_conflicting, newFeeRate, hash)}) { + // Even though this is a fee-related failure, this result is TX_MEMPOOL_POLICY, not + // TX_RECONSIDERABLE, because it cannot be bypassed using package validation. + // This must be changed if package RBF is enabled. return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", *err_string); } @@ -1001,6 +1006,9 @@ bool MemPoolAccept::ReplacementChecks(Workspace& ws) } if (const auto err_string{PaysForRBF(ws.m_conflicting_fees, ws.m_modified_fees, ws.m_vsize, m_pool.m_incremental_relay_feerate, hash)}) { + // Even though this is a fee-related failure, this result is TX_MEMPOOL_POLICY, not + // TX_RECONSIDERABLE, because it cannot be bypassed using package validation. + // This must be changed if package RBF is enabled. return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", *err_string); } return true; @@ -1017,10 +1025,10 @@ bool MemPoolAccept::PackageMempoolChecks(const std::vector<CTransactionRef>& txn assert(std::all_of(txns.cbegin(), txns.cend(), [this](const auto& tx) { return !m_pool.exists(GenTxid::Txid(tx->GetHash()));})); - std::string err_string; - if (!m_pool.CheckPackageLimits(txns, total_vsize, err_string)) { + auto result = m_pool.CheckPackageLimits(txns, total_vsize); + if (!result) { // This is a package-wide error, separate from an individual transaction error. - return package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package-mempool-limits", err_string); + return package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package-mempool-limits", util::ErrorString(result).original); } return true; } @@ -1119,17 +1127,8 @@ bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) ws.m_replaced_transactions.push_back(it->GetSharedTx()); } m_pool.RemoveStaged(ws.m_all_conflicting, false, MemPoolRemovalReason::REPLACED); - - // This transaction should only count for fee estimation if: - // - it's not being re-added during a reorg which bypasses typical mempool fee limits - // - the node is not behind - // - the transaction is not dependent on any other transactions in the mempool - // - it's not part of a package. Since package relay is not currently supported, this - // transaction has not necessarily been accepted to miners' mempools. - bool validForFeeEstimation = !bypass_limits && !args.m_package_submission && IsCurrentForFeeEstimation(m_active_chainstate) && m_pool.HasNoInputsOf(tx); - // Store transaction in memory - m_pool.addUnchecked(*entry, ws.m_ancestors, validForFeeEstimation); + m_pool.addUnchecked(*entry, ws.m_ancestors); // trim mempool and check if tx was trimmed // If we are validating a package, don't trim here because we could evict a previous transaction @@ -1138,7 +1137,8 @@ bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) if (!args.m_package_submission && !bypass_limits) { LimitMempoolSize(m_pool, m_active_chainstate.CoinsTip()); if (!m_pool.exists(GenTxid::Txid(hash))) - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool full"); + // The tx no longer meets our (new) mempool minimum feerate but could be reconsidered in a package. + return state.Invalid(TxValidationResult::TX_RECONSIDERABLE, "mempool full"); } return true; } @@ -1200,7 +1200,7 @@ bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& } } - std::vector<uint256> all_package_wtxids; + std::vector<Wtxid> all_package_wtxids; all_package_wtxids.reserve(workspaces.size()); std::transform(workspaces.cbegin(), workspaces.cend(), std::back_inserter(all_package_wtxids), [](const auto& ws) { return ws.m_ptx->GetWitnessHash(); }); @@ -1210,11 +1210,17 @@ bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& const auto effective_feerate = args.m_package_feerates ? ws.m_package_feerate : CFeeRate{ws.m_modified_fees, static_cast<uint32_t>(ws.m_vsize)}; const auto effective_feerate_wtxids = args.m_package_feerates ? all_package_wtxids : - std::vector<uint256>({ws.m_ptx->GetWitnessHash()}); + std::vector<Wtxid>{ws.m_ptx->GetWitnessHash()}; results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, ws.m_base_fees, effective_feerate, effective_feerate_wtxids)); - GetMainSignals().TransactionAddedToMempool(ws.m_ptx, m_pool.GetAndIncrementSequence()); + const CTransaction& tx = *ws.m_ptx; + const auto tx_info = NewMempoolTransactionInfo(ws.m_ptx, ws.m_base_fees, + ws.m_vsize, ws.m_entry->GetHeight(), + args.m_bypass_limits, args.m_package_submission, + IsCurrentForFeeEstimation(m_active_chainstate), + m_pool.HasNoInputsOf(tx)); + GetMainSignals().TransactionAddedToMempool(tx_info, m_pool.GetAndIncrementSequence()); } return all_submitted; } @@ -1225,8 +1231,15 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef LOCK(m_pool.cs); // mempool "read lock" (held through GetMainSignals().TransactionAddedToMempool()) Workspace ws(ptx); + const std::vector<Wtxid> single_wtxid{ws.m_ptx->GetWitnessHash()}; - if (!PreChecks(args, ws)) return MempoolAcceptResult::Failure(ws.m_state); + if (!PreChecks(args, ws)) { + if (ws.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE) { + // Failed for fee reasons. Provide the effective feerate and which tx was included. + return MempoolAcceptResult::FeeFailure(ws.m_state, CFeeRate(ws.m_modified_fees, ws.m_vsize), single_wtxid); + } + return MempoolAcceptResult::Failure(ws.m_state); + } if (m_rbf && !ReplacementChecks(ws)) return MempoolAcceptResult::Failure(ws.m_state); @@ -1237,16 +1250,26 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef if (!ConsensusScriptChecks(args, ws)) return MempoolAcceptResult::Failure(ws.m_state); const CFeeRate effective_feerate{ws.m_modified_fees, static_cast<uint32_t>(ws.m_vsize)}; - const std::vector<uint256> single_wtxid{ws.m_ptx->GetWitnessHash()}; // Tx was accepted, but not added if (args.m_test_accept) { return MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, ws.m_base_fees, effective_feerate, single_wtxid); } - if (!Finalize(args, ws)) return MempoolAcceptResult::Failure(ws.m_state); + if (!Finalize(args, ws)) { + // The only possible failure reason is fee-related (mempool full). + // Failed for fee reasons. Provide the effective feerate and which txns were included. + Assume(ws.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE); + return MempoolAcceptResult::FeeFailure(ws.m_state, CFeeRate(ws.m_modified_fees, ws.m_vsize), {ws.m_ptx->GetWitnessHash()}); + } - GetMainSignals().TransactionAddedToMempool(ptx, m_pool.GetAndIncrementSequence()); + const CTransaction& tx = *ws.m_ptx; + const auto tx_info = NewMempoolTransactionInfo(ws.m_ptx, ws.m_base_fees, + ws.m_vsize, ws.m_entry->GetHeight(), + args.m_bypass_limits, args.m_package_submission, + IsCurrentForFeeEstimation(m_active_chainstate), + m_pool.HasNoInputsOf(tx)); + GetMainSignals().TransactionAddedToMempool(tx_info, m_pool.GetAndIncrementSequence()); return MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, ws.m_base_fees, effective_feerate, single_wtxid); @@ -1258,7 +1281,7 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: // These context-free package limits can be done before taking the mempool lock. PackageValidationState package_state; - if (!CheckPackage(txns, package_state)) return PackageMempoolAcceptResult(package_state, {}); + if (!IsWellFormedPackage(txns, package_state, /*require_sorted=*/true)) return PackageMempoolAcceptResult(package_state, {}); std::vector<Workspace> workspaces{}; workspaces.reserve(txns.size()); @@ -1298,11 +1321,16 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: const auto m_total_modified_fees = std::accumulate(workspaces.cbegin(), workspaces.cend(), CAmount{0}, [](CAmount sum, auto& ws) { return sum + ws.m_modified_fees; }); const CFeeRate package_feerate(m_total_modified_fees, m_total_vsize); + std::vector<Wtxid> all_package_wtxids; + all_package_wtxids.reserve(workspaces.size()); + std::transform(workspaces.cbegin(), workspaces.cend(), std::back_inserter(all_package_wtxids), + [](const auto& ws) { return ws.m_ptx->GetWitnessHash(); }); TxValidationState placeholder_state; if (args.m_package_feerates && !CheckFeeRate(m_total_vsize, m_total_modified_fees, placeholder_state)) { - package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package-fee-too-low"); - return PackageMempoolAcceptResult(package_state, {}); + package_state.Invalid(PackageValidationResult::PCKG_TX, "transaction failed"); + return PackageMempoolAcceptResult(package_state, {{workspaces.back().m_ptx->GetWitnessHash(), + MempoolAcceptResult::FeeFailure(placeholder_state, CFeeRate(m_total_modified_fees, m_total_vsize), all_package_wtxids)}}); } // Apply package mempool ancestor/descendant limits. Skip if there is only one transaction, @@ -1313,10 +1341,6 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: return PackageMempoolAcceptResult(package_state, std::move(results)); } - std::vector<uint256> all_package_wtxids; - all_package_wtxids.reserve(workspaces.size()); - std::transform(workspaces.cbegin(), workspaces.cend(), std::back_inserter(all_package_wtxids), - [](const auto& ws) { return ws.m_ptx->GetWitnessHash(); }); for (Workspace& ws : workspaces) { ws.m_package_feerate = package_feerate; if (!PolicyScriptChecks(args, ws)) { @@ -1329,7 +1353,7 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: const auto effective_feerate = args.m_package_feerates ? ws.m_package_feerate : CFeeRate{ws.m_modified_fees, static_cast<uint32_t>(ws.m_vsize)}; const auto effective_feerate_wtxids = args.m_package_feerates ? all_package_wtxids : - std::vector<uint256>{ws.m_ptx->GetWitnessHash()}; + std::vector<Wtxid>{ws.m_ptx->GetWitnessHash()}; results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, ws.m_base_fees, effective_feerate, @@ -1347,26 +1371,8 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: return PackageMempoolAcceptResult(package_state, std::move(results)); } -PackageMempoolAcceptResult MemPoolAccept::AcceptSubPackage(const std::vector<CTransactionRef>& subpackage, ATMPArgs& args) +void MemPoolAccept::CleanupTemporaryCoins() { - AssertLockHeld(::cs_main); - AssertLockHeld(m_pool.cs); - auto result = [&]() EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_pool.cs) { - if (subpackage.size() > 1) { - return AcceptMultipleTransactions(subpackage, args); - } - const auto& tx = subpackage.front(); - ATMPArgs single_args = ATMPArgs::SingleInPackageAccept(args); - const auto single_res = AcceptSingleTransaction(tx, single_args); - PackageValidationState package_state_wrapped; - if (single_res.m_result_type != MempoolAcceptResult::ResultType::VALID) { - package_state_wrapped.Invalid(PackageValidationResult::PCKG_TX, "transaction failed"); - } - return PackageMempoolAcceptResult(package_state_wrapped, {{tx->GetWitnessHash(), single_res}}); - }(); - // Clean up m_view and m_viewmempool so that other subpackage evaluations don't have access to - // coins they shouldn't. Keep some coins in order to minimize re-fetching coins from the UTXO set. - // // There are 3 kinds of coins in m_view: // (1) Temporary coins from the transactions in subpackage, constructed by m_viewmempool. // (2) Mempool coins from transactions in the mempool, constructed by m_viewmempool. @@ -1392,6 +1398,30 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptSubPackage(const std::vector<CTr } // This deletes the temporary and mempool coins. m_viewmempool.Reset(); +} + +PackageMempoolAcceptResult MemPoolAccept::AcceptSubPackage(const std::vector<CTransactionRef>& subpackage, ATMPArgs& args) +{ + AssertLockHeld(::cs_main); + AssertLockHeld(m_pool.cs); + auto result = [&]() EXCLUSIVE_LOCKS_REQUIRED(::cs_main, m_pool.cs) { + if (subpackage.size() > 1) { + return AcceptMultipleTransactions(subpackage, args); + } + const auto& tx = subpackage.front(); + ATMPArgs single_args = ATMPArgs::SingleInPackageAccept(args); + const auto single_res = AcceptSingleTransaction(tx, single_args); + PackageValidationState package_state_wrapped; + if (single_res.m_result_type != MempoolAcceptResult::ResultType::VALID) { + package_state_wrapped.Invalid(PackageValidationResult::PCKG_TX, "transaction failed"); + } + return PackageMempoolAcceptResult(package_state_wrapped, {{tx->GetWitnessHash(), single_res}}); + }(); + + // Clean up m_view and m_viewmempool so that other subpackage evaluations don't have access to + // coins they shouldn't. Keep some coins in order to minimize re-fetching coins from the UTXO set. + CleanupTemporaryCoins(); + return result; } @@ -1405,7 +1435,9 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, // transactions and thus won't return any MempoolAcceptResults, just a package-wide error. // Context-free package checks. - if (!CheckPackage(package, package_state_quit_early)) return PackageMempoolAcceptResult(package_state_quit_early, {}); + if (!IsWellFormedPackage(package, package_state_quit_early, /*require_sorted=*/true)) { + return PackageMempoolAcceptResult(package_state_quit_early, {}); + } // All transactions in the package must be a parent of the last transaction. This is just an // opportunity for us to fail fast on a context-free check without taking the mempool lock. @@ -1476,9 +1508,8 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, // transactions that are already in the mempool, and only call AcceptMultipleTransactions() with // the new transactions. This ensures we don't double-count transaction counts and sizes when // checking ancestor/descendant limits, or double-count transaction fees for fee-related policy. - auto iter = m_pool.GetIter(txid); - assert(iter != std::nullopt); - results_final.emplace(wtxid, MempoolAcceptResult::MempoolTx(iter.value()->GetTxSize(), iter.value()->GetFee())); + const auto& entry{*Assert(m_pool.GetEntry(txid))}; + results_final.emplace(wtxid, MempoolAcceptResult::MempoolTx(entry.GetTxSize(), entry.GetFee())); } else if (m_pool.exists(GenTxid::Txid(txid))) { // Transaction with the same non-witness data but different witness (same txid, // different wtxid) already exists in the mempool. @@ -1487,10 +1518,9 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, // transaction for the mempool one. Note that we are ignoring the validity of the // package transaction passed in. // TODO: allow witness replacement in packages. - auto iter = m_pool.GetIter(txid); - assert(iter != std::nullopt); + const auto& entry{*Assert(m_pool.GetEntry(txid))}; // Provide the wtxid of the mempool tx so that the caller can look it up in the mempool. - results_final.emplace(wtxid, MempoolAcceptResult::MempoolTxDifferentWitness(iter.value()->GetTx().GetWitnessHash())); + results_final.emplace(wtxid, MempoolAcceptResult::MempoolTxDifferentWitness(entry.GetTx().GetWitnessHash())); } else { // Transaction does not already exist in the mempool. // Try submitting the transaction on its own. @@ -1501,7 +1531,7 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, // in package validation, because its fees should only be "used" once. assert(m_pool.exists(GenTxid::Wtxid(wtxid))); results_final.emplace(wtxid, single_res); - } else if (single_res.m_state.GetResult() != TxValidationResult::TX_MEMPOOL_POLICY && + } else if (single_res.m_state.GetResult() != TxValidationResult::TX_RECONSIDERABLE && single_res.m_state.GetResult() != TxValidationResult::TX_MISSING_INPUTS) { // Package validation policy only differs from individual policy in its evaluation // of feerate. For example, if a transaction fails here due to violation of a @@ -1870,7 +1900,7 @@ bool CheckInputScripts(const CTransaction& tx, TxValidationState& state, // transaction). uint256 hashCacheEntry; CSHA256 hasher = g_scriptExecutionCacheHasher; - hasher.Write(tx.GetWitnessHash().begin(), 32).Write((unsigned char*)&flags, sizeof(flags)).Finalize(hashCacheEntry.begin()); + hasher.Write(UCharCast(tx.GetWitnessHash().begin()), 32).Write((unsigned char*)&flags, sizeof(flags)).Finalize(hashCacheEntry.begin()); AssertLockHeld(cs_main); //TODO: Remove this requirement by making CuckooCache not require external locks if (g_scriptExecutionCache.contains(hashCacheEntry, !cacheFullScriptStore)) { return true; @@ -2010,7 +2040,7 @@ DisconnectResult Chainstate::DisconnectBlock(const CBlock& block, const CBlockIn // undo transactions in reverse order for (int i = block.vtx.size() - 1; i >= 0; i--) { const CTransaction &tx = *(block.vtx[i]); - uint256 hash = tx.GetHash(); + Txid hash = tx.GetHash(); bool is_coinbase = tx.IsCoinBase(); bool is_bip30_exception = (is_coinbase && !fEnforceBIP30); @@ -2053,18 +2083,6 @@ DisconnectResult Chainstate::DisconnectBlock(const CBlock& block, const CBlockIn return fClean ? DISCONNECT_OK : DISCONNECT_UNCLEAN; } -static CCheckQueue<CScriptCheck> scriptcheckqueue(128); - -void StartScriptCheckWorkerThreads(int threads_num) -{ - scriptcheckqueue.StartWorkerThreads(threads_num); -} - -void StopScriptCheckWorkerThreads() -{ - scriptcheckqueue.StopWorkerThreads(); -} - /** * Threshold condition checker that triggers when unknown versionbits are seen on the network. */ @@ -2153,7 +2171,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, uint256 block_hash{block.GetHash()}; assert(*pindex->phashBlock == block_hash); - const bool parallel_script_checks{scriptcheckqueue.HasThreads()}; + const bool parallel_script_checks{m_chainman.GetCheckQueue().HasThreads()}; const auto time_start{SteadyClock::now()}; const CChainParams& params{m_chainman.GetParams()}; @@ -2342,7 +2360,7 @@ bool Chainstate::ConnectBlock(const CBlock& block, BlockValidationState& state, // in multiple threads). Preallocate the vector size so a new allocation // doesn't invalidate pointers into the vector, and keep txsdata in scope // for as long as `control`. - CCheckQueueControl<CScriptCheck> control(fScriptChecks && parallel_script_checks ? &scriptcheckqueue : nullptr); + CCheckQueueControl<CScriptCheck> control(fScriptChecks && parallel_script_checks ? &m_chainman.GetCheckQueue() : nullptr); std::vector<PrecomputedTransactionData> txsdata(block.vtx.size()); std::vector<int> prevheights; @@ -3064,7 +3082,7 @@ bool Chainstate::ActivateBestChainStep(BlockValidationState& state, CBlockIndex* // Disconnect active blocks which are no longer in the best chain. bool fBlocksDisconnected = false; - DisconnectedBlockTransactions disconnectpool{MAX_DISCONNECTED_TX_POOL_SIZE * 1000}; + DisconnectedBlockTransactions disconnectpool{MAX_DISCONNECTED_TX_POOL_BYTES}; while (m_chain.Tip() && m_chain.Tip() != pindexFork) { if (!DisconnectTip(state, &disconnectpool)) { // This is likely a fatal error, but keep the mempool consistent, @@ -3422,7 +3440,7 @@ bool Chainstate::InvalidateBlock(BlockValidationState& state, CBlockIndex* pinde // ActivateBestChain considers blocks already in m_chain // unconditionally valid already, so force disconnect away from it. - DisconnectedBlockTransactions disconnectpool{MAX_DISCONNECTED_TX_POOL_SIZE * 1000}; + DisconnectedBlockTransactions disconnectpool{MAX_DISCONNECTED_TX_POOL_BYTES}; bool ret = DisconnectTip(state, &disconnectpool); // DisconnectTip will add transactions to disconnectpool. // Adjust the mempool to be consistent with the new tip, adding @@ -3659,7 +3677,7 @@ bool CheckBlock(const CBlock& block, BlockValidationState& state, const Consensu // checks that use witness data may be performed here. // Size limits - if (block.vtx.empty() || block.vtx.size() * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT || ::GetSerializeSize(block, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT) + if (block.vtx.empty() || block.vtx.size() * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT || ::GetSerializeSize(TX_NO_WITNESS(block)) * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT) return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-blk-length", "size limits failed"); // First transaction must be coinbase, the rest must not be @@ -4623,7 +4641,7 @@ bool Chainstate::LoadGenesisBlock() } void ChainstateManager::LoadExternalBlockFile( - CAutoFile& file_in, + AutoFile& file_in, FlatFilePos* dbp, std::multimap<uint256, FlatFilePos>* blocks_with_unknown_parent) { @@ -4698,7 +4716,7 @@ void ChainstateManager::LoadExternalBlockFile( // This block can be processed immediately; rewind to its start, read and deserialize it. blkdat.SetPos(nBlockPos); pblock = std::make_shared<CBlock>(); - blkdat >> *pblock; + blkdat >> TX_WITH_WITNESS(*pblock); nRewind = blkdat.GetPos(); BlockValidationState state; @@ -5766,9 +5784,12 @@ static ChainstateManager::Options&& Flatten(ChainstateManager::Options&& opts) } ChainstateManager::ChainstateManager(const util::SignalInterrupt& interrupt, Options options, node::BlockManager::Options blockman_options) - : m_interrupt{interrupt}, + : m_script_check_queue{/*batch_size=*/128, options.worker_threads_num}, + m_interrupt{interrupt}, m_options{Flatten(std::move(options))}, - m_blockman{interrupt, std::move(blockman_options)} {} + m_blockman{interrupt, std::move(blockman_options)} +{ +} ChainstateManager::~ChainstateManager() { @@ -5953,8 +5974,8 @@ bool ChainstateManager::ValidatedSnapshotCleanup() fs::path p_old, fs::path p_new, const fs::filesystem_error& err) { - LogPrintf("%s: error renaming file (%s): %s\n", - __func__, fs::PathToString(p_old), err.what()); + LogPrintf("Error renaming path (%s) -> (%s): %s\n", + fs::PathToString(p_old), fs::PathToString(p_new), err.what()); GetNotifications().fatalError(strprintf( "Rename of '%s' -> '%s' failed. " "Cannot clean up the background chainstate leveldb directory.", diff --git a/src/validation.h b/src/validation.h index 7ce60da634..093cecfcd1 100644 --- a/src/validation.h +++ b/src/validation.h @@ -13,6 +13,7 @@ #include <arith_uint256.h> #include <attributes.h> #include <chain.h> +#include <checkqueue.h> #include <kernel/chain.h> #include <consensus/amount.h> #include <deploymentstatus.h> @@ -65,10 +66,6 @@ namespace util { class SignalInterrupt; } // namespace util -/** Maximum number of dedicated script-checking threads allowed */ -static const int MAX_SCRIPTCHECK_THREADS = 15; -/** -par default (number of script-checking threads, 0 = auto) */ -static const int DEFAULT_SCRIPTCHECK_THREADS = 0; /** Block files containing a block-height within MIN_BLOCKS_TO_KEEP of ActiveChain().Tip() will not be pruned. */ static const unsigned int MIN_BLOCKS_TO_KEEP = 288; static const signed int DEFAULT_CHECKBLOCKS = 6; @@ -98,11 +95,6 @@ extern uint256 g_best_block; /** Documentation for argument 'checklevel'. */ extern const std::vector<std::string> CHECKLEVEL_DOC; -/** Run instances of script checking worker threads */ -void StartScriptCheckWorkerThreads(int threads_num); -/** Stop all of the script checking worker threads */ -void StopScriptCheckWorkerThreads(); - CAmount GetBlockSubsidy(int nHeight, const Consensus::Params& consensusParams); bool FatalError(kernel::Notifications& notifications, BlockValidationState& state, const std::string& strMessage, const bilingual_str& userMessage = {}); @@ -114,7 +106,27 @@ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex* pin void PruneBlockFilesManual(Chainstate& active_chainstate, int nManualPruneHeight); /** -* Validation result for a single transaction mempool acceptance. +* Validation result for a transaction evaluated by MemPoolAccept (single or package). +* Here are the expected fields and properties of a result depending on its ResultType, applicable to +* results returned from package evaluation: +*+---------------------------+----------------+-------------------+------------------+----------------+-------------------+ +*| Field or property | VALID | INVALID | MEMPOOL_ENTRY | DIFFERENT_WITNESS | +*| | |--------------------------------------| | | +*| | | TX_RECONSIDERABLE | Other | | | +*+---------------------------+----------------+-------------------+------------------+----------------+-------------------+ +*| txid in mempool? | yes | no | no* | yes | yes | +*| wtxid in mempool? | yes | no | no* | yes | no | +*| m_state | yes, IsValid() | yes, IsInvalid() | yes, IsInvalid() | yes, IsValid() | yes, IsValid() | +*| m_replaced_transactions | yes | no | no | no | no | +*| m_vsize | yes | no | no | yes | no | +*| m_base_fees | yes | no | no | yes | no | +*| m_effective_feerate | yes | yes | no | no | no | +*| m_wtxids_fee_calculations | yes | yes | no | no | no | +*| m_other_wtxid | no | no | no | no | yes | +*+---------------------------+----------------+-------------------+------------------+----------------+-------------------+ +* (*) Individual transaction acceptance doesn't return MEMPOOL_ENTRY and DIFFERENT_WITNESS. It returns +* INVALID, with the errors txn-already-in-mempool and txn-same-nonwitness-data-in-mempool +* respectively. In those cases, the txid or wtxid may be in the mempool for a TX_CONFLICT. */ struct MempoolAcceptResult { /** Used to indicate the results of mempool validation. */ @@ -130,7 +142,6 @@ struct MempoolAcceptResult { /** Contains information about why the transaction failed. */ const TxValidationState m_state; - // The following fields are only present when m_result_type = ResultType::VALID or MEMPOOL_ENTRY /** Mempool transactions replaced by the tx. */ const std::optional<std::list<CTransactionRef>> m_replaced_transactions; /** Virtual size as used by the mempool, calculated using serialized size and sigops. */ @@ -141,7 +152,6 @@ struct MempoolAcceptResult { * using prioritisetransaction (i.e. modified fees). If this transaction was submitted as a * package, this is the package feerate, which may also include its descendants and/or * ancestors (see m_wtxids_fee_calculations below). - * Only present when m_result_type = ResultType::VALID. */ const std::optional<CFeeRate> m_effective_feerate; /** Contains the wtxids of the transactions used for fee-related checks. Includes this @@ -149,9 +159,8 @@ struct MempoolAcceptResult { * package. This is not necessarily equivalent to the list of transactions passed to * ProcessNewPackage(). * Only present when m_result_type = ResultType::VALID. */ - const std::optional<std::vector<uint256>> m_wtxids_fee_calculations; + const std::optional<std::vector<Wtxid>> m_wtxids_fee_calculations; - // The following field is only present when m_result_type = ResultType::DIFFERENT_WITNESS /** The wtxid of the transaction in the mempool which has the same txid but different witness. */ const std::optional<uint256> m_other_wtxid; @@ -159,11 +168,17 @@ struct MempoolAcceptResult { return MempoolAcceptResult(state); } + static MempoolAcceptResult FeeFailure(TxValidationState state, + CFeeRate effective_feerate, + const std::vector<Wtxid>& wtxids_fee_calculations) { + return MempoolAcceptResult(state, effective_feerate, wtxids_fee_calculations); + } + static MempoolAcceptResult Success(std::list<CTransactionRef>&& replaced_txns, int64_t vsize, CAmount fees, CFeeRate effective_feerate, - const std::vector<uint256>& wtxids_fee_calculations) { + const std::vector<Wtxid>& wtxids_fee_calculations) { return MempoolAcceptResult(std::move(replaced_txns), vsize, fees, effective_feerate, wtxids_fee_calculations); } @@ -189,7 +204,7 @@ private: int64_t vsize, CAmount fees, CFeeRate effective_feerate, - const std::vector<uint256>& wtxids_fee_calculations) + const std::vector<Wtxid>& wtxids_fee_calculations) : m_result_type(ResultType::VALID), m_replaced_transactions(std::move(replaced_txns)), m_vsize{vsize}, @@ -197,6 +212,15 @@ private: m_effective_feerate(effective_feerate), m_wtxids_fee_calculations(wtxids_fee_calculations) {} + /** Constructor for fee-related failure case */ + explicit MempoolAcceptResult(TxValidationState state, + CFeeRate effective_feerate, + const std::vector<Wtxid>& wtxids_fee_calculations) + : m_result_type(ResultType::INVALID), + m_state(state), + m_effective_feerate(effective_feerate), + m_wtxids_fee_calculations(wtxids_fee_calculations) {} + /** Constructor for already-in-mempool case. It wouldn't replace any transactions. */ explicit MempoolAcceptResult(int64_t vsize, CAmount fees) : m_result_type(ResultType::MEMPOOL_ENTRY), m_vsize{vsize}, m_base_fees(fees) {} @@ -894,6 +918,9 @@ private: return cs && !cs->m_disabled; } + //! A queue for script verifications that have to be performed by worker threads. + CCheckQueue<CScriptCheck> m_script_check_queue; + public: using Options = kernel::ChainstateManagerOpts; @@ -1105,7 +1132,7 @@ public: * (only used for reindex) * */ void LoadExternalBlockFile( - CAutoFile& file_in, + AutoFile& file_in, FlatFilePos* dbp = nullptr, std::multimap<uint256, FlatFilePos>* blocks_with_unknown_parent = nullptr); @@ -1244,6 +1271,8 @@ public: //! nullopt. std::optional<int> GetSnapshotBaseHeight() const EXCLUSIVE_LOCKS_REQUIRED(::cs_main); + CCheckQueue<CScriptCheck>& GetCheckQueue() { return m_script_check_queue; } + ~ChainstateManager(); }; diff --git a/src/validationinterface.cpp b/src/validationinterface.cpp index 9241395ad5..5e944a7c47 100644 --- a/src/validationinterface.cpp +++ b/src/validationinterface.cpp @@ -9,6 +9,7 @@ #include <chain.h> #include <consensus/validation.h> #include <kernel/chain.h> +#include <kernel/mempool_entry.h> #include <logging.h> #include <primitives/block.h> #include <primitives/transaction.h> @@ -205,13 +206,14 @@ void CMainSignals::UpdatedBlockTip(const CBlockIndex *pindexNew, const CBlockInd fInitialDownload); } -void CMainSignals::TransactionAddedToMempool(const CTransactionRef& tx, uint64_t mempool_sequence) { +void CMainSignals::TransactionAddedToMempool(const NewMempoolTransactionInfo& tx, uint64_t mempool_sequence) +{ auto event = [tx, mempool_sequence, this] { m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.TransactionAddedToMempool(tx, mempool_sequence); }); }; ENQUEUE_AND_LOG_EVENT(event, "%s: txid=%s wtxid=%s", __func__, - tx->GetHash().ToString(), - tx->GetWitnessHash().ToString()); + tx.info.m_tx->GetHash().ToString(), + tx.info.m_tx->GetWitnessHash().ToString()); } void CMainSignals::TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason, uint64_t mempool_sequence) { @@ -233,6 +235,16 @@ void CMainSignals::BlockConnected(ChainstateRole role, const std::shared_ptr<con pindex->nHeight); } +void CMainSignals::MempoolTransactionsRemovedForBlock(const std::vector<RemovedMempoolTransactionInfo>& txs_removed_for_block, unsigned int nBlockHeight) +{ + auto event = [txs_removed_for_block, nBlockHeight, this] { + m_internals->Iterate([&](CValidationInterface& callbacks) { callbacks.MempoolTransactionsRemovedForBlock(txs_removed_for_block, nBlockHeight); }); + }; + ENQUEUE_AND_LOG_EVENT(event, "%s: block height=%s txs removed=%s", __func__, + nBlockHeight, + txs_removed_for_block.size()); +} + void CMainSignals::BlockDisconnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindex) { auto event = [pblock, pindex, this] { diff --git a/src/validationinterface.h b/src/validationinterface.h index eb15aa4d5f..d9292ae2c9 100644 --- a/src/validationinterface.h +++ b/src/validationinterface.h @@ -21,6 +21,8 @@ struct CBlockLocator; class CValidationInterface; class CScheduler; enum class MemPoolRemovalReason; +struct RemovedMempoolTransactionInfo; +struct NewMempoolTransactionInfo; /** Register subscriber */ void RegisterValidationInterface(CValidationInterface* callbacks); @@ -60,10 +62,10 @@ void CallFunctionInValidationInterfaceQueue(std::function<void ()> func); void SyncWithValidationInterfaceQueue() LOCKS_EXCLUDED(cs_main); /** - * Implement this to subscribe to events generated in validation + * Implement this to subscribe to events generated in validation and mempool * * Each CValidationInterface() subscriber will receive event callbacks - * in the order in which the events were generated by validation. + * in the order in which the events were generated by validation and mempool. * Furthermore, each ValidationInterface() subscriber may assume that * callbacks effectively run in a single thread with single-threaded * memory consistency. That is, for a given ValidationInterface() @@ -96,7 +98,7 @@ protected: * * Called on a background thread. */ - virtual void TransactionAddedToMempool(const CTransactionRef& tx, uint64_t mempool_sequence) {} + virtual void TransactionAddedToMempool(const NewMempoolTransactionInfo& tx, uint64_t mempool_sequence) {} /** * Notifies listeners of a transaction leaving mempool. @@ -113,7 +115,7 @@ protected: * This does not fire for transactions that are removed from the mempool * because they have been included in a block. Any client that is interested * in transactions removed from the mempool for inclusion in a block can learn - * about those transactions from the BlockConnected notification. + * about those transactions from the MempoolTransactionsRemovedForBlock notification. * * Transactions that are removed from the mempool because they conflict * with a transaction in the new block will have @@ -131,6 +133,14 @@ protected: * Called on a background thread. */ virtual void TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason, uint64_t mempool_sequence) {} + /* + * Notifies listeners of transactions removed from the mempool as + * as a result of new block being connected. + * MempoolTransactionsRemovedForBlock will be fired before BlockConnected. + * + * Called on a background thread. + */ + virtual void MempoolTransactionsRemovedForBlock(const std::vector<RemovedMempoolTransactionInfo>& txs_removed_for_block, unsigned int nBlockHeight) {} /** * Notifies listeners of a block being connected. * Provides a vector of transactions evicted from the mempool as a result. @@ -140,6 +150,7 @@ protected: virtual void BlockConnected(ChainstateRole role, const std::shared_ptr<const CBlock> &block, const CBlockIndex *pindex) {} /** * Notifies listeners of a block being disconnected + * Provides the block that was disconnected. * * Called on a background thread. Only called for the active chainstate, since * background chainstates should never disconnect blocks. @@ -200,8 +211,9 @@ public: void UpdatedBlockTip(const CBlockIndex *, const CBlockIndex *, bool fInitialDownload); - void TransactionAddedToMempool(const CTransactionRef&, uint64_t mempool_sequence); + void TransactionAddedToMempool(const NewMempoolTransactionInfo&, uint64_t mempool_sequence); void TransactionRemovedFromMempool(const CTransactionRef&, MemPoolRemovalReason, uint64_t mempool_sequence); + void MempoolTransactionsRemovedForBlock(const std::vector<RemovedMempoolTransactionInfo>&, unsigned int nBlockHeight); void BlockConnected(ChainstateRole, const std::shared_ptr<const CBlock> &, const CBlockIndex *pindex); void BlockDisconnected(const std::shared_ptr<const CBlock> &, const CBlockIndex* pindex); void ChainStateFlushed(ChainstateRole, const CBlockLocator &); diff --git a/src/wallet/bdb.cpp b/src/wallet/bdb.cpp index 9ea43ca67c..cbf6c9b1ea 100644 --- a/src/wallet/bdb.cpp +++ b/src/wallet/bdb.cpp @@ -149,7 +149,7 @@ bool BerkeleyEnvironment::Open(bilingual_str& err) fs::path pathIn = fs::PathFromString(strPath); TryCreateDirectories(pathIn); - if (!LockDirectory(pathIn, ".walletlock")) { + if (util::LockDirectory(pathIn, ".walletlock") != util::LockResult::Success) { LogPrintf("Cannot obtain a lock on wallet directory %s. Another instance may be using it.\n", strPath); err = strprintf(_("Error initializing wallet database environment %s!"), fs::quoted(fs::PathToString(Directory()))); return false; diff --git a/src/wallet/coincontrol.cpp b/src/wallet/coincontrol.cpp index 2087119db9..873c5ab383 100644 --- a/src/wallet/coincontrol.cpp +++ b/src/wallet/coincontrol.cpp @@ -14,69 +14,142 @@ CCoinControl::CCoinControl() bool CCoinControl::HasSelected() const { - return !m_selected_inputs.empty(); + return !m_selected.empty(); } -bool CCoinControl::IsSelected(const COutPoint& output) const +bool CCoinControl::IsSelected(const COutPoint& outpoint) const { - return m_selected_inputs.count(output) > 0; + return m_selected.count(outpoint) > 0; } -bool CCoinControl::IsExternalSelected(const COutPoint& output) const +bool CCoinControl::IsExternalSelected(const COutPoint& outpoint) const { - return m_external_txouts.count(output) > 0; + const auto it = m_selected.find(outpoint); + return it != m_selected.end() && it->second.HasTxOut(); } std::optional<CTxOut> CCoinControl::GetExternalOutput(const COutPoint& outpoint) const { - const auto ext_it = m_external_txouts.find(outpoint); - if (ext_it == m_external_txouts.end()) { + const auto it = m_selected.find(outpoint); + if (it == m_selected.end() || !it->second.HasTxOut()) { return std::nullopt; } + return it->second.GetTxOut(); +} - return std::make_optional(ext_it->second); +PreselectedInput& CCoinControl::Select(const COutPoint& outpoint) +{ + auto& input = m_selected[outpoint]; + input.SetPosition(m_selection_pos); + ++m_selection_pos; + return input; +} +void CCoinControl::UnSelect(const COutPoint& outpoint) +{ + m_selected.erase(outpoint); } -void CCoinControl::Select(const COutPoint& output) +void CCoinControl::UnSelectAll() { - m_selected_inputs.insert(output); + m_selected.clear(); } -void CCoinControl::SelectExternal(const COutPoint& outpoint, const CTxOut& txout) +std::vector<COutPoint> CCoinControl::ListSelected() const { - m_selected_inputs.insert(outpoint); - m_external_txouts.emplace(outpoint, txout); + std::vector<COutPoint> outpoints; + std::transform(m_selected.begin(), m_selected.end(), std::back_inserter(outpoints), + [](const std::map<COutPoint, PreselectedInput>::value_type& pair) { + return pair.first; + }); + return outpoints; } -void CCoinControl::UnSelect(const COutPoint& output) +void CCoinControl::SetInputWeight(const COutPoint& outpoint, int64_t weight) { - m_selected_inputs.erase(output); + m_selected[outpoint].SetInputWeight(weight); } -void CCoinControl::UnSelectAll() +std::optional<int64_t> CCoinControl::GetInputWeight(const COutPoint& outpoint) const { - m_selected_inputs.clear(); + const auto it = m_selected.find(outpoint); + return it != m_selected.end() ? it->second.GetInputWeight() : std::nullopt; } -std::vector<COutPoint> CCoinControl::ListSelected() const +std::optional<uint32_t> CCoinControl::GetSequence(const COutPoint& outpoint) const { - return {m_selected_inputs.begin(), m_selected_inputs.end()}; + const auto it = m_selected.find(outpoint); + return it != m_selected.end() ? it->second.GetSequence() : std::nullopt; } -void CCoinControl::SetInputWeight(const COutPoint& outpoint, int64_t weight) +std::pair<std::optional<CScript>, std::optional<CScriptWitness>> CCoinControl::GetScripts(const COutPoint& outpoint) const +{ + const auto it = m_selected.find(outpoint); + return it != m_selected.end() ? m_selected.at(outpoint).GetScripts() : std::make_pair(std::nullopt, std::nullopt); +} + +void PreselectedInput::SetTxOut(const CTxOut& txout) +{ + m_txout = txout; +} + +CTxOut PreselectedInput::GetTxOut() const +{ + assert(m_txout.has_value()); + return m_txout.value(); +} + +bool PreselectedInput::HasTxOut() const +{ + return m_txout.has_value(); +} + +void PreselectedInput::SetInputWeight(int64_t weight) +{ + m_weight = weight; +} + +std::optional<int64_t> PreselectedInput::GetInputWeight() const +{ + return m_weight; +} + +void PreselectedInput::SetSequence(uint32_t sequence) +{ + m_sequence = sequence; +} + +std::optional<uint32_t> PreselectedInput::GetSequence() const +{ + return m_sequence; +} + +void PreselectedInput::SetScriptSig(const CScript& script) +{ + m_script_sig = script; +} + +void PreselectedInput::SetScriptWitness(const CScriptWitness& script_wit) +{ + m_script_witness = script_wit; +} + +bool PreselectedInput::HasScripts() const +{ + return m_script_sig.has_value() || m_script_witness.has_value(); +} + +std::pair<std::optional<CScript>, std::optional<CScriptWitness>> PreselectedInput::GetScripts() const { - m_input_weights[outpoint] = weight; + return {m_script_sig, m_script_witness}; } -bool CCoinControl::HasInputWeight(const COutPoint& outpoint) const +void PreselectedInput::SetPosition(unsigned int pos) { - return m_input_weights.count(outpoint) > 0; + m_pos = pos; } -int64_t CCoinControl::GetInputWeight(const COutPoint& outpoint) const +std::optional<unsigned int> PreselectedInput::GetPosition() const { - auto it = m_input_weights.find(outpoint); - assert(it != m_input_weights.end()); - return it->second; + return m_pos; } } // namespace wallet diff --git a/src/wallet/coincontrol.h b/src/wallet/coincontrol.h index 71593e236f..b2f813383d 100644 --- a/src/wallet/coincontrol.h +++ b/src/wallet/coincontrol.h @@ -24,6 +24,58 @@ const int DEFAULT_MAX_DEPTH = 9999999; //! Default for -avoidpartialspends static constexpr bool DEFAULT_AVOIDPARTIALSPENDS = false; +class PreselectedInput +{ +private: + //! The previous output being spent by this input + std::optional<CTxOut> m_txout; + //! The input weight for spending this input + std::optional<int64_t> m_weight; + //! The sequence number for this input + std::optional<uint32_t> m_sequence; + //! The scriptSig for this input + std::optional<CScript> m_script_sig; + //! The scriptWitness for this input + std::optional<CScriptWitness> m_script_witness; + //! The position in the inputs vector for this input + std::optional<unsigned int> m_pos; + +public: + /** + * Set the previous output for this input. + * Only necessary if the input is expected to be an external input. + */ + void SetTxOut(const CTxOut& txout); + /** Retrieve the previous output for this input. */ + CTxOut GetTxOut() const; + /** Return whether the previous output is set for this input. */ + bool HasTxOut() const; + + /** Set the weight for this input. */ + void SetInputWeight(int64_t weight); + /** Retrieve the input weight for this input. */ + std::optional<int64_t> GetInputWeight() const; + + /** Set the sequence for this input. */ + void SetSequence(uint32_t sequence); + /** Retrieve the sequence for this input. */ + std::optional<uint32_t> GetSequence() const; + + /** Set the scriptSig for this input. */ + void SetScriptSig(const CScript& script); + /** Set the scriptWitness for this input. */ + void SetScriptWitness(const CScriptWitness& script_wit); + /** Return whether either the scriptSig or scriptWitness are set for this input. */ + bool HasScripts() const; + /** Retrieve both the scriptSig and the scriptWitness. */ + std::pair<std::optional<CScript>, std::optional<CScriptWitness>> GetScripts() const; + + /** Store the position of this input. */ + void SetPosition(unsigned int pos); + /** Retrieve the position of this input. */ + std::optional<unsigned int> GetPosition() const; +}; + /** Coin Control Features. */ class CCoinControl { @@ -59,6 +111,10 @@ public: int m_max_depth = DEFAULT_MAX_DEPTH; //! SigningProvider that has pubkeys and scripts to do spend size estimation for external inputs FlatSigningProvider m_external_provider; + //! Locktime + std::optional<uint32_t> m_locktime; + //! Version + std::optional<uint32_t> m_version; CCoinControl(); @@ -69,11 +125,11 @@ public: /** * Returns true if the given output is pre-selected. */ - bool IsSelected(const COutPoint& output) const; + bool IsSelected(const COutPoint& outpoint) const; /** * Returns true if the given output is selected as an external input. */ - bool IsExternalSelected(const COutPoint& output) const; + bool IsExternalSelected(const COutPoint& outpoint) const; /** * Returns the external output for the given outpoint if it exists. */ @@ -82,16 +138,11 @@ public: * Lock-in the given output for spending. * The output will be included in the transaction even if it's not the most optimal choice. */ - void Select(const COutPoint& output); - /** - * Lock-in the given output as an external input for spending because it is not in the wallet. - * The output will be included in the transaction even if it's not the most optimal choice. - */ - void SelectExternal(const COutPoint& outpoint, const CTxOut& txout); + PreselectedInput& Select(const COutPoint& outpoint); /** * Unselects the given output. */ - void UnSelect(const COutPoint& output); + void UnSelect(const COutPoint& outpoint); /** * Unselects all outputs. */ @@ -105,22 +156,32 @@ public: */ void SetInputWeight(const COutPoint& outpoint, int64_t weight); /** - * Returns true if the input weight is set. - */ - bool HasInputWeight(const COutPoint& outpoint) const; - /** * Returns the input weight. */ - int64_t GetInputWeight(const COutPoint& outpoint) const; + std::optional<int64_t> GetInputWeight(const COutPoint& outpoint) const; + /** Retrieve the sequence for an input */ + std::optional<uint32_t> GetSequence(const COutPoint& outpoint) const; + /** Retrieves the scriptSig and scriptWitness for an input. */ + std::pair<std::optional<CScript>, std::optional<CScriptWitness>> GetScripts(const COutPoint& outpoint) const; + + bool HasSelectedOrder() const + { + return m_selection_pos > 0; + } + + std::optional<unsigned int> GetSelectionPos(const COutPoint& outpoint) const + { + const auto it = m_selected.find(outpoint); + if (it == m_selected.end()) { + return std::nullopt; + } + return it->second.GetPosition(); + } private: //! Selected inputs (inputs that will be used, regardless of whether they're optimal or not) - std::set<COutPoint> m_selected_inputs; - //! Map of external inputs to include in the transaction - //! These are not in the wallet, so we need to track them separately - std::map<COutPoint, CTxOut> m_external_txouts; - //! Map of COutPoints to the maximum weight for that input - std::map<COutPoint, int64_t> m_input_weights; + std::map<COutPoint, PreselectedInput> m_selected; + unsigned int m_selection_pos{0}; }; } // namespace wallet diff --git a/src/wallet/context.h b/src/wallet/context.h index 57a6ed77f7..58b9924fb4 100644 --- a/src/wallet/context.h +++ b/src/wallet/context.h @@ -13,6 +13,7 @@ #include <vector> class ArgsManager; +class CScheduler; namespace interfaces { class Chain; class Wallet; @@ -34,6 +35,7 @@ using LoadWalletFn = std::function<void(std::unique_ptr<interfaces::Wallet> wall //! behavior. struct WalletContext { interfaces::Chain* chain{nullptr}; + CScheduler* scheduler{nullptr}; ArgsManager* args{nullptr}; // Currently a raw pointer because the memory is not managed by this struct // It is unsafe to lock this after locking a CWallet::cs_wallet mutex because // this could introduce inconsistent lock ordering and cause deadlocks. diff --git a/src/wallet/db.cpp b/src/wallet/db.cpp index 0ee39d2b5a..ea06767e9b 100644 --- a/src/wallet/db.cpp +++ b/src/wallet/db.cpp @@ -129,9 +129,9 @@ bool IsSQLiteFile(const fs::path& path) file.close(); - // Check the magic, see https://sqlite.org/fileformat2.html + // Check the magic, see https://sqlite.org/fileformat.html std::string magic_str(magic, 16); - if (magic_str != std::string("SQLite format 3", 16)) { + if (magic_str != std::string{"SQLite format 3\000", 16}) { return false; } diff --git a/src/wallet/db.h b/src/wallet/db.h index 9d684225c3..c263f54144 100644 --- a/src/wallet/db.h +++ b/src/wallet/db.h @@ -67,7 +67,7 @@ public: ssKey.reserve(1000); ssKey << key; - CDataStream ssValue(SER_DISK, CLIENT_VERSION); + DataStream ssValue{}; if (!ReadKey(std::move(ssKey), ssValue)) return false; try { ssValue >> value; @@ -84,7 +84,7 @@ public: ssKey.reserve(1000); ssKey << key; - CDataStream ssValue(SER_DISK, CLIENT_VERSION); + DataStream ssValue{}; ssValue.reserve(10000); ssValue << value; diff --git a/src/wallet/dump.cpp b/src/wallet/dump.cpp index 3ac5cf03b1..7a36910dc1 100644 --- a/src/wallet/dump.cpp +++ b/src/wallet/dump.cpp @@ -8,6 +8,7 @@ #include <util/fs.h> #include <util/translation.h> #include <wallet/wallet.h> +#include <wallet/walletdb.h> #include <algorithm> #include <fstream> @@ -20,7 +21,7 @@ namespace wallet { static const std::string DUMP_MAGIC = "BITCOIN_CORE_WALLET_DUMP"; uint32_t DUMP_VERSION = 1; -bool DumpWallet(const ArgsManager& args, CWallet& wallet, bilingual_str& error) +bool DumpWallet(const ArgsManager& args, WalletDatabase& db, bilingual_str& error) { // Get the dumpfile std::string dump_filename = args.GetArg("-dumpfile", ""); @@ -44,7 +45,6 @@ bool DumpWallet(const ArgsManager& args, CWallet& wallet, bilingual_str& error) HashWriter hasher{}; - WalletDatabase& db = wallet.GetDatabase(); std::unique_ptr<DatabaseBatch> batch = db.MakeBatch(); bool ret = true; @@ -90,9 +90,6 @@ bool DumpWallet(const ArgsManager& args, CWallet& wallet, bilingual_str& error) cursor.reset(); batch.reset(); - // Close the wallet after we're done with it. The caller won't be doing this - wallet.Close(); - if (ret) { // Write the hash tfm::format(dump_file, "checksum,%s\n", HexStr(hasher.GetHash())); diff --git a/src/wallet/dump.h b/src/wallet/dump.h index 5034f95479..9b44af922e 100644 --- a/src/wallet/dump.h +++ b/src/wallet/dump.h @@ -14,8 +14,9 @@ struct bilingual_str; class ArgsManager; namespace wallet { -class CWallet; -bool DumpWallet(const ArgsManager& args, CWallet& wallet, bilingual_str& error); +class WalletDatabase; + +bool DumpWallet(const ArgsManager& args, WalletDatabase& db, bilingual_str& error); bool CreateFromDump(const ArgsManager& args, const std::string& name, const fs::path& wallet_path, bilingual_str& error, std::vector<bilingual_str>& warnings); } // namespace wallet diff --git a/src/wallet/external_signer_scriptpubkeyman.cpp b/src/wallet/external_signer_scriptpubkeyman.cpp index 6d026d02c1..a71f8f9fbc 100644 --- a/src/wallet/external_signer_scriptpubkeyman.cpp +++ b/src/wallet/external_signer_scriptpubkeyman.cpp @@ -16,7 +16,7 @@ #include <vector> namespace wallet { -bool ExternalSignerScriptPubKeyMan::SetupDescriptor(std::unique_ptr<Descriptor> desc) +bool ExternalSignerScriptPubKeyMan::SetupDescriptor(WalletBatch& batch, std::unique_ptr<Descriptor> desc) { LOCK(cs_desc_man); assert(m_storage.IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); @@ -29,13 +29,12 @@ bool ExternalSignerScriptPubKeyMan::SetupDescriptor(std::unique_ptr<Descriptor> m_wallet_descriptor = w_desc; // Store the descriptor - WalletBatch batch(m_storage.GetDatabase()); if (!batch.WriteDescriptor(GetID(), m_wallet_descriptor)) { throw std::runtime_error(std::string(__func__) + ": writing descriptor failed"); } // TopUp - TopUp(); + TopUpWithDB(batch); m_storage.UnsetBlankWalletFlag(batch); return true; diff --git a/src/wallet/external_signer_scriptpubkeyman.h b/src/wallet/external_signer_scriptpubkeyman.h index 01dc80b1ca..c052ce6129 100644 --- a/src/wallet/external_signer_scriptpubkeyman.h +++ b/src/wallet/external_signer_scriptpubkeyman.h @@ -23,7 +23,7 @@ class ExternalSignerScriptPubKeyMan : public DescriptorScriptPubKeyMan /** Provide a descriptor at setup time * Returns false if already setup or setup fails, true if setup is successful */ - bool SetupDescriptor(std::unique_ptr<Descriptor>desc); + bool SetupDescriptor(WalletBatch& batch, std::unique_ptr<Descriptor>desc); static ExternalSigner GetExternalSigner(); diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index f4cb4bbd66..6a8453965b 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -86,7 +86,7 @@ static feebumper::Result CheckFeeRate(const CWallet& wallet, const CMutableTrans reused_inputs.push_back(txin.prevout); } - std::optional<CAmount> combined_bump_fee = wallet.chain().CalculateCombinedBumpFee(reused_inputs, newFeerate); + std::optional<CAmount> combined_bump_fee = wallet.chain().calculateCombinedBumpFee(reused_inputs, newFeerate); if (!combined_bump_fee.has_value()) { errors.push_back(strprintf(Untranslated("Failed to calculate bump fees, because unconfirmed UTXOs depend on enormous cluster of unconfirmed transactions."))); } @@ -170,7 +170,7 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo return Result::INVALID_PARAMETER; } - // We are going to modify coin control later, copy to re-use + // We are going to modify coin control later, copy to reuse CCoinControl new_coin_control(coin_control); LOCK(wallet.cs_wallet); @@ -203,10 +203,9 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo errors.push_back(Untranslated(strprintf("%s:%u is already spent", txin.prevout.hash.GetHex(), txin.prevout.n))); return Result::MISC_ERROR; } - if (wallet.IsMine(txin.prevout)) { - new_coin_control.Select(txin.prevout); - } else { - new_coin_control.SelectExternal(txin.prevout, coin.out); + PreselectedInput& preset_txin = new_coin_control.Select(txin.prevout); + if (!wallet.IsMine(txin.prevout)) { + preset_txin.SetTxOut(coin.out); } input_value += coin.out.nValue; spent_outputs.push_back(coin.out); @@ -317,8 +316,7 @@ Result CreateRateBumpTransaction(CWallet& wallet, const uint256& txid, const CCo // We cannot source new unconfirmed inputs(bip125 rule 2) new_coin_control.m_min_depth = 1; - constexpr int RANDOM_CHANGE_POSITION = -1; - auto res = CreateTransaction(wallet, recipients, RANDOM_CHANGE_POSITION, new_coin_control, false); + auto res = CreateTransaction(wallet, recipients, /*change_pos=*/std::nullopt, new_coin_control, false); if (!res) { errors.push_back(Untranslated("Unable to create transaction.") + Untranslated(" ") + util::ErrorString(res)); return Result::WALLET_ERROR; diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 0d0a8650ac..088343458c 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -95,8 +95,6 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const argsman.AddArg("-walletrejectlongchains", strprintf("Wallet will not create transactions that violate mempool chain limits (default: %u)", DEFAULT_WALLET_REJECT_LONG_CHAINS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); argsman.AddArg("-walletcrosschain", strprintf("Allow reusing wallet files across chains (default: %u)", DEFAULT_WALLETCROSSCHAIN), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); - - argsman.AddHiddenArgs({"-zapwallettxes"}); } bool WalletInit::ParameterInteraction() const @@ -118,10 +116,6 @@ bool WalletInit::ParameterInteraction() const LogPrintf("%s: parameter interaction: -blocksonly=1 -> setting -walletbroadcast=0\n", __func__); } - if (gArgs.IsArgSet("-zapwallettxes")) { - return InitError(Untranslated("-zapwallettxes has been removed. If you are attempting to remove a stuck transaction from your wallet, please use abandontransaction instead.")); - } - return true; } diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index 65285187f4..d15273dfc9 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -11,6 +11,7 @@ #include <policy/fees.h> #include <primitives/transaction.h> #include <rpc/server.h> +#include <scheduler.h> #include <support/allocators/secure.h> #include <sync.h> #include <uint256.h> @@ -212,7 +213,7 @@ public: } return true; } - std::vector<WalletAddress> getAddresses() const override + std::vector<WalletAddress> getAddresses() override { LOCK(m_wallet->cs_wallet); std::vector<WalletAddress> result; @@ -280,12 +281,12 @@ public: CAmount& fee) override { LOCK(m_wallet->cs_wallet); - auto res = CreateTransaction(*m_wallet, recipients, change_pos, + auto res = CreateTransaction(*m_wallet, recipients, change_pos == -1 ? std::nullopt : std::make_optional(change_pos), coin_control, sign); if (!res) return util::Error{util::ErrorString(res)}; const auto& txr = *res; fee = txr.fee; - change_pos = txr.change_pos; + change_pos = txr.change_pos ? *txr.change_pos : -1; return txr.tx; } @@ -585,10 +586,15 @@ public: } bool verify() override { return VerifyWallets(m_context); } bool load() override { return LoadWallets(m_context); } - void start(CScheduler& scheduler) override { return StartWallets(m_context, scheduler); } + void start(CScheduler& scheduler) override + { + m_context.scheduler = &scheduler; + return StartWallets(m_context); + } void flush() override { return FlushWallets(m_context); } void stop() override { return StopWallets(m_context); } void setMockTime(int64_t time) override { return SetMockTime(time); } + void schedulerMockForward(std::chrono::seconds delta) override { Assert(m_context.scheduler)->MockForward(delta); } //! WalletLoader methods util::Result<std::unique_ptr<Wallet>> createWallet(const std::string& name, const SecureString& passphrase, uint64_t wallet_creation_flags, std::vector<bilingual_str>& warnings) override @@ -602,7 +608,7 @@ public: bilingual_str error; std::unique_ptr<Wallet> wallet{MakeWallet(m_context, CreateWallet(m_context, name, /*load_on_start=*/true, options, status, error, warnings))}; if (wallet) { - return {std::move(wallet)}; + return wallet; } else { return util::Error{error}; } @@ -616,7 +622,7 @@ public: bilingual_str error; std::unique_ptr<Wallet> wallet{MakeWallet(m_context, LoadWallet(m_context, name, /*load_on_start=*/true, options, status, error, warnings))}; if (wallet) { - return {std::move(wallet)}; + return wallet; } else { return util::Error{error}; } @@ -627,7 +633,7 @@ public: bilingual_str error; std::unique_ptr<Wallet> wallet{MakeWallet(m_context, RestoreWallet(m_context, backup_file, wallet_name, /*load_on_start=*/true, status, error, warnings))}; if (wallet) { - return {std::move(wallet)}; + return wallet; } else { return util::Error{error}; } @@ -642,7 +648,7 @@ public: .solvables_wallet_name = res->solvables_wallet ? std::make_optional(res->solvables_wallet->GetName()) : std::nullopt, .backup_path = res->backup_path, }; - return {std::move(out)}; // std::move to work around clang bug + return out; } std::string getWalletDir() override { diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 4cdfadbee2..8b78a670e4 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -141,7 +141,7 @@ bool LoadWallets(WalletContext& context) } } -void StartWallets(WalletContext& context, CScheduler& scheduler) +void StartWallets(WalletContext& context) { for (const std::shared_ptr<CWallet>& pwallet : GetWallets(context)) { pwallet->postInitProcess(); @@ -149,9 +149,9 @@ void StartWallets(WalletContext& context, CScheduler& scheduler) // Schedule periodic wallet flushes and tx rebroadcasts if (context.args->GetBoolArg("-flushwallet", DEFAULT_FLUSHWALLET)) { - scheduler.scheduleEvery([&context] { MaybeCompactWalletDB(context); }, std::chrono::milliseconds{500}); + context.scheduler->scheduleEvery([&context] { MaybeCompactWalletDB(context); }, 500ms); } - scheduler.scheduleEvery([&context] { MaybeResendWalletTxs(context); }, 1min); + context.scheduler->scheduleEvery([&context] { MaybeResendWalletTxs(context); }, 1min); } void FlushWallets(WalletContext& context) diff --git a/src/wallet/load.h b/src/wallet/load.h index 0882f7f8ad..c079cad955 100644 --- a/src/wallet/load.h +++ b/src/wallet/load.h @@ -26,7 +26,7 @@ bool VerifyWallets(WalletContext& context); bool LoadWallets(WalletContext& context); //! Complete startup of wallets. -void StartWallets(WalletContext& context, CScheduler& scheduler); +void StartWallets(WalletContext& context); //! Flush all wallets in preparation for shutdown. void FlushWallets(WalletContext& context); diff --git a/src/wallet/receive.cpp b/src/wallet/receive.cpp index 0a75bb6d92..b9d8d9abc9 100644 --- a/src/wallet/receive.cpp +++ b/src/wallet/receive.cpp @@ -173,7 +173,7 @@ CAmount CachedTxGetAvailableCredit(const CWallet& wallet, const CWalletTx& wtx, bool allow_used_addresses = (filter & ISMINE_USED) || !wallet.IsWalletFlagSet(WALLET_FLAG_AVOID_REUSE); CAmount nCredit = 0; - uint256 hashTx = wtx.GetHash(); + Txid hashTx = wtx.GetHash(); for (unsigned int i = 0; i < wtx.tx->vout.size(); i++) { const CTxOut& txout = wtx.tx->vout[i]; if (!wallet.IsSpent(COutPoint(hashTx, i)) && (allow_used_addresses || !wallet.IsSpentKey(txout.scriptPubKey))) { @@ -348,7 +348,7 @@ std::map<CTxDestination, CAmount> GetAddressBalances(const CWallet& wallet) if(!ExtractDestination(output.scriptPubKey, addr)) continue; - CAmount n = wallet.IsSpent(COutPoint(walletEntry.first, i)) ? 0 : output.nValue; + CAmount n = wallet.IsSpent(COutPoint(Txid::FromUint256(walletEntry.first), i)) ? 0 : output.nValue; balances[addr] += n; } } diff --git a/src/wallet/rpc/backup.cpp b/src/wallet/rpc/backup.cpp index 3e1ec667e0..99c548bf1d 100644 --- a/src/wallet/rpc/backup.cpp +++ b/src/wallet/rpc/backup.cpp @@ -734,7 +734,7 @@ RPCHelpMan dumpwallet() * It may also avoid other security issues. */ if (fs::exists(filepath)) { - throw JSONRPCError(RPC_INVALID_PARAMETER, filepath.u8string() + " already exists. If you are sure this is what you want, move it out of the way first"); + throw JSONRPCError(RPC_INVALID_PARAMETER, filepath.utf8string() + " already exists. If you are sure this is what you want, move it out of the way first"); } std::ofstream file; @@ -828,7 +828,7 @@ RPCHelpMan dumpwallet() file.close(); UniValue reply(UniValue::VOBJ); - reply.pushKV("filename", filepath.u8string()); + reply.pushKV("filename", filepath.utf8string()); return reply; }, diff --git a/src/wallet/rpc/coins.cpp b/src/wallet/rpc/coins.cpp index fdc6ee055d..0cb0891141 100644 --- a/src/wallet/rpc/coins.cpp +++ b/src/wallet/rpc/coins.cpp @@ -320,7 +320,7 @@ RPCHelpMan lockunspent() {"vout", UniValueType(UniValue::VNUM)}, }); - const uint256 txid(ParseHashO(o, "txid")); + const Txid txid = Txid::FromUint256(ParseHashO(o, "txid")); const int nOutput = o.find_value("vout").getInt<int>(); if (nOutput < 0) { throw JSONRPCError(RPC_INVALID_PARAMETER, "Invalid parameter, vout cannot be negative"); diff --git a/src/wallet/rpc/encrypt.cpp b/src/wallet/rpc/encrypt.cpp index 0226d15698..e237286603 100644 --- a/src/wallet/rpc/encrypt.cpp +++ b/src/wallet/rpc/encrypt.cpp @@ -220,7 +220,11 @@ RPCHelpMan encryptwallet() "After this, any calls that interact with private keys such as sending or signing \n" "will require the passphrase to be set prior the making these calls.\n" "Use the walletpassphrase call for this, and then walletlock call.\n" - "If the wallet is already encrypted, use the walletpassphrasechange call.\n", + "If the wallet is already encrypted, use the walletpassphrasechange call.\n" + "** IMPORTANT **\n" + "For security reasons, the encryption process will generate a new HD seed, resulting\n" + "in the creation of a fresh set of active descriptors. Therefore, it is crucial to\n" + "securely back up the newly generated wallet file using the backupwallet RPC.\n", { {"passphrase", RPCArg::Type::STR, RPCArg::Optional::NO, "The pass phrase to encrypt the wallet with. It must be at least 1 character, but should be long."}, }, @@ -268,7 +272,7 @@ RPCHelpMan encryptwallet() throw JSONRPCError(RPC_WALLET_ENCRYPTION_FAILED, "Error: Failed to encrypt the wallet."); } - return "wallet encrypted; The keypool has been flushed and a new HD seed was generated (if you are using HD). You need to make a new backup."; + return "wallet encrypted; The keypool has been flushed and a new HD seed was generated. You need to make a new backup with the backupwallet RPC."; }, }; } diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index 6b96fc4e49..5a13b5ac8e 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -99,7 +99,7 @@ static UniValue FinishTransaction(const std::shared_ptr<CWallet> pwallet, const bool add_to_wallet{options.exists("add_to_wallet") ? options["add_to_wallet"].get_bool() : true}; if (psbt_opt_in || !complete || !add_to_wallet) { // Serialize the PSBT - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssTx{}; ssTx << psbtx; result.pushKV("psbt", EncodeBase64(ssTx.str())); } @@ -155,8 +155,7 @@ UniValue SendMoney(CWallet& wallet, const CCoinControl &coin_control, std::vecto std::shuffle(recipients.begin(), recipients.end(), FastRandomContext()); // Send - constexpr int RANDOM_CHANGE_POSITION = -1; - auto res = CreateTransaction(wallet, recipients, RANDOM_CHANGE_POSITION, coin_control, true); + auto res = CreateTransaction(wallet, recipients, /*change_pos=*/std::nullopt, coin_control, true); if (!res) { throw JSONRPCError(RPC_WALLET_INSUFFICIENT_FUNDS, util::ErrorString(res).original); } @@ -489,13 +488,13 @@ static std::vector<RPCArg> FundTxDoc(bool solving_data = true) return args; } -void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, int& change_position, const UniValue& options, CCoinControl& coinControl, bool override_min_fee) +CreatedTransactionResult FundTransaction(CWallet& wallet, const CMutableTransaction& tx, const UniValue& options, CCoinControl& coinControl, bool override_min_fee) { // Make sure the results are valid at least up to the most recent block // the user could have gotten from another RPC command prior to now wallet.BlockUntilSyncedToCurrentChain(); - change_position = -1; + std::optional<unsigned int> change_position; bool lockUnspents = false; UniValue subtractFeeFromOutputs; std::set<int> setSubtractFeeFromOutputs; @@ -553,7 +552,11 @@ void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, } if (options.exists("changePosition") || options.exists("change_position")) { - change_position = (options.exists("change_position") ? options["change_position"] : options["changePosition"]).getInt<int>(); + int pos = (options.exists("change_position") ? options["change_position"] : options["changePosition"]).getInt<int>(); + if (pos < 0 || (unsigned int)pos > tx.vout.size()) { + throw JSONRPCError(RPC_INVALID_PARAMETER, "changePosition out of bounds"); + } + change_position = (unsigned int)pos; } if (options.exists("change_type")) { @@ -671,7 +674,7 @@ void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, if (options.exists("input_weights")) { for (const UniValue& input : options["input_weights"].get_array().getValues()) { - uint256 txid = ParseHashO(input, "txid"); + Txid txid = Txid::FromUint256(ParseHashO(input, "txid")); const UniValue& vout_v = input.find_value("vout"); if (!vout_v.isNum()) { @@ -703,9 +706,6 @@ void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, if (tx.vout.size() == 0) throw JSONRPCError(RPC_INVALID_PARAMETER, "TX must have at least one output"); - if (change_position != -1 && (change_position < 0 || (unsigned int)change_position > tx.vout.size())) - throw JSONRPCError(RPC_INVALID_PARAMETER, "changePosition out of bounds"); - for (unsigned int idx = 0; idx < subtractFeeFromOutputs.size(); idx++) { int pos = subtractFeeFromOutputs[idx].getInt<int>(); if (setSubtractFeeFromOutputs.count(pos)) @@ -717,11 +717,11 @@ void FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& fee_out, setSubtractFeeFromOutputs.insert(pos); } - bilingual_str error; - - if (!FundTransaction(wallet, tx, fee_out, change_position, error, lockUnspents, setSubtractFeeFromOutputs, coinControl)) { - throw JSONRPCError(RPC_WALLET_ERROR, error.original); + auto txr = FundTransaction(wallet, tx, change_position, lockUnspents, setSubtractFeeFromOutputs, coinControl); + if (!txr) { + throw JSONRPCError(RPC_WALLET_ERROR, ErrorString(txr).original); } + return *txr; } static void SetOptionsInputWeights(const UniValue& inputs, UniValue& options) @@ -844,17 +844,15 @@ RPCHelpMan fundrawtransaction() throw JSONRPCError(RPC_DESERIALIZATION_ERROR, "TX decode failed"); } - CAmount fee; - int change_position; CCoinControl coin_control; // Automatically select (additional) coins. Can be overridden by options.add_inputs. coin_control.m_allow_other_inputs = true; - FundTransaction(*pwallet, tx, fee, change_position, request.params[1], coin_control, /*override_min_fee=*/true); + auto txr = FundTransaction(*pwallet, tx, request.params[1], coin_control, /*override_min_fee=*/true); UniValue result(UniValue::VOBJ); - result.pushKV("hex", EncodeHexTx(CTransaction(tx))); - result.pushKV("fee", ValueFromAmount(fee)); - result.pushKV("changepos", change_position); + result.pushKV("hex", EncodeHexTx(*txr.tx)); + result.pushKV("fee", ValueFromAmount(txr.fee)); + result.pushKV("changepos", txr.change_pos ? (int)*txr.change_pos : -1); return result; }, @@ -1164,7 +1162,7 @@ static RPCHelpMan bumpfee_helper(std::string method_name) const TransactionError err = pwallet->FillPSBT(psbtx, complete, SIGHASH_DEFAULT, /*sign=*/false, /*bip32derivs=*/true); CHECK_NONFATAL(err == TransactionError::OK); CHECK_NONFATAL(!complete); - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssTx{}; ssTx << psbtx; result.pushKV("psbt", EncodeBase64(ssTx.str())); } @@ -1276,8 +1274,6 @@ RPCHelpMan send() PreventOutdatedOptions(options); - CAmount fee; - int change_position; bool rbf{options.exists("replaceable") ? options["replaceable"].get_bool() : pwallet->m_signal_rbf}; CMutableTransaction rawTx = ConstructTransaction(options["inputs"], request.params[0], options["locktime"], rbf); CCoinControl coin_control; @@ -1285,9 +1281,9 @@ RPCHelpMan send() // be overridden by options.add_inputs. coin_control.m_allow_other_inputs = rawTx.vin.size() == 0; SetOptionsInputWeights(options["inputs"], options); - FundTransaction(*pwallet, rawTx, fee, change_position, options, coin_control, /*override_min_fee=*/false); + auto txr = FundTransaction(*pwallet, rawTx, options, coin_control, /*override_min_fee=*/false); - return FinishTransaction(pwallet, options, rawTx); + return FinishTransaction(pwallet, options, CMutableTransaction(*txr.tx)); } }; } @@ -1609,7 +1605,7 @@ RPCHelpMan walletprocesspsbt() } UniValue result(UniValue::VOBJ); - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssTx{}; ssTx << psbtx; result.pushKV("psbt", EncodeBase64(ssTx.str())); result.pushKV("complete", complete); @@ -1617,8 +1613,8 @@ RPCHelpMan walletprocesspsbt() CMutableTransaction mtx; // Returns true if complete, which we already think it is. CHECK_NONFATAL(FinalizeAndExtractPSBT(psbtx, mtx)); - CDataStream ssTx_final(SER_NETWORK, PROTOCOL_VERSION); - ssTx_final << mtx; + DataStream ssTx_final; + ssTx_final << TX_WITH_WITNESS(mtx); result.pushKV("hex", HexStr(ssTx_final)); } @@ -1712,8 +1708,6 @@ RPCHelpMan walletcreatefundedpsbt() UniValue options{request.params[3].isNull() ? UniValue::VOBJ : request.params[3]}; - CAmount fee; - int change_position; const UniValue &replaceable_arg = options["replaceable"]; const bool rbf{replaceable_arg.isNull() ? wallet.m_signal_rbf : replaceable_arg.get_bool()}; CMutableTransaction rawTx = ConstructTransaction(request.params[0], request.params[1], request.params[2], rbf); @@ -1722,10 +1716,10 @@ RPCHelpMan walletcreatefundedpsbt() // be overridden by options.add_inputs. coin_control.m_allow_other_inputs = rawTx.vin.size() == 0; SetOptionsInputWeights(request.params[0], options); - FundTransaction(wallet, rawTx, fee, change_position, options, coin_control, /*override_min_fee=*/true); + auto txr = FundTransaction(wallet, rawTx, options, coin_control, /*override_min_fee=*/true); // Make a blank psbt - PartiallySignedTransaction psbtx(rawTx); + PartiallySignedTransaction psbtx(CMutableTransaction(*txr.tx)); // Fill transaction with out data but don't sign bool bip32derivs = request.params[4].isNull() ? true : request.params[4].get_bool(); @@ -1736,13 +1730,13 @@ RPCHelpMan walletcreatefundedpsbt() } // Serialize the PSBT - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssTx{}; ssTx << psbtx; UniValue result(UniValue::VOBJ); result.pushKV("psbt", EncodeBase64(ssTx.str())); - result.pushKV("fee", ValueFromAmount(fee)); - result.pushKV("changepos", change_position); + result.pushKV("fee", ValueFromAmount(txr.fee)); + result.pushKV("changepos", txr.change_pos ? (int)*txr.change_pos : -1); return result; }, }; diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index c34391e6e8..0e30d3e0fe 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -783,8 +783,7 @@ RPCHelpMan gettransaction() ListTransactions(*pwallet, wtx, 0, false, details, filter, /*filter_label=*/std::nullopt); entry.pushKV("details", details); - std::string strHex = EncodeHexTx(*wtx.tx, pwallet->chain().rpcSerializationFlags()); - entry.pushKV("hex", strHex); + entry.pushKV("hex", EncodeHexTx(*wtx.tx)); if (verbose) { UniValue decoded(UniValue::VOBJ); diff --git a/src/wallet/rpc/wallet.cpp b/src/wallet/rpc/wallet.cpp index 164ce9afed..391153a2a1 100644 --- a/src/wallet/rpc/wallet.cpp +++ b/src/wallet/rpc/wallet.cpp @@ -69,6 +69,7 @@ static RPCHelpMan getwalletinfo() {RPCResult::Type::BOOL, "descriptors", "whether this wallet uses descriptors for scriptPubKey management"}, {RPCResult::Type::BOOL, "external_signer", "whether this wallet is configured to use an external signer such as a hardware wallet"}, {RPCResult::Type::BOOL, "blank", "Whether this wallet intentionally does not contain any keys, scripts, or descriptors"}, + {RPCResult::Type::NUM_TIME, "birthtime", /*optional=*/true, "The start time for blocks scanning. It could be modified by (re)importing any descriptor with an earlier timestamp."}, RESULT_LAST_PROCESSED_BLOCK, }}, }, @@ -132,6 +133,9 @@ static RPCHelpMan getwalletinfo() obj.pushKV("descriptors", pwallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); obj.pushKV("external_signer", pwallet->IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER)); obj.pushKV("blank", pwallet->IsWalletFlagSet(WALLET_FLAG_BLANK_WALLET)); + if (int64_t birthtime = pwallet->GetBirthTime(); birthtime != UNKNOWN_TIME) { + obj.pushKV("birthtime", birthtime); + } AppendLastProcessedBlock(obj, *pwallet); return obj; @@ -165,7 +169,7 @@ static RPCHelpMan listwalletdir() UniValue wallets(UniValue::VARR); for (const auto& path : ListDatabases(GetWalletDir())) { UniValue wallet(UniValue::VOBJ); - wallet.pushKV("name", path.u8string()); + wallet.pushKV("name", path.utf8string()); wallets.push_back(wallet); } @@ -802,7 +806,7 @@ static RPCHelpMan migratewallet() if (res->solvables_wallet) { r.pushKV("solvables_name", res->solvables_wallet->GetName()); } - r.pushKV("backup_path", res->backup_path.u8string()); + r.pushKV("backup_path", res->backup_path.utf8string()); return r; }, diff --git a/src/wallet/scriptpubkeyman.cpp b/src/wallet/scriptpubkeyman.cpp index bc3327cdb2..8f0b72c2b0 100644 --- a/src/wallet/scriptpubkeyman.cpp +++ b/src/wallet/scriptpubkeyman.cpp @@ -333,7 +333,8 @@ bool LegacyScriptPubKeyMan::TopUpInactiveHDChain(const CKeyID seed_id, int64_t i chain.m_next_external_index = std::max(chain.m_next_external_index, index + 1); } - TopUpChain(chain, 0); + WalletBatch batch(m_storage.GetDatabase()); + TopUpChain(batch, chain, 0); return true; } @@ -709,7 +710,7 @@ void LegacyScriptPubKeyMan::UpdateTimeFirstKey(int64_t nCreateTime) // Cannot determine birthday information, so set the wallet birthday to // the beginning of time. nTimeFirstKey = 1; - } else if (!nTimeFirstKey || nCreateTime < nTimeFirstKey) { + } else if (nTimeFirstKey == UNKNOWN_TIME || nCreateTime < nTimeFirstKey) { nTimeFirstKey = nCreateTime; } @@ -1182,8 +1183,7 @@ bool LegacyScriptPubKeyMan::CanGenerateKeys() const CPubKey LegacyScriptPubKeyMan::GenerateNewSeed() { assert(!m_storage.IsWalletFlagSet(WALLET_FLAG_DISABLE_PRIVATE_KEYS)); - CKey key; - key.MakeNewKey(true); + CKey key = GenerateRandomKey(); return DeriveNewSeed(key); } @@ -1274,19 +1274,22 @@ bool LegacyScriptPubKeyMan::TopUp(unsigned int kpSize) return false; } - if (!TopUpChain(m_hd_chain, kpSize)) { + WalletBatch batch(m_storage.GetDatabase()); + if (!batch.TxnBegin()) return false; + if (!TopUpChain(batch, m_hd_chain, kpSize)) { return false; } for (auto& [chain_id, chain] : m_inactive_hd_chains) { - if (!TopUpChain(chain, kpSize)) { + if (!TopUpChain(batch, chain, kpSize)) { return false; } } + if (!batch.TxnCommit()) throw std::runtime_error(strprintf("Error during keypool top up. Cannot commit changes for wallet %s", m_storage.GetDisplayName())); NotifyCanGetAddressesChanged(); return true; } -bool LegacyScriptPubKeyMan::TopUpChain(CHDChain& chain, unsigned int kpSize) +bool LegacyScriptPubKeyMan::TopUpChain(WalletBatch& batch, CHDChain& chain, unsigned int kpSize) { LOCK(cs_KeyStore); @@ -1318,7 +1321,6 @@ bool LegacyScriptPubKeyMan::TopUpChain(CHDChain& chain, unsigned int kpSize) missingInternal = 0; } bool internal = false; - WalletBatch batch(m_storage.GetDatabase()); for (int64_t i = missingInternal + missingExternal; i--;) { if (i < missingInternal) { internal = true; @@ -2136,6 +2138,15 @@ std::map<CKeyID, CKey> DescriptorScriptPubKeyMan::GetKeys() const bool DescriptorScriptPubKeyMan::TopUp(unsigned int size) { + WalletBatch batch(m_storage.GetDatabase()); + if (!batch.TxnBegin()) return false; + bool res = TopUpWithDB(batch, size); + if (!batch.TxnCommit()) throw std::runtime_error(strprintf("Error during descriptors keypool top up. Cannot commit changes for wallet %s", m_storage.GetDisplayName())); + return res; +} + +bool DescriptorScriptPubKeyMan::TopUpWithDB(WalletBatch& batch, unsigned int size) +{ LOCK(cs_desc_man); unsigned int target_size; if (size > 0) { @@ -2157,7 +2168,6 @@ bool DescriptorScriptPubKeyMan::TopUp(unsigned int size) FlatSigningProvider provider; provider.keys = GetKeys(); - WalletBatch batch(m_storage.GetDatabase()); uint256 id = GetID(); for (int32_t i = m_max_cached_index + 1; i < new_range_end; ++i) { FlatSigningProvider out_keys; @@ -2264,7 +2274,7 @@ bool DescriptorScriptPubKeyMan::AddDescriptorKeyWithDB(WalletBatch& batch, const } } -bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_key, OutputType addr_type, bool internal) +bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(WalletBatch& batch, const CExtKey& master_key, OutputType addr_type, bool internal) { LOCK(cs_desc_man); assert(m_storage.IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); @@ -2325,7 +2335,6 @@ bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_ m_wallet_descriptor = w_desc; // Store the master private key, and descriptor - WalletBatch batch(m_storage.GetDatabase()); if (!AddDescriptorKeyWithDB(batch, master_key.key, master_key.key.GetPubKey())) { throw std::runtime_error(std::string(__func__) + ": writing descriptor master private key failed"); } @@ -2334,7 +2343,7 @@ bool DescriptorScriptPubKeyMan::SetupDescriptorGeneration(const CExtKey& master_ } // TopUp - TopUp(); + TopUpWithDB(batch); m_storage.UnsetBlankWalletFlag(batch); return true; @@ -2601,7 +2610,7 @@ std::unique_ptr<CKeyMetadata> DescriptorScriptPubKeyMan::GetMetadata(const CTxDe uint256 DescriptorScriptPubKeyMan::GetID() const { LOCK(cs_desc_man); - return DescriptorID(*m_wallet_descriptor.descriptor); + return m_wallet_descriptor.id; } void DescriptorScriptPubKeyMan::SetCache(const DescriptorCache& cache) @@ -2655,7 +2664,7 @@ bool DescriptorScriptPubKeyMan::AddCryptedKey(const CKeyID& key_id, const CPubKe bool DescriptorScriptPubKeyMan::HasWalletDescriptor(const WalletDescriptor& desc) const { LOCK(cs_desc_man); - return m_wallet_descriptor.descriptor != nullptr && desc.descriptor != nullptr && m_wallet_descriptor.descriptor->ToString() == desc.descriptor->ToString(); + return !m_wallet_descriptor.id.IsNull() && !desc.id.IsNull() && m_wallet_descriptor.id == desc.id; } void DescriptorScriptPubKeyMan::WriteDescriptor() diff --git a/src/wallet/scriptpubkeyman.h b/src/wallet/scriptpubkeyman.h index 7c0eca1475..449a75eb6b 100644 --- a/src/wallet/scriptpubkeyman.h +++ b/src/wallet/scriptpubkeyman.h @@ -51,6 +51,9 @@ public: virtual bool IsLocked() const = 0; }; +//! Constant representing an unknown spkm creation time +static constexpr int64_t UNKNOWN_TIME = std::numeric_limits<int64_t>::max(); + //! Default for -keypool static const unsigned int DEFAULT_KEYPOOL_SIZE = 1000; @@ -286,7 +289,8 @@ private: WatchOnlySet setWatchOnly GUARDED_BY(cs_KeyStore); WatchKeyMap mapWatchKeys GUARDED_BY(cs_KeyStore); - int64_t nTimeFirstKey GUARDED_BY(cs_KeyStore) = 0; + // By default, do not scan any block until keys/scripts are generated/imported + int64_t nTimeFirstKey GUARDED_BY(cs_KeyStore) = UNKNOWN_TIME; //! Number of pre-generated keys/scripts (part of the look-ahead process, used to detect payments) int64_t m_keypool_size GUARDED_BY(cs_KeyStore){DEFAULT_KEYPOOL_SIZE}; @@ -366,7 +370,7 @@ private: */ bool TopUpInactiveHDChain(const CKeyID seed_id, int64_t index, bool internal); - bool TopUpChain(CHDChain& chain, unsigned int size); + bool TopUpChain(WalletBatch& batch, CHDChain& chain, unsigned int size); public: LegacyScriptPubKeyMan(WalletStorage& storage, int64_t keypool_size) : ScriptPubKeyMan(storage), m_keypool_size(keypool_size) {} @@ -583,7 +587,10 @@ private: std::unique_ptr<FlatSigningProvider> GetSigningProvider(int32_t index, bool include_private = false) const EXCLUSIVE_LOCKS_REQUIRED(cs_desc_man); protected: - WalletDescriptor m_wallet_descriptor GUARDED_BY(cs_desc_man); + WalletDescriptor m_wallet_descriptor GUARDED_BY(cs_desc_man); + + //! Same as 'TopUp' but designed for use within a batch transaction context + bool TopUpWithDB(WalletBatch& batch, unsigned int size = 0); public: DescriptorScriptPubKeyMan(WalletStorage& storage, WalletDescriptor& descriptor, int64_t keypool_size) @@ -618,12 +625,7 @@ public: bool IsHDEnabled() const override; //! Setup descriptors based on the given CExtkey - bool SetupDescriptorGeneration(const CExtKey& master_key, OutputType addr_type, bool internal); - - /** Provide a descriptor at setup time - * Returns false if already setup or setup fails, true if setup is successful - */ - bool SetupDescriptor(std::unique_ptr<Descriptor>desc); + bool SetupDescriptorGeneration(WalletBatch& batch, const CExtKey& master_key, OutputType addr_type, bool internal); bool HavePrivateKeys() const override; diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index 8314a2ddfa..b51cd6332f 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -117,8 +117,9 @@ static std::optional<int64_t> GetSignedTxinWeight(const CWallet* wallet, const C const bool can_grind_r) { // If weight was provided, use that. - if (coin_control && coin_control->HasInputWeight(txin.prevout)) { - return coin_control->GetInputWeight(txin.prevout); + std::optional<int64_t> weight; + if (coin_control && (weight = coin_control->GetInputWeight(txin.prevout))) { + return weight.value(); } // Otherwise, use the maximum satisfaction size provided by the descriptor. @@ -259,9 +260,12 @@ util::Result<PreSelectedInputs> FetchSelectedInputs(const CWallet& wallet, const { PreSelectedInputs result; const bool can_grind_r = wallet.CanGrindR(); - std::map<COutPoint, CAmount> map_of_bump_fees = wallet.chain().CalculateIndividualBumpFees(coin_control.ListSelected(), coin_selection_params.m_effective_feerate); + std::map<COutPoint, CAmount> map_of_bump_fees = wallet.chain().calculateIndividualBumpFees(coin_control.ListSelected(), coin_selection_params.m_effective_feerate); for (const COutPoint& outpoint : coin_control.ListSelected()) { - int input_bytes = -1; + int64_t input_bytes = coin_control.GetInputWeight(outpoint).value_or(-1); + if (input_bytes != -1) { + input_bytes = GetVirtualTransactionSize(input_bytes, 0, 0); + } CTxOut txout; if (auto ptr_wtx = wallet.GetWalletTx(outpoint.hash)) { // Clearly invalid input, fail @@ -269,7 +273,9 @@ util::Result<PreSelectedInputs> FetchSelectedInputs(const CWallet& wallet, const return util::Error{strprintf(_("Invalid pre-selected input %s"), outpoint.ToString())}; } txout = ptr_wtx->tx->vout.at(outpoint.n); - input_bytes = CalculateMaximumSignedInputSize(txout, &wallet, &coin_control); + if (input_bytes == -1) { + input_bytes = CalculateMaximumSignedInputSize(txout, &wallet, &coin_control); + } } else { // The input is external. We did not find the tx in mapWallet. const auto out{coin_control.GetExternalOutput(outpoint)}; @@ -284,11 +290,6 @@ util::Result<PreSelectedInputs> FetchSelectedInputs(const CWallet& wallet, const input_bytes = CalculateMaximumSignedInputSize(txout, outpoint, &coin_control.m_external_provider, can_grind_r, &coin_control); } - // If available, override calculated size with coin control specified size - if (coin_control.HasInputWeight(outpoint)) { - input_bytes = GetVirtualTransactionSize(coin_control.GetInputWeight(outpoint), 0, 0); - } - if (input_bytes == -1) { return util::Error{strprintf(_("Not solvable pre-selected input %s"), outpoint.ToString())}; // Not solvable, can't estimate size for fee } @@ -321,7 +322,7 @@ CoinsResult AvailableCoins(const CWallet& wallet, std::set<uint256> trusted_parents; for (const auto& entry : wallet.mapWallet) { - const uint256& wtxid = entry.first; + const uint256& txid = entry.first; const CWalletTx& wtx = entry.second; if (wallet.IsTxImmatureCoinBase(wtx) && !params.include_immature_coinbase) @@ -381,7 +382,7 @@ CoinsResult AvailableCoins(const CWallet& wallet, for (unsigned int i = 0; i < wtx.tx->vout.size(); i++) { const CTxOut& output = wtx.tx->vout[i]; - const COutPoint outpoint(wtxid, i); + const COutPoint outpoint(Txid::FromUint256(txid), i); if (output.nValue < params.min_amount || output.nValue > params.max_amount) continue; @@ -453,7 +454,7 @@ CoinsResult AvailableCoins(const CWallet& wallet, } if (feerate.has_value()) { - std::map<COutPoint, CAmount> map_of_bump_fees = wallet.chain().CalculateIndividualBumpFees(outpoints, feerate.value()); + std::map<COutPoint, CAmount> map_of_bump_fees = wallet.chain().calculateIndividualBumpFees(outpoints, feerate.value()); for (auto& [_, outputs] : result.coins) { for (auto& output : outputs) { @@ -693,9 +694,12 @@ util::Result<SelectionResult> ChooseSelectionResult(interfaces::Chain& chain, co // Maximum allowed weight int max_inputs_weight = MAX_STANDARD_TX_WEIGHT - (coin_selection_params.tx_noinputs_size * WITNESS_SCALE_FACTOR); - if (auto bnb_result{SelectCoinsBnB(groups.positive_group, nTargetValue, coin_selection_params.m_cost_of_change, max_inputs_weight)}) { - results.push_back(*bnb_result); - } else append_error(bnb_result); + // SFFO frequently causes issues in the context of changeless input sets: skip BnB when SFFO is active + if (!coin_selection_params.m_subtract_fee_outputs) { + if (auto bnb_result{SelectCoinsBnB(groups.positive_group, nTargetValue, coin_selection_params.m_cost_of_change, max_inputs_weight)}) { + results.push_back(*bnb_result); + } else append_error(bnb_result); + } // As Knapsack and SRD can create change, also deduce change weight. max_inputs_weight -= (coin_selection_params.change_output_size * WITNESS_SCALE_FACTOR); @@ -725,7 +729,7 @@ util::Result<SelectionResult> ChooseSelectionResult(interfaces::Chain& chain, co outpoints.push_back(coin->outpoint); summed_bump_fees += coin->ancestor_bump_fees; } - std::optional<CAmount> combined_bump_fee = chain.CalculateCombinedBumpFee(outpoints, coin_selection_params.m_effective_feerate); + std::optional<CAmount> combined_bump_fee = chain.calculateCombinedBumpFee(outpoints, coin_selection_params.m_effective_feerate); if (!combined_bump_fee.has_value()) { return util::Error{_("Failed to calculate bump fees, because unconfirmed UTXOs depend on enormous cluster of unconfirmed transactions.")}; } @@ -964,18 +968,19 @@ static void DiscourageFeeSniping(CMutableTransaction& tx, FastRandomContext& rng static util::Result<CreatedTransactionResult> CreateTransactionInternal( CWallet& wallet, const std::vector<CRecipient>& vecSend, - int change_pos, + std::optional<unsigned int> change_pos, const CCoinControl& coin_control, bool sign) EXCLUSIVE_LOCKS_REQUIRED(wallet.cs_wallet) { AssertLockHeld(wallet.cs_wallet); - // out variables, to be packed into returned result structure - int nChangePosInOut = change_pos; - FastRandomContext rng_fast; CMutableTransaction txNew; // The resulting transaction that we make + if (coin_control.m_version) { + txNew.nVersion = coin_control.m_version.value(); + } + CoinSelectionParams coin_selection_params{rng_fast}; // Parameters for coin selection, init with dummy coin_selection_params.m_avoid_partial_spends = coin_control.m_avoid_partial_spends; coin_selection_params.m_include_unsafe_inputs = coin_control.m_include_unsafe_inputs; @@ -1081,7 +1086,7 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( CTxOut txout(recipient.nAmount, GetScriptForDestination(recipient.dest)); // Include the fee cost for outputs. - coin_selection_params.tx_noinputs_size += ::GetSerializeSize(txout, PROTOCOL_VERSION); + coin_selection_params.tx_noinputs_size += ::GetSerializeSize(txout); if (IsDust(txout, wallet.chain().relayDustFee())) { return util::Error{_("Transaction amount too small")}; @@ -1127,20 +1132,39 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( const CAmount change_amount = result.GetChange(coin_selection_params.min_viable_change, coin_selection_params.m_change_fee); if (change_amount > 0) { CTxOut newTxOut(change_amount, scriptChange); - if (nChangePosInOut == -1) { + if (!change_pos) { // Insert change txn at random position: - nChangePosInOut = rng_fast.randrange(txNew.vout.size() + 1); - } else if ((unsigned int)nChangePosInOut > txNew.vout.size()) { + change_pos = rng_fast.randrange(txNew.vout.size() + 1); + } else if ((unsigned int)*change_pos > txNew.vout.size()) { return util::Error{_("Transaction change output index out of range")}; } - txNew.vout.insert(txNew.vout.begin() + nChangePosInOut, newTxOut); + txNew.vout.insert(txNew.vout.begin() + *change_pos, newTxOut); } else { - nChangePosInOut = -1; + change_pos = std::nullopt; } // Shuffle selected coins and fill in final vin std::vector<std::shared_ptr<COutput>> selected_coins = result.GetShuffledInputVector(); + if (coin_control.HasSelected() && coin_control.HasSelectedOrder()) { + // When there are preselected inputs, we need to move them to be the first UTXOs + // and have them be in the order selected. We can use stable_sort for this, where we + // compare with the positions stored in coin_control. The COutputs that have positions + // will be placed before those that don't, and those positions will be in order. + std::stable_sort(selected_coins.begin(), selected_coins.end(), + [&coin_control](const std::shared_ptr<COutput>& a, const std::shared_ptr<COutput>& b) { + auto a_pos = coin_control.GetSelectionPos(a->outpoint); + auto b_pos = coin_control.GetSelectionPos(b->outpoint); + if (a_pos.has_value() && b_pos.has_value()) { + return a_pos.value() < b_pos.value(); + } else if (a_pos.has_value() && !b_pos.has_value()) { + return true; + } else { + return false; + } + }); + } + // The sequence number is set to non-maxint so that DiscourageFeeSniping // works. // @@ -1149,11 +1173,32 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( // to avoid conflicting with other possible uses of nSequence, // and in the spirit of "smallest possible change from prior // behavior." - const uint32_t nSequence{coin_control.m_signal_bip125_rbf.value_or(wallet.m_signal_rbf) ? MAX_BIP125_RBF_SEQUENCE : CTxIn::MAX_SEQUENCE_NONFINAL}; + bool use_anti_fee_sniping = true; + const uint32_t default_sequence{coin_control.m_signal_bip125_rbf.value_or(wallet.m_signal_rbf) ? MAX_BIP125_RBF_SEQUENCE : CTxIn::MAX_SEQUENCE_NONFINAL}; for (const auto& coin : selected_coins) { - txNew.vin.emplace_back(coin->outpoint, CScript(), nSequence); + std::optional<uint32_t> sequence = coin_control.GetSequence(coin->outpoint); + if (sequence) { + // If an input has a preset sequence, we can't do anti-fee-sniping + use_anti_fee_sniping = false; + } + txNew.vin.emplace_back(coin->outpoint, CScript{}, sequence.value_or(default_sequence)); + + auto scripts = coin_control.GetScripts(coin->outpoint); + if (scripts.first) { + txNew.vin.back().scriptSig = *scripts.first; + } + if (scripts.second) { + txNew.vin.back().scriptWitness = *scripts.second; + } + } + if (coin_control.m_locktime) { + txNew.nLockTime = coin_control.m_locktime.value(); + // If we have a locktime set, we can't use anti-fee-sniping + use_anti_fee_sniping = false; + } + if (use_anti_fee_sniping) { + DiscourageFeeSniping(txNew, rng_fast, wallet.chain(), wallet.GetLastBlockHash(), wallet.GetLastBlockHeight()); } - DiscourageFeeSniping(txNew, rng_fast, wallet.chain(), wallet.GetLastBlockHash(), wallet.GetLastBlockHeight()); // Calculate the transaction fee TxSize tx_sizes = CalculateMaximumSignedTxSize(CTransaction(txNew), &wallet, &coin_control); @@ -1172,8 +1217,8 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( } // If there is a change output and we overpay the fees then increase the change to match the fee needed - if (nChangePosInOut != -1 && fee_needed < current_fee) { - auto& change = txNew.vout.at(nChangePosInOut); + if (change_pos && fee_needed < current_fee) { + auto& change = txNew.vout.at(*change_pos); change.nValue += current_fee - fee_needed; current_fee = result.GetSelectedValue() - CalculateOutputValue(txNew); if (fee_needed != current_fee) { @@ -1184,11 +1229,11 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( // Reduce output values for subtractFeeFromAmount if (coin_selection_params.m_subtract_fee_outputs) { CAmount to_reduce = fee_needed - current_fee; - int i = 0; + unsigned int i = 0; bool fFirst = true; for (const auto& recipient : vecSend) { - if (i == nChangePosInOut) { + if (change_pos && i == *change_pos) { ++i; } CTxOut& txout = txNew.vout[i]; @@ -1227,7 +1272,7 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( } // Give up if change keypool ran out and change is required - if (scriptChange.empty() && nChangePosInOut != -1) { + if (scriptChange.empty() && change_pos) { return util::Error{error}; } @@ -1251,15 +1296,17 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( if (gArgs.GetBoolArg("-walletrejectlongchains", DEFAULT_WALLET_REJECT_LONG_CHAINS)) { // Lastly, ensure this tx will pass the mempool's chain limits - if (!wallet.chain().checkChainLimits(tx)) { - return util::Error{_("Transaction has too long of a mempool chain")}; + auto result = wallet.chain().checkChainLimits(tx); + if (!result) { + return util::Error{util::ErrorString(result)}; } } // Before we return success, we assume any change key will be used to prevent - // accidental re-use. + // accidental reuse. reservedest.KeepDestination(); + wallet.WalletLogPrintf("Coin Selection: Algorithm:%s, Waste Metric Score:%d\n", GetAlgorithmName(result.GetAlgo()), result.GetWaste()); wallet.WalletLogPrintf("Fee Calculation: Fee:%d Bytes:%u Tgt:%d (requested %d) Reason:\"%s\" Decay %.5f: Estimation: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out) Fail: (%g - %g) %.2f%% %.1f/(%.1f %d mem %.1f out)\n", current_fee, nBytes, feeCalc.returnedTarget, feeCalc.desiredTarget, StringForFeeReason(feeCalc.reason), feeCalc.est.decay, feeCalc.est.pass.start, feeCalc.est.pass.end, @@ -1268,13 +1315,13 @@ static util::Result<CreatedTransactionResult> CreateTransactionInternal( feeCalc.est.fail.start, feeCalc.est.fail.end, (feeCalc.est.fail.totalConfirmed + feeCalc.est.fail.inMempool + feeCalc.est.fail.leftMempool) > 0.0 ? 100 * feeCalc.est.fail.withinTarget / (feeCalc.est.fail.totalConfirmed + feeCalc.est.fail.inMempool + feeCalc.est.fail.leftMempool) : 0.0, feeCalc.est.fail.withinTarget, feeCalc.est.fail.totalConfirmed, feeCalc.est.fail.inMempool, feeCalc.est.fail.leftMempool); - return CreatedTransactionResult(tx, current_fee, nChangePosInOut, feeCalc); + return CreatedTransactionResult(tx, current_fee, change_pos, feeCalc); } util::Result<CreatedTransactionResult> CreateTransaction( CWallet& wallet, const std::vector<CRecipient>& vecSend, - int change_pos, + std::optional<unsigned int> change_pos, const CCoinControl& coin_control, bool sign) { @@ -1290,7 +1337,7 @@ util::Result<CreatedTransactionResult> CreateTransaction( auto res = CreateTransactionInternal(wallet, vecSend, change_pos, coin_control, sign); TRACE4(coin_selection, normal_create_tx_internal, wallet.GetName().c_str(), bool(res), - res ? res->fee : 0, res ? res->change_pos : 0); + res ? res->fee : 0, res && res->change_pos.has_value() ? *res->change_pos : 0); if (!res) return res; const auto& txr_ungrouped = *res; // try with avoidpartialspends unless it's enabled already @@ -1299,17 +1346,16 @@ util::Result<CreatedTransactionResult> CreateTransaction( CCoinControl tmp_cc = coin_control; tmp_cc.m_avoid_partial_spends = true; - // Re-use the change destination from the first creation attempt to avoid skipping BIP44 indexes - const int ungrouped_change_pos = txr_ungrouped.change_pos; - if (ungrouped_change_pos != -1) { - ExtractDestination(txr_ungrouped.tx->vout[ungrouped_change_pos].scriptPubKey, tmp_cc.destChange); + // Reuse the change destination from the first creation attempt to avoid skipping BIP44 indexes + if (txr_ungrouped.change_pos) { + ExtractDestination(txr_ungrouped.tx->vout[*txr_ungrouped.change_pos].scriptPubKey, tmp_cc.destChange); } auto txr_grouped = CreateTransactionInternal(wallet, vecSend, change_pos, tmp_cc, sign); // if fee of this alternative one is within the range of the max fee, we use this one const bool use_aps{txr_grouped.has_value() ? (txr_grouped->fee <= txr_ungrouped.fee + wallet.m_max_aps_fee) : false}; TRACE5(coin_selection, aps_create_tx_internal, wallet.GetName().c_str(), use_aps, txr_grouped.has_value(), - txr_grouped.has_value() ? txr_grouped->fee : 0, txr_grouped.has_value() ? txr_grouped->change_pos : 0); + txr_grouped.has_value() ? txr_grouped->fee : 0, txr_grouped.has_value() && txr_grouped->change_pos.has_value() ? *txr_grouped->change_pos : 0); if (txr_grouped) { wallet.WalletLogPrintf("Fee non-grouped = %lld, grouped = %lld, using %s\n", txr_ungrouped.fee, txr_grouped->fee, use_aps ? "grouped" : "non-grouped"); @@ -1319,7 +1365,7 @@ util::Result<CreatedTransactionResult> CreateTransaction( return res; } -bool FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, CCoinControl coinControl) +util::Result<CreatedTransactionResult> FundTransaction(CWallet& wallet, const CMutableTransaction& tx, std::optional<unsigned int> change_pos, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, CCoinControl coinControl) { std::vector<CRecipient> vecSend; @@ -1332,6 +1378,12 @@ bool FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& nFeeRet, vecSend.push_back(recipient); } + // Set the user desired locktime + coinControl.m_locktime = tx.nLockTime; + + // Set the user desired version + coinControl.m_version = tx.nVersion; + // Acquire the locks to prevent races to the new locked unspents between the // CreateTransaction call and LockCoin calls (when lockUnspents is true). LOCK(wallet.cs_wallet); @@ -1346,50 +1398,31 @@ bool FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& nFeeRet, for (const CTxIn& txin : tx.vin) { const auto& outPoint = txin.prevout; - if (wallet.IsMine(outPoint)) { - // The input was found in the wallet, so select as internal - coinControl.Select(outPoint); - } else if (coins[outPoint].out.IsNull()) { - error = _("Unable to find UTXO for external input"); - return false; - } else { + PreselectedInput& preset_txin = coinControl.Select(outPoint); + if (!wallet.IsMine(outPoint)) { + if (coins[outPoint].out.IsNull()) { + return util::Error{_("Unable to find UTXO for external input")}; + } + // The input was not in the wallet, but is in the UTXO set, so select as external - coinControl.SelectExternal(outPoint, coins[outPoint].out); + preset_txin.SetTxOut(coins[outPoint].out); } + preset_txin.SetSequence(txin.nSequence); + preset_txin.SetScriptSig(txin.scriptSig); + preset_txin.SetScriptWitness(txin.scriptWitness); } - auto res = CreateTransaction(wallet, vecSend, nChangePosInOut, coinControl, false); + auto res = CreateTransaction(wallet, vecSend, change_pos, coinControl, false); if (!res) { - error = util::ErrorString(res); - return false; - } - const auto& txr = *res; - CTransactionRef tx_new = txr.tx; - nFeeRet = txr.fee; - nChangePosInOut = txr.change_pos; - - if (nChangePosInOut != -1) { - tx.vout.insert(tx.vout.begin() + nChangePosInOut, tx_new->vout[nChangePosInOut]); - } - - // Copy output sizes from new transaction; they may have had the fee - // subtracted from them. - for (unsigned int idx = 0; idx < tx.vout.size(); idx++) { - tx.vout[idx].nValue = tx_new->vout[idx].nValue; + return res; } - // Add new txins while keeping original txin scriptSig/order. - for (const CTxIn& txin : tx_new->vin) { - if (!coinControl.IsSelected(txin.prevout)) { - tx.vin.push_back(txin); - - } - if (lockUnspents) { + if (lockUnspents) { + for (const CTxIn& txin : res->tx->vin) { wallet.LockCoin(txin.prevout); } - } - return true; + return res; } } // namespace wallet diff --git a/src/wallet/spend.h b/src/wallet/spend.h index 407627b5f1..3bd37cfd0e 100644 --- a/src/wallet/spend.h +++ b/src/wallet/spend.h @@ -207,24 +207,24 @@ struct CreatedTransactionResult CTransactionRef tx; CAmount fee; FeeCalculation fee_calc; - int change_pos; + std::optional<unsigned int> change_pos; - CreatedTransactionResult(CTransactionRef _tx, CAmount _fee, int _change_pos, const FeeCalculation& _fee_calc) + CreatedTransactionResult(CTransactionRef _tx, CAmount _fee, std::optional<unsigned int> _change_pos, const FeeCalculation& _fee_calc) : tx(_tx), fee(_fee), fee_calc(_fee_calc), change_pos(_change_pos) {} }; /** * Create a new transaction paying the recipients with a set of coins * selected by SelectCoins(); Also create the change output, when needed - * @note passing change_pos as -1 will result in setting a random position + * @note passing change_pos as std::nullopt will result in setting a random position */ -util::Result<CreatedTransactionResult> CreateTransaction(CWallet& wallet, const std::vector<CRecipient>& vecSend, int change_pos, const CCoinControl& coin_control, bool sign = true); +util::Result<CreatedTransactionResult> CreateTransaction(CWallet& wallet, const std::vector<CRecipient>& vecSend, std::optional<unsigned int> change_pos, const CCoinControl& coin_control, bool sign = true); /** * Insert additional inputs into the transaction by * calling CreateTransaction(); */ -bool FundTransaction(CWallet& wallet, CMutableTransaction& tx, CAmount& nFeeRet, int& nChangePosInOut, bilingual_str& error, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, CCoinControl); +util::Result<CreatedTransactionResult> FundTransaction(CWallet& wallet, const CMutableTransaction& tx, std::optional<unsigned int> change_pos, bool lockUnspents, const std::set<int>& setSubtractFeeFromOutputs, CCoinControl); } // namespace wallet #endif // BITCOIN_WALLET_SPEND_H diff --git a/src/wallet/test/coinselector_tests.cpp b/src/wallet/test/coinselector_tests.cpp index 9569210ba0..9fea14145f 100644 --- a/src/wallet/test/coinselector_tests.cpp +++ b/src/wallet/test/coinselector_tests.cpp @@ -320,7 +320,6 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) coin_selection_params_bnb.m_change_fee = coin_selection_params_bnb.m_effective_feerate.GetFee(coin_selection_params_bnb.change_output_size); coin_selection_params_bnb.m_cost_of_change = coin_selection_params_bnb.m_effective_feerate.GetFee(coin_selection_params_bnb.change_spend_size) + coin_selection_params_bnb.m_change_fee; coin_selection_params_bnb.min_viable_change = coin_selection_params_bnb.m_effective_feerate.GetFee(coin_selection_params_bnb.change_spend_size); - coin_selection_params_bnb.m_subtract_fee_outputs = true; { std::unique_ptr<CWallet> wallet = NewWallet(m_node); @@ -345,6 +344,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) CoinsResult available_coins; + coin_selection_params_bnb.m_effective_feerate = CFeeRate(0); add_coin(available_coins, *wallet, 5 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); add_coin(available_coins, *wallet, 3 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); add_coin(available_coins, *wallet, 2 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); @@ -355,7 +355,7 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) PreSelectedInputs selected_input; selected_input.Insert(select_coin, coin_selection_params_bnb.m_subtract_fee_outputs); available_coins.Erase({available_coins.coins[OutputType::BECH32].begin()->outpoint}); - coin_selection_params_bnb.m_effective_feerate = CFeeRate(0); + LOCK(wallet->cs_wallet); const auto result10 = SelectCoins(*wallet, available_coins, selected_input, 10 * CENT, coin_control, coin_selection_params_bnb); BOOST_CHECK(result10); @@ -370,12 +370,14 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) coin_selection_params_bnb.m_effective_feerate = CFeeRate(5000); coin_selection_params_bnb.m_long_term_feerate = CFeeRate(3000); - add_coin(available_coins, *wallet, 10 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); - add_coin(available_coins, *wallet, 9 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); - add_coin(available_coins, *wallet, 1 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + // Add selectable outputs, increasing their raw amounts by their input fee to make the effective value equal to the raw amount + CAmount input_fee = coin_selection_params_bnb.m_effective_feerate.GetFee(/*num_bytes=*/68); // bech32 input size (default test output type) + add_coin(available_coins, *wallet, 10 * CENT + input_fee, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + add_coin(available_coins, *wallet, 9 * CENT + input_fee, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + add_coin(available_coins, *wallet, 1 * CENT + input_fee, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); expected_result.Clear(); - add_coin(10 * CENT, 2, expected_result); + add_coin(10 * CENT + input_fee, 2, expected_result); CCoinControl coin_control; const auto result11 = SelectCoins(*wallet, available_coins, /*pre_set_inputs=*/{}, 10 * CENT, coin_control, coin_selection_params_bnb); BOOST_CHECK(EquivalentResult(expected_result, *result11)); @@ -385,13 +387,15 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) coin_selection_params_bnb.m_effective_feerate = CFeeRate(3000); coin_selection_params_bnb.m_long_term_feerate = CFeeRate(5000); - add_coin(available_coins, *wallet, 10 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); - add_coin(available_coins, *wallet, 9 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); - add_coin(available_coins, *wallet, 1 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + // Add selectable outputs, increasing their raw amounts by their input fee to make the effective value equal to the raw amount + input_fee = coin_selection_params_bnb.m_effective_feerate.GetFee(/*num_bytes=*/68); // bech32 input size (default test output type) + add_coin(available_coins, *wallet, 10 * CENT + input_fee, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + add_coin(available_coins, *wallet, 9 * CENT + input_fee, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + add_coin(available_coins, *wallet, 1 * CENT + input_fee, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); expected_result.Clear(); - add_coin(9 * CENT, 2, expected_result); - add_coin(1 * CENT, 2, expected_result); + add_coin(9 * CENT + input_fee, 2, expected_result); + add_coin(1 * CENT + input_fee, 2, expected_result); const auto result12 = SelectCoins(*wallet, available_coins, /*pre_set_inputs=*/{}, 10 * CENT, coin_control, coin_selection_params_bnb); BOOST_CHECK(EquivalentResult(expected_result, *result12)); available_coins.Clear(); @@ -400,13 +404,15 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) coin_selection_params_bnb.m_effective_feerate = CFeeRate(5000); coin_selection_params_bnb.m_long_term_feerate = CFeeRate(3000); - add_coin(available_coins, *wallet, 10 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); - add_coin(available_coins, *wallet, 9 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); - add_coin(available_coins, *wallet, 1 * CENT, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + // Add selectable outputs, increasing their raw amounts by their input fee to make the effective value equal to the raw amount + input_fee = coin_selection_params_bnb.m_effective_feerate.GetFee(/*num_bytes=*/68); // bech32 input size (default test output type) + add_coin(available_coins, *wallet, 10 * CENT + input_fee, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + add_coin(available_coins, *wallet, 9 * CENT + input_fee, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); + add_coin(available_coins, *wallet, 1 * CENT + input_fee, coin_selection_params_bnb.m_effective_feerate, 6 * 24, false, 0, true); expected_result.Clear(); - add_coin(9 * CENT, 2, expected_result); - add_coin(1 * CENT, 2, expected_result); + add_coin(9 * CENT + input_fee, 2, expected_result); + add_coin(1 * CENT + input_fee, 2, expected_result); coin_control.m_allow_other_inputs = true; COutput select_coin = available_coins.All().at(1); // pre select 9 coin coin_control.Select(select_coin.outpoint); @@ -449,6 +455,44 @@ BOOST_AUTO_TEST_CASE(bnb_search_test) } } +BOOST_AUTO_TEST_CASE(bnb_sffo_restriction) +{ + // Verify the coin selection process does not produce a BnB solution when SFFO is enabled. + // This is currently problematic because it could require a change output. And BnB is specialized on changeless solutions. + std::unique_ptr<CWallet> wallet = NewWallet(m_node); + WITH_LOCK(wallet->cs_wallet, wallet->SetLastBlockProcessed(300, uint256{})); // set a high block so internal UTXOs are selectable + + FastRandomContext rand{}; + CoinSelectionParams params{ + rand, + /*change_output_size=*/ 31, // unused value, p2wpkh output size (wallet default change type) + /*change_spend_size=*/ 68, // unused value, p2wpkh input size (high-r signature) + /*min_change_target=*/ 0, // dummy, set later + /*effective_feerate=*/ CFeeRate(3000), + /*long_term_feerate=*/ CFeeRate(1000), + /*discard_feerate=*/ CFeeRate(1000), + /*tx_noinputs_size=*/ 0, + /*avoid_partial=*/ false, + }; + params.m_subtract_fee_outputs = true; + params.m_change_fee = params.m_effective_feerate.GetFee(params.change_output_size); + params.m_cost_of_change = params.m_discard_feerate.GetFee(params.change_spend_size) + params.m_change_fee; + params.m_min_change_target = params.m_cost_of_change + 1; + // Add spendable coin at the BnB selection upper bound + CoinsResult available_coins; + add_coin(available_coins, *wallet, COIN + params.m_cost_of_change, /*feerate=*/params.m_effective_feerate, /*nAge=*/6, /*fIsFromMe=*/true, /*nInput=*/0, /*spendable=*/true); + add_coin(available_coins, *wallet, 0.5 * COIN + params.m_cost_of_change, /*feerate=*/params.m_effective_feerate, /*nAge=*/6, /*fIsFromMe=*/true, /*nInput=*/0, /*spendable=*/true); + add_coin(available_coins, *wallet, 0.5 * COIN, /*feerate=*/params.m_effective_feerate, /*nAge=*/6, /*fIsFromMe=*/true, /*nInput=*/0, /*spendable=*/true); + // Knapsack will only find a changeless solution on an exact match to the satoshi, SRD doesn’t look for changeless + // If BnB were run, it would produce a single input solution with the best waste score + auto result = WITH_LOCK(wallet->cs_wallet, return SelectCoins(*wallet, available_coins, /*pre_set_inputs=*/{}, COIN, /*coin_control=*/{}, params)); + BOOST_CHECK(result.has_value()); + BOOST_CHECK_NE(result->GetAlgo(), SelectionAlgorithm::BNB); + BOOST_CHECK(result->GetInputSet().size() == 2); + // We have only considered BnB, SRD, and Knapsack. Test needs to be reevaluated if new algo is added + BOOST_CHECK(result->GetAlgo() == SelectionAlgorithm::SRD || result->GetAlgo() == SelectionAlgorithm::KNAPSACK); +} + BOOST_AUTO_TEST_CASE(knapsack_solver_test) { FastRandomContext rand{}; @@ -1282,7 +1326,7 @@ BOOST_AUTO_TEST_CASE(SelectCoins_effective_value_test) cc.m_allow_other_inputs = false; COutput output = available_coins.All().at(0); cc.SetInputWeight(output.outpoint, 148); - cc.SelectExternal(output.outpoint, output.txout); + cc.Select(output.outpoint).SetTxOut(output.txout); LOCK(wallet->cs_wallet); const auto preset_inputs = *Assert(FetchSelectedInputs(*wallet, cc, cs_params)); diff --git a/src/wallet/test/feebumper_tests.cpp b/src/wallet/test/feebumper_tests.cpp index 2480a9b4e1..80fd778419 100644 --- a/src/wallet/test/feebumper_tests.cpp +++ b/src/wallet/test/feebumper_tests.cpp @@ -21,7 +21,7 @@ static void CheckMaxWeightComputation(const std::string& script_str, const std:: { std::vector script_data(ParseHex(script_str)); CScript script(script_data.begin(), script_data.end()); - CTxIn input(uint256(), 0, script); + CTxIn input(Txid{}, 0, script); for (const auto& s : witness_str_stack) { input.scriptWitness.stack.push_back(ParseHex(s)); diff --git a/src/wallet/test/fuzz/coincontrol.cpp b/src/wallet/test/fuzz/coincontrol.cpp index 0f71f28df2..f1efbc1cb8 100644 --- a/src/wallet/test/fuzz/coincontrol.cpp +++ b/src/wallet/test/fuzz/coincontrol.cpp @@ -60,7 +60,7 @@ FUZZ_TARGET(coincontrol, .init = initialize_coincontrol) }, [&] { const CTxOut tx_out{ConsumeMoney(fuzzed_data_provider), ConsumeScript(fuzzed_data_provider)}; - (void)coin_control.SelectExternal(out_point, tx_out); + (void)coin_control.Select(out_point).SetTxOut(tx_out); }, [&] { (void)coin_control.UnSelect(out_point); @@ -76,10 +76,7 @@ FUZZ_TARGET(coincontrol, .init = initialize_coincontrol) (void)coin_control.SetInputWeight(out_point, weight); }, [&] { - // Condition to avoid the assertion in GetInputWeight - if (coin_control.HasInputWeight(out_point)) { - (void)coin_control.GetInputWeight(out_point); - } + (void)coin_control.GetInputWeight(out_point); }); } } diff --git a/src/wallet/test/fuzz/coinselection.cpp b/src/wallet/test/fuzz/coinselection.cpp index 4caf96b18d..87d419493b 100644 --- a/src/wallet/test/fuzz/coinselection.cpp +++ b/src/wallet/test/fuzz/coinselection.cpp @@ -86,7 +86,6 @@ FUZZ_TARGET(coinselection) const CFeeRate effective_fee_rate{ConsumeMoney(fuzzed_data_provider, /*max=*/COIN)}; // Discard feerate must be at least dust relay feerate const CFeeRate discard_fee_rate{fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(DUST_RELAY_TX_FEE, COIN)}; - const CAmount min_viable_change{ConsumeMoney(fuzzed_data_provider, /*max=*/COIN)}; const CAmount target{fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(1, MAX_MONEY)}; const bool subtract_fee_outputs{fuzzed_data_provider.ConsumeBool()}; @@ -95,12 +94,15 @@ FUZZ_TARGET(coinselection) coin_params.m_subtract_fee_outputs = subtract_fee_outputs; coin_params.m_long_term_feerate = long_term_fee_rate; coin_params.m_effective_feerate = effective_fee_rate; - coin_params.min_viable_change = min_viable_change; - coin_params.change_output_size = fuzzed_data_provider.ConsumeIntegralInRange<int>(10, 1000); + coin_params.change_output_size = fuzzed_data_provider.ConsumeIntegralInRange(1, MAX_SCRIPT_SIZE); coin_params.m_change_fee = effective_fee_rate.GetFee(coin_params.change_output_size); coin_params.m_discard_feerate = discard_fee_rate; coin_params.change_spend_size = fuzzed_data_provider.ConsumeIntegralInRange<int>(41, 1000); - coin_params.m_cost_of_change = coin_params.m_change_fee + coin_params.m_discard_feerate.GetFee(coin_params.change_spend_size); + const auto change_spend_fee{coin_params.m_discard_feerate.GetFee(coin_params.change_spend_size)}; + coin_params.m_cost_of_change = coin_params.m_change_fee + change_spend_fee; + CScript change_out_script = CScript() << std::vector<unsigned char>(coin_params.change_output_size, OP_TRUE); + const auto dust{GetDustThreshold(CTxOut{/*nValueIn=*/0, change_out_script}, coin_params.m_discard_feerate)}; + coin_params.min_viable_change = std::max(change_spend_fee + 1, dust); int next_locktime{0}; CAmount total_balance{CreateCoins(fuzzed_data_provider, utxo_pool, coin_params, next_locktime)}; @@ -116,9 +118,10 @@ FUZZ_TARGET(coinselection) } // Run coinselection algorithms - auto result_bnb = SelectCoinsBnB(group_pos, target, coin_params.m_cost_of_change, MAX_STANDARD_TX_WEIGHT); + auto result_bnb = coin_params.m_subtract_fee_outputs ? util::Error{Untranslated("BnB disabled when SFFO is enabled")} : + SelectCoinsBnB(group_pos, target, coin_params.m_cost_of_change, MAX_STANDARD_TX_WEIGHT); if (result_bnb) { - assert(result_bnb->GetChange(coin_params.m_cost_of_change, CAmount{0}) == 0); + assert(result_bnb->GetChange(coin_params.min_viable_change, coin_params.m_change_fee) == 0); assert(result_bnb->GetSelectedValue() >= target); (void)result_bnb->GetShuffledInputVector(); (void)result_bnb->GetInputSet(); diff --git a/src/wallet/test/fuzz/fees.cpp b/src/wallet/test/fuzz/fees.cpp index 2f7892dc0a..c2e785651a 100644 --- a/src/wallet/test/fuzz/fees.cpp +++ b/src/wallet/test/fuzz/fees.cpp @@ -37,6 +37,10 @@ FUZZ_TARGET(wallet_fees, .init = initialize_setup) } if (fuzzed_data_provider.ConsumeBool()) { + wallet.m_fallback_fee = CFeeRate{ConsumeMoney(fuzzed_data_provider, /*max=*/COIN)}; + } + + if (fuzzed_data_provider.ConsumeBool()) { wallet.m_discard_rate = CFeeRate{ConsumeMoney(fuzzed_data_provider, /*max=*/COIN)}; } (void)GetDiscardRate(wallet); @@ -58,6 +62,9 @@ FUZZ_TARGET(wallet_fees, .init = initialize_setup) if (fuzzed_data_provider.ConsumeBool()) { coin_control.m_confirm_target = fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(0, 999'000); } + if (fuzzed_data_provider.ConsumeBool()) { + coin_control.m_fee_mode = fuzzed_data_provider.ConsumeBool() ? FeeEstimateMode::CONSERVATIVE : FeeEstimateMode::ECONOMICAL; + } FeeCalculation fee_calculation; FeeCalculation* maybe_fee_calculation{fuzzed_data_provider.ConsumeBool() ? nullptr : &fee_calculation}; diff --git a/src/wallet/test/fuzz/notifications.cpp b/src/wallet/test/fuzz/notifications.cpp index abd788f96f..203ab5f606 100644 --- a/src/wallet/test/fuzz/notifications.cpp +++ b/src/wallet/test/fuzz/notifications.cpp @@ -1,22 +1,47 @@ -// Copyright (c) 2021-2022 The Bitcoin Core developers +// Copyright (c) 2021-present The Bitcoin Core developers // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. +#include <addresstype.h> +#include <common/args.h> +#include <consensus/amount.h> +#include <interfaces/chain.h> #include <kernel/chain.h> +#include <outputtype.h> +#include <policy/feerate.h> +#include <policy/policy.h> +#include <primitives/block.h> +#include <primitives/transaction.h> +#include <script/descriptor.h> +#include <script/script.h> +#include <script/signingprovider.h> +#include <sync.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <test/util/setup_common.h> +#include <tinyformat.h> +#include <uint256.h> +#include <util/check.h> +#include <util/result.h> #include <util/translation.h> +#include <wallet/coincontrol.h> #include <wallet/context.h> +#include <wallet/fees.h> #include <wallet/receive.h> +#include <wallet/spend.h> +#include <wallet/test/util.h> #include <wallet/wallet.h> -#include <wallet/walletdb.h> #include <wallet/walletutil.h> -#include <cassert> +#include <cstddef> #include <cstdint> +#include <limits> +#include <numeric> +#include <set> #include <string> +#include <tuple> +#include <utility> #include <vector> namespace wallet { @@ -29,45 +54,59 @@ void initialize_setup() g_setup = testing_setup.get(); } +void ImportDescriptors(CWallet& wallet, const std::string& seed_insecure) +{ + const std::vector<std::string> DESCS{ + "pkh(%s/%s/*)", + "sh(wpkh(%s/%s/*))", + "tr(%s/%s/*)", + "wpkh(%s/%s/*)", + }; + + for (const std::string& desc_fmt : DESCS) { + for (bool internal : {true, false}) { + const auto descriptor{(strprintf)(desc_fmt, "[5aa9973a/66h/4h/2h]" + seed_insecure, int{internal})}; + + FlatSigningProvider keys; + std::string error; + auto parsed_desc = Parse(descriptor, keys, error, /*require_checksum=*/false); + assert(parsed_desc); + assert(error.empty()); + assert(parsed_desc->IsRange()); + assert(parsed_desc->IsSingleType()); + assert(!keys.keys.empty()); + WalletDescriptor w_desc{std::move(parsed_desc), /*creation_time=*/0, /*range_start=*/0, /*range_end=*/1, /*next_index=*/0}; + assert(!wallet.GetDescriptorScriptPubKeyMan(w_desc)); + LOCK(wallet.cs_wallet); + auto spk_manager{wallet.AddWalletDescriptor(w_desc, keys, /*label=*/"", internal)}; + assert(spk_manager); + wallet.AddActiveScriptPubKeyMan(spk_manager->GetID(), *Assert(w_desc.descriptor->GetOutputType()), internal); + } + } +} + /** - * Wraps a descriptor wallet for fuzzing. The constructor writes the sqlite db - * to disk, the destructor deletes it. + * Wraps a descriptor wallet for fuzzing. */ struct FuzzedWallet { ArgsManager args; WalletContext context; std::shared_ptr<CWallet> wallet; - FuzzedWallet(const std::string& name) + FuzzedWallet(const std::string& name, const std::string& seed_insecure) { - context.args = &args; - context.chain = g_setup->m_node.chain.get(); - - DatabaseOptions options; - options.require_create = true; - options.create_flags = WALLET_FLAG_DESCRIPTORS; - const std::optional<bool> load_on_start; - gArgs.ForceSetArg("-keypool", "0"); // Avoid timeout in TopUp() - - DatabaseStatus status; - bilingual_str error; - std::vector<bilingual_str> warnings; - wallet = CreateWallet(context, name, load_on_start, options, status, error, warnings); - assert(wallet); - assert(error.empty()); - assert(warnings.empty()); + auto& chain{*Assert(g_setup->m_node.chain)}; + wallet = std::make_shared<CWallet>(&chain, name, CreateMockableWalletDatabase()); + { + LOCK(wallet->cs_wallet); + wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + auto height{*Assert(chain.getHeight())}; + wallet->SetLastBlockProcessed(height, chain.getBlockHash(height)); + } + wallet->m_keypool_size = 1; // Avoid timeout in TopUp() assert(wallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)); + ImportDescriptors(*wallet, seed_insecure); } - ~FuzzedWallet() - { - const auto name{wallet->GetName()}; - std::vector<bilingual_str> warnings; - std::optional<bool> load_on_start; - assert(RemoveWallet(context, wallet, load_on_start, warnings)); - assert(warnings.empty()); - UnloadWallet(std::move(wallet)); - fs::remove_all(GetWalletDir() / fs::PathFromString(name)); - } - CScript GetScriptPubKey(FuzzedDataProvider& fuzzed_data_provider) + CTxDestination GetDestination(FuzzedDataProvider& fuzzed_data_provider) { auto type{fuzzed_data_provider.PickValueInArray(OUTPUT_TYPES)}; util::Result<CTxDestination> op_dest{util::Error{}}; @@ -76,7 +115,50 @@ struct FuzzedWallet { } else { op_dest = wallet->GetNewChangeDestination(type); } - return GetScriptForDestination(*Assert(op_dest)); + return *Assert(op_dest); + } + CScript GetScriptPubKey(FuzzedDataProvider& fuzzed_data_provider) { return GetScriptForDestination(GetDestination(fuzzed_data_provider)); } + void FundTx(FuzzedDataProvider& fuzzed_data_provider, CMutableTransaction tx) + { + // The fee of "tx" is 0, so this is the total input and output amount + const CAmount total_amt{ + std::accumulate(tx.vout.begin(), tx.vout.end(), CAmount{}, [](CAmount t, const CTxOut& out) { return t + out.nValue; })}; + const uint32_t tx_size(GetVirtualTransactionSize(CTransaction{tx})); + std::set<int> subtract_fee_from_outputs; + if (fuzzed_data_provider.ConsumeBool()) { + for (size_t i{}; i < tx.vout.size(); ++i) { + if (fuzzed_data_provider.ConsumeBool()) { + subtract_fee_from_outputs.insert(i); + } + } + } + CCoinControl coin_control; + coin_control.m_allow_other_inputs = fuzzed_data_provider.ConsumeBool(); + CallOneOf( + fuzzed_data_provider, [&] { coin_control.destChange = GetDestination(fuzzed_data_provider); }, + [&] { coin_control.m_change_type.emplace(fuzzed_data_provider.PickValueInArray(OUTPUT_TYPES)); }, + [&] { /* no op (leave uninitialized) */ }); + coin_control.fAllowWatchOnly = fuzzed_data_provider.ConsumeBool(); + coin_control.m_include_unsafe_inputs = fuzzed_data_provider.ConsumeBool(); + { + auto& r{coin_control.m_signal_bip125_rbf}; + CallOneOf( + fuzzed_data_provider, [&] { r = true; }, [&] { r = false; }, [&] { r = std::nullopt; }); + } + coin_control.m_feerate = CFeeRate{ + // A fee of this range should cover all cases + fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(0, 2 * total_amt), + tx_size, + }; + if (fuzzed_data_provider.ConsumeBool()) { + *coin_control.m_feerate += GetMinimumFeeRate(*wallet, coin_control, nullptr); + } + coin_control.fOverrideFeeRate = fuzzed_data_provider.ConsumeBool(); + // Add solving data (m_external_provider and SelectExternal)? + + int change_position{fuzzed_data_provider.ConsumeIntegralInRange<int>(-1, tx.vout.size() - 1)}; + bilingual_str error; + (void)FundTransaction(*wallet, tx, change_position, /*lockUnspents=*/false, subtract_fee_from_outputs, coin_control); } }; @@ -86,9 +168,15 @@ FUZZ_TARGET(wallet_notifications, .init = initialize_setup) // The total amount, to be distributed to the wallets a and b in txs // without fee. Thus, the balance of the wallets should always equal the // total amount. - const auto total_amount{ConsumeMoney(fuzzed_data_provider)}; - FuzzedWallet a{"fuzzed_wallet_a"}; - FuzzedWallet b{"fuzzed_wallet_b"}; + const auto total_amount{ConsumeMoney(fuzzed_data_provider, /*max=*/MAX_MONEY / 100000)}; + FuzzedWallet a{ + "fuzzed_wallet_a", + "tprv8ZgxMBicQKsPd1QwsGgzfu2pcPYbBosZhJknqreRHgsWx32nNEhMjGQX2cgFL8n6wz9xdDYwLcs78N4nsCo32cxEX8RBtwGsEGgybLiQJfk", + }; + FuzzedWallet b{ + "fuzzed_wallet_b", + "tprv8ZgxMBicQKsPfCunYTF18sEmEyjz8TfhGnZ3BoVAhkqLv7PLkQgmoG2Ecsp4JuqciWnkopuEwShit7st743fdmB9cMD4tznUkcs33vK51K9", + }; // Keep track of all coins in this test. // Each tuple in the chain represents the coins and the block created with @@ -100,7 +188,7 @@ FUZZ_TARGET(wallet_notifications, .init = initialize_setup) // Add the initial entry chain.emplace_back(); auto& [coins, block]{chain.back()}; - coins.emplace(total_amount, COutPoint{uint256::ONE, 1}); + coins.emplace(total_amount, COutPoint{Txid::FromUint256(uint256::ONE), 1}); } LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 200) { @@ -123,7 +211,7 @@ FUZZ_TARGET(wallet_notifications, .init = initialize_setup) coins.erase(coins.begin()); } // Create some outputs spending all inputs, without fee - LIMITED_WHILE(in > 0 && fuzzed_data_provider.ConsumeBool(), 100) + LIMITED_WHILE(in > 0 && fuzzed_data_provider.ConsumeBool(), 10) { const auto out_value{ConsumeMoney(fuzzed_data_provider, in)}; in -= out_value; @@ -135,6 +223,9 @@ FUZZ_TARGET(wallet_notifications, .init = initialize_setup) tx.vout.emplace_back(in, wallet.GetScriptPubKey(fuzzed_data_provider)); // Add tx to block block.vtx.emplace_back(MakeTransactionRef(tx)); + // Check that funding the tx doesn't crash the wallet + a.FundTx(fuzzed_data_provider, tx); + b.FundTx(fuzzed_data_provider, tx); } // Mine block const uint256& hash = block.GetHash(); diff --git a/src/wallet/test/fuzz/scriptpubkeyman.cpp b/src/wallet/test/fuzz/scriptpubkeyman.cpp new file mode 100644 index 0000000000..228e9629ed --- /dev/null +++ b/src/wallet/test/fuzz/scriptpubkeyman.cpp @@ -0,0 +1,200 @@ +// Copyright (c) 2023-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <addresstype.h> +#include <chainparams.h> +#include <coins.h> +#include <key.h> +#include <primitives/transaction.h> +#include <psbt.h> +#include <script/descriptor.h> +#include <script/interpreter.h> +#include <script/script.h> +#include <script/signingprovider.h> +#include <sync.h> +#include <test/fuzz/FuzzedDataProvider.h> +#include <test/fuzz/fuzz.h> +#include <test/fuzz/util.h> +#include <test/fuzz/util/descriptor.h> +#include <test/util/setup_common.h> +#include <util/check.h> +#include <util/translation.h> +#include <validation.h> +#include <wallet/scriptpubkeyman.h> +#include <wallet/test/util.h> +#include <wallet/types.h> +#include <wallet/wallet.h> +#include <wallet/walletutil.h> + +#include <map> +#include <memory> +#include <optional> +#include <string> +#include <utility> +#include <variant> + +namespace wallet { +namespace { +const TestingSetup* g_setup; + +//! The converter of mocked descriptors, needs to be initialized when the target is. +MockedDescriptorConverter MOCKED_DESC_CONVERTER; + +void initialize_spkm() +{ + static const auto testing_setup{MakeNoLogFileContext<const TestingSetup>()}; + g_setup = testing_setup.get(); + SelectParams(ChainType::MAIN); + MOCKED_DESC_CONVERTER.Init(); +} + +/** + * Key derivation is expensive. Deriving deep derivation paths take a lot of compute and we'd rather spend time + * elsewhere in this target, like on actually fuzzing the DescriptorScriptPubKeyMan. So rule out strings which could + * correspond to a descriptor containing a too large derivation path. + */ +static bool TooDeepDerivPath(std::string_view desc) +{ + const FuzzBufferType desc_buf{reinterpret_cast<const unsigned char *>(desc.data()), desc.size()}; + return HasDeepDerivPath(desc_buf); +} + +static std::optional<std::pair<WalletDescriptor, FlatSigningProvider>> CreateWalletDescriptor(FuzzedDataProvider& fuzzed_data_provider) +{ + const std::string mocked_descriptor{fuzzed_data_provider.ConsumeRandomLengthString()}; + if (TooDeepDerivPath(mocked_descriptor)) return {}; + const auto desc_str{MOCKED_DESC_CONVERTER.GetDescriptor(mocked_descriptor)}; + if (!desc_str.has_value()) return std::nullopt; + + FlatSigningProvider keys; + std::string error; + std::unique_ptr<Descriptor> parsed_desc{Parse(desc_str.value(), keys, error, false)}; + if (!parsed_desc) return std::nullopt; + + WalletDescriptor w_desc{std::move(parsed_desc), /*creation_time=*/0, /*range_start=*/0, /*range_end=*/1, /*next_index=*/1}; + return std::make_pair(w_desc, keys); +} + +static DescriptorScriptPubKeyMan* CreateDescriptor(WalletDescriptor& wallet_desc, FlatSigningProvider& keys, CWallet& keystore) +{ + LOCK(keystore.cs_wallet); + keystore.AddWalletDescriptor(wallet_desc, keys, /*label=*/"", /*internal=*/false); + return keystore.GetDescriptorScriptPubKeyMan(wallet_desc); +}; + +FUZZ_TARGET(scriptpubkeyman, .init = initialize_spkm) +{ + FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + const auto& node{g_setup->m_node}; + Chainstate& chainstate{node.chainman->ActiveChainstate()}; + std::unique_ptr<CWallet> wallet_ptr{std::make_unique<CWallet>(node.chain.get(), "", CreateMockableWalletDatabase())}; + CWallet& wallet{*wallet_ptr}; + { + LOCK(wallet.cs_wallet); + wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); + wallet.SetLastBlockProcessed(chainstate.m_chain.Height(), chainstate.m_chain.Tip()->GetBlockHash()); + } + + auto wallet_desc{CreateWalletDescriptor(fuzzed_data_provider)}; + if (!wallet_desc.has_value()) return; + auto spk_manager{CreateDescriptor(wallet_desc->first, wallet_desc->second, wallet)}; + if (spk_manager == nullptr) return; + + bool good_data{true}; + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 300) { + CallOneOf( + fuzzed_data_provider, + [&] { + auto wallet_desc{CreateWalletDescriptor(fuzzed_data_provider)}; + if (!wallet_desc.has_value()) { + good_data = false; + return; + } + std::string error; + if (spk_manager->CanUpdateToWalletDescriptor(wallet_desc->first, error)) { + auto new_spk_manager{CreateDescriptor(wallet_desc->first, wallet_desc->second, wallet)}; + if (new_spk_manager != nullptr) spk_manager = new_spk_manager; + } + }, + [&] { + const CScript script{ConsumeScript(fuzzed_data_provider)}; + auto is_mine{spk_manager->IsMine(script)}; + if (is_mine == isminetype::ISMINE_SPENDABLE) { + assert(spk_manager->GetScriptPubKeys().count(script)); + } + }, + [&] { + auto spks{spk_manager->GetScriptPubKeys()}; + for (const CScript& spk : spks) { + assert(spk_manager->IsMine(spk) == ISMINE_SPENDABLE); + CTxDestination dest; + bool extract_dest{ExtractDestination(spk, dest)}; + if (extract_dest) { + const std::string msg{fuzzed_data_provider.ConsumeRandomLengthString()}; + PKHash pk_hash{std::get_if<PKHash>(&dest) && fuzzed_data_provider.ConsumeBool() ? + *std::get_if<PKHash>(&dest) : + PKHash{ConsumeUInt160(fuzzed_data_provider)}}; + std::string str_sig; + (void)spk_manager->SignMessage(msg, pk_hash, str_sig); + } + } + }, + [&] { + CKey key{ConsumePrivateKey(fuzzed_data_provider, /*compressed=*/fuzzed_data_provider.ConsumeBool())}; + if (!key.IsValid()) { + good_data = false; + return; + } + spk_manager->AddDescriptorKey(key, key.GetPubKey()); + spk_manager->TopUp(); + }, + [&] { + std::string descriptor; + (void)spk_manager->GetDescriptorString(descriptor, /*priv=*/fuzzed_data_provider.ConsumeBool()); + }, + [&] { + LOCK(spk_manager->cs_desc_man); + auto wallet_desc{spk_manager->GetWalletDescriptor()}; + if (wallet_desc.descriptor->IsSingleType()) { + auto output_type{wallet_desc.descriptor->GetOutputType()}; + if (output_type.has_value()) { + auto dest{spk_manager->GetNewDestination(*output_type)}; + if (dest) { + assert(IsValidDestination(*dest)); + assert(spk_manager->IsHDEnabled()); + } + } + } + }, + [&] { + CMutableTransaction tx_to; + const std::optional<CMutableTransaction> opt_tx_to{ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS)}; + if (!opt_tx_to) { + good_data = false; + return; + } + tx_to = *opt_tx_to; + + std::map<COutPoint, Coin> coins{ConsumeCoins(fuzzed_data_provider)}; + const int sighash{fuzzed_data_provider.ConsumeIntegral<int>()}; + std::map<int, bilingual_str> input_errors; + (void)spk_manager->SignTransaction(tx_to, coins, sighash, input_errors); + }, + [&] { + std::optional<PartiallySignedTransaction> opt_psbt{ConsumeDeserializable<PartiallySignedTransaction>(fuzzed_data_provider)}; + if (!opt_psbt) { + good_data = false; + return; + } + auto psbt{*opt_psbt}; + const PrecomputedTransactionData txdata{PrecomputePSBTData(psbt)}; + const int sighash_type{fuzzed_data_provider.ConsumeIntegralInRange<int>(0, 150)}; + (void)spk_manager->FillPSBT(psbt, txdata, sighash_type, fuzzed_data_provider.ConsumeBool(), fuzzed_data_provider.ConsumeBool(), nullptr, fuzzed_data_provider.ConsumeBool()); + } + ); + } +} + +} // namespace +} // namespace wallet diff --git a/src/wallet/test/group_outputs_tests.cpp b/src/wallet/test/group_outputs_tests.cpp index e6b25cc216..b32dc8ed5a 100644 --- a/src/wallet/test/group_outputs_tests.cpp +++ b/src/wallet/test/group_outputs_tests.cpp @@ -40,7 +40,7 @@ static void addCoin(CoinsResult& coins, tx.vout[0].nValue = nValue; tx.vout[0].scriptPubKey = GetScriptForDestination(dest); - const uint256& txid = tx.GetHash(); + const auto txid{tx.GetHash().ToUint256()}; LOCK(wallet.cs_wallet); auto ret = wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(txid), std::forward_as_tuple(MakeTransactionRef(std::move(tx)), TxStateInactive{})); assert(ret.second); diff --git a/src/wallet/test/ismine_tests.cpp b/src/wallet/test/ismine_tests.cpp index 95d5c1b9ce..dfad0e2126 100644 --- a/src/wallet/test/ismine_tests.cpp +++ b/src/wallet/test/ismine_tests.cpp @@ -47,8 +47,7 @@ BOOST_AUTO_TEST_CASE(ismine_standard) pubkeys[i] = keys[i].GetPubKey(); } - CKey uncompressedKey; - uncompressedKey.MakeNewKey(false); + CKey uncompressedKey = GenerateRandomKey(/*compressed=*/false); CPubKey uncompressedPubkey = uncompressedKey.GetPubKey(); std::unique_ptr<interfaces::Chain>& chain = m_node.chain; diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 9510f28282..9f533bf6ed 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -32,14 +32,18 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) m_wallet.SetWalletFlag(WALLET_FLAG_DESCRIPTORS); // Create prevtxs and add to wallet - CDataStream s_prev_tx1(ParseHex("0200000000010158e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd7501000000171600145f275f436b09a8cc9a2eb2a2f528485c68a56323feffffff02d8231f1b0100000017a914aed962d6654f9a2b36608eb9d64d2b260db4f1118700c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88702483045022100a22edcc6e5bc511af4cc4ae0de0fcd75c7e04d8c1c3a8aa9d820ed4b967384ec02200642963597b9b1bc22c75e9f3e117284a962188bf5e8a74c895089046a20ad770121035509a48eb623e10aace8bfd0212fdb8a8e5af3c94b0b133b95e114cab89e4f7965000000"), SER_NETWORK, PROTOCOL_VERSION); + DataStream s_prev_tx1{ + ParseHex("0200000000010158e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd7501000000171600145f275f436b09a8cc9a2eb2a2f528485c68a56323feffffff02d8231f1b0100000017a914aed962d6654f9a2b36608eb9d64d2b260db4f1118700c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88702483045022100a22edcc6e5bc511af4cc4ae0de0fcd75c7e04d8c1c3a8aa9d820ed4b967384ec02200642963597b9b1bc22c75e9f3e117284a962188bf5e8a74c895089046a20ad770121035509a48eb623e10aace8bfd0212fdb8a8e5af3c94b0b133b95e114cab89e4f7965000000"), + }; CTransactionRef prev_tx1; - s_prev_tx1 >> prev_tx1; + s_prev_tx1 >> TX_WITH_WITNESS(prev_tx1); m_wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(prev_tx1->GetHash()), std::forward_as_tuple(prev_tx1, TxStateInactive{})); - CDataStream s_prev_tx2(ParseHex("0200000001aad73931018bd25f84ae400b68848be09db706eac2ac18298babee71ab656f8b0000000048473044022058f6fc7c6a33e1b31548d481c826c015bd30135aad42cd67790dab66d2ad243b02204a1ced2604c6735b6393e5b41691dd78b00f0c5942fb9f751856faa938157dba01feffffff0280f0fa020000000017a9140fb9463421696b82c833af241c78c17ddbde493487d0f20a270100000017a91429ca74f8a08f81999428185c97b5d852e4063f618765000000"), SER_NETWORK, PROTOCOL_VERSION); + DataStream s_prev_tx2{ + ParseHex("0200000001aad73931018bd25f84ae400b68848be09db706eac2ac18298babee71ab656f8b0000000048473044022058f6fc7c6a33e1b31548d481c826c015bd30135aad42cd67790dab66d2ad243b02204a1ced2604c6735b6393e5b41691dd78b00f0c5942fb9f751856faa938157dba01feffffff0280f0fa020000000017a9140fb9463421696b82c833af241c78c17ddbde493487d0f20a270100000017a91429ca74f8a08f81999428185c97b5d852e4063f618765000000"), + }; CTransactionRef prev_tx2; - s_prev_tx2 >> prev_tx2; + s_prev_tx2 >> TX_WITH_WITNESS(prev_tx2); m_wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(prev_tx2->GetHash()), std::forward_as_tuple(prev_tx2, TxStateInactive{})); // Import descriptors for keys and scripts @@ -49,7 +53,9 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) // Call FillPSBT PartiallySignedTransaction psbtx; - CDataStream ssData(ParseHex("70736274ff01009a020000000258e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd750000000000ffffffff838d0427d0ec650a68aa46bb0b098aea4422c071b2ca78352a077959d07cea1d0100000000ffffffff0270aaf00800000000160014d85c2b71d0060b09c9886aeb815e50991dda124d00e1f5050000000016001400aea9a2e5f0f876a588df5546e8742d1d87008f000000000000000000"), SER_NETWORK, PROTOCOL_VERSION); + DataStream ssData{ + ParseHex("70736274ff01009a020000000258e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd750000000000ffffffff838d0427d0ec650a68aa46bb0b098aea4422c071b2ca78352a077959d07cea1d0100000000ffffffff0270aaf00800000000160014d85c2b71d0060b09c9886aeb815e50991dda124d00e1f5050000000016001400aea9a2e5f0f876a588df5546e8742d1d87008f000000000000000000"), + }; ssData >> psbtx; // Fill transaction with our data @@ -57,7 +63,7 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) BOOST_REQUIRE_EQUAL(TransactionError::OK, m_wallet.FillPSBT(psbtx, complete, SIGHASH_ALL, false, true)); // Get the final tx - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); + DataStream ssTx{}; ssTx << psbtx; std::string final_hex = HexStr(ssTx); BOOST_CHECK_EQUAL(final_hex, "70736274ff01009a020000000258e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd750000000000ffffffff838d0427d0ec650a68aa46bb0b098aea4422c071b2ca78352a077959d07cea1d0100000000ffffffff0270aaf00800000000160014d85c2b71d0060b09c9886aeb815e50991dda124d00e1f5050000000016001400aea9a2e5f0f876a588df5546e8742d1d87008f00000000000100bb0200000001aad73931018bd25f84ae400b68848be09db706eac2ac18298babee71ab656f8b0000000048473044022058f6fc7c6a33e1b31548d481c826c015bd30135aad42cd67790dab66d2ad243b02204a1ced2604c6735b6393e5b41691dd78b00f0c5942fb9f751856faa938157dba01feffffff0280f0fa020000000017a9140fb9463421696b82c833af241c78c17ddbde493487d0f20a270100000017a91429ca74f8a08f81999428185c97b5d852e4063f6187650000000104475221029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f2102dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d752ae2206029583bf39ae0a609747ad199addd634fa6108559d6c5cd39b4c2183f1ab96e07f10d90c6a4f000000800000008000000080220602dab61ff49a14db6a7d02b0cd1fbb78fc4b18312b5b4e54dae4dba2fbfef536d710d90c6a4f0000008000000080010000800001008a020000000158e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd7501000000171600145f275f436b09a8cc9a2eb2a2f528485c68a56323feffffff02d8231f1b0100000017a914aed962d6654f9a2b36608eb9d64d2b260db4f1118700c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e8876500000001012000c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88701042200208c2353173743b595dfb4a07b72ba8e42e3797da74e87fe7d9d7497e3b2028903010547522103089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc21023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7352ae2206023add904f3d6dcf59ddb906b0dee23529b7ffb9ed50e5e86151926860221f0e7310d90c6a4f000000800000008003000080220603089dc10c7ac6db54f91329af617333db388cead0c231f723379d1b99030b02dc10d90c6a4f00000080000000800200008000220203a9a4c37f5996d3aa25dbac6b570af0650394492942460b354753ed9eeca5877110d90c6a4f000000800000008004000080002202027f6399757d2eff55a136ad02c684b1838b6556e5f1b6b34282a94b6b5005109610d90c6a4f00000080000000800500008000"); diff --git a/src/wallet/test/spend_tests.cpp b/src/wallet/test/spend_tests.cpp index 68c98ae6b9..3509bc116f 100644 --- a/src/wallet/test/spend_tests.cpp +++ b/src/wallet/test/spend_tests.cpp @@ -28,13 +28,12 @@ BOOST_FIXTURE_TEST_CASE(SubtractFee, TestChain100Setup) // instead of the miner. auto check_tx = [&wallet](CAmount leftover_input_amount) { CRecipient recipient{PubKeyDestination({}), 50 * COIN - leftover_input_amount, /*subtract_fee=*/true}; - constexpr int RANDOM_CHANGE_POSITION = -1; CCoinControl coin_control; coin_control.m_feerate.emplace(10000); coin_control.fOverrideFeeRate = true; // We need to use a change type with high cost of change so that the leftover amount will be dropped to fee instead of added as a change output coin_control.m_change_type = OutputType::LEGACY; - auto res = CreateTransaction(*wallet, {recipient}, RANDOM_CHANGE_POSITION, coin_control); + auto res = CreateTransaction(*wallet, {recipient}, /*change_pos=*/std::nullopt, coin_control); BOOST_CHECK(res); const auto& txr = *res; BOOST_CHECK_EQUAL(txr.tx->vout.size(), 1); @@ -78,7 +77,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_duplicated_preset_inputs_test, TestChain100Setup) // Try to create a tx that spends more than what preset inputs + wallet selected inputs are covering for. // The wallet can cover up to 200 BTC, and the tx target is 299 BTC. - std::vector<CRecipient> recipients = {{GetScriptForDestination(*Assert(wallet->GetNewDestination(OutputType::BECH32, "dummy"))), + std::vector<CRecipient> recipients{{*Assert(wallet->GetNewDestination(OutputType::BECH32, "dummy")), /*nAmount=*/299 * COIN, /*fSubtractFeeFromAmount=*/true}}; CCoinControl coin_control; coin_control.m_allow_other_inputs = true; @@ -98,12 +97,12 @@ BOOST_FIXTURE_TEST_CASE(wallet_duplicated_preset_inputs_test, TestChain100Setup) // so that the recipient's amount is no longer equal to the user's selected target of 299 BTC. // First case, use 'subtract_fee_from_outputs=true' - util::Result<CreatedTransactionResult> res_tx = CreateTransaction(*wallet, recipients, /*change_pos*/-1, coin_control); + util::Result<CreatedTransactionResult> res_tx = CreateTransaction(*wallet, recipients, /*change_pos=*/std::nullopt, coin_control); BOOST_CHECK(!res_tx.has_value()); // Second case, don't use 'subtract_fee_from_outputs'. recipients[0].fSubtractFeeFromAmount = false; - res_tx = CreateTransaction(*wallet, recipients, /*change_pos*/-1, coin_control); + res_tx = CreateTransaction(*wallet, recipients, /*change_pos=*/std::nullopt, coin_control); BOOST_CHECK(!res_tx.has_value()); } diff --git a/src/wallet/test/util.cpp b/src/wallet/test/util.cpp index ad8613d515..cbf3ccd1ec 100644 --- a/src/wallet/test/util.cpp +++ b/src/wallet/test/util.cpp @@ -24,7 +24,6 @@ std::unique_ptr<CWallet> CreateSyncedWallet(interfaces::Chain& chain, CChain& cc LOCK2(wallet->cs_wallet, ::cs_main); wallet->SetLastBlockProcessed(cchain.Height(), cchain.Tip()->GetBlockHash()); } - wallet->LoadWallet(); { LOCK(wallet->cs_wallet); wallet->SetWalletFlag(WALLET_FLAG_DESCRIPTORS); diff --git a/src/wallet/test/wallet_tests.cpp b/src/wallet/test/wallet_tests.cpp index ad4bb3a9d2..65297054df 100644 --- a/src/wallet/test/wallet_tests.cpp +++ b/src/wallet/test/wallet_tests.cpp @@ -231,8 +231,7 @@ BOOST_FIXTURE_TEST_CASE(importmulti_rescan, TestChain100Setup) keys.push_back(key); key.clear(); key.setObject(); - CKey futureKey; - futureKey.MakeNewKey(true); + CKey futureKey = GenerateRandomKey(); key.pushKV("scriptPubKey", HexStr(GetScriptForRawPubKey(futureKey.GetPubKey()))); key.pushKV("timestamp", newTip->GetBlockTimeMax() + TIMESTAMP_WINDOW + 1); key.pushKV("internal", UniValue(true)); @@ -558,8 +557,7 @@ public: CTransactionRef tx; CCoinControl dummy; { - constexpr int RANDOM_CHANGE_POSITION = -1; - auto res = CreateTransaction(*wallet, {recipient}, RANDOM_CHANGE_POSITION, dummy); + auto res = CreateTransaction(*wallet, {recipient}, /*change_pos=*/std::nullopt, dummy); BOOST_CHECK(res); tx = res->tx; } @@ -605,7 +603,7 @@ BOOST_FIXTURE_TEST_CASE(ListCoinsTest, ListCoinsTestingSetup) // returns the coin associated with the change address underneath the // coinbaseKey pubkey, even though the change address has a different // pubkey. - AddTx(CRecipient{GetScriptForRawPubKey({}), 1 * COIN, /*subtract_fee=*/false}); + AddTx(CRecipient{PubKeyDestination{{}}, 1 * COIN, /*subtract_fee=*/false}); { LOCK(wallet->cs_wallet); list = ListCoins(*wallet); @@ -705,8 +703,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_disableprivkeys, TestChain100Setup) static size_t CalculateNestedKeyhashInputSize(bool use_max_sig) { // Generate ephemeral valid pubkey - CKey key; - key.MakeNewKey(true); + CKey key = GenerateRandomKey(); CPubKey pubkey = key.GetPubKey(); // Generate pubkey hash @@ -752,14 +749,14 @@ bool malformed_descriptor(std::ios_base::failure e) BOOST_FIXTURE_TEST_CASE(wallet_descriptor_test, BasicTestingSetup) { std::vector<unsigned char> malformed_record; - CVectorWriter vw{0, malformed_record, 0}; + VectorWriter vw{malformed_record, 0}; vw << std::string("notadescriptor"); vw << uint64_t{0}; vw << int32_t{0}; vw << int32_t{0}; vw << int32_t{1}; - SpanReader vr{0, malformed_record}; + SpanReader vr{malformed_record}; WalletDescriptor w_desc; BOOST_CHECK_EXCEPTION(vr >> w_desc, std::ios_base::failure, malformed_descriptor); } @@ -790,8 +787,7 @@ BOOST_FIXTURE_TEST_CASE(CreateWallet, TestChain100Setup) context.args = &m_args; context.chain = m_node.chain.get(); auto wallet = TestLoadWallet(context); - CKey key; - key.MakeNewKey(true); + CKey key = GenerateRandomKey(); AddKey(*wallet, key); TestUnloadWallet(std::move(wallet)); @@ -899,8 +895,7 @@ BOOST_FIXTURE_TEST_CASE(ZapSelectTx, TestChain100Setup) context.args = &m_args; context.chain = m_node.chain.get(); auto wallet = TestLoadWallet(context); - CKey key; - key.MakeNewKey(true); + CKey key = GenerateRandomKey(); AddKey(*wallet, key); std::string error; @@ -946,7 +941,7 @@ BOOST_FIXTURE_TEST_CASE(wallet_sync_tx_invalid_state_test, TestingSetup) CMutableTransaction mtx; mtx.vout.emplace_back(COIN, GetScriptForDestination(op_dest)); - mtx.vin.emplace_back(g_insecure_rand_ctx.rand256(), 0); + mtx.vin.emplace_back(Txid::FromUint256(g_insecure_rand_ctx.rand256()), 0); const auto& tx_id_to_spend = wallet.AddToWallet(MakeTransactionRef(mtx), TxStateInMempool{})->GetHash(); { @@ -963,12 +958,12 @@ BOOST_FIXTURE_TEST_CASE(wallet_sync_tx_invalid_state_test, TestingSetup) mtx.vin.clear(); mtx.vin.emplace_back(tx_id_to_spend, 0); wallet.transactionAddedToMempool(MakeTransactionRef(mtx)); - const uint256& good_tx_id = mtx.GetHash(); + const auto good_tx_id{mtx.GetHash()}; { // Verify balance update for the new tx and the old one LOCK(wallet.cs_wallet); - const CWalletTx* new_wtx = wallet.GetWalletTx(good_tx_id); + const CWalletTx* new_wtx = wallet.GetWalletTx(good_tx_id.ToUint256()); BOOST_CHECK_EQUAL(CachedTxGetAvailableCredit(wallet, *new_wtx), 1 * COIN); // Now the old wtx diff --git a/src/wallet/test/walletdb_tests.cpp b/src/wallet/test/walletdb_tests.cpp index 17b6c4f7ed..fee8c85873 100644 --- a/src/wallet/test/walletdb_tests.cpp +++ b/src/wallet/test/walletdb_tests.cpp @@ -17,14 +17,14 @@ BOOST_FIXTURE_TEST_SUITE(walletdb_tests, BasicTestingSetup) BOOST_AUTO_TEST_CASE(walletdb_readkeyvalue) { /** - * When ReadKeyValue() reads from either a "key" or "wkey" it first reads the CDataStream steam into a + * When ReadKeyValue() reads from either a "key" or "wkey" it first reads the DataStream into a * CPrivKey or CWalletKey respectively and then reads a hash of the pubkey and privkey into a uint256. * Wallets from 0.8 or before do not store the pubkey/privkey hash, trying to read the hash from old * wallets throws an exception, for backwards compatibility this read is wrapped in a try block to - * silently fail. The test here makes sure the type of exception thrown from CDataStream::read() + * silently fail. The test here makes sure the type of exception thrown from DataStream::read() * matches the type we expect, otherwise we need to update the "key"/"wkey" exception type caught. */ - CDataStream ssValue(SER_DISK, CLIENT_VERSION); + DataStream ssValue{}; uint256 dummy; BOOST_CHECK_THROW(ssValue >> dummy, std::ios_base::failure); } diff --git a/src/wallet/test/walletload_tests.cpp b/src/wallet/test/walletload_tests.cpp index 302db455b1..3dba2231f0 100644 --- a/src/wallet/test/walletload_tests.cpp +++ b/src/wallet/test/walletload_tests.cpp @@ -104,7 +104,7 @@ bool HasAnyRecordOfType(WalletDatabase& db, const std::string& key) template<typename... Args> SerializeData MakeSerializeData(const Args&... args) { - CDataStream s(0, 0); + DataStream s{}; SerializeMany(s, args...); return {s.begin(), s.end()}; } diff --git a/src/wallet/transaction.cpp b/src/wallet/transaction.cpp index 4f78fe7520..6777257e53 100644 --- a/src/wallet/transaction.cpp +++ b/src/wallet/transaction.cpp @@ -4,6 +4,10 @@ #include <wallet/transaction.h> +#include <interfaces/chain.h> + +using interfaces::FoundBlock; + namespace wallet { bool CWalletTx::IsEquivalentTo(const CWalletTx& _tx) const { @@ -25,6 +29,27 @@ int64_t CWalletTx::GetTxTime() const return n ? n : nTimeReceived; } +void CWalletTx::updateState(interfaces::Chain& chain) +{ + bool active; + auto lookup_block = [&](const uint256& hash, int& height, TxState& state) { + // If tx block (or conflicting block) was reorged out of chain + // while the wallet was shutdown, change tx status to UNCONFIRMED + // and reset block height, hash, and index. ABANDONED tx don't have + // associated blocks and don't need to be updated. The case where a + // transaction was reorged out while online and then reconfirmed + // while offline is covered by the rescan logic. + if (!chain.findBlock(hash, FoundBlock().inActiveChain(active).height(height)) || !active) { + state = TxStateInactive{}; + } + }; + if (auto* conf = state<TxStateConfirmed>()) { + lookup_block(conf->confirmed_block_hash, conf->confirmed_block_height, m_state); + } else if (auto* conf = state<TxStateConflicted>()) { + lookup_block(conf->conflicting_block_hash, conf->conflicting_block_height, m_state); + } +} + void CWalletTx::CopyFrom(const CWalletTx& _tx) { *this = _tx; diff --git a/src/wallet/transaction.h b/src/wallet/transaction.h index 0d0821e857..ddeb931112 100644 --- a/src/wallet/transaction.h +++ b/src/wallet/transaction.h @@ -5,22 +5,27 @@ #ifndef BITCOIN_WALLET_TRANSACTION_H #define BITCOIN_WALLET_TRANSACTION_H -#include <bitset> -#include <cstdint> +#include <attributes.h> #include <consensus/amount.h> #include <primitives/transaction.h> -#include <serialize.h> -#include <wallet/types.h> -#include <threadsafety.h> #include <tinyformat.h> +#include <uint256.h> #include <util/overloaded.h> #include <util/strencodings.h> #include <util/string.h> +#include <wallet/types.h> -#include <list> +#include <bitset> +#include <cstdint> +#include <map> +#include <utility> #include <variant> #include <vector> +namespace interfaces { +class Chain; +} // namespace interfaces + namespace wallet { //! State of transaction confirmed in a block. struct TxStateConfirmed { @@ -160,7 +165,7 @@ public: std::vector<uint256> vMerkleBranch; int nIndex; - s >> tx >> hashBlock >> vMerkleBranch >> nIndex; + s >> TX_WITH_WITNESS(tx) >> hashBlock >> vMerkleBranch >> nIndex; } }; @@ -271,7 +276,7 @@ public: bool dummy_bool = false; //!< Used to be fSpent uint256 serializedHash = TxStateSerializedBlockHash(m_state); int serializedIndex = TxStateSerializedIndex(m_state); - s << tx << serializedHash << dummy_vector1 << serializedIndex << dummy_vector2 << mapValueCopy << vOrderForm << fTimeReceivedIsTxTime << nTimeReceived << fFromMe << dummy_bool; + s << TX_WITH_WITNESS(tx) << serializedHash << dummy_vector1 << serializedIndex << dummy_vector2 << mapValueCopy << vOrderForm << fTimeReceivedIsTxTime << nTimeReceived << fFromMe << dummy_bool; } template<typename Stream> @@ -284,7 +289,7 @@ public: bool dummy_bool; //! Used to be fSpent uint256 serialized_block_hash; int serializedIndex; - s >> tx >> serialized_block_hash >> dummy_vector1 >> serializedIndex >> dummy_vector2 >> mapValue >> vOrderForm >> fTimeReceivedIsTxTime >> nTimeReceived >> fFromMe >> dummy_bool; + s >> TX_WITH_WITNESS(tx) >> serialized_block_hash >> dummy_vector1 >> serializedIndex >> dummy_vector2 >> mapValue >> vOrderForm >> fTimeReceivedIsTxTime >> nTimeReceived >> fFromMe >> dummy_bool; m_state = TxStateInterpretSerialized({serialized_block_hash, serializedIndex}); @@ -325,13 +330,17 @@ public: template<typename T> const T* state() const { return std::get_if<T>(&m_state); } template<typename T> T* state() { return std::get_if<T>(&m_state); } + //! Update transaction state when attaching to a chain, filling in heights + //! of conflicted and confirmed blocks + void updateState(interfaces::Chain& chain); + bool isAbandoned() const { return state<TxStateInactive>() && state<TxStateInactive>()->abandoned; } bool isConflicted() const { return state<TxStateConflicted>(); } bool isInactive() const { return state<TxStateInactive>(); } bool isUnconfirmed() const { return !isAbandoned() && !isConflicted() && !isConfirmed(); } bool isConfirmed() const { return state<TxStateConfirmed>(); } - const uint256& GetHash() const { return tx->GetHash(); } - const uint256& GetWitnessHash() const { return tx->GetWitnessHash(); } + const Txid& GetHash() const LIFETIMEBOUND { return tx->GetHash(); } + const Wtxid& GetWitnessHash() const LIFETIMEBOUND { return tx->GetWitnessHash(); } bool IsCoinBase() const { return tx->IsCoinBase(); } private: diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 162d7f9ec7..70fb375efa 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -681,7 +681,7 @@ std::set<uint256> CWallet::GetConflicts(const uint256& txid) const bool CWallet::HasWalletSpend(const CTransactionRef& tx) const { AssertLockHeld(cs_wallet); - const uint256& txid = tx->GetHash(); + const Txid& txid = tx->GetHash(); for (unsigned int i = 0; i < tx->vout.size(); ++i) { if (IsSpent(COutPoint(txid, i))) { return true; @@ -1080,6 +1080,9 @@ CWalletTx* CWallet::AddToWallet(CTransactionRef tx, const TxState& state, const wtx.m_it_wtxOrdered = wtxOrdered.insert(std::make_pair(wtx.nOrderPos, &wtx)); wtx.nTimeSmart = ComputeTimeSmart(wtx, rescanning_old_block); AddToSpends(wtx, &batch); + + // Update birth time when tx time is older than it. + MaybeUpdateBirthTime(wtx.GetTxTime()); } if (!fInsertedNew) @@ -1184,23 +1187,7 @@ bool CWallet::LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx // If wallet doesn't have a chain (e.g when using bitcoin-wallet tool), // don't bother to update txn. if (HaveChain()) { - bool active; - auto lookup_block = [&](const uint256& hash, int& height, TxState& state) { - // If tx block (or conflicting block) was reorged out of chain - // while the wallet was shutdown, change tx status to UNCONFIRMED - // and reset block height, hash, and index. ABANDONED tx don't have - // associated blocks and don't need to be updated. The case where a - // transaction was reorged out while online and then reconfirmed - // while offline is covered by the rescan logic. - if (!chain().findBlock(hash, FoundBlock().inActiveChain(active).height(height)) || !active) { - state = TxStateInactive{}; - } - }; - if (auto* conf = wtx.state<TxStateConfirmed>()) { - lookup_block(conf->confirmed_block_hash, conf->confirmed_block_height, wtx.m_state); - } else if (auto* conf = wtx.state<TxStateConflicted>()) { - lookup_block(conf->conflicting_block_hash, conf->conflicting_block_height, wtx.m_state); - } + wtx.updateState(chain()); } if (/* insertion took place */ ins.second) { wtx.m_it_wtxOrdered = wtxOrdered.insert(std::make_pair(wtx.nOrderPos, &wtx)); @@ -1215,6 +1202,10 @@ bool CWallet::LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx } } } + + // Update birth time when tx time is older than it. + MaybeUpdateBirthTime(wtx.GetTxTime()); + return true; } @@ -1389,7 +1380,7 @@ void CWallet::RecursiveUpdateTxState(const uint256& tx_hash, const TryUpdatingSt batch.WriteTx(wtx); // Iterate over all its outputs, and update those tx states as well (if applicable) for (unsigned int i = 0; i < wtx.tx->vout.size(); ++i) { - std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(COutPoint(now, i)); + std::pair<TxSpends::const_iterator, TxSpends::const_iterator> range = mapTxSpends.equal_range(COutPoint(Txid::FromUint256(now), i)); for (TxSpends::const_iterator iter = range.first; iter != range.second; ++iter) { if (!done.count(iter->second)) { todo.insert(iter->second); @@ -1763,11 +1754,11 @@ bool CWallet::ImportScriptPubKeys(const std::string& label, const std::set<CScri return true; } -void CWallet::FirstKeyTimeChanged(const ScriptPubKeyMan* spkm, int64_t new_birth_time) +void CWallet::MaybeUpdateBirthTime(int64_t time) { int64_t birthtime = m_birth_time.load(); - if (new_birth_time < birthtime) { - m_birth_time = new_birth_time; + if (time < birthtime) { + m_birth_time = time; } } @@ -2308,6 +2299,8 @@ DBErrors CWallet::LoadWallet() { LOCK(cs_wallet); + Assert(m_spk_managers.empty()); + Assert(m_wallet_flags == 0); DBErrors nLoadWalletRet = WalletBatch(GetDatabase()).LoadWallet(this); if (nLoadWalletRet == DBErrors::NEED_REWRITE) { @@ -3103,7 +3096,7 @@ std::shared_ptr<CWallet> CWallet::Create(WalletContext& context, const std::stri int64_t time = spk_man->GetTimeFirstKey(); if (!time_first_key || time < *time_first_key) time_first_key = time; } - if (time_first_key) walletInstance->m_birth_time = *time_first_key; + if (time_first_key) walletInstance->MaybeUpdateBirthTime(*time_first_key); if (chain && !AttachChain(walletInstance, *chain, rescan_required, error, warnings)) { return nullptr; @@ -3319,8 +3312,10 @@ int CWallet::GetTxDepthInMainChain(const CWalletTx& wtx) const { AssertLockHeld(cs_wallet); if (auto* conf = wtx.state<TxStateConfirmed>()) { + assert(conf->confirmed_block_height >= 0); return GetLastBlockHeight() - conf->confirmed_block_height + 1; } else if (auto* conf = wtx.state<TxStateConflicted>()) { + assert(conf->conflicting_block_height >= 0); return -1 * (GetLastBlockHeight() - conf->conflicting_block_height + 1); } else { return 0; @@ -3494,10 +3489,12 @@ LegacyScriptPubKeyMan* CWallet::GetOrCreateLegacyScriptPubKeyMan() void CWallet::AddScriptPubKeyMan(const uint256& id, std::unique_ptr<ScriptPubKeyMan> spkm_man) { + // Add spkm_man to m_spk_managers before calling any method + // that might access it. const auto& spkm = m_spk_managers[id] = std::move(spkm_man); // Update birth time if needed - FirstKeyTimeChanged(spkm.get(), spkm->GetTimeFirstKey()); + MaybeUpdateBirthTime(spkm->GetTimeFirstKey()); } void CWallet::SetupLegacyScriptPubKeyMan() @@ -3530,7 +3527,7 @@ void CWallet::ConnectScriptPubKeyManNotifiers() for (const auto& spk_man : GetActiveScriptPubKeyMans()) { spk_man->NotifyWatchonlyChanged.connect(NotifyWatchonlyChanged); spk_man->NotifyCanGetAddressesChanged.connect(NotifyCanGetAddressesChanged); - spk_man->NotifyFirstKeyTimeChanged.connect(std::bind(&CWallet::FirstKeyTimeChanged, this, std::placeholders::_1, std::placeholders::_2)); + spk_man->NotifyFirstKeyTimeChanged.connect(std::bind(&CWallet::MaybeUpdateBirthTime, this, std::placeholders::_2)); } } @@ -3549,6 +3546,10 @@ void CWallet::SetupDescriptorScriptPubKeyMans(const CExtKey& master_key) { AssertLockHeld(cs_wallet); + // Create single batch txn + WalletBatch batch(GetDatabase()); + if (!batch.TxnBegin()) throw std::runtime_error("Error: cannot create db transaction for descriptors setup"); + for (bool internal : {false, true}) { for (OutputType t : OUTPUT_TYPES) { auto spk_manager = std::unique_ptr<DescriptorScriptPubKeyMan>(new DescriptorScriptPubKeyMan(*this, m_keypool_size)); @@ -3556,16 +3557,19 @@ void CWallet::SetupDescriptorScriptPubKeyMans(const CExtKey& master_key) if (IsLocked()) { throw std::runtime_error(std::string(__func__) + ": Wallet is locked, cannot setup new descriptors"); } - if (!spk_manager->CheckDecryptionKey(vMasterKey) && !spk_manager->Encrypt(vMasterKey, nullptr)) { + if (!spk_manager->CheckDecryptionKey(vMasterKey) && !spk_manager->Encrypt(vMasterKey, &batch)) { throw std::runtime_error(std::string(__func__) + ": Could not encrypt new descriptors"); } } - spk_manager->SetupDescriptorGeneration(master_key, t, internal); + spk_manager->SetupDescriptorGeneration(batch, master_key, t, internal); uint256 id = spk_manager->GetID(); AddScriptPubKeyMan(id, std::move(spk_manager)); - AddActiveScriptPubKeyMan(id, t, internal); + AddActiveScriptPubKeyManWithDb(batch, id, t, internal); } } + + // Ensure information is committed to disk + if (!batch.TxnCommit()) throw std::runtime_error("Error: cannot commit db transaction for descriptors setup"); } void CWallet::SetupDescriptorScriptPubKeyMans() @@ -3574,8 +3578,7 @@ void CWallet::SetupDescriptorScriptPubKeyMans() if (!IsWalletFlagSet(WALLET_FLAG_EXTERNAL_SIGNER)) { // Make a seed - CKey seed_key; - seed_key.MakeNewKey(true); + CKey seed_key = GenerateRandomKey(); CPubKey seed = seed_key.GetPubKey(); assert(seed_key.VerifyPubKey(seed)); @@ -3592,6 +3595,10 @@ void CWallet::SetupDescriptorScriptPubKeyMans() UniValue signer_res = signer.GetDescriptors(account); if (!signer_res.isObject()) throw std::runtime_error(std::string(__func__) + ": Unexpected result"); + + WalletBatch batch(GetDatabase()); + if (!batch.TxnBegin()) throw std::runtime_error("Error: cannot create db transaction for descriptors import"); + for (bool internal : {false, true}) { const UniValue& descriptor_vals = signer_res.find_value(internal ? "internal" : "receive"); if (!descriptor_vals.isArray()) throw std::runtime_error(std::string(__func__) + ": Unexpected result"); @@ -3608,18 +3615,26 @@ void CWallet::SetupDescriptorScriptPubKeyMans() } OutputType t = *desc->GetOutputType(); auto spk_manager = std::unique_ptr<ExternalSignerScriptPubKeyMan>(new ExternalSignerScriptPubKeyMan(*this, m_keypool_size)); - spk_manager->SetupDescriptor(std::move(desc)); + spk_manager->SetupDescriptor(batch, std::move(desc)); uint256 id = spk_manager->GetID(); AddScriptPubKeyMan(id, std::move(spk_manager)); - AddActiveScriptPubKeyMan(id, t, internal); + AddActiveScriptPubKeyManWithDb(batch, id, t, internal); } } + + // Ensure imported descriptors are committed to disk + if (!batch.TxnCommit()) throw std::runtime_error("Error: cannot commit db transaction for descriptors import"); } } void CWallet::AddActiveScriptPubKeyMan(uint256 id, OutputType type, bool internal) { WalletBatch batch(GetDatabase()); + return AddActiveScriptPubKeyManWithDb(batch, id, type, internal); +} + +void CWallet::AddActiveScriptPubKeyManWithDb(WalletBatch& batch, uint256 id, OutputType type, bool internal) +{ if (!batch.WriteActiveScriptPubKeyMan(static_cast<uint8_t>(type), id, internal)) { throw std::runtime_error(std::string(__func__) + ": writing active ScriptPubKeyMan id failed"); } diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 9333493a6e..487921443f 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -178,7 +178,7 @@ static const std::map<std::string,WalletFlags> WALLET_FLAG_MAP{ * Instantiating a ReserveDestination does not reserve an address. To do so, * GetReservedDestination() needs to be called on the object. Once an address has been * reserved, call KeepDestination() on the ReserveDestination object to make sure it is not - * returned. Call ReturnDestination() to return the address so it can be re-used (for + * returned. Call ReturnDestination() to return the address so it can be reused (for * example, if the address was used in a new transaction * and that transaction was not completed and needed to be aborted). * @@ -419,6 +419,9 @@ private: // Must be the only method adding data to it. void AddScriptPubKeyMan(const uint256& id, std::unique_ptr<ScriptPubKeyMan> spkm_man); + // Same as 'AddActiveScriptPubKeyMan' but designed for use within a batch transaction context + void AddActiveScriptPubKeyManWithDb(WalletBatch& batch, uint256 id, OutputType type, bool internal); + /** * Catch wallet up to current chain, scanning new blocks, updating the best * block locator and m_last_block_processed, and registering for @@ -503,6 +506,13 @@ public: * <0 : conflicts with a transaction this deep in the blockchain * 0 : in memory pool, waiting to be included in a block * >=1 : this many blocks deep in the main chain + * + * Preconditions: it is only valid to call this function when the wallet is + * online and the block index is loaded. So this cannot be called by + * bitcoin-wallet tool code or by wallet migration code. If this is called + * without the wallet being online, it won't be able able to determine the + * the height of the last block processed, or the heights of blocks + * referenced in transaction, and might cause assert failures. */ int GetTxDepthInMainChain(const CWalletTx& wtx) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool IsTxInMainChain(const CWalletTx& wtx) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) @@ -678,8 +688,8 @@ public: bool ImportPubKeys(const std::vector<CKeyID>& ordered_pubkeys, const std::map<CKeyID, CPubKey>& pubkey_map, const std::map<CKeyID, std::pair<CPubKey, KeyOriginInfo>>& key_origins, const bool add_keypool, const bool internal, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool ImportScriptPubKeys(const std::string& label, const std::set<CScript>& script_pub_keys, const bool have_solving_data, const bool apply_label, const int64_t timestamp) EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); - /** Updates wallet birth time if 'new_birth_time' is below it */ - void FirstKeyTimeChanged(const ScriptPubKeyMan* spkm, int64_t new_birth_time); + /** Updates wallet birth time if 'time' is below it */ + void MaybeUpdateBirthTime(int64_t time); CFeeRate m_pay_tx_fee{DEFAULT_PAY_TX_FEE}; unsigned int m_confirm_target{DEFAULT_TX_CONFIRM_TARGET}; @@ -877,6 +887,9 @@ public: /* Returns true if the wallet can give out new addresses. This means it has keys in the keypool or can generate new keys */ bool CanGetAddresses(bool internal = false) const; + /* Returns the time of the first created key or, in case of an import, it could be the time of the first received transaction */ + int64_t GetBirthTime() const { return m_birth_time; } + /** * Blocks until the wallet state is up-to-date to /at least/ the current * chain at the time this function is entered diff --git a/src/wallet/walletdb.cpp b/src/wallet/walletdb.cpp index 92eca46f05..9820c7c0ee 100644 --- a/src/wallet/walletdb.cpp +++ b/src/wallet/walletdb.cpp @@ -478,12 +478,12 @@ struct LoadResult int m_records{0}; }; -using LoadFunc = std::function<DBErrors(CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err)>; +using LoadFunc = std::function<DBErrors(CWallet* pwallet, DataStream& key, DataStream& value, std::string& err)>; static LoadResult LoadRecords(CWallet* pwallet, DatabaseBatch& batch, const std::string& key, DataStream& prefix, LoadFunc load_func) { LoadResult result; DataStream ssKey; - CDataStream ssValue(SER_DISK, CLIENT_VERSION); + DataStream ssValue{}; Assume(!prefix.empty()); std::unique_ptr<DatabaseCursor> cursor = batch.GetNewPrefixCursor(prefix); @@ -532,7 +532,7 @@ static DBErrors LoadLegacyWalletRecords(CWallet* pwallet, DatabaseBatch& batch, if (pwallet->IsWalletFlagSet(WALLET_FLAG_DESCRIPTORS)) { for (const auto& type : DBKeys::LEGACY_TYPES) { DataStream key; - CDataStream value(SER_DISK, CLIENT_VERSION); + DataStream value{}; DataStream prefix; prefix << type; @@ -555,28 +555,28 @@ static DBErrors LoadLegacyWalletRecords(CWallet* pwallet, DatabaseBatch& batch, // Load HD Chain // Note: There should only be one HDCHAIN record with no data following the type LoadResult hd_chain_res = LoadRecords(pwallet, batch, DBKeys::HDCHAIN, - [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + [] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& err) { return LoadHDChain(pwallet, value, err) ? DBErrors:: LOAD_OK : DBErrors::CORRUPT; }); result = std::max(result, hd_chain_res.m_result); // Load unencrypted keys LoadResult key_res = LoadRecords(pwallet, batch, DBKeys::KEY, - [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + [] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& err) { return LoadKey(pwallet, key, value, err) ? DBErrors::LOAD_OK : DBErrors::CORRUPT; }); result = std::max(result, key_res.m_result); // Load encrypted keys LoadResult ckey_res = LoadRecords(pwallet, batch, DBKeys::CRYPTED_KEY, - [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + [] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& err) { return LoadCryptedKey(pwallet, key, value, err) ? DBErrors::LOAD_OK : DBErrors::CORRUPT; }); result = std::max(result, ckey_res.m_result); // Load scripts LoadResult script_res = LoadRecords(pwallet, batch, DBKeys::CSCRIPT, - [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& strErr) { + [] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& strErr) { uint160 hash; key >> hash; CScript script; @@ -599,7 +599,7 @@ static DBErrors LoadLegacyWalletRecords(CWallet* pwallet, DatabaseBatch& batch, // Load keymeta std::map<uint160, CHDChain> hd_chains; LoadResult keymeta_res = LoadRecords(pwallet, batch, DBKeys::KEYMETA, - [&hd_chains] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& strErr) { + [&hd_chains] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& strErr) { CPubKey vchPubKey; key >> vchPubKey; CKeyMetadata keyMeta; @@ -686,7 +686,7 @@ static DBErrors LoadLegacyWalletRecords(CWallet* pwallet, DatabaseBatch& batch, // Load watchonly scripts LoadResult watch_script_res = LoadRecords(pwallet, batch, DBKeys::WATCHS, - [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + [] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& err) { CScript script; key >> script; uint8_t fYes; @@ -700,7 +700,7 @@ static DBErrors LoadLegacyWalletRecords(CWallet* pwallet, DatabaseBatch& batch, // Load watchonly meta LoadResult watch_meta_res = LoadRecords(pwallet, batch, DBKeys::WATCHMETA, - [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + [] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& err) { CScript script; key >> script; CKeyMetadata keyMeta; @@ -712,7 +712,7 @@ static DBErrors LoadLegacyWalletRecords(CWallet* pwallet, DatabaseBatch& batch, // Load keypool LoadResult pool_res = LoadRecords(pwallet, batch, DBKeys::POOL, - [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + [] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& err) { int64_t nIndex; key >> nIndex; CKeyPool keypool; @@ -729,7 +729,7 @@ static DBErrors LoadLegacyWalletRecords(CWallet* pwallet, DatabaseBatch& batch, // we want to make sure that it is valid so that we can detect corruption // Note: There should only be one DEFAULTKEY with nothing trailing the type LoadResult default_key_res = LoadRecords(pwallet, batch, DBKeys::DEFAULTKEY, - [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + [] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& err) { CPubKey default_pubkey; try { value >> default_pubkey; @@ -747,7 +747,7 @@ static DBErrors LoadLegacyWalletRecords(CWallet* pwallet, DatabaseBatch& batch, // "wkey" records are unsupported, if we see any, throw an error LoadResult wkey_res = LoadRecords(pwallet, batch, DBKeys::OLD_KEY, - [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + [] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& err) { err = "Found unsupported 'wkey' record, try loading with version 0.18"; return DBErrors::LOAD_FAIL; }); @@ -787,7 +787,7 @@ static DBErrors LoadDescriptorWalletRecords(CWallet* pwallet, DatabaseBatch& bat int num_keys = 0; int num_ckeys= 0; LoadResult desc_res = LoadRecords(pwallet, batch, DBKeys::WALLETDESCRIPTOR, - [&batch, &num_keys, &num_ckeys, &last_client] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& strErr) { + [&batch, &num_keys, &num_ckeys, &last_client] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& strErr) { DBErrors result = DBErrors::LOAD_OK; uint256 id; @@ -817,7 +817,7 @@ static DBErrors LoadDescriptorWalletRecords(CWallet* pwallet, DatabaseBatch& bat // Get key cache for this descriptor DataStream prefix = PrefixStream(DBKeys::WALLETDESCRIPTORCACHE, id); LoadResult key_cache_res = LoadRecords(pwallet, batch, DBKeys::WALLETDESCRIPTORCACHE, prefix, - [&id, &cache] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + [&id, &cache] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& err) { bool parent = true; uint256 desc_id; uint32_t key_exp_index; @@ -850,7 +850,7 @@ static DBErrors LoadDescriptorWalletRecords(CWallet* pwallet, DatabaseBatch& bat // Get last hardened cache for this descriptor prefix = PrefixStream(DBKeys::WALLETDESCRIPTORLHCACHE, id); LoadResult lh_cache_res = LoadRecords(pwallet, batch, DBKeys::WALLETDESCRIPTORLHCACHE, prefix, - [&id, &cache] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + [&id, &cache] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& err) { uint256 desc_id; uint32_t key_exp_index; key >> desc_id; @@ -874,7 +874,7 @@ static DBErrors LoadDescriptorWalletRecords(CWallet* pwallet, DatabaseBatch& bat // Get unencrypted keys prefix = PrefixStream(DBKeys::WALLETDESCRIPTORKEY, id); LoadResult key_res = LoadRecords(pwallet, batch, DBKeys::WALLETDESCRIPTORKEY, prefix, - [&id, &spk_man] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& strErr) { + [&id, &spk_man] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& strErr) { uint256 desc_id; CPubKey pubkey; key >> desc_id; @@ -918,7 +918,7 @@ static DBErrors LoadDescriptorWalletRecords(CWallet* pwallet, DatabaseBatch& bat // Get encrypted keys prefix = PrefixStream(DBKeys::WALLETDESCRIPTORCKEY, id); LoadResult ckey_res = LoadRecords(pwallet, batch, DBKeys::WALLETDESCRIPTORCKEY, prefix, - [&id, &spk_man] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + [&id, &spk_man] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& err) { uint256 desc_id; CPubKey pubkey; key >> desc_id; @@ -957,7 +957,7 @@ static DBErrors LoadAddressBookRecords(CWallet* pwallet, DatabaseBatch& batch) E // Load name record LoadResult name_res = LoadRecords(pwallet, batch, DBKeys::NAME, - [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { + [] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& err) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { std::string strAddress; key >> strAddress; std::string label; @@ -969,7 +969,7 @@ static DBErrors LoadAddressBookRecords(CWallet* pwallet, DatabaseBatch& batch) E // Load purpose record LoadResult purpose_res = LoadRecords(pwallet, batch, DBKeys::PURPOSE, - [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { + [] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& err) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { std::string strAddress; key >> strAddress; std::string purpose_str; @@ -985,7 +985,7 @@ static DBErrors LoadAddressBookRecords(CWallet* pwallet, DatabaseBatch& batch) E // Load destination data record LoadResult dest_res = LoadRecords(pwallet, batch, DBKeys::DESTDATA, - [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { + [] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& err) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { std::string strAddress, strKey, strValue; key >> strAddress; key >> strKey; @@ -1019,7 +1019,7 @@ static DBErrors LoadTxRecords(CWallet* pwallet, DatabaseBatch& batch, std::vecto // Load tx record any_unordered = false; LoadResult tx_res = LoadRecords(pwallet, batch, DBKeys::TX, - [&any_unordered, &upgraded_txs] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { + [&any_unordered, &upgraded_txs] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& err) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { DBErrors result = DBErrors::LOAD_OK; uint256 hash; key >> hash; @@ -1072,8 +1072,8 @@ static DBErrors LoadTxRecords(CWallet* pwallet, DatabaseBatch& batch, std::vecto // Load locked utxo record LoadResult locked_utxo_res = LoadRecords(pwallet, batch, DBKeys::LOCKED_UTXO, - [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { - uint256 hash; + [] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& err) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { + Txid hash; uint32_t n; key >> hash; key >> n; @@ -1085,7 +1085,7 @@ static DBErrors LoadTxRecords(CWallet* pwallet, DatabaseBatch& batch, std::vecto // Load orderposnext record // Note: There should only be one ORDERPOSNEXT record with nothing trailing the type LoadResult order_pos_res = LoadRecords(pwallet, batch, DBKeys::ORDERPOSNEXT, - [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { + [] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& err) EXCLUSIVE_LOCKS_REQUIRED(pwallet->cs_wallet) { try { value >> pwallet->nOrderPosNext; } catch (const std::exception& e) { @@ -1108,7 +1108,7 @@ static DBErrors LoadActiveSPKMs(CWallet* pwallet, DatabaseBatch& batch) EXCLUSIV std::set<std::pair<OutputType, bool>> seen_spks; for (const auto& spk_key : {DBKeys::ACTIVEEXTERNALSPK, DBKeys::ACTIVEINTERNALSPK}) { LoadResult spkm_res = LoadRecords(pwallet, batch, spk_key, - [&seen_spks, &spk_key] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& strErr) { + [&seen_spks, &spk_key] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& strErr) { uint8_t output_type; key >> output_type; uint256 id; @@ -1134,7 +1134,7 @@ static DBErrors LoadDecryptionKeys(CWallet* pwallet, DatabaseBatch& batch) EXCLU // Load decryption key (mkey) records LoadResult mkey_res = LoadRecords(pwallet, batch, DBKeys::MASTER_KEY, - [] (CWallet* pwallet, DataStream& key, CDataStream& value, std::string& err) { + [] (CWallet* pwallet, DataStream& key, DataStream& value, std::string& err) { if (!LoadEncryptionKey(pwallet, key, value, err)) { return DBErrors::CORRUPT; } @@ -1401,13 +1401,13 @@ bool WalletBatch::EraseRecords(const std::unordered_set<std::string>& types) } // Make a copy of key to avoid data being deleted by the following read of the type - Span key_data{key}; + const SerializeData key_data{key.begin(), key.end()}; std::string type; key >> type; if (types.count(type) > 0) { - if (!m_batch->Erase(key_data)) { + if (!m_batch->Erase(Span{key_data})) { cursor.reset(nullptr); m_batch->TxnAbort(); return false; // erase failed @@ -1499,17 +1499,19 @@ std::unique_ptr<WalletDatabase> MakeDatabase(const fs::path& path, const Databas if (format == DatabaseFormat::SQLITE) { #ifdef USE_SQLITE return MakeSQLiteDatabase(path, options, status, error); -#endif +#else error = Untranslated(strprintf("Failed to open database path '%s'. Build does not support SQLite database format.", fs::PathToString(path))); status = DatabaseStatus::FAILED_BAD_FORMAT; return nullptr; +#endif } #ifdef USE_BDB return MakeBerkeleyDatabase(path, options, status, error); -#endif +#else error = Untranslated(strprintf("Failed to open database path '%s'. Build does not support Berkeley DB database format.", fs::PathToString(path))); status = DatabaseStatus::FAILED_BAD_FORMAT; return nullptr; +#endif } } // namespace wallet diff --git a/src/wallet/wallettool.cpp b/src/wallet/wallettool.cpp index 2f3f8ef77d..cda344ab19 100644 --- a/src/wallet/wallettool.cpp +++ b/src/wallet/wallettool.cpp @@ -68,7 +68,6 @@ static std::shared_ptr<CWallet> MakeWallet(const std::string& name, const fs::pa } if (load_wallet_ret != DBErrors::LOAD_OK) { - wallet_instance = nullptr; if (load_wallet_ret == DBErrors::CORRUPT) { tfm::format(std::cerr, "Error loading %s: Wallet corrupted", name); return nullptr; @@ -194,10 +193,15 @@ bool ExecuteWalletToolFunc(const ArgsManager& args, const std::string& command) DatabaseOptions options; ReadDatabaseArgs(args, options); options.require_existing = true; - const std::shared_ptr<CWallet> wallet_instance = MakeWallet(name, path, options); - if (!wallet_instance) return false; + DatabaseStatus status; bilingual_str error; - bool ret = DumpWallet(args, *wallet_instance, error); + std::unique_ptr<WalletDatabase> database = MakeDatabase(path, options, status, error); + if (!database) { + tfm::format(std::cerr, "%s\n", error.original); + return false; + } + + bool ret = DumpWallet(args, *database, error); if (!ret && !error.empty()) { tfm::format(std::cerr, "%s\n", error.original); return ret; diff --git a/src/wallet/walletutil.h b/src/wallet/walletutil.h index c5975144c1..7ad3ffe9e4 100644 --- a/src/wallet/walletutil.h +++ b/src/wallet/walletutil.h @@ -85,6 +85,7 @@ class WalletDescriptor { public: std::shared_ptr<Descriptor> descriptor; + uint256 id; // Descriptor ID (calculated once at descriptor initialization/deserialization) uint64_t creation_time = 0; int32_t range_start = 0; // First item in range; start of range, inclusive, i.e. [range_start, range_end). This never changes. int32_t range_end = 0; // Item after the last; end of range, exclusive, i.e. [range_start, range_end). This will increment with each TopUp() @@ -99,6 +100,7 @@ public: if (!descriptor) { throw std::ios_base::failure("Invalid descriptor: " + error); } + id = DescriptorID(*descriptor); } SERIALIZE_METHODS(WalletDescriptor, obj) @@ -110,7 +112,7 @@ public: } WalletDescriptor() {} - WalletDescriptor(std::shared_ptr<Descriptor> descriptor, uint64_t creation_time, int32_t range_start, int32_t range_end, int32_t next_index) : descriptor(descriptor), creation_time(creation_time), range_start(range_start), range_end(range_end), next_index(next_index) {} + WalletDescriptor(std::shared_ptr<Descriptor> descriptor, uint64_t creation_time, int32_t range_start, int32_t range_end, int32_t next_index) : descriptor(descriptor), id(DescriptorID(*descriptor)), creation_time(creation_time), range_start(range_start), range_end(range_end), next_index(next_index) { } }; } // namespace wallet diff --git a/src/zmq/zmqnotificationinterface.cpp b/src/zmq/zmqnotificationinterface.cpp index 03aae86577..63c2737706 100644 --- a/src/zmq/zmqnotificationinterface.cpp +++ b/src/zmq/zmqnotificationinterface.cpp @@ -6,6 +6,7 @@ #include <common/args.h> #include <kernel/chain.h> +#include <kernel/mempool_entry.h> #include <logging.h> #include <primitives/block.h> #include <primitives/transaction.h> @@ -152,9 +153,9 @@ void CZMQNotificationInterface::UpdatedBlockTip(const CBlockIndex *pindexNew, co }); } -void CZMQNotificationInterface::TransactionAddedToMempool(const CTransactionRef& ptx, uint64_t mempool_sequence) +void CZMQNotificationInterface::TransactionAddedToMempool(const NewMempoolTransactionInfo& ptx, uint64_t mempool_sequence) { - const CTransaction& tx = *ptx; + const CTransaction& tx = *(ptx.info.m_tx); TryForEachAndRemoveFailed(notifiers, [&tx, mempool_sequence](CZMQAbstractNotifier* notifier) { return notifier->NotifyTransaction(tx) && notifier->NotifyTransactionAcceptance(tx, mempool_sequence); diff --git a/src/zmq/zmqnotificationinterface.h b/src/zmq/zmqnotificationinterface.h index 4246c53bd3..45d0982bd3 100644 --- a/src/zmq/zmqnotificationinterface.h +++ b/src/zmq/zmqnotificationinterface.h @@ -16,6 +16,7 @@ class CBlock; class CBlockIndex; class CZMQAbstractNotifier; +struct NewMempoolTransactionInfo; class CZMQNotificationInterface final : public CValidationInterface { @@ -31,7 +32,7 @@ protected: void Shutdown(); // CValidationInterface - void TransactionAddedToMempool(const CTransactionRef& tx, uint64_t mempool_sequence) override; + void TransactionAddedToMempool(const NewMempoolTransactionInfo& tx, uint64_t mempool_sequence) override; void TransactionRemovedFromMempool(const CTransactionRef& tx, MemPoolRemovalReason reason, uint64_t mempool_sequence) override; void BlockConnected(ChainstateRole role, const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexConnected) override; void BlockDisconnected(const std::shared_ptr<const CBlock>& pblock, const CBlockIndex* pindexDisconnected) override; diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index 1241431523..0f20706364 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -19,7 +19,6 @@ #include <streams.h> #include <sync.h> #include <uint256.h> -#include <version.h> #include <zmq/zmqutil.h> #include <zmq.h> @@ -244,14 +243,14 @@ bool CZMQPublishRawBlockNotifier::NotifyBlock(const CBlockIndex *pindex) { LogPrint(BCLog::ZMQ, "Publish rawblock %s to %s\n", pindex->GetBlockHash().GetHex(), this->address); - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); + DataStream ss; CBlock block; if (!m_get_block_by_index(block, *pindex)) { zmqError("Can't read block from disk"); return false; } - ss << block; + ss << TX_WITH_WITNESS(block); return SendZmqMessage(MSG_RAWBLOCK, &(*ss.begin()), ss.size()); } @@ -260,8 +259,8 @@ bool CZMQPublishRawTransactionNotifier::NotifyTransaction(const CTransaction &tr { uint256 hash = transaction.GetHash(); LogPrint(BCLog::ZMQ, "Publish rawtx %s to %s\n", hash.GetHex(), this->address); - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); - ss << transaction; + DataStream ss; + ss << TX_WITH_WITNESS(transaction); return SendZmqMessage(MSG_RAWTX, &(*ss.begin()), ss.size()); } |