diff options
Diffstat (limited to 'src')
130 files changed, 1556 insertions, 1010 deletions
diff --git a/src/Makefile.test.include b/src/Makefile.test.include index 4a42557372..e9d3bff1de 100644 --- a/src/Makefile.test.include +++ b/src/Makefile.test.include @@ -112,6 +112,7 @@ BITCOIN_TESTS =\ test/miniscript_tests.cpp \ test/minisketch_tests.cpp \ test/multisig_tests.cpp \ + test/net_peer_connection_tests.cpp \ test/net_peer_eviction_tests.cpp \ test/net_tests.cpp \ test/netbase_tests.cpp \ @@ -145,6 +146,7 @@ BITCOIN_TESTS =\ test/sigopcount_tests.cpp \ test/skiplist_tests.cpp \ test/sock_tests.cpp \ + test/span_tests.cpp \ test/streams_tests.cpp \ test/sync_tests.cpp \ test/system_tests.cpp \ diff --git a/src/bench/checkblock.cpp b/src/bench/checkblock.cpp index 269ac847a5..9f52a22672 100644 --- a/src/bench/checkblock.cpp +++ b/src/bench/checkblock.cpp @@ -24,7 +24,7 @@ static void DeserializeBlockTest(benchmark::Bench& bench) bench.unit("block").run([&] { CBlock block; - stream >> block; + stream >> TX_WITH_WITNESS(block); bool rewound = stream.Rewind(benchmark::data::block413567.size()); assert(rewound); }); @@ -41,7 +41,7 @@ static void DeserializeAndCheckBlockTest(benchmark::Bench& bench) bench.unit("block").run([&] { CBlock block; // Note that CBlock caches its checked state, so we need to recreate it here - stream >> block; + stream >> TX_WITH_WITNESS(block); bool rewound = stream.Rewind(benchmark::data::block413567.size()); assert(rewound); diff --git a/src/bench/nanobench.h b/src/bench/nanobench.h index 8b3dc6c71c..127240d3c7 100644 --- a/src/bench/nanobench.h +++ b/src/bench/nanobench.h @@ -33,7 +33,7 @@ // see https://semver.org/ #define ANKERL_NANOBENCH_VERSION_MAJOR 4 // incompatible API changes #define ANKERL_NANOBENCH_VERSION_MINOR 3 // backwards-compatible changes -#define ANKERL_NANOBENCH_VERSION_PATCH 10 // backwards-compatible bug fixes +#define ANKERL_NANOBENCH_VERSION_PATCH 11 // backwards-compatible bug fixes /////////////////////////////////////////////////////////////////////////////////////////////////// // public facing api - as minimal as possible @@ -120,6 +120,10 @@ # define ANKERL_NANOBENCH_IS_TRIVIALLY_COPYABLE(...) std::is_trivially_copyable<__VA_ARGS__>::value #endif +// noexcept may be missing for std::string. +// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=58265 +#define ANKERL_NANOBENCH_PRIVATE_NOEXCEPT_STRING_MOVE() std::is_nothrow_move_assignable<std::string>::value + // declarations /////////////////////////////////////////////////////////////////////////////////// namespace ankerl { @@ -404,7 +408,7 @@ struct Config { Config(); ~Config(); Config& operator=(Config const& other); - Config& operator=(Config&& other) noexcept; + Config& operator=(Config&& other) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)); Config(Config const& other); Config(Config&& other) noexcept; }; @@ -430,7 +434,7 @@ public: ~Result(); Result& operator=(Result const& other); - Result& operator=(Result&& other) noexcept; + Result& operator=(Result&& other) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)); Result(Result const& other); Result(Result&& other) noexcept; @@ -596,7 +600,7 @@ public: * * @return Vector containing the full state: */ - std::vector<uint64_t> state() const; + ANKERL_NANOBENCH(NODISCARD) std::vector<uint64_t> state() const; private: static constexpr uint64_t rotl(uint64_t x, unsigned k) noexcept; @@ -628,7 +632,7 @@ public: Bench(); Bench(Bench&& other) noexcept; - Bench& operator=(Bench&& other) noexcept; + Bench& operator=(Bench&& other) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)); Bench(Bench const& other); Bench& operator=(Bench const& other); ~Bench() noexcept; @@ -818,7 +822,7 @@ public: * Default is zero, so we are fully relying on clockResolutionMultiple(). In most cases this is exactly what you want. If you see * that the evaluation is unreliable with a high `err%`, you can increase either minEpochTime() or minEpochIterations(). * - * @see maxEpochTim), minEpochIterations + * @see maxEpochTime, minEpochIterations * * @param t Minimum time each epoch should take. */ @@ -1030,7 +1034,7 @@ void doNotOptimizeAway(T const& val); // These assembly magic is directly from what Google Benchmark is doing. I have previously used what facebook's folly was doing, but // this seemed to have compilation problems in some cases. Google Benchmark seemed to be the most well tested anyways. -// see https://github.com/google/benchmark/blob/master/include/benchmark/benchmark.h#L307 +// see https://github.com/google/benchmark/blob/v1.7.1/include/benchmark/benchmark.h#L443-L446 template <typename T> void doNotOptimizeAway(T const& val) { // NOLINTNEXTLINE(hicpp-no-assembler) @@ -1781,7 +1785,7 @@ bool isEndlessRunning(std::string const& name); bool isWarningsEnabled(); template <typename T> -T parseFile(std::string const& filename); +T parseFile(std::string const& filename, bool* fail); void gatherStabilityInformation(std::vector<std::string>& warnings, std::vector<std::string>& recommendations); void printStabilityInformationOnce(std::ostream* outStream); @@ -1839,7 +1843,7 @@ class Number { public: Number(int width, int precision, double value); Number(int width, int precision, int64_t value); - std::string to_s() const; + ANKERL_NANOBENCH(NODISCARD) std::string to_s() const; private: friend std::ostream& operator<<(std::ostream& os, Number const& n); @@ -1857,11 +1861,11 @@ std::ostream& operator<<(std::ostream& os, Number const& n); class MarkDownColumn { public: - MarkDownColumn(int w, int prec, std::string tit, std::string suff, double val); - std::string title() const; - std::string separator() const; - std::string invalid() const; - std::string value() const; + MarkDownColumn(int w, int prec, std::string tit, std::string suff, double val) noexcept; + ANKERL_NANOBENCH(NODISCARD) std::string title() const; + ANKERL_NANOBENCH(NODISCARD) std::string separator() const; + ANKERL_NANOBENCH(NODISCARD) std::string invalid() const; + ANKERL_NANOBENCH(NODISCARD) std::string value() const; private: int mWidth; @@ -1976,9 +1980,9 @@ PerformanceCounters& performanceCounters() { } // Windows version of doNotOptimizeAway -// see https://github.com/google/benchmark/blob/master/include/benchmark/benchmark.h#L307 -// see https://github.com/facebook/folly/blob/master/folly/Benchmark.h#L280 -// see https://docs.microsoft.com/en-us/cpp/preprocessor/optimize +// see https://github.com/google/benchmark/blob/v1.7.1/include/benchmark/benchmark.h#L514 +// see https://github.com/facebook/folly/blob/v2023.01.30.00/folly/lang/Hint-inl.h#L54-L58 +// see https://learn.microsoft.com/en-us/cpp/preprocessor/optimize # if defined(_MSC_VER) # pragma optimize("", off) void doNotOptimizeAwaySink(void const*) {} @@ -1986,10 +1990,13 @@ void doNotOptimizeAwaySink(void const*) {} # endif template <typename T> -T parseFile(std::string const& filename) { +T parseFile(std::string const& filename, bool* fail) { std::ifstream fin(filename); // NOLINT(misc-const-correctness) T num{}; fin >> num; + if (fail != nullptr) { + *fail = fin.fail(); + } return num; } @@ -2032,16 +2039,15 @@ void gatherStabilityInformation(std::vector<std::string>& warnings, std::vector< if (nprocs <= 0) { warnings.emplace_back("couldn't figure out number of processors - no governor, turbo check possible"); } else { - // check frequency scaling for (long id = 0; id < nprocs; ++id) { auto idStr = detail::fmt::to_s(static_cast<uint64_t>(id)); auto sysCpu = "/sys/devices/system/cpu/cpu" + idStr; - auto minFreq = parseFile<int64_t>(sysCpu + "/cpufreq/scaling_min_freq"); - auto maxFreq = parseFile<int64_t>(sysCpu + "/cpufreq/scaling_max_freq"); + auto minFreq = parseFile<int64_t>(sysCpu + "/cpufreq/scaling_min_freq", nullptr); + auto maxFreq = parseFile<int64_t>(sysCpu + "/cpufreq/scaling_max_freq", nullptr); if (minFreq != maxFreq) { - auto minMHz = static_cast<double>(minFreq) / 1000.0; - auto maxMHz = static_cast<double>(maxFreq) / 1000.0; + auto minMHz = d(minFreq) / 1000.0; + auto maxMHz = d(maxFreq) / 1000.0; warnings.emplace_back("CPU frequency scaling enabled: CPU " + idStr + " between " + detail::fmt::Number(1, 1, minMHz).to_s() + " and " + detail::fmt::Number(1, 1, maxMHz).to_s() + " MHz"); @@ -2050,13 +2056,15 @@ void gatherStabilityInformation(std::vector<std::string>& warnings, std::vector< } } - auto currentGovernor = parseFile<std::string>("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor"); - if ("performance" != currentGovernor) { + auto fail = false; + auto currentGovernor = parseFile<std::string>("/sys/devices/system/cpu/cpu0/cpufreq/scaling_governor", &fail); + if (!fail && "performance" != currentGovernor) { warnings.emplace_back("CPU governor is '" + currentGovernor + "' but should be 'performance'"); recommendPyPerf = true; } - if (0 == parseFile<int>("/sys/devices/system/cpu/intel_pstate/no_turbo")) { + auto noTurbo = parseFile<int>("/sys/devices/system/cpu/intel_pstate/no_turbo", &fail); + if (!fail && noTurbo == 0) { warnings.emplace_back("Turbo is enabled, CPU frequency will fluctuate"); recommendPyPerf = true; } @@ -2250,10 +2258,9 @@ struct IterationLogic::Impl { mNumIters = 0; } - ANKERL_NANOBENCH_LOG(mBench.name() << ": " << detail::fmt::Number(20, 3, static_cast<double>(elapsed.count())) << " elapsed, " - << detail::fmt::Number(20, 3, static_cast<double>(mTargetRuntimePerEpoch.count())) - << " target. oldIters=" << oldIters << ", mNumIters=" << mNumIters - << ", mState=" << static_cast<int>(mState)); + ANKERL_NANOBENCH_LOG(mBench.name() << ": " << detail::fmt::Number(20, 3, d(elapsed.count())) << " elapsed, " + << detail::fmt::Number(20, 3, d(mTargetRuntimePerEpoch.count())) << " target. oldIters=" + << oldIters << ", mNumIters=" << mNumIters << ", mState=" << static_cast<int>(mState)); } // NOLINTNEXTLINE(readability-function-cognitive-complexity) @@ -2357,7 +2364,7 @@ struct IterationLogic::Impl { } os << fmt::MarkDownCode(mBench.name()); if (showUnstable) { - auto avgIters = static_cast<double>(mTotalNumIters) / static_cast<double>(mBench.epochs()); + auto avgIters = d(mTotalNumIters) / d(mBench.epochs()); // NOLINTNEXTLINE(bugprone-incorrect-roundings) auto suggestedIters = static_cast<uint64_t>(avgIters * 10 + 0.5); @@ -2435,7 +2442,7 @@ public: bool monitor(perf_sw_ids swId, Target target); bool monitor(perf_hw_id hwId, Target target); - bool hasError() const noexcept { + ANKERL_NANOBENCH(NODISCARD) bool hasError() const noexcept { return mHasError; } @@ -2691,16 +2698,23 @@ PerformanceCounters::PerformanceCounters() , mVal() , mHas() { - mHas.pageFaults = mPc->monitor(PERF_COUNT_SW_PAGE_FAULTS, LinuxPerformanceCounters::Target(&mVal.pageFaults, true, false)); + // HW events mHas.cpuCycles = mPc->monitor(PERF_COUNT_HW_REF_CPU_CYCLES, LinuxPerformanceCounters::Target(&mVal.cpuCycles, true, false)); - mHas.contextSwitches = - mPc->monitor(PERF_COUNT_SW_CONTEXT_SWITCHES, LinuxPerformanceCounters::Target(&mVal.contextSwitches, true, false)); + if (!mHas.cpuCycles) { + // Fallback to cycles counter, reference cycles not available in many systems. + mHas.cpuCycles = mPc->monitor(PERF_COUNT_HW_CPU_CYCLES, LinuxPerformanceCounters::Target(&mVal.cpuCycles, true, false)); + } mHas.instructions = mPc->monitor(PERF_COUNT_HW_INSTRUCTIONS, LinuxPerformanceCounters::Target(&mVal.instructions, true, true)); mHas.branchInstructions = mPc->monitor(PERF_COUNT_HW_BRANCH_INSTRUCTIONS, LinuxPerformanceCounters::Target(&mVal.branchInstructions, true, false)); mHas.branchMisses = mPc->monitor(PERF_COUNT_HW_BRANCH_MISSES, LinuxPerformanceCounters::Target(&mVal.branchMisses, true, false)); // mHas.branchMisses = false; + // SW events + mHas.pageFaults = mPc->monitor(PERF_COUNT_SW_PAGE_FAULTS, LinuxPerformanceCounters::Target(&mVal.pageFaults, true, false)); + mHas.contextSwitches = + mPc->monitor(PERF_COUNT_SW_CONTEXT_SWITCHES, LinuxPerformanceCounters::Target(&mVal.contextSwitches, true, false)); + mPc->start(); mPc->calibrate([] { auto before = ankerl::nanobench::Clock::now(); @@ -2789,7 +2803,7 @@ void StreamStateRestorer::restore() { Number::Number(int width, int precision, int64_t value) : mWidth(width) , mPrecision(precision) - , mValue(static_cast<double>(value)) {} + , mValue(d(value)) {} Number::Number(int width, int precision, double value) : mWidth(width) @@ -2823,7 +2837,7 @@ std::ostream& operator<<(std::ostream& os, Number const& n) { return n.write(os); } -MarkDownColumn::MarkDownColumn(int w, int prec, std::string tit, std::string suff, double val) +MarkDownColumn::MarkDownColumn(int w, int prec, std::string tit, std::string suff, double val) noexcept : mWidth(w) , mPrecision(prec) , mTitle(std::move(tit)) @@ -2884,14 +2898,14 @@ std::ostream& operator<<(std::ostream& os, MarkDownCode const& mdCode) { Config::Config() = default; Config::~Config() = default; Config& Config::operator=(Config const&) = default; -Config& Config::operator=(Config&&) noexcept = default; +Config& Config::operator=(Config&&) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)) = default; Config::Config(Config const&) = default; Config::Config(Config&&) noexcept = default; // provide implementation here so it's only generated once Result::~Result() = default; Result& Result::operator=(Result const&) = default; -Result& Result::operator=(Result&&) noexcept = default; +Result& Result::operator=(Result&&) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)) = default; Result::Result(Result const&) = default; Result::Result(Result&&) noexcept = default; @@ -2992,7 +3006,7 @@ double Result::medianAbsolutePercentError(Measure m) const { auto data = mNameToMeasurements[detail::u(m)]; // calculates MdAPE which is the median of percentage error - // see https://www.spiderfinancial.com/support/documentation/numxl/reference-manual/forecasting-performance/mdape + // see https://support.numxl.com/hc/en-us/articles/115001223503-MdAPE-Median-Absolute-Percentage-Error auto med = calcMedian(data); // transform the data to absolute error @@ -3106,7 +3120,7 @@ Bench::Bench() { } Bench::Bench(Bench&&) noexcept = default; -Bench& Bench::operator=(Bench&&) noexcept = default; +Bench& Bench::operator=(Bench&&) noexcept(ANKERL_NANOBENCH(NOEXCEPT_STRING_MOVE)) = default; Bench::Bench(Bench const&) = default; Bench& Bench::operator=(Bench const&) = default; Bench::~Bench() noexcept = default; @@ -3423,7 +3437,7 @@ BigO::BigO(std::string bigOName, RangeMeasure const& rangeMeasure) sumMeasure += rm.second; } - auto n = static_cast<double>(rangeMeasure.size()); + auto n = detail::d(rangeMeasure.size()); auto mean = sumMeasure / n; mNormalizedRootMeanSquare = std::sqrt(err / n) / mean; } diff --git a/src/bench/rpc_blockchain.cpp b/src/bench/rpc_blockchain.cpp index a9b197b190..0fb222f3b5 100644 --- a/src/bench/rpc_blockchain.cpp +++ b/src/bench/rpc_blockchain.cpp @@ -27,7 +27,7 @@ struct TestBlockAndIndex { std::byte a{0}; stream.write({&a, 1}); // Prevent compaction - stream >> block; + stream >> TX_WITH_WITNESS(block); blockHash = block.GetHash(); blockindex.phashBlock = &blockHash; diff --git a/src/bench/verify_script.cpp b/src/bench/verify_script.cpp index 11f96b1005..e1a0bf138e 100644 --- a/src/bench/verify_script.cpp +++ b/src/bench/verify_script.cpp @@ -62,7 +62,7 @@ static void VerifyScriptBench(benchmark::Bench& bench) #if defined(HAVE_CONSENSUS_LIB) CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << txSpend; + stream << TX_WITH_WITNESS(txSpend); int csuccess = bitcoinconsensus_verify_script_with_amount( txCredit.vout[0].scriptPubKey.data(), txCredit.vout[0].scriptPubKey.size(), diff --git a/src/bitcoin-chainstate.cpp b/src/bitcoin-chainstate.cpp index fc83a4ad3a..995b4781fc 100644 --- a/src/bitcoin-chainstate.cpp +++ b/src/bitcoin-chainstate.cpp @@ -26,13 +26,13 @@ #include <scheduler.h> #include <script/sigcache.h> #include <util/chaintype.h> +#include <util/fs.h> #include <util/thread.h> #include <validation.h> #include <validationinterface.h> #include <cassert> #include <cstdint> -#include <filesystem> #include <functional> #include <iosfwd> #include <memory> @@ -50,8 +50,8 @@ int main(int argc, char* argv[]) << " BREAK IN FUTURE VERSIONS. DO NOT USE ON YOUR ACTUAL DATADIR." << std::endl; return 1; } - std::filesystem::path abs_datadir = std::filesystem::absolute(argv[1]); - std::filesystem::create_directories(abs_datadir); + fs::path abs_datadir{fs::absolute(argv[1])}; + fs::create_directories(abs_datadir); // SETUP: Context diff --git a/src/blockencodings.cpp b/src/blockencodings.cpp index 211b4740be..29306b2229 100644 --- a/src/blockencodings.cpp +++ b/src/blockencodings.cpp @@ -107,12 +107,12 @@ ReadStatus PartiallyDownloadedBlock::InitData(const CBlockHeaderAndShortTxIDs& c std::vector<bool> have_txn(txn_available.size()); { LOCK(pool->cs); - for (size_t i = 0; i < pool->vTxHashes.size(); i++) { - uint64_t shortid = cmpctblock.GetShortID(pool->vTxHashes[i].first); + for (const auto& tx : pool->txns_randomized) { + uint64_t shortid = cmpctblock.GetShortID(tx->GetWitnessHash()); std::unordered_map<uint64_t, uint16_t>::iterator idit = shorttxids.find(shortid); if (idit != shorttxids.end()) { if (!have_txn[idit->second]) { - txn_available[idit->second] = pool->vTxHashes[i].second->GetSharedTx(); + txn_available[idit->second] = tx; have_txn[idit->second] = true; mempool_count++; } else { diff --git a/src/blockencodings.h b/src/blockencodings.h index afdfa426f1..fb0f734ff8 100644 --- a/src/blockencodings.h +++ b/src/blockencodings.h @@ -65,7 +65,7 @@ public: SERIALIZE_METHODS(BlockTransactions, obj) { - READWRITE(obj.blockhash, Using<VectorFormatter<TransactionCompression>>(obj.txn)); + READWRITE(obj.blockhash, TX_WITH_WITNESS(Using<VectorFormatter<TransactionCompression>>(obj.txn))); } }; @@ -76,7 +76,7 @@ struct PrefilledTransaction { uint16_t index; CTransactionRef tx; - SERIALIZE_METHODS(PrefilledTransaction, obj) { READWRITE(COMPACTSIZE(obj.index), Using<TransactionCompression>(obj.tx)); } + SERIALIZE_METHODS(PrefilledTransaction, obj) { READWRITE(COMPACTSIZE(obj.index), TX_WITH_WITNESS(Using<TransactionCompression>(obj.tx))); } }; typedef enum ReadStatus_t diff --git a/src/common/args.cpp b/src/common/args.cpp index ca04175696..1f25d13bee 100644 --- a/src/common/args.cpp +++ b/src/common/args.cpp @@ -28,7 +28,6 @@ #include <cstdint> #include <cstdlib> #include <cstring> -#include <filesystem> #include <map> #include <optional> #include <stdexcept> @@ -720,6 +719,13 @@ fs::path ArgsManager::GetConfigFilePath() const return *Assert(m_config_path); } +void ArgsManager::SetConfigFilePath(fs::path path) +{ + LOCK(cs_args); + assert(!m_config_path); + m_config_path = path; +} + ChainType ArgsManager::GetChainType() const { std::variant<ChainType, std::string> arg = GetChainArg(); diff --git a/src/common/args.h b/src/common/args.h index ae3ed02bc7..1c5db718f4 100644 --- a/src/common/args.h +++ b/src/common/args.h @@ -180,6 +180,7 @@ protected: * Return config file path (read-only) */ fs::path GetConfigFilePath() const; + void SetConfigFilePath(fs::path); [[nodiscard]] bool ReadConfigFiles(std::string& error, bool ignore_invalid_keys = false); /** diff --git a/src/consensus/tx_check.cpp b/src/consensus/tx_check.cpp index f949655909..b3fee1e8b1 100644 --- a/src/consensus/tx_check.cpp +++ b/src/consensus/tx_check.cpp @@ -16,8 +16,9 @@ bool CheckTransaction(const CTransaction& tx, TxValidationState& state) if (tx.vout.empty()) return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-vout-empty"); // Size limits (this doesn't take the witness into account, as that hasn't been checked for malleability) - if (::GetSerializeSize(tx, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT) + if (::GetSerializeSize(TX_NO_WITNESS(tx)) * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT) { return state.Invalid(TxValidationResult::TX_CONSENSUS, "bad-txns-oversize"); + } // Check for negative or overflow output values (see CVE-2010-5139) CAmount nValueOut = 0; diff --git a/src/consensus/validation.h b/src/consensus/validation.h index d5bf08cd61..3fdc01e66b 100644 --- a/src/consensus/validation.h +++ b/src/consensus/validation.h @@ -54,6 +54,8 @@ enum class TxValidationResult { TX_CONFLICT, TX_MEMPOOL_POLICY, //!< violated mempool's fee/size/descendant/RBF/etc limits TX_NO_MEMPOOL, //!< this node does not have a mempool so can't validate the transaction + TX_RECONSIDERABLE, //!< fails some policy, but might be acceptable if submitted in a (different) package + TX_UNKNOWN, //!< transaction was not validated because package failed }; /** A "reason" why a block was invalid, suitable for determining whether the @@ -147,16 +149,16 @@ class BlockValidationState : public ValidationState<BlockValidationResult> {}; // weight = (stripped_size * 3) + total_size. static inline int32_t GetTransactionWeight(const CTransaction& tx) { - return ::GetSerializeSize(tx, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(tx, PROTOCOL_VERSION); + return ::GetSerializeSize(TX_NO_WITNESS(tx)) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(TX_WITH_WITNESS(tx)); } static inline int64_t GetBlockWeight(const CBlock& block) { - return ::GetSerializeSize(block, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(block, PROTOCOL_VERSION); + return ::GetSerializeSize(TX_NO_WITNESS(block)) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(TX_WITH_WITNESS(block)); } static inline int64_t GetTransactionInputWeight(const CTxIn& txin) { // scriptWitness size is added here because witnesses and txins are split up in segwit serialization. - return ::GetSerializeSize(txin, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(txin, PROTOCOL_VERSION) + ::GetSerializeSize(txin.scriptWitness.stack, PROTOCOL_VERSION); + return ::GetSerializeSize(TX_NO_WITNESS(txin)) * (WITNESS_SCALE_FACTOR - 1) + ::GetSerializeSize(TX_WITH_WITNESS(txin)) + ::GetSerializeSize(txin.scriptWitness.stack); } /** Compute at which vout of the block's coinbase transaction the witness commitment occurs, or -1 if not found */ diff --git a/src/core_io.h b/src/core_io.h index 1f5ecbaea6..a104f240c1 100644 --- a/src/core_io.h +++ b/src/core_io.h @@ -51,9 +51,9 @@ bool ParseHashStr(const std::string& strHex, uint256& result); // core_write.cpp UniValue ValueFromAmount(const CAmount amount); std::string FormatScript(const CScript& script); -std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags = 0); +std::string EncodeHexTx(const CTransaction& tx, const bool without_witness = false); std::string SighashToStr(unsigned char sighash_type); void ScriptToUniv(const CScript& script, UniValue& out, bool include_hex = true, bool include_address = false, const SigningProvider* provider = nullptr); -void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex = true, int serialize_flags = 0, const CTxUndo* txundo = nullptr, TxVerbosity verbosity = TxVerbosity::SHOW_DETAILS); +void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex = true, bool without_witness = false, const CTxUndo* txundo = nullptr, TxVerbosity verbosity = TxVerbosity::SHOW_DETAILS); #endif // BITCOIN_CORE_IO_H diff --git a/src/core_read.cpp b/src/core_read.cpp index dfabf3a0c2..bffb9e9042 100644 --- a/src/core_read.cpp +++ b/src/core_read.cpp @@ -142,9 +142,9 @@ static bool DecodeTx(CMutableTransaction& tx, const std::vector<unsigned char>& // Try decoding with extended serialization support, and remember if the result successfully // consumes the entire input. if (try_witness) { - CDataStream ssData(tx_data, SER_NETWORK, PROTOCOL_VERSION); + DataStream ssData(tx_data); try { - ssData >> tx_extended; + ssData >> TX_WITH_WITNESS(tx_extended); if (ssData.empty()) ok_extended = true; } catch (const std::exception&) { // Fall through. @@ -160,9 +160,9 @@ static bool DecodeTx(CMutableTransaction& tx, const std::vector<unsigned char>& // Try decoding with legacy serialization, and remember if the result successfully consumes the entire input. if (try_no_witness) { - CDataStream ssData(tx_data, SER_NETWORK, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS); + DataStream ssData(tx_data); try { - ssData >> tx_legacy; + ssData >> TX_NO_WITNESS(tx_legacy); if (ssData.empty()) ok_legacy = true; } catch (const std::exception&) { // Fall through. @@ -222,9 +222,9 @@ bool DecodeHexBlk(CBlock& block, const std::string& strHexBlk) return false; std::vector<unsigned char> blockData(ParseHex(strHexBlk)); - CDataStream ssBlock(blockData, SER_NETWORK, PROTOCOL_VERSION); + DataStream ssBlock(blockData); try { - ssBlock >> block; + ssBlock >> TX_WITH_WITNESS(block); } catch (const std::exception&) { return false; diff --git a/src/core_write.cpp b/src/core_write.cpp index 7cf019a42e..a2be17d1aa 100644 --- a/src/core_write.cpp +++ b/src/core_write.cpp @@ -140,10 +140,14 @@ std::string ScriptToAsmStr(const CScript& script, const bool fAttemptSighashDeco return str; } -std::string EncodeHexTx(const CTransaction& tx, const int serializeFlags) +std::string EncodeHexTx(const CTransaction& tx, const bool without_witness) { - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | serializeFlags); - ssTx << tx; + DataStream ssTx; + if (without_witness) { + ssTx << TX_NO_WITNESS(tx); + } else { + ssTx << TX_WITH_WITNESS(tx); + } return HexStr(ssTx); } @@ -168,7 +172,7 @@ void ScriptToUniv(const CScript& script, UniValue& out, bool include_hex, bool i out.pushKV("type", GetTxnOutputType(type)); } -void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex, int serialize_flags, const CTxUndo* txundo, TxVerbosity verbosity) +void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry, bool include_hex, bool without_witness, const CTxUndo* txundo, TxVerbosity verbosity) { CHECK_NONFATAL(verbosity >= TxVerbosity::SHOW_DETAILS); @@ -177,7 +181,7 @@ void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry // Transaction version is actually unsigned in consensus checks, just signed in memory, // so cast to unsigned before giving it to the user. entry.pushKV("version", static_cast<int64_t>(static_cast<uint32_t>(tx.nVersion))); - entry.pushKV("size", (int)::GetSerializeSize(tx, PROTOCOL_VERSION)); + entry.pushKV("size", tx.GetTotalSize()); entry.pushKV("vsize", (GetTransactionWeight(tx) + WITNESS_SCALE_FACTOR - 1) / WITNESS_SCALE_FACTOR); entry.pushKV("weight", GetTransactionWeight(tx)); entry.pushKV("locktime", (int64_t)tx.nLockTime); @@ -264,6 +268,6 @@ void TxToUniv(const CTransaction& tx, const uint256& block_hash, UniValue& entry } if (include_hex) { - entry.pushKV("hex", EncodeHexTx(tx, serialize_flags)); // The hex-encoded transaction. Used the name "hex" to be consistent with the verbose output of "getrawtransaction". + entry.pushKV("hex", EncodeHexTx(tx, without_witness)); // The hex-encoded transaction. Used the name "hex" to be consistent with the verbose output of "getrawtransaction". } } diff --git a/src/hash.h b/src/hash.h index d355b703ff..132e1a9fb3 100644 --- a/src/hash.h +++ b/src/hash.h @@ -146,23 +146,6 @@ public: } }; -class CHashWriter : public HashWriter -{ -private: - const int nVersion; - -public: - CHashWriter(int nVersionIn) : nVersion{nVersionIn} {} - - int GetVersion() const { return nVersion; } - - template<typename T> - CHashWriter& operator<<(const T& obj) { - ::Serialize(*this, obj); - return (*this); - } -}; - /** Reads data from an underlying stream, while hashing the read data. */ template <typename Source> class HashVerifier : public HashWriter diff --git a/src/httprpc.cpp b/src/httprpc.cpp index 661406e122..c72dbf10bc 100644 --- a/src/httprpc.cpp +++ b/src/httprpc.cpp @@ -8,6 +8,7 @@ #include <crypto/hmac_sha256.h> #include <httpserver.h> #include <logging.h> +#include <netaddress.h> #include <rpc/protocol.h> #include <rpc/server.h> #include <util/strencodings.h> diff --git a/src/index/txindex.cpp b/src/index/txindex.cpp index e16dd0f8bd..5d37fd0d8f 100644 --- a/src/index/txindex.cpp +++ b/src/index/txindex.cpp @@ -65,7 +65,7 @@ bool TxIndex::CustomAppend(const interfaces::BlockInfo& block) vPos.reserve(block.data->vtx.size()); for (const auto& tx : block.data->vtx) { vPos.emplace_back(tx->GetHash(), pos); - pos.nTxOffset += ::GetSerializeSize(*tx, CLIENT_VERSION); + pos.nTxOffset += ::GetSerializeSize(TX_WITH_WITNESS(*tx)); } return m_db->WriteTxs(vPos); } @@ -89,7 +89,7 @@ bool TxIndex::FindTx(const uint256& tx_hash, uint256& block_hash, CTransactionRe if (fseek(file.Get(), postx.nTxOffset, SEEK_CUR)) { return error("%s: fseek(...) failed", __func__); } - file >> tx; + file >> TX_WITH_WITNESS(tx); } catch (const std::exception& e) { return error("%s: Deserialize or I/O error - %s", __func__, e.what()); } diff --git a/src/init.cpp b/src/init.cpp index 42331d37e8..d6dc62f707 100644 --- a/src/init.cpp +++ b/src/init.cpp @@ -343,11 +343,11 @@ void Shutdown(NodeContext& node) node.chain_clients.clear(); UnregisterAllValidationInterfaces(); GetMainSignals().UnregisterBackgroundSignalScheduler(); - node.kernel.reset(); node.mempool.reset(); node.fee_estimator.reset(); node.chainman.reset(); node.scheduler.reset(); + node.kernel.reset(); try { if (!fs::remove(GetPidFile(*node.args))) { @@ -455,9 +455,14 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-maxorphantx=<n>", strprintf("Keep at most <n> unconnectable transactions in memory (default: %u)", DEFAULT_MAX_ORPHAN_TRANSACTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-mempoolexpiry=<n>", strprintf("Do not keep transactions in the mempool longer than <n> hours (default: %u)", DEFAULT_MEMPOOL_EXPIRY_HOURS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-minimumchainwork=<hex>", strprintf("Minimum work assumed to exist on a valid chain in hex (default: %s, testnet: %s, signet: %s)", defaultChainParams->GetConsensus().nMinimumChainWork.GetHex(), testnetChainParams->GetConsensus().nMinimumChainWork.GetHex(), signetChainParams->GetConsensus().nMinimumChainWork.GetHex()), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::OPTIONS); - argsman.AddArg("-par=<n>", strprintf("Set the number of script verification threads (%u to %d, 0 = auto, <0 = leave that many cores free, default: %d)", - -GetNumCores(), MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-par=<n>", strprintf("Set the number of script verification threads (0 = auto, up to %d, <0 = leave that many cores free, default: %d)", + MAX_SCRIPTCHECK_THREADS, DEFAULT_SCRIPTCHECK_THREADS), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-persistmempool", strprintf("Whether to save the mempool on shutdown and load on restart (default: %u)", DEFAULT_PERSIST_MEMPOOL), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); + argsman.AddArg("-persistmempoolv1", + strprintf("Whether a mempool.dat file created by -persistmempool or the savemempool RPC will be written in the legacy format " + "(version 1) or the current format (version 2). This temporary option will be removed in the future. (default: %u)", + DEFAULT_PERSIST_V1_DAT), + ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-pid=<file>", strprintf("Specify pid file. Relative paths will be prefixed by a net-specific datadir location. (default: %s)", BITCOIN_PID_FILENAME), ArgsManager::ALLOW_ANY, OptionsCategory::OPTIONS); argsman.AddArg("-prune=<n>", strprintf("Reduce storage requirements by enabling pruning (deleting) of old blocks. This allows the pruneblockchain RPC to be called to delete specific blocks and enables automatic pruning of old blocks if a target size in MiB is provided. This mode is incompatible with -txindex. " "Warning: Reverting this setting requires re-downloading the entire blockchain. " @@ -489,7 +494,7 @@ void SetupServerArgs(ArgsManager& argsman) argsman.AddArg("-forcednsseed", strprintf("Always query for peer addresses via DNS lookup (default: %u)", DEFAULT_FORCEDNSSEED), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-listen", strprintf("Accept connections from outside (default: %u if no -proxy, -connect or -maxconnections=0)", DEFAULT_LISTEN), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-listenonion", strprintf("Automatically create Tor onion service (default: %d)", DEFAULT_LISTEN_ONION), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); - argsman.AddArg("-maxconnections=<n>", strprintf("Maintain at most <n> connections to peers (default: %u). This limit does not apply to connections manually added via -addnode or the addnode RPC, which have a separate limit of %u.", DEFAULT_MAX_PEER_CONNECTIONS, MAX_ADDNODE_CONNECTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); + argsman.AddArg("-maxconnections=<n>", strprintf("Maintain at most <n> automatic connections to peers (default: %u). This limit does not apply to connections manually added via -addnode or the addnode RPC, which have a separate limit of %u.", DEFAULT_MAX_PEER_CONNECTIONS, MAX_ADDNODE_CONNECTIONS), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxreceivebuffer=<n>", strprintf("Maximum per-connection receive buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXRECEIVEBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxsendbuffer=<n>", strprintf("Maximum per-connection memory usage for the send buffer, <n>*1000 bytes (default: %u)", DEFAULT_MAXSENDBUFFER), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); argsman.AddArg("-maxtimeadjustment", strprintf("Maximum allowed median peer time offset adjustment. Local perspective of time may be influenced by outbound peers forward or backward by this amount (default: %u seconds).", DEFAULT_MAX_TIME_ADJUSTMENT), ArgsManager::ALLOW_ANY, OptionsCategory::CONNECTION); @@ -1751,11 +1756,7 @@ bool AppInitMain(NodeContext& node, interfaces::BlockAndHeaderTipInfo* tip_info) CConnman::Options connOptions; connOptions.nLocalServices = nLocalServices; - connOptions.nMaxConnections = nMaxConnections; - connOptions.m_max_outbound_full_relay = std::min(MAX_OUTBOUND_FULL_RELAY_CONNECTIONS, connOptions.nMaxConnections); - connOptions.m_max_outbound_block_relay = std::min(MAX_BLOCK_RELAY_ONLY_CONNECTIONS, connOptions.nMaxConnections-connOptions.m_max_outbound_full_relay); - connOptions.nMaxAddnode = MAX_ADDNODE_CONNECTIONS; - connOptions.nMaxFeeler = MAX_FEELER_CONNECTIONS; + connOptions.m_max_automatic_connections = nMaxConnections; connOptions.uiInterface = &uiInterface; connOptions.m_banman = node.banman.get(); connOptions.m_msgproc = node.peerman.get(); diff --git a/src/interfaces/chain.h b/src/interfaces/chain.h index dea868f844..aeb3797392 100644 --- a/src/interfaces/chain.h +++ b/src/interfaces/chain.h @@ -244,7 +244,7 @@ public: // outputs in the same transaction) or have shared ancestry, the bump fees are calculated // independently, i.e. as if only one of them is spent. This may result in double-fee-bumping. This // caveat can be rectified per use of the sister-function CalculateCombinedBumpFee(…). - virtual std::map<COutPoint, CAmount> CalculateIndividualBumpFees(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) = 0; + virtual std::map<COutPoint, CAmount> calculateIndividualBumpFees(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) = 0; //! Calculate the combined bump fee for an input set per the same strategy // as in CalculateIndividualBumpFees(…). @@ -252,7 +252,7 @@ public: // bump fees per outpoint, but a single bump fee for the shared ancestry. // The combined bump fee may be used to correct overestimation due to // shared ancestry by multiple UTXOs after coin selection. - virtual std::optional<CAmount> CalculateCombinedBumpFee(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) = 0; + virtual std::optional<CAmount> calculateCombinedBumpFee(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) = 0; //! Get the node's package limits. //! Currently only returns the ancestor and descendant count limits, but could be enhanced to @@ -335,7 +335,7 @@ public: virtual void rpcRunLater(const std::string& name, std::function<void()> fn, int64_t seconds) = 0; //! Current RPC serialization flags. - virtual int rpcSerializationFlags() = 0; + virtual bool rpcSerializationWithoutWitness() = 0; //! Get settings value. virtual common::SettingsValue getSetting(const std::string& arg) = 0; @@ -395,6 +395,9 @@ public: //! Set mock time. virtual void setMockTime(int64_t time) = 0; + + //! Mock the scheduler to fast forward in time. + virtual void schedulerMockForward(std::chrono::seconds delta_seconds) = 0; }; //! Return implementation of Chain interface. diff --git a/src/interfaces/node.h b/src/interfaces/node.h index 3f8df57124..aeb2612c07 100644 --- a/src/interfaces/node.h +++ b/src/interfaces/node.h @@ -204,8 +204,8 @@ public: //! Unset RPC timer interface. virtual void rpcUnsetTimerInterface(RPCTimerInterface* iface) = 0; - //! Get unspent outputs associated with a transaction. - virtual bool getUnspentOutput(const COutPoint& output, Coin& coin) = 0; + //! Get unspent output associated with a transaction. + virtual std::optional<Coin> getUnspentOutput(const COutPoint& output) = 0; //! Broadcast transaction. virtual TransactionError broadcastTransaction(CTransactionRef tx, CAmount max_tx_fee, std::string& err_string) = 0; diff --git a/src/interfaces/wallet.h b/src/interfaces/wallet.h index 4b896c11a3..6114236623 100644 --- a/src/interfaces/wallet.h +++ b/src/interfaces/wallet.h @@ -118,7 +118,7 @@ public: wallet::AddressPurpose* purpose) = 0; //! Get wallet address list. - virtual std::vector<WalletAddress> getAddresses() const = 0; + virtual std::vector<WalletAddress> getAddresses() = 0; //! Get receive requests. virtual std::vector<std::string> getAddressReceiveRequests() = 0; diff --git a/src/kernel/mempool_entry.h b/src/kernel/mempool_entry.h index 1f175a5ccf..7c905ca4f4 100644 --- a/src/kernel/mempool_entry.h +++ b/src/kernel/mempool_entry.h @@ -83,7 +83,7 @@ private: const bool spendsCoinbase; //!< keep track of transactions that spend a coinbase const int64_t sigOpCost; //!< Total sigop cost CAmount m_modified_fee; //!< Used for determining the priority of the transaction for mining in a block - LockPoints lockPoints; //!< Track the height and time at which tx was final + mutable LockPoints lockPoints; //!< Track the height and time at which tx was final // Information about descendants of this transaction that are in the // mempool; if we remove this transaction we must remove all of these @@ -151,7 +151,7 @@ public: } // Update the LockPoints after a reorg - void UpdateLockPoints(const LockPoints& lp) + void UpdateLockPoints(const LockPoints& lp) const { lockPoints = lp; } @@ -172,8 +172,10 @@ public: Parents& GetMemPoolParents() const { return m_parents; } Children& GetMemPoolChildren() const { return m_children; } - mutable size_t vTxHashesIdx; //!< Index in mempool's vTxHashes + mutable size_t idx_randomized; //!< Index in mempool's txns_randomized mutable Epoch::Marker m_epoch_marker; //!< epoch when last touched, useful for graph algorithms }; +using CTxMemPoolEntryRef = CTxMemPoolEntry::CTxMemPoolEntryRef; + #endif // BITCOIN_KERNEL_MEMPOOL_ENTRY_H diff --git a/src/kernel/mempool_options.h b/src/kernel/mempool_options.h index 757be41b3c..d09fd2ba35 100644 --- a/src/kernel/mempool_options.h +++ b/src/kernel/mempool_options.h @@ -23,6 +23,8 @@ static constexpr unsigned int DEFAULT_BLOCKSONLY_MAX_MEMPOOL_SIZE_MB{5}; static constexpr unsigned int DEFAULT_MEMPOOL_EXPIRY_HOURS{336}; /** Default for -mempoolfullrbf, if the transaction replaceability signaling is ignored */ static constexpr bool DEFAULT_MEMPOOL_FULL_RBF{false}; +/** Whether to fall back to legacy V1 serialization when writing mempool.dat */ +static constexpr bool DEFAULT_PERSIST_V1_DAT{false}; /** Default for -acceptnonstdtxn */ static constexpr bool DEFAULT_ACCEPT_NON_STD_TXN{false}; @@ -56,6 +58,7 @@ struct MemPoolOptions { bool permit_bare_multisig{DEFAULT_PERMIT_BAREMULTISIG}; bool require_standard{true}; bool full_rbf{DEFAULT_MEMPOOL_FULL_RBF}; + bool persist_v1_dat{DEFAULT_PERSIST_V1_DAT}; MemPoolLimits limits{}; }; } // namespace kernel diff --git a/src/kernel/mempool_persist.cpp b/src/kernel/mempool_persist.cpp index ff655c5ffa..0808d42452 100644 --- a/src/kernel/mempool_persist.cpp +++ b/src/kernel/mempool_persist.cpp @@ -8,6 +8,7 @@ #include <consensus/amount.h> #include <logging.h> #include <primitives/transaction.h> +#include <random.h> #include <serialize.h> #include <streams.h> #include <sync.h> @@ -34,14 +35,14 @@ using fsbridge::FopenFn; namespace kernel { -static const uint64_t MEMPOOL_DUMP_VERSION = 1; +static const uint64_t MEMPOOL_DUMP_VERSION_NO_XOR_KEY{1}; +static const uint64_t MEMPOOL_DUMP_VERSION{2}; bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active_chainstate, ImportMempoolOptions&& opts) { if (load_path.empty()) return false; - FILE* filestr{opts.mockable_fopen_function(load_path, "rb")}; - CAutoFile file{filestr, CLIENT_VERSION}; + AutoFile file{opts.mockable_fopen_function(load_path, "rb")}; if (file.IsNull()) { LogPrintf("Failed to open mempool file from disk. Continuing anyway.\n"); return false; @@ -57,9 +58,15 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active try { uint64_t version; file >> version; - if (version != MEMPOOL_DUMP_VERSION) { + std::vector<std::byte> xor_key; + if (version == MEMPOOL_DUMP_VERSION_NO_XOR_KEY) { + // Leave XOR-key empty + } else if (version == MEMPOOL_DUMP_VERSION) { + file >> xor_key; + } else { return false; } + file.SetXor(xor_key); uint64_t num; file >> num; while (num) { @@ -67,7 +74,7 @@ bool LoadMempool(CTxMemPool& pool, const fs::path& load_path, Chainstate& active CTransactionRef tx; int64_t nTime; int64_t nFeeDelta; - file >> tx; + file >> TX_WITH_WITNESS(tx); file >> nTime; file >> nFeeDelta; @@ -151,20 +158,25 @@ bool DumpMempool(const CTxMemPool& pool, const fs::path& dump_path, FopenFn mock auto mid = SteadyClock::now(); - try { - FILE* filestr{mockable_fopen_function(dump_path + ".new", "wb")}; - if (!filestr) { - return false; - } - - CAutoFile file{filestr, CLIENT_VERSION}; + AutoFile file{mockable_fopen_function(dump_path + ".new", "wb")}; + if (file.IsNull()) { + return false; + } - uint64_t version = MEMPOOL_DUMP_VERSION; + try { + const uint64_t version{pool.m_persist_v1_dat ? MEMPOOL_DUMP_VERSION_NO_XOR_KEY : MEMPOOL_DUMP_VERSION}; file << version; + std::vector<std::byte> xor_key(8); + if (!pool.m_persist_v1_dat) { + FastRandomContext{}.fillrand(xor_key); + file << xor_key; + } + file.SetXor(xor_key); + file << (uint64_t)vinfo.size(); for (const auto& i : vinfo) { - file << *(i.tx); + file << TX_WITH_WITNESS(*(i.tx)); file << int64_t{count_seconds(i.m_time)}; file << int64_t{i.nFeeDelta}; mapDeltas.erase(i.tx->GetHash()); diff --git a/src/net.cpp b/src/net.cpp index 16075efdbb..a2f80cbcf7 100644 --- a/src/net.cpp +++ b/src/net.cpp @@ -417,21 +417,25 @@ CNode* CConnman::ConnectNode(CAddress addrConnect, const char *pszDest, bool fCo const uint16_t default_port{pszDest != nullptr ? GetDefaultPort(pszDest) : m_params.GetDefaultPort()}; if (pszDest) { - const std::vector<CService> resolved{Lookup(pszDest, default_port, fNameLookup && !HaveNameProxy(), 256)}; + std::vector<CService> resolved{Lookup(pszDest, default_port, fNameLookup && !HaveNameProxy(), 256)}; if (!resolved.empty()) { - const CService& rnd{resolved[GetRand(resolved.size())]}; - addrConnect = CAddress{MaybeFlipIPv6toCJDNS(rnd), NODE_NONE}; - if (!addrConnect.IsValid()) { - LogPrint(BCLog::NET, "Resolver returned invalid address %s for %s\n", addrConnect.ToStringAddrPort(), pszDest); - return nullptr; - } - // It is possible that we already have a connection to the IP/port pszDest resolved to. - // In that case, drop the connection that was just created. - LOCK(m_nodes_mutex); - CNode* pnode = FindNode(static_cast<CService>(addrConnect)); - if (pnode) { - LogPrintf("Failed to open new connection, already connected\n"); - return nullptr; + Shuffle(resolved.begin(), resolved.end(), FastRandomContext()); + // If the connection is made by name, it can be the case that the name resolves to more than one address. + // We don't want to connect any more of them if we are already connected to one + for (const auto& r : resolved) { + addrConnect = CAddress{MaybeFlipIPv6toCJDNS(r), NODE_NONE}; + if (!addrConnect.IsValid()) { + LogPrint(BCLog::NET, "Resolver returned invalid address %s for %s\n", addrConnect.ToStringAddrPort(), pszDest); + return nullptr; + } + // It is possible that we already have a connection to the IP/port pszDest resolved to. + // In that case, drop the connection that was just created. + LOCK(m_nodes_mutex); + CNode* pnode = FindNode(static_cast<CService>(addrConnect)); + if (pnode) { + LogPrintf("Not opening a connection to %s, already connected to %s\n", pszDest, addrConnect.ToStringAddrPort()); + return nullptr; + } } } } @@ -680,10 +684,8 @@ bool CNode::ReceiveMsgBytes(Span<const uint8_t> msg_bytes, bool& complete) } V1Transport::V1Transport(const NodeId node_id, int nTypeIn, int nVersionIn) noexcept : - m_node_id(node_id), hdrbuf(nTypeIn, nVersionIn), vRecv(nTypeIn, nVersionIn) + m_magic_bytes{Params().MessageStart()}, m_node_id(node_id), hdrbuf(nTypeIn, nVersionIn), vRecv(nTypeIn, nVersionIn) { - assert(std::size(Params().MessageStart()) == std::size(m_magic_bytes)); - m_magic_bytes = Params().MessageStart(); LOCK(m_recv_mutex); Reset(); } @@ -1730,7 +1732,6 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, const CAddress& addr) { int nInbound = 0; - int nMaxInbound = nMaxConnections - m_max_outbound; AddWhitelistPermissionFlags(permission_flags, addr); if (NetPermissions::HasFlag(permission_flags, NetPermissionFlags::Implicit)) { @@ -1776,13 +1777,13 @@ void CConnman::CreateNodeFromAcceptedSocket(std::unique_ptr<Sock>&& sock, // Only accept connections from discouraged peers if our inbound slots aren't (almost) full. bool discouraged = m_banman && m_banman->IsDiscouraged(addr); - if (!NetPermissions::HasFlag(permission_flags, NetPermissionFlags::NoBan) && nInbound + 1 >= nMaxInbound && discouraged) + if (!NetPermissions::HasFlag(permission_flags, NetPermissionFlags::NoBan) && nInbound + 1 >= m_max_inbound && discouraged) { LogPrint(BCLog::NET, "connection from %s dropped (discouraged)\n", addr.ToStringAddrPort()); return; } - if (nInbound >= nMaxInbound) + if (nInbound >= m_max_inbound) { if (!AttemptToEvictConnection()) { // No connection to evict, disconnect the new connection @@ -2733,7 +2734,7 @@ void CConnman::ThreadOpenConnections(const std::vector<std::string> connect) // different netgroups in ipv4/ipv6 networks + all peers in Tor/I2P/CJDNS networks. // Don't record addrman failure attempts when node is offline. This can be identified since all local // network connections (if any) belong in the same netgroup, and the size of `outbound_ipv46_peer_netgroups` would only be 1. - const bool count_failures{((int)outbound_ipv46_peer_netgroups.size() + outbound_privacy_network_peers) >= std::min(nMaxConnections - 1, 2)}; + const bool count_failures{((int)outbound_ipv46_peer_netgroups.size() + outbound_privacy_network_peers) >= std::min(m_max_automatic_connections - 1, 2)}; // Use BIP324 transport when both us and them have NODE_V2_P2P set. const bool use_v2transport(addrConnect.nServices & GetLocalServices() & NODE_P2P_V2); OpenNetworkConnection(addrConnect, count_failures, std::move(grant), /*strDest=*/nullptr, conn_type, use_v2transport); @@ -2754,7 +2755,7 @@ std::vector<CAddress> CConnman::GetCurrentBlockRelayOnlyConns() const return ret; } -std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const +std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo(bool include_connected) const { std::vector<AddedNodeInfo> ret; @@ -2789,6 +2790,9 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const // strAddNode is an IP:port auto it = mapConnected.find(service); if (it != mapConnected.end()) { + if (!include_connected) { + continue; + } addedNode.resolvedAddress = service; addedNode.fConnected = true; addedNode.fInbound = it->second; @@ -2797,6 +2801,9 @@ std::vector<AddedNodeInfo> CConnman::GetAddedNodeInfo() const // strAddNode is a name auto it = mapConnectedByName.find(addr.m_added_node); if (it != mapConnectedByName.end()) { + if (!include_connected) { + continue; + } addedNode.resolvedAddress = it->second.second; addedNode.fConnected = true; addedNode.fInbound = it->second.first; @@ -2815,21 +2822,19 @@ void CConnman::ThreadOpenAddedConnections() while (true) { CSemaphoreGrant grant(*semAddnode); - std::vector<AddedNodeInfo> vInfo = GetAddedNodeInfo(); + std::vector<AddedNodeInfo> vInfo = GetAddedNodeInfo(/*include_connected=*/false); bool tried = false; for (const AddedNodeInfo& info : vInfo) { - if (!info.fConnected) { - if (!grant) { - // If we've used up our semaphore and need a new one, let's not wait here since while we are waiting - // the addednodeinfo state might change. - break; - } - tried = true; - CAddress addr(CService(), NODE_NONE); - OpenNetworkConnection(addr, false, std::move(grant), info.m_params.m_added_node.c_str(), ConnectionType::MANUAL, info.m_params.m_use_v2transport); - if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) return; - grant = CSemaphoreGrant(*semAddnode, /*fTry=*/true); + if (!grant) { + // If we've used up our semaphore and need a new one, let's not wait here since while we are waiting + // the addednodeinfo state might change. + break; } + tried = true; + CAddress addr(CService(), NODE_NONE); + OpenNetworkConnection(addr, false, std::move(grant), info.m_params.m_added_node.c_str(), ConnectionType::MANUAL, info.m_params.m_use_v2transport); + if (!interruptNet.sleep_for(std::chrono::milliseconds(500))) return; + grant = CSemaphoreGrant(*semAddnode, /*fTry=*/true); } // Retry every 60 seconds if a connection was attempted, otherwise two seconds if (!interruptNet.sleep_for(std::chrono::seconds(tried ? 60 : 2))) @@ -3208,18 +3213,17 @@ bool CConnman::Start(CScheduler& scheduler, const Options& connOptions) if (semOutbound == nullptr) { // initialize semaphore - semOutbound = std::make_unique<CSemaphore>(std::min(m_max_outbound, nMaxConnections)); + semOutbound = std::make_unique<CSemaphore>(std::min(m_max_automatic_outbound, m_max_automatic_connections)); } if (semAddnode == nullptr) { // initialize semaphore - semAddnode = std::make_unique<CSemaphore>(nMaxAddnode); + semAddnode = std::make_unique<CSemaphore>(m_max_addnode); } // // Start threads // assert(m_msgproc); - InterruptSocks5(false); interruptNet.reset(); flagInterruptMsgProc = false; @@ -3291,16 +3295,16 @@ void CConnman::Interrupt() condMsgProc.notify_all(); interruptNet(); - InterruptSocks5(true); + g_socks5_interrupt(); if (semOutbound) { - for (int i=0; i<m_max_outbound; i++) { + for (int i=0; i<m_max_automatic_outbound; i++) { semOutbound->post(); } } if (semAddnode) { - for (int i=0; i<nMaxAddnode; i++) { + for (int i=0; i<m_max_addnode; i++) { semAddnode->post(); } } @@ -3426,9 +3430,12 @@ std::vector<CAddress> CConnman::GetAddresses(CNode& requestor, size_t max_addres bool CConnman::AddNode(const AddedNodeParams& add) { + const CService resolved(LookupNumeric(add.m_added_node, GetDefaultPort(add.m_added_node))); + const bool resolved_is_valid{resolved.IsValid()}; + LOCK(m_added_nodes_mutex); for (const auto& it : m_added_node_params) { - if (add.m_added_node == it.m_added_node) return false; + if (add.m_added_node == it.m_added_node || (resolved_is_valid && resolved == LookupNumeric(it.m_added_node, GetDefaultPort(it.m_added_node)))) return false; } m_added_node_params.push_back(add); @@ -372,7 +372,7 @@ public: class V1Transport final : public Transport { private: - MessageStartChars m_magic_bytes; + const MessageStartChars m_magic_bytes; const NodeId m_node_id; // Only for logging mutable Mutex m_recv_mutex; //!< Lock for receive state mutable CHash256 hasher GUARDED_BY(m_recv_mutex); @@ -1045,11 +1045,7 @@ public: struct Options { ServiceFlags nLocalServices = NODE_NONE; - int nMaxConnections = 0; - int m_max_outbound_full_relay = 0; - int m_max_outbound_block_relay = 0; - int nMaxAddnode = 0; - int nMaxFeeler = 0; + int m_max_automatic_connections = 0; CClientUIInterface* uiInterface = nullptr; NetEventsInterface* m_msgproc = nullptr; BanMan* m_banman = nullptr; @@ -1076,13 +1072,12 @@ public: AssertLockNotHeld(m_total_bytes_sent_mutex); nLocalServices = connOptions.nLocalServices; - nMaxConnections = connOptions.nMaxConnections; - m_max_outbound_full_relay = std::min(connOptions.m_max_outbound_full_relay, connOptions.nMaxConnections); - m_max_outbound_block_relay = connOptions.m_max_outbound_block_relay; + m_max_automatic_connections = connOptions.m_max_automatic_connections; + m_max_outbound_full_relay = std::min(MAX_OUTBOUND_FULL_RELAY_CONNECTIONS, m_max_automatic_connections); + m_max_outbound_block_relay = std::min(MAX_BLOCK_RELAY_ONLY_CONNECTIONS, m_max_automatic_connections - m_max_outbound_full_relay); + m_max_automatic_outbound = m_max_outbound_full_relay + m_max_outbound_block_relay + m_max_feeler; + m_max_inbound = std::max(0, m_max_automatic_connections - m_max_automatic_outbound); m_use_addrman_outgoing = connOptions.m_use_addrman_outgoing; - nMaxAddnode = connOptions.nMaxAddnode; - nMaxFeeler = connOptions.nMaxFeeler; - m_max_outbound = m_max_outbound_full_relay + m_max_outbound_block_relay + nMaxFeeler; m_client_interface = connOptions.uiInterface; m_banman = connOptions.m_banman; m_msgproc = connOptions.m_msgproc; @@ -1189,7 +1184,7 @@ public: bool AddNode(const AddedNodeParams& add) EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); bool RemoveAddedNode(const std::string& node) EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); - std::vector<AddedNodeInfo> GetAddedNodeInfo() const EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); + std::vector<AddedNodeInfo> GetAddedNodeInfo(bool include_connected) const EXCLUSIVE_LOCKS_REQUIRED(!m_added_nodes_mutex); /** * Attempts to open a connection. Currently only used from tests. @@ -1466,7 +1461,18 @@ private: std::unique_ptr<CSemaphore> semOutbound; std::unique_ptr<CSemaphore> semAddnode; - int nMaxConnections; + + /** + * Maximum number of automatic connections permitted, excluding manual + * connections but including inbounds. May be changed by the user and is + * potentially limited by the operating system (number of file descriptors). + */ + int m_max_automatic_connections; + + /* + * Maximum number of peers by connection type. Might vary from defaults + * based on -maxconnections init value. + */ // How many full-relay (tx, block, addr) outbound peers we want int m_max_outbound_full_relay; @@ -1475,9 +1481,11 @@ private: // We do not relay tx or addr messages with these peers int m_max_outbound_block_relay; - int nMaxAddnode; - int nMaxFeeler; - int m_max_outbound; + int m_max_addnode{MAX_ADDNODE_CONNECTIONS}; + int m_max_feeler{MAX_FEELER_CONNECTIONS}; + int m_max_automatic_outbound; + int m_max_inbound; + bool m_use_addrman_outgoing; CClientUIInterface* m_client_interface; NetEventsInterface* m_msgproc; diff --git a/src/net_processing.cpp b/src/net_processing.cpp index 611bd47e7f..1556dc7e67 100644 --- a/src/net_processing.cpp +++ b/src/net_processing.cpp @@ -1820,6 +1820,8 @@ bool PeerManagerImpl::MaybePunishNodeForTx(NodeId nodeid, const TxValidationStat case TxValidationResult::TX_CONFLICT: case TxValidationResult::TX_MEMPOOL_POLICY: case TxValidationResult::TX_NO_MEMPOOL: + case TxValidationResult::TX_RECONSIDERABLE: + case TxValidationResult::TX_UNKNOWN: break; } return false; @@ -2306,9 +2308,9 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& } if (pblock) { if (inv.IsMsgBlk()) { - m_connman.PushMessage(&pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::BLOCK, *pblock)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, TX_NO_WITNESS(*pblock))); } else if (inv.IsMsgWitnessBlk()) { - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, TX_WITH_WITNESS(*pblock))); } else if (inv.IsMsgFilteredBlk()) { bool sendMerkleBlock = false; CMerkleBlock merkleBlock; @@ -2329,7 +2331,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& // however we MUST always provide at least what the remote peer needs typedef std::pair<unsigned int, uint256> PairType; for (PairType& pair : merkleBlock.vMatchedTxn) - m_connman.PushMessage(&pfrom, msgMaker.Make(SERIALIZE_TRANSACTION_NO_WITNESS, NetMsgType::TX, *pblock->vtx[pair.first])); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::TX, TX_NO_WITNESS(*pblock->vtx[pair.first]))); } // else // no response @@ -2346,7 +2348,7 @@ void PeerManagerImpl::ProcessGetBlockData(CNode& pfrom, Peer& peer, const CInv& m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::CMPCTBLOCK, cmpctblock)); } } else { - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, *pblock)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::BLOCK, TX_WITH_WITNESS(*pblock))); } } } @@ -2416,8 +2418,8 @@ void PeerManagerImpl::ProcessGetData(CNode& pfrom, Peer& peer, const std::atomic CTransactionRef tx = FindTxForGetData(*tx_relay, ToGenTxid(inv)); if (tx) { // WTX and WITNESS_TX imply we serialize with witness - int nSendFlags = (inv.IsMsgTx() ? SERIALIZE_TRANSACTION_NO_WITNESS : 0); - m_connman.PushMessage(&pfrom, msgMaker.Make(nSendFlags, NetMsgType::TX, *tx)); + const auto maybe_with_witness = (inv.IsMsgTx() ? TX_NO_WITNESS : TX_WITH_WITNESS); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::TX, maybe_with_witness(*tx))); m_mempool.RemoveUnbroadcastTx(tx->GetHash()); } else { vNotFound.push_back(inv); @@ -4117,7 +4119,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, LogPrint(BCLog::NET, "Ignoring getheaders from peer=%d because active chain has too little work; sending empty response\n", pfrom.GetId()); // Just respond with an empty headers message, to tell the peer to // go away but not treat us as unresponsive. - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, std::vector<CBlock>())); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, std::vector<CBlockHeader>())); return; } @@ -4167,7 +4169,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, // will re-announce the new block via headers (or compact blocks again) // in the SendMessages logic. nodestate->pindexBestHeaderSent = pindex ? pindex : m_chainman.ActiveChain().Tip(); - m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + m_connman.PushMessage(&pfrom, msgMaker.Make(NetMsgType::HEADERS, TX_WITH_WITNESS(vHeaders))); return; } @@ -4184,7 +4186,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, if (m_chainman.IsInitialBlockDownload()) return; CTransactionRef ptx; - vRecv >> ptx; + vRecv >> TX_WITH_WITNESS(ptx); const CTransaction& tx = *ptx; const uint256& txid = ptx->GetHash(); @@ -4685,7 +4687,7 @@ void PeerManagerImpl::ProcessMessage(CNode& pfrom, const std::string& msg_type, } std::shared_ptr<CBlock> pblock = std::make_shared<CBlock>(); - vRecv >> *pblock; + vRecv >> TX_WITH_WITNESS(*pblock); LogPrint(BCLog::NET, "received block %s peer=%d\n", pblock->GetHash().ToString(), pfrom.GetId()); @@ -5696,7 +5698,7 @@ bool PeerManagerImpl::SendMessages(CNode* pto) LogPrint(BCLog::NET, "%s: sending header %s to peer=%d\n", __func__, vHeaders.front().GetHash().ToString(), pto->GetId()); } - m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, vHeaders)); + m_connman.PushMessage(pto, msgMaker.Make(NetMsgType::HEADERS, TX_WITH_WITNESS(vHeaders))); state.pindexBestHeaderSent = pBestIndex; } else fRevertToInv = true; diff --git a/src/netbase.cpp b/src/netbase.cpp index 5a609a872e..9fbd9f7dea 100644 --- a/src/netbase.cpp +++ b/src/netbase.cpp @@ -30,7 +30,7 @@ bool fNameLookup = DEFAULT_NAME_LOOKUP; // Need ample time for negotiation for very slow proxies such as Tor std::chrono::milliseconds g_socks5_recv_timeout = 20s; -static std::atomic<bool> interruptSocks5Recv(false); +CThreadInterrupt g_socks5_interrupt; ReachableNets g_reachable_nets; @@ -271,7 +271,7 @@ enum class IntrRecvError { * IntrRecvError::OK only if all of the specified number of bytes were * read. * - * @see This function can be interrupted by calling InterruptSocks5(bool). + * @see This function can be interrupted by calling g_socks5_interrupt(). * Sockets can be made non-blocking with Sock::SetNonBlocking(). */ static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, std::chrono::milliseconds timeout, const Sock& sock) @@ -299,8 +299,9 @@ static IntrRecvError InterruptibleRecv(uint8_t* data, size_t len, std::chrono::m return IntrRecvError::NetworkError; } } - if (interruptSocks5Recv) + if (g_socks5_interrupt) { return IntrRecvError::Interrupted; + } curTime = Now<SteadyMilliseconds>(); } return len == 0 ? IntrRecvError::OK : IntrRecvError::Timeout; @@ -333,103 +334,93 @@ static std::string Socks5ErrorString(uint8_t err) bool Socks5(const std::string& strDest, uint16_t port, const ProxyCredentials* auth, const Sock& sock) { - IntrRecvError recvr; - LogPrint(BCLog::NET, "SOCKS5 connecting %s\n", strDest); - if (strDest.size() > 255) { - return error("Hostname too long"); - } - // Construct the version identifier/method selection message - std::vector<uint8_t> vSocks5Init; - vSocks5Init.push_back(SOCKSVersion::SOCKS5); // We want the SOCK5 protocol - if (auth) { - vSocks5Init.push_back(0x02); // 2 method identifiers follow... - vSocks5Init.push_back(SOCKS5Method::NOAUTH); - vSocks5Init.push_back(SOCKS5Method::USER_PASS); - } else { - vSocks5Init.push_back(0x01); // 1 method identifier follows... - vSocks5Init.push_back(SOCKS5Method::NOAUTH); - } - ssize_t ret = sock.Send(vSocks5Init.data(), vSocks5Init.size(), MSG_NOSIGNAL); - if (ret != (ssize_t)vSocks5Init.size()) { - return error("Error sending to proxy"); - } - uint8_t pchRet1[2]; - if (InterruptibleRecv(pchRet1, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { - LogPrintf("Socks5() connect to %s:%d failed: InterruptibleRecv() timeout or other failure\n", strDest, port); - return false; - } - if (pchRet1[0] != SOCKSVersion::SOCKS5) { - return error("Proxy failed to initialize"); - } - if (pchRet1[1] == SOCKS5Method::USER_PASS && auth) { - // Perform username/password authentication (as described in RFC1929) - std::vector<uint8_t> vAuth; - vAuth.push_back(0x01); // Current (and only) version of user/pass subnegotiation - if (auth->username.size() > 255 || auth->password.size() > 255) - return error("Proxy username or password too long"); - vAuth.push_back(auth->username.size()); - vAuth.insert(vAuth.end(), auth->username.begin(), auth->username.end()); - vAuth.push_back(auth->password.size()); - vAuth.insert(vAuth.end(), auth->password.begin(), auth->password.end()); - ret = sock.Send(vAuth.data(), vAuth.size(), MSG_NOSIGNAL); - if (ret != (ssize_t)vAuth.size()) { - return error("Error sending authentication to proxy"); - } - LogPrint(BCLog::PROXY, "SOCKS5 sending proxy authentication %s:%s\n", auth->username, auth->password); - uint8_t pchRetA[2]; - if (InterruptibleRecv(pchRetA, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { - return error("Error reading proxy authentication response"); + try { + IntrRecvError recvr; + LogPrint(BCLog::NET, "SOCKS5 connecting %s\n", strDest); + if (strDest.size() > 255) { + return error("Hostname too long"); } - if (pchRetA[0] != 0x01 || pchRetA[1] != 0x00) { - return error("Proxy authentication unsuccessful"); + // Construct the version identifier/method selection message + std::vector<uint8_t> vSocks5Init; + vSocks5Init.push_back(SOCKSVersion::SOCKS5); // We want the SOCK5 protocol + if (auth) { + vSocks5Init.push_back(0x02); // 2 method identifiers follow... + vSocks5Init.push_back(SOCKS5Method::NOAUTH); + vSocks5Init.push_back(SOCKS5Method::USER_PASS); + } else { + vSocks5Init.push_back(0x01); // 1 method identifier follows... + vSocks5Init.push_back(SOCKS5Method::NOAUTH); } - } else if (pchRet1[1] == SOCKS5Method::NOAUTH) { - // Perform no authentication - } else { - return error("Proxy requested wrong authentication method %02x", pchRet1[1]); - } - std::vector<uint8_t> vSocks5; - vSocks5.push_back(SOCKSVersion::SOCKS5); // VER protocol version - vSocks5.push_back(SOCKS5Command::CONNECT); // CMD CONNECT - vSocks5.push_back(0x00); // RSV Reserved must be 0 - vSocks5.push_back(SOCKS5Atyp::DOMAINNAME); // ATYP DOMAINNAME - vSocks5.push_back(strDest.size()); // Length<=255 is checked at beginning of function - vSocks5.insert(vSocks5.end(), strDest.begin(), strDest.end()); - vSocks5.push_back((port >> 8) & 0xFF); - vSocks5.push_back((port >> 0) & 0xFF); - ret = sock.Send(vSocks5.data(), vSocks5.size(), MSG_NOSIGNAL); - if (ret != (ssize_t)vSocks5.size()) { - return error("Error sending to proxy"); - } - uint8_t pchRet2[4]; - if ((recvr = InterruptibleRecv(pchRet2, 4, g_socks5_recv_timeout, sock)) != IntrRecvError::OK) { - if (recvr == IntrRecvError::Timeout) { - /* If a timeout happens here, this effectively means we timed out while connecting - * to the remote node. This is very common for Tor, so do not print an - * error message. */ + sock.SendComplete(vSocks5Init, g_socks5_recv_timeout, g_socks5_interrupt); + uint8_t pchRet1[2]; + if (InterruptibleRecv(pchRet1, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { + LogPrintf("Socks5() connect to %s:%d failed: InterruptibleRecv() timeout or other failure\n", strDest, port); return false; + } + if (pchRet1[0] != SOCKSVersion::SOCKS5) { + return error("Proxy failed to initialize"); + } + if (pchRet1[1] == SOCKS5Method::USER_PASS && auth) { + // Perform username/password authentication (as described in RFC1929) + std::vector<uint8_t> vAuth; + vAuth.push_back(0x01); // Current (and only) version of user/pass subnegotiation + if (auth->username.size() > 255 || auth->password.size() > 255) + return error("Proxy username or password too long"); + vAuth.push_back(auth->username.size()); + vAuth.insert(vAuth.end(), auth->username.begin(), auth->username.end()); + vAuth.push_back(auth->password.size()); + vAuth.insert(vAuth.end(), auth->password.begin(), auth->password.end()); + sock.SendComplete(vAuth, g_socks5_recv_timeout, g_socks5_interrupt); + LogPrint(BCLog::PROXY, "SOCKS5 sending proxy authentication %s:%s\n", auth->username, auth->password); + uint8_t pchRetA[2]; + if (InterruptibleRecv(pchRetA, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { + return error("Error reading proxy authentication response"); + } + if (pchRetA[0] != 0x01 || pchRetA[1] != 0x00) { + return error("Proxy authentication unsuccessful"); + } + } else if (pchRet1[1] == SOCKS5Method::NOAUTH) { + // Perform no authentication } else { - return error("Error while reading proxy response"); + return error("Proxy requested wrong authentication method %02x", pchRet1[1]); } - } - if (pchRet2[0] != SOCKSVersion::SOCKS5) { - return error("Proxy failed to accept request"); - } - if (pchRet2[1] != SOCKS5Reply::SUCCEEDED) { - // Failures to connect to a peer that are not proxy errors - LogPrintf("Socks5() connect to %s:%d failed: %s\n", strDest, port, Socks5ErrorString(pchRet2[1])); - return false; - } - if (pchRet2[2] != 0x00) { // Reserved field must be 0 - return error("Error: malformed proxy response"); - } - uint8_t pchRet3[256]; - switch (pchRet2[3]) - { + std::vector<uint8_t> vSocks5; + vSocks5.push_back(SOCKSVersion::SOCKS5); // VER protocol version + vSocks5.push_back(SOCKS5Command::CONNECT); // CMD CONNECT + vSocks5.push_back(0x00); // RSV Reserved must be 0 + vSocks5.push_back(SOCKS5Atyp::DOMAINNAME); // ATYP DOMAINNAME + vSocks5.push_back(strDest.size()); // Length<=255 is checked at beginning of function + vSocks5.insert(vSocks5.end(), strDest.begin(), strDest.end()); + vSocks5.push_back((port >> 8) & 0xFF); + vSocks5.push_back((port >> 0) & 0xFF); + sock.SendComplete(vSocks5, g_socks5_recv_timeout, g_socks5_interrupt); + uint8_t pchRet2[4]; + if ((recvr = InterruptibleRecv(pchRet2, 4, g_socks5_recv_timeout, sock)) != IntrRecvError::OK) { + if (recvr == IntrRecvError::Timeout) { + /* If a timeout happens here, this effectively means we timed out while connecting + * to the remote node. This is very common for Tor, so do not print an + * error message. */ + return false; + } else { + return error("Error while reading proxy response"); + } + } + if (pchRet2[0] != SOCKSVersion::SOCKS5) { + return error("Proxy failed to accept request"); + } + if (pchRet2[1] != SOCKS5Reply::SUCCEEDED) { + // Failures to connect to a peer that are not proxy errors + LogPrintf("Socks5() connect to %s:%d failed: %s\n", strDest, port, Socks5ErrorString(pchRet2[1])); + return false; + } + if (pchRet2[2] != 0x00) { // Reserved field must be 0 + return error("Error: malformed proxy response"); + } + uint8_t pchRet3[256]; + switch (pchRet2[3]) { case SOCKS5Atyp::IPV4: recvr = InterruptibleRecv(pchRet3, 4, g_socks5_recv_timeout, sock); break; case SOCKS5Atyp::IPV6: recvr = InterruptibleRecv(pchRet3, 16, g_socks5_recv_timeout, sock); break; - case SOCKS5Atyp::DOMAINNAME: - { + case SOCKS5Atyp::DOMAINNAME: { recvr = InterruptibleRecv(pchRet3, 1, g_socks5_recv_timeout, sock); if (recvr != IntrRecvError::OK) { return error("Error reading from proxy"); @@ -439,15 +430,18 @@ bool Socks5(const std::string& strDest, uint16_t port, const ProxyCredentials* a break; } default: return error("Error: malformed proxy response"); + } + if (recvr != IntrRecvError::OK) { + return error("Error reading from proxy"); + } + if (InterruptibleRecv(pchRet3, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { + return error("Error reading from proxy"); + } + LogPrint(BCLog::NET, "SOCKS5 connected %s\n", strDest); + return true; + } catch (const std::runtime_error& e) { + return error("Error during SOCKS5 proxy handshake: %s", e.what()); } - if (recvr != IntrRecvError::OK) { - return error("Error reading from proxy"); - } - if (InterruptibleRecv(pchRet3, 2, g_socks5_recv_timeout, sock) != IntrRecvError::OK) { - return error("Error reading from proxy"); - } - LogPrint(BCLog::NET, "SOCKS5 connected %s\n", strDest); - return true; } std::unique_ptr<Sock> CreateSockTCP(const CService& address_family) @@ -681,11 +675,6 @@ CSubNet LookupSubNet(const std::string& subnet_str) return subnet; } -void InterruptSocks5(bool interrupt) -{ - interruptSocks5Recv = interrupt; -} - bool IsBadPort(uint16_t port) { /* Don't forget to update doc/p2p-bad-ports.md if you change this list. */ diff --git a/src/netbase.h b/src/netbase.h index d51f63fd81..8523f59b4d 100644 --- a/src/netbase.h +++ b/src/netbase.h @@ -13,6 +13,7 @@ #include <netaddress.h> #include <serialize.h> #include <util/sock.h> +#include <util/threadinterrupt.h> #include <functional> #include <memory> @@ -274,7 +275,10 @@ bool ConnectSocketDirectly(const CService &addrConnect, const Sock& sock, int nT */ bool ConnectThroughProxy(const Proxy& proxy, const std::string& strDest, uint16_t port, const Sock& sock, int nTimeout, bool& outProxyConnectionFailed); -void InterruptSocks5(bool interrupt); +/** + * Interrupt SOCKS5 reads or writes. + */ +extern CThreadInterrupt g_socks5_interrupt; /** * Connect to a specified destination service through an already connected diff --git a/src/node/blockstorage.cpp b/src/node/blockstorage.cpp index 058e524745..1108cc676b 100644 --- a/src/node/blockstorage.cpp +++ b/src/node/blockstorage.cpp @@ -956,7 +956,7 @@ bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const } // Write index header - unsigned int nSize = GetSerializeSize(block, fileout.GetVersion()); + unsigned int nSize = GetSerializeSize(TX_WITH_WITNESS(block)); fileout << GetParams().MessageStart() << nSize; // Write block @@ -965,7 +965,7 @@ bool BlockManager::WriteBlockToDisk(const CBlock& block, FlatFilePos& pos) const return error("WriteBlockToDisk: ftell failed"); } pos.nPos = (unsigned int)fileOutPos; - fileout << block; + fileout << TX_WITH_WITNESS(block); return true; } @@ -1023,7 +1023,7 @@ bool BlockManager::ReadBlockFromDisk(CBlock& block, const FlatFilePos& pos) cons // Read block try { - filein >> block; + filein >> TX_WITH_WITNESS(block); } catch (const std::exception& e) { return error("%s: Deserialize or I/O error - %s at %s", __func__, e.what(), pos.ToString()); } @@ -1092,7 +1092,7 @@ bool BlockManager::ReadRawBlockFromDisk(std::vector<uint8_t>& block, const FlatF FlatFilePos BlockManager::SaveBlockToDisk(const CBlock& block, int nHeight, const FlatFilePos* dbp) { - unsigned int nBlockSize = ::GetSerializeSize(block, CLIENT_VERSION); + unsigned int nBlockSize = ::GetSerializeSize(TX_WITH_WITNESS(block)); FlatFilePos blockPos; const auto position_known {dbp != nullptr}; if (position_known) { diff --git a/src/node/interfaces.cpp b/src/node/interfaces.cpp index f6dbe4f008..f4ecfeb9d5 100644 --- a/src/node/interfaces.cpp +++ b/src/node/interfaces.cpp @@ -328,10 +328,12 @@ public: std::vector<std::string> listRpcCommands() override { return ::tableRPC.listCommands(); } void rpcSetTimerInterfaceIfUnset(RPCTimerInterface* iface) override { RPCSetTimerInterfaceIfUnset(iface); } void rpcUnsetTimerInterface(RPCTimerInterface* iface) override { RPCUnsetTimerInterface(iface); } - bool getUnspentOutput(const COutPoint& output, Coin& coin) override + std::optional<Coin> getUnspentOutput(const COutPoint& output) override { LOCK(::cs_main); - return chainman().ActiveChainstate().CoinsTip().GetCoin(output, coin); + Coin coin; + if (chainman().ActiveChainstate().CoinsTip().GetCoin(output, coin)) return coin; + return {}; } TransactionError broadcastTransaction(CTransactionRef tx, CAmount max_tx_fee, std::string& err_string) override { @@ -648,8 +650,9 @@ public: { if (!m_node.mempool) return false; LOCK(m_node.mempool->cs); - auto it = m_node.mempool->GetIter(txid); - return it && (*it)->GetCountWithDescendants() > 1; + const auto entry{m_node.mempool->GetEntry(Txid::FromUint256(txid))}; + if (entry == nullptr) return false; + return entry->GetCountWithDescendants() > 1; } bool broadcastTransaction(const CTransactionRef& tx, const CAmount& max_tx_fee, @@ -669,7 +672,7 @@ public: m_node.mempool->GetTransactionAncestry(txid, ancestors, descendants, ancestorsize, ancestorfees); } - std::map<COutPoint, CAmount> CalculateIndividualBumpFees(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) override + std::map<COutPoint, CAmount> calculateIndividualBumpFees(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) override { if (!m_node.mempool) { std::map<COutPoint, CAmount> bump_fees; @@ -681,7 +684,7 @@ public: return MiniMiner(*m_node.mempool, outpoints).CalculateBumpFees(target_feerate); } - std::optional<CAmount> CalculateCombinedBumpFee(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) override + std::optional<CAmount> calculateCombinedBumpFee(const std::vector<COutPoint>& outpoints, const CFeeRate& target_feerate) override { if (!m_node.mempool) { return 0; @@ -702,9 +705,9 @@ public: if (!m_node.mempool) return true; LockPoints lp; CTxMemPoolEntry entry(tx, 0, 0, 0, 0, false, 0, lp); - const CTxMemPool::Limits& limits{m_node.mempool->m_limits}; LOCK(m_node.mempool->cs); - return m_node.mempool->CalculateMemPoolAncestors(entry, limits).has_value(); + std::string err_string; + return m_node.mempool->CheckPackageLimits({tx}, entry.GetTxSize(), err_string); } CFeeRate estimateSmartFee(int num_blocks, bool conservative, FeeCalculation* calc) override { @@ -772,7 +775,7 @@ public: { RPCRunLater(name, std::move(fn), seconds); } - int rpcSerializationFlags() override { return RPCSerializationFlags(); } + bool rpcSerializationWithoutWitness() override { return RPCSerializationWithoutWitness(); } common::SettingsValue getSetting(const std::string& name) override { return args().GetSetting(name); @@ -806,7 +809,7 @@ public: { if (!m_node.mempool) return; LOCK2(::cs_main, m_node.mempool->cs); - for (const CTxMemPoolEntry& entry : m_node.mempool->mapTx) { + for (const CTxMemPoolEntry& entry : m_node.mempool->entryAll()) { notifications.transactionAddedToMempool(entry.GetSharedTx()); } } diff --git a/src/node/mempool_args.cpp b/src/node/mempool_args.cpp index f63d9875fc..ac26600919 100644 --- a/src/node/mempool_args.cpp +++ b/src/node/mempool_args.cpp @@ -93,6 +93,8 @@ util::Result<void> ApplyArgsManOptions(const ArgsManager& argsman, const CChainP mempool_opts.full_rbf = argsman.GetBoolArg("-mempoolfullrbf", mempool_opts.full_rbf); + mempool_opts.persist_v1_dat = argsman.GetBoolArg("-persistmempoolv1", mempool_opts.persist_v1_dat); + ApplyArgsManOptions(argsman, mempool_opts.limits); return {}; diff --git a/src/node/miner.cpp b/src/node/miner.cpp index caa2991819..ce5452d1f9 100644 --- a/src/node/miner.cpp +++ b/src/node/miner.cpp @@ -188,7 +188,7 @@ void BlockAssembler::onlyUnconfirmed(CTxMemPool::setEntries& testSet) { for (CTxMemPool::setEntries::iterator iit = testSet.begin(); iit != testSet.end(); ) { // Only test txs not already in the block - if (inBlock.count(*iit)) { + if (inBlock.count((*iit)->GetSharedTx()->GetHash())) { testSet.erase(iit++); } else { iit++; @@ -229,7 +229,7 @@ void BlockAssembler::AddToBlock(CTxMemPool::txiter iter) ++nBlockTx; nBlockSigOpsCost += iter->GetSigOpCost(); nFees += iter->GetFee(); - inBlock.insert(iter); + inBlock.insert(iter->GetSharedTx()->GetHash()); bool fPrintPriority = gArgs.GetBoolArg("-printpriority", DEFAULT_PRINTPRIORITY); if (fPrintPriority) { @@ -298,7 +298,7 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele // because some of their txs are already in the block indexed_modified_transaction_set mapModifiedTx; // Keep track of entries that failed inclusion, to avoid duplicate work - CTxMemPool::setEntries failedTx; + std::set<Txid> failedTx; CTxMemPool::indexed_transaction_set::index<ancestor_score>::type::iterator mi = mempool.mapTx.get<ancestor_score>().begin(); CTxMemPool::txiter iter; @@ -326,7 +326,7 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele if (mi != mempool.mapTx.get<ancestor_score>().end()) { auto it = mempool.mapTx.project<0>(mi); assert(it != mempool.mapTx.end()); - if (mapModifiedTx.count(it) || inBlock.count(it) || failedTx.count(it)) { + if (mapModifiedTx.count(it) || inBlock.count(it->GetSharedTx()->GetHash()) || failedTx.count(it->GetSharedTx()->GetHash())) { ++mi; continue; } @@ -360,7 +360,7 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele // We skip mapTx entries that are inBlock, and mapModifiedTx shouldn't // contain anything that is inBlock. - assert(!inBlock.count(iter)); + assert(!inBlock.count(iter->GetSharedTx()->GetHash())); uint64_t packageSize = iter->GetSizeWithAncestors(); CAmount packageFees = iter->GetModFeesWithAncestors(); @@ -382,7 +382,7 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele // we must erase failed entries so that we can consider the // next best entry on the next loop iteration mapModifiedTx.get<ancestor_score>().erase(modit); - failedTx.insert(iter); + failedTx.insert(iter->GetSharedTx()->GetHash()); } ++nConsecutiveFailed; @@ -404,7 +404,7 @@ void BlockAssembler::addPackageTxs(const CTxMemPool& mempool, int& nPackagesSele if (!TestPackageTransactions(ancestors)) { if (fUsingModified) { mapModifiedTx.get<ancestor_score>().erase(modit); - failedTx.insert(iter); + failedTx.insert(iter->GetSharedTx()->GetHash()); } continue; } diff --git a/src/node/miner.h b/src/node/miner.h index 4173521585..06a917228d 100644 --- a/src/node/miner.h +++ b/src/node/miner.h @@ -142,7 +142,7 @@ private: uint64_t nBlockTx; uint64_t nBlockSigOpsCost; CAmount nFees; - CTxMemPool::setEntries inBlock; + std::unordered_set<Txid, SaltedTxidHasher> inBlock; // Chain context for the block int nHeight; diff --git a/src/node/mini_miner.cpp b/src/node/mini_miner.cpp index 3d24a3f58e..58422c4439 100644 --- a/src/node/mini_miner.cpp +++ b/src/node/mini_miner.cpp @@ -78,11 +78,11 @@ MiniMiner::MiniMiner(const CTxMemPool& mempool, const std::vector<COutPoint>& ou for (const auto& txiter : cluster) { if (!m_to_be_replaced.count(txiter->GetTx().GetHash())) { auto [mapiter, success] = m_entries_by_txid.emplace(txiter->GetTx().GetHash(), - MiniMinerMempoolEntry{/*fee_self=*/txiter->GetModifiedFee(), - /*fee_ancestor=*/txiter->GetModFeesWithAncestors(), + MiniMinerMempoolEntry{/*tx_in=*/txiter->GetSharedTx(), /*vsize_self=*/txiter->GetTxSize(), /*vsize_ancestor=*/txiter->GetSizeWithAncestors(), - /*tx_in=*/txiter->GetSharedTx()}); + /*fee_self=*/txiter->GetModifiedFee(), + /*fee_ancestor=*/txiter->GetModFeesWithAncestors()}); m_entries.push_back(mapiter); } else { auto outpoints_it = m_requested_outpoints_by_txid.find(txiter->GetTx().GetHash()); @@ -154,7 +154,7 @@ MiniMiner::MiniMiner(const std::vector<MiniMinerMempoolEntry>& manual_entries, m_ready_to_calculate = false; return; } - std::vector<MockEntryMap::iterator> cached_descendants; + std::vector<MockEntryMap::iterator> descendants; for (const auto& desc_txid : desc_txids) { auto desc_it{m_entries_by_txid.find(desc_txid)}; // Descendants should only include transactions with corresponding entries. @@ -162,10 +162,10 @@ MiniMiner::MiniMiner(const std::vector<MiniMinerMempoolEntry>& manual_entries, m_ready_to_calculate = false; return; } else { - cached_descendants.emplace_back(desc_it); + descendants.emplace_back(desc_it); } } - m_descendant_set_by_txid.emplace(txid, cached_descendants); + m_descendant_set_by_txid.emplace(txid, descendants); } Assume(m_to_be_replaced.empty()); Assume(m_requested_outpoints_by_txid.empty()); diff --git a/src/node/mini_miner.h b/src/node/mini_miner.h index 8f86709ae4..de62c0af75 100644 --- a/src/node/mini_miner.h +++ b/src/node/mini_miner.h @@ -34,11 +34,11 @@ class MiniMinerMempoolEntry // methods can be called without holding that lock. public: - explicit MiniMinerMempoolEntry(CAmount fee_self, - CAmount fee_ancestor, + explicit MiniMinerMempoolEntry(const CTransactionRef& tx_in, int64_t vsize_self, int64_t vsize_ancestor, - const CTransactionRef& tx_in): + CAmount fee_self, + CAmount fee_ancestor): tx{tx_in}, vsize_individual{vsize_self}, vsize_with_ancestors{vsize_ancestor}, @@ -137,10 +137,10 @@ public: */ MiniMiner(const CTxMemPool& mempool, const std::vector<COutPoint>& outpoints); - /** Constructor in which the MiniMinerMempoolEntry entries have been constructed manually, - * presumably because these transactions are not in the mempool (yet). It is assumed that all - * entries are unique and their values are correct, otherwise results computed by MiniMiner may - * be incorrect. Callers should check IsReadyToCalculate() after construction. + /** Constructor in which the MiniMinerMempoolEntry entries have been constructed manually. + * It is assumed that all entries are unique and their values are correct, otherwise results + * computed by MiniMiner may be incorrect. Callers should check IsReadyToCalculate() after + * construction. * @param[in] descendant_caches A map from each transaction to the set of txids of this * transaction's descendant set, including itself. Each tx in * manual_entries must have a corresponding entry in this map, and diff --git a/src/policy/fees.cpp b/src/policy/fees.cpp index 9557594622..654c4cf0ce 100644 --- a/src/policy/fees.cpp +++ b/src/policy/fees.cpp @@ -1044,8 +1044,8 @@ void CBlockPolicyEstimator::FlushUnconfirmed() std::chrono::hours CBlockPolicyEstimator::GetFeeEstimatorFileAge() { - auto file_time = std::filesystem::last_write_time(m_estimation_filepath); - auto now = std::filesystem::file_time_type::clock::now(); + auto file_time{fs::last_write_time(m_estimation_filepath)}; + auto now{fs::file_time_type::clock::now()}; return std::chrono::duration_cast<std::chrono::hours>(now - file_time); } diff --git a/src/policy/rbf.cpp b/src/policy/rbf.cpp index d032b74008..76ab2b1a96 100644 --- a/src/policy/rbf.cpp +++ b/src/policy/rbf.cpp @@ -12,6 +12,7 @@ #include <tinyformat.h> #include <txmempool.h> #include <uint256.h> +#include <util/check.h> #include <util/moneystr.h> #include <util/rbf.h> @@ -35,7 +36,7 @@ RBFTransactionState IsRBFOptIn(const CTransaction& tx, const CTxMemPool& pool) // If all the inputs have nSequence >= maxint-1, it still might be // signaled for RBF if any unconfirmed parents have signaled. - const CTxMemPoolEntry entry{*pool.mapTx.find(tx.GetHash())}; + const auto& entry{*Assert(pool.GetEntry(tx.GetHash()))}; auto ancestors{pool.AssumeCalculateMemPoolAncestors(__func__, entry, CTxMemPool::Limits::NoLimits(), /*fSearchForParents=*/false)}; diff --git a/src/primitives/block.cpp b/src/primitives/block.cpp index 3d21708820..27da161e1e 100644 --- a/src/primitives/block.cpp +++ b/src/primitives/block.cpp @@ -10,7 +10,7 @@ uint256 CBlockHeader::GetHash() const { - return (CHashWriter{PROTOCOL_VERSION} << *this).GetHash(); + return (HashWriter{} << *this).GetHash(); } std::string CBlock::ToString() const diff --git a/src/primitives/transaction.cpp b/src/primitives/transaction.cpp index 1ad8345fcb..6650277c2b 100644 --- a/src/primitives/transaction.cpp +++ b/src/primitives/transaction.cpp @@ -68,12 +68,12 @@ CMutableTransaction::CMutableTransaction(const CTransaction& tx) : vin(tx.vin), Txid CMutableTransaction::GetHash() const { - return Txid::FromUint256((CHashWriter{SERIALIZE_TRANSACTION_NO_WITNESS} << *this).GetHash()); + return Txid::FromUint256((HashWriter{} << TX_NO_WITNESS(*this)).GetHash()); } Txid CTransaction::ComputeHash() const { - return Txid::FromUint256((CHashWriter{SERIALIZE_TRANSACTION_NO_WITNESS} << *this).GetHash()); + return Txid::FromUint256((HashWriter{} << TX_NO_WITNESS(*this)).GetHash()); } Wtxid CTransaction::ComputeWitnessHash() const @@ -82,7 +82,7 @@ Wtxid CTransaction::ComputeWitnessHash() const return Wtxid::FromUint256(hash.ToUint256()); } - return Wtxid::FromUint256((CHashWriter{0} << *this).GetHash()); + return Wtxid::FromUint256((HashWriter{} << TX_WITH_WITNESS(*this)).GetHash()); } CTransaction::CTransaction(const CMutableTransaction& tx) : vin(tx.vin), vout(tx.vout), nVersion(tx.nVersion), nLockTime(tx.nLockTime), hash{ComputeHash()}, m_witness_hash{ComputeWitnessHash()} {} @@ -102,7 +102,7 @@ CAmount CTransaction::GetValueOut() const unsigned int CTransaction::GetTotalSize() const { - return ::GetSerializeSize(*this, PROTOCOL_VERSION); + return ::GetSerializeSize(TX_WITH_WITNESS(*this)); } std::string CTransaction::ToString() const diff --git a/src/primitives/transaction.h b/src/primitives/transaction.h index 89deb9de4d..ad190f7d7f 100644 --- a/src/primitives/transaction.h +++ b/src/primitives/transaction.h @@ -24,14 +24,6 @@ #include <utility> #include <vector> -/** - * A flag that is ORed into the protocol version to designate that a transaction - * should be (un)serialized without witness data. - * Make sure that this does not collide with any of the values in `version.h` - * or with `ADDRV2_FORMAT`. - */ -static const int SERIALIZE_TRANSACTION_NO_WITNESS = 0x40000000; - /** An outpoint - a combination of a transaction hash and an index n into its vout */ class COutPoint { @@ -197,6 +189,13 @@ public: struct CMutableTransaction; +struct TransactionSerParams { + const bool allow_witness; + SER_PARAMS_OPFUNC +}; +static constexpr TransactionSerParams TX_WITH_WITNESS{.allow_witness = true}; +static constexpr TransactionSerParams TX_NO_WITNESS{.allow_witness = false}; + /** * Basic transaction serialization format: * - int32_t nVersion @@ -215,8 +214,9 @@ struct CMutableTransaction; * - uint32_t nLockTime */ template<typename Stream, typename TxType> -inline void UnserializeTransaction(TxType& tx, Stream& s) { - const bool fAllowWitness = !(s.GetVersion() & SERIALIZE_TRANSACTION_NO_WITNESS); +void UnserializeTransaction(TxType& tx, Stream& s, const TransactionSerParams& params) +{ + const bool fAllowWitness = params.allow_witness; s >> tx.nVersion; unsigned char flags = 0; @@ -254,8 +254,9 @@ inline void UnserializeTransaction(TxType& tx, Stream& s) { } template<typename Stream, typename TxType> -inline void SerializeTransaction(const TxType& tx, Stream& s) { - const bool fAllowWitness = !(s.GetVersion() & SERIALIZE_TRANSACTION_NO_WITNESS); +void SerializeTransaction(const TxType& tx, Stream& s, const TransactionSerParams& params) +{ + const bool fAllowWitness = params.allow_witness; s << tx.nVersion; unsigned char flags = 0; @@ -323,13 +324,15 @@ public: template <typename Stream> inline void Serialize(Stream& s) const { - SerializeTransaction(*this, s); + SerializeTransaction(*this, s, s.GetParams()); } /** This deserializing constructor is provided instead of an Unserialize method. * Unserialize is not possible, since it would require overwriting const fields. */ template <typename Stream> - CTransaction(deserialize_type, Stream& s) : CTransaction(CMutableTransaction(deserialize, s)) {} + CTransaction(deserialize_type, const TransactionSerParams& params, Stream& s) : CTransaction(CMutableTransaction(deserialize, params, s)) {} + template <typename Stream> + CTransaction(deserialize_type, ParamsStream<TransactionSerParams,Stream>& s) : CTransaction(CMutableTransaction(deserialize, s)) {} bool IsNull() const { return vin.empty() && vout.empty(); @@ -389,17 +392,21 @@ struct CMutableTransaction template <typename Stream> inline void Serialize(Stream& s) const { - SerializeTransaction(*this, s); + SerializeTransaction(*this, s, s.GetParams()); } - template <typename Stream> inline void Unserialize(Stream& s) { - UnserializeTransaction(*this, s); + UnserializeTransaction(*this, s, s.GetParams()); + } + + template <typename Stream> + CMutableTransaction(deserialize_type, const TransactionSerParams& params, Stream& s) { + UnserializeTransaction(*this, s, params); } template <typename Stream> - CMutableTransaction(deserialize_type, Stream& s) { + CMutableTransaction(deserialize_type, ParamsStream<TransactionSerParams,Stream>& s) { Unserialize(s); } diff --git a/src/protocol.cpp b/src/protocol.cpp index f956728af2..27a0a2ffc1 100644 --- a/src/protocol.cpp +++ b/src/protocol.cpp @@ -91,9 +91,8 @@ const static std::vector<std::string> g_all_net_message_types{ }; CMessageHeader::CMessageHeader(const MessageStartChars& pchMessageStartIn, const char* pszCommand, unsigned int nMessageSizeIn) + : pchMessageStart{pchMessageStartIn} { - pchMessageStart = pchMessageStartIn; - // Copy the command name size_t i = 0; for (; i < COMMAND_SIZE && pszCommand[i] != 0; ++i) pchCommand[i] = pszCommand[i]; diff --git a/src/psbt.h b/src/psbt.h index 5b4daafed5..1fd186f6a5 100644 --- a/src/psbt.h +++ b/src/psbt.h @@ -95,7 +95,7 @@ void SerializeToVector(Stream& s, const X&... args) // Takes a stream and multiple arguments and unserializes them first as a vector then each object individually in the order provided in the arguments template<typename Stream, typename... X> -void UnserializeFromVector(Stream& s, X&... args) +void UnserializeFromVector(Stream& s, X&&... args) { size_t expected_size = ReadCompactSize(s); size_t remaining_before = s.size(); @@ -226,8 +226,7 @@ struct PSBTInput // Write the utxo if (non_witness_utxo) { SerializeToVector(s, CompactSizeWriter(PSBT_IN_NON_WITNESS_UTXO)); - OverrideStream<Stream> os{&s, s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS}; - SerializeToVector(os, non_witness_utxo); + SerializeToVector(s, TX_NO_WITNESS(non_witness_utxo)); } if (!witness_utxo.IsNull()) { SerializeToVector(s, CompactSizeWriter(PSBT_IN_WITNESS_UTXO)); @@ -394,8 +393,7 @@ struct PSBTInput throw std::ios_base::failure("Non-witness utxo key is more than one byte type"); } // Set the stream to unserialize with witness since this is always a valid network transaction - OverrideStream<Stream> os{&s, s.GetVersion() & ~SERIALIZE_TRANSACTION_NO_WITNESS}; - UnserializeFromVector(os, non_witness_utxo); + UnserializeFromVector(s, TX_WITH_WITNESS(non_witness_utxo)); break; } case PSBT_IN_WITNESS_UTXO: @@ -984,8 +982,7 @@ struct PartiallySignedTransaction SerializeToVector(s, CompactSizeWriter(PSBT_GLOBAL_UNSIGNED_TX)); // Write serialized tx to a stream - OverrideStream<Stream> os{&s, s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS}; - SerializeToVector(os, *tx); + SerializeToVector(s, TX_NO_WITNESS(*tx)); // Write xpubs for (const auto& xpub_pair : m_xpubs) { @@ -1075,8 +1072,7 @@ struct PartiallySignedTransaction } CMutableTransaction mtx; // Set the stream to serialize with non-witness since this should always be non-witness - OverrideStream<Stream> os{&s, s.GetVersion() | SERIALIZE_TRANSACTION_NO_WITNESS}; - UnserializeFromVector(os, mtx); + UnserializeFromVector(s, TX_NO_WITNESS(mtx)); tx = std::move(mtx); // Make sure that all scriptSigs and scriptWitnesses are empty for (const CTxIn& txin : tx->vin) { diff --git a/src/qt/transactiondesc.cpp b/src/qt/transactiondesc.cpp index 51f6f44923..a916e4ead6 100644 --- a/src/qt/transactiondesc.cpp +++ b/src/qt/transactiondesc.cpp @@ -360,12 +360,10 @@ QString TransactionDesc::toHTML(interfaces::Node& node, interfaces::Wallet& wall { COutPoint prevout = txin.prevout; - Coin prev; - if(node.getUnspentOutput(prevout, prev)) - { + if (auto prev{node.getUnspentOutput(prevout)}) { { strHTML += "<li>"; - const CTxOut &vout = prev.out; + const CTxOut& vout = prev->out; CTxDestination address; if (ExtractDestination(vout.scriptPubKey, address)) { diff --git a/src/qt/walletmodel.cpp b/src/qt/walletmodel.cpp index a45579fa0d..445495897d 100644 --- a/src/qt/walletmodel.cpp +++ b/src/qt/walletmodel.cpp @@ -260,8 +260,8 @@ void WalletModel::sendCoins(WalletModelTransaction& transaction) auto& newTx = transaction.getWtx(); wallet().commitTransaction(newTx, /*value_map=*/{}, std::move(vOrderForm)); - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION); - ssTx << *newTx; + DataStream ssTx; + ssTx << TX_WITH_WITNESS(*newTx); transaction_array.append((const char*)ssTx.data(), ssTx.size()); } diff --git a/src/random.cpp b/src/random.cpp index 9bd8ff9f1a..ce4266a567 100644 --- a/src/random.cpp +++ b/src/random.cpp @@ -38,6 +38,9 @@ #ifdef HAVE_SYSCTL_ARND #include <sys/sysctl.h> #endif +#if defined(HAVE_STRONG_GETAUXVAL) && defined(__aarch64__) +#include <sys/auxv.h> +#endif [[noreturn]] static void RandFailure() { @@ -173,6 +176,62 @@ static uint64_t GetRdSeed() noexcept #endif } +#elif defined(__aarch64__) && defined(HWCAP2_RNG) + +static bool g_rndr_supported = false; + +static void InitHardwareRand() +{ + if (getauxval(AT_HWCAP2) & HWCAP2_RNG) { + g_rndr_supported = true; + } +} + +static void ReportHardwareRand() +{ + // This must be done in a separate function, as InitHardwareRand() may be indirectly called + // from global constructors, before logging is initialized. + if (g_rndr_supported) { + LogPrintf("Using RNDR and RNDRRS as additional entropy sources\n"); + } +} + +/** Read 64 bits of entropy using rndr. + * + * Must only be called when RNDR is supported. + */ +static uint64_t GetRNDR() noexcept +{ + uint8_t ok; + uint64_t r1; + do { + // https://developer.arm.com/documentation/ddi0601/2022-12/AArch64-Registers/RNDR--Random-Number + __asm__ volatile("mrs %0, s3_3_c2_c4_0; cset %w1, ne;" + : "=r"(r1), "=r"(ok)::"cc"); + if (ok) break; + __asm__ volatile("yield"); + } while (true); + return r1; +} + +/** Read 64 bits of entropy using rndrrs. + * + * Must only be called when RNDRRS is supported. + */ +static uint64_t GetRNDRRS() noexcept +{ + uint8_t ok; + uint64_t r1; + do { + // https://developer.arm.com/documentation/ddi0601/2022-12/AArch64-Registers/RNDRRS--Reseeded-Random-Number + __asm__ volatile("mrs %0, s3_3_c2_c4_1; cset %w1, ne;" + : "=r"(r1), "=r"(ok)::"cc"); + if (ok) break; + __asm__ volatile("yield"); + } while (true); + return r1; +} + #else /* Access to other hardware random number generators could be added here later, * assuming it is sufficiently fast (in the order of a few hundred CPU cycles). @@ -191,6 +250,12 @@ static void SeedHardwareFast(CSHA512& hasher) noexcept { hasher.Write((const unsigned char*)&out, sizeof(out)); return; } +#elif defined(__aarch64__) && defined(HWCAP2_RNG) + if (g_rndr_supported) { + uint64_t out = GetRNDR(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + return; + } #endif } @@ -216,6 +281,14 @@ static void SeedHardwareSlow(CSHA512& hasher) noexcept { } return; } +#elif defined(__aarch64__) && defined(HWCAP2_RNG) + if (g_rndr_supported) { + for (int i = 0; i < 4; ++i) { + uint64_t out = GetRNDRRS(); + hasher.Write((const unsigned char*)&out, sizeof(out)); + } + return; + } #endif } diff --git a/src/rest.cpp b/src/rest.cpp index e094039366..f86c47ee6b 100644 --- a/src/rest.cpp +++ b/src/rest.cpp @@ -26,6 +26,7 @@ #include <txmempool.h> #include <util/any.h> #include <util/check.h> +#include <util/strencodings.h> #include <validation.h> #include <version.h> @@ -316,8 +317,8 @@ static bool rest_block(const std::any& context, switch (rf) { case RESTResponseFormat::BINARY: { - CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); - ssBlock << block; + DataStream ssBlock; + ssBlock << RPCTxSerParams(block); std::string binaryBlock = ssBlock.str(); req->WriteHeader("Content-Type", "application/octet-stream"); req->WriteReply(HTTP_OK, binaryBlock); @@ -325,8 +326,8 @@ static bool rest_block(const std::any& context, } case RESTResponseFormat::HEX: { - CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); - ssBlock << block; + DataStream ssBlock; + ssBlock << RPCTxSerParams(block); std::string strHex = HexStr(ssBlock) + "\n"; req->WriteHeader("Content-Type", "text/plain"); req->WriteReply(HTTP_OK, strHex); @@ -722,8 +723,8 @@ static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string switch (rf) { case RESTResponseFormat::BINARY: { - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); - ssTx << tx; + DataStream ssTx; + ssTx << RPCTxSerParams(tx); std::string binaryTx = ssTx.str(); req->WriteHeader("Content-Type", "application/octet-stream"); @@ -732,8 +733,8 @@ static bool rest_tx(const std::any& context, HTTPRequest* req, const std::string } case RESTResponseFormat::HEX: { - CDataStream ssTx(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); - ssTx << tx; + DataStream ssTx; + ssTx << RPCTxSerParams(tx); std::string strHex = HexStr(ssTx) + "\n"; req->WriteHeader("Content-Type", "text/plain"); diff --git a/src/rpc/blockchain.cpp b/src/rpc/blockchain.cpp index 7b84747a3f..9a29b73bee 100644 --- a/src/rpc/blockchain.cpp +++ b/src/rpc/blockchain.cpp @@ -166,8 +166,8 @@ UniValue blockToJSON(BlockManager& blockman, const CBlock& block, const CBlockIn { UniValue result = blockheaderToJSON(tip, blockindex); - result.pushKV("strippedsize", (int)::GetSerializeSize(block, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS)); - result.pushKV("size", (int)::GetSerializeSize(block, PROTOCOL_VERSION)); + result.pushKV("strippedsize", (int)::GetSerializeSize(TX_NO_WITNESS(block))); + result.pushKV("size", (int)::GetSerializeSize(TX_WITH_WITNESS(block))); result.pushKV("weight", (int)::GetBlockWeight(block)); UniValue txs(UniValue::VARR); @@ -189,7 +189,7 @@ UniValue blockToJSON(BlockManager& blockman, const CBlock& block, const CBlockIn // coinbase transaction (i.e. i == 0) doesn't have undo data const CTxUndo* txundo = (have_undo && i > 0) ? &blockUndo.vtxundo.at(i - 1) : nullptr; UniValue objTx(UniValue::VOBJ); - TxToUniv(*tx, /*block_hash=*/uint256(), /*entry=*/objTx, /*include_hex=*/true, RPCSerializationFlags(), txundo, verbosity); + TxToUniv(*tx, /*block_hash=*/uint256(), /*entry=*/objTx, /*include_hex=*/true, /*without_witness=*/RPCSerializationWithoutWitness(), txundo, verbosity); txs.push_back(objTx); } break; @@ -740,8 +740,8 @@ static RPCHelpMan getblock() if (verbosity <= 0) { - CDataStream ssBlock(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); - ssBlock << block; + DataStream ssBlock; + ssBlock << RPCTxSerParams(block); std::string strHex = HexStr(ssBlock); return strHex; } diff --git a/src/rpc/mempool.cpp b/src/rpc/mempool.cpp index 23cef42d30..bf60eae433 100644 --- a/src/rpc/mempool.cpp +++ b/src/rpc/mempool.cpp @@ -21,6 +21,7 @@ #include <univalue.h> #include <util/fs.h> #include <util/moneystr.h> +#include <util/strencodings.h> #include <util/time.h> #include <utility> @@ -289,7 +290,7 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool info.pushKV("descendantsize", e.GetSizeWithDescendants()); info.pushKV("ancestorcount", e.GetCountWithAncestors()); info.pushKV("ancestorsize", e.GetSizeWithAncestors()); - info.pushKV("wtxid", pool.vTxHashes[e.vTxHashesIdx].first.ToString()); + info.pushKV("wtxid", e.GetTx().GetWitnessHash().ToString()); UniValue fees(UniValue::VOBJ); fees.pushKV("base", ValueFromAmount(e.GetFee())); @@ -315,9 +316,7 @@ static void entryToJSON(const CTxMemPool& pool, UniValue& info, const CTxMemPool info.pushKV("depends", depends); UniValue spent(UniValue::VARR); - const CTxMemPool::txiter& it = pool.mapTx.find(tx.GetHash()); - const CTxMemPoolEntry::Children& children = it->GetMemPoolChildrenConst(); - for (const CTxMemPoolEntry& child : children) { + for (const CTxMemPoolEntry& child : e.GetMemPoolChildrenConst()) { spent.push_back(child.GetTx().GetHash().ToString()); } @@ -344,14 +343,13 @@ UniValue MempoolToJSON(const CTxMemPool& pool, bool verbose, bool include_mempoo } LOCK(pool.cs); UniValue o(UniValue::VOBJ); - for (const CTxMemPoolEntry& e : pool.mapTx) { - const uint256& hash = e.GetTx().GetHash(); + for (const CTxMemPoolEntry& e : pool.entryAll()) { UniValue info(UniValue::VOBJ); entryToJSON(pool, info, e); // Mempool has unique entries so there is no advantage in using // UniValue::pushKV, which checks if the key already exists in O(N). // UniValue::pushKVEnd is used instead which currently is O(1). - o.pushKVEnd(hash.ToString(), info); + o.pushKVEnd(e.GetTx().GetHash().ToString(), info); } return o; } else { @@ -460,12 +458,12 @@ static RPCHelpMan getmempoolancestors() const CTxMemPool& mempool = EnsureAnyMemPool(request.context); LOCK(mempool.cs); - CTxMemPool::txiter it = mempool.mapTx.find(hash); - if (it == mempool.mapTx.end()) { + const auto entry{mempool.GetEntry(Txid::FromUint256(hash))}; + if (entry == nullptr) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); } - auto ancestors{mempool.AssumeCalculateMemPoolAncestors(self.m_name, *it, CTxMemPool::Limits::NoLimits(), /*fSearchForParents=*/false)}; + auto ancestors{mempool.AssumeCalculateMemPoolAncestors(self.m_name, *entry, CTxMemPool::Limits::NoLimits(), /*fSearchForParents=*/false)}; if (!fVerbose) { UniValue o(UniValue::VARR); @@ -521,15 +519,15 @@ static RPCHelpMan getmempooldescendants() const CTxMemPool& mempool = EnsureAnyMemPool(request.context); LOCK(mempool.cs); - CTxMemPool::txiter it = mempool.mapTx.find(hash); - if (it == mempool.mapTx.end()) { + const auto it{mempool.GetIter(hash)}; + if (!it) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); } CTxMemPool::setEntries setDescendants; - mempool.CalculateDescendants(it, setDescendants); + mempool.CalculateDescendants(*it, setDescendants); // CTxMemPool::CalculateDescendants will include the given tx - setDescendants.erase(it); + setDescendants.erase(*it); if (!fVerbose) { UniValue o(UniValue::VARR); @@ -573,14 +571,13 @@ static RPCHelpMan getmempoolentry() const CTxMemPool& mempool = EnsureAnyMemPool(request.context); LOCK(mempool.cs); - CTxMemPool::txiter it = mempool.mapTx.find(hash); - if (it == mempool.mapTx.end()) { + const auto entry{mempool.GetEntry(Txid::FromUint256(hash))}; + if (entry == nullptr) { throw JSONRPCError(RPC_INVALID_ADDRESS_OR_KEY, "Transaction not in mempool"); } - const CTxMemPoolEntry &e = *it; UniValue info(UniValue::VOBJ); - entryToJSON(mempool, info, e); + entryToJSON(mempool, info, *entry); return info; }, }; diff --git a/src/rpc/mining.cpp b/src/rpc/mining.cpp index 76170c3201..1bdc1e1029 100644 --- a/src/rpc/mining.cpp +++ b/src/rpc/mining.cpp @@ -390,8 +390,8 @@ static RPCHelpMan generateblock() UniValue obj(UniValue::VOBJ); obj.pushKV("hash", block_out->GetHash().GetHex()); if (!process_new_block) { - CDataStream block_ser{SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()}; - block_ser << *block_out; + DataStream block_ser; + block_ser << RPCTxSerParams(*block_out); obj.pushKV("hex", HexStr(block_ser)); } return obj; diff --git a/src/rpc/net.cpp b/src/rpc/net.cpp index 1a9052e008..c631132df2 100644 --- a/src/rpc/net.cpp +++ b/src/rpc/net.cpp @@ -16,6 +16,7 @@ #include <netbase.h> #include <node/context.h> #include <policy/settings.h> +#include <protocol.h> #include <rpc/blockchain.h> #include <rpc/protocol.h> #include <rpc/server_util.h> @@ -98,6 +99,18 @@ static RPCHelpMan ping() }; } +/** Returns, given services flags, a list of humanly readable (known) network services */ +static UniValue GetServicesNames(ServiceFlags services) +{ + UniValue servicesNames(UniValue::VARR); + + for (const auto& flag : serviceFlagsToStr(services)) { + servicesNames.push_back(flag); + } + + return servicesNames; +} + static RPCHelpMan getpeerinfo() { return RPCHelpMan{ @@ -487,7 +500,7 @@ static RPCHelpMan getaddednodeinfo() NodeContext& node = EnsureAnyNodeContext(request.context); const CConnman& connman = EnsureConnman(node); - std::vector<AddedNodeInfo> vInfo = connman.GetAddedNodeInfo(); + std::vector<AddedNodeInfo> vInfo = connman.GetAddedNodeInfo(/*include_connected=*/true); if (!request.params[0].isNull()) { bool found = false; diff --git a/src/rpc/node.cpp b/src/rpc/node.cpp index 6b3662996c..b085828215 100644 --- a/src/rpc/node.cpp +++ b/src/rpc/node.cpp @@ -91,6 +91,9 @@ static RPCHelpMan mockscheduler() const NodeContext& node_context{EnsureAnyNodeContext(request.context)}; CHECK_NONFATAL(node_context.scheduler)->MockForward(std::chrono::seconds{delta_seconds}); SyncWithValidationInterfaceQueue(); + for (const auto& chain_client : node_context.chain_clients) { + chain_client->schedulerMockForward(std::chrono::seconds(delta_seconds)); + } return UniValue::VNULL; }, diff --git a/src/rpc/rawtransaction.cpp b/src/rpc/rawtransaction.cpp index 16705b3ce2..397ece08c0 100644 --- a/src/rpc/rawtransaction.cpp +++ b/src/rpc/rawtransaction.cpp @@ -62,7 +62,7 @@ static void TxToJSON(const CTransaction& tx, const uint256 hashBlock, UniValue& // Blockchain contextual information (confirmations and blocktime) is not // available to code in bitcoin-common, so we query them here and push the // data into the returned UniValue. - TxToUniv(tx, /*block_hash=*/uint256(), entry, /*include_hex=*/true, RPCSerializationFlags(), txundo, verbosity); + TxToUniv(tx, /*block_hash=*/uint256(), entry, /*include_hex=*/true, RPCSerializationWithoutWitness(), txundo, verbosity); if (!hashBlock.IsNull()) { LOCK(cs_main); @@ -383,7 +383,7 @@ static RPCHelpMan getrawtransaction() } if (verbosity <= 0) { - return EncodeHexTx(*tx, RPCSerializationFlags()); + return EncodeHexTx(*tx, /*without_witness=*/RPCSerializationWithoutWitness()); } UniValue result(UniValue::VOBJ); @@ -1541,7 +1541,7 @@ static RPCHelpMan finalizepsbt() std::string result_str; if (complete && extract) { - ssTx << mtx; + ssTx << TX_WITH_WITNESS(mtx); result_str = HexStr(ssTx); result.pushKV("hex", result_str); } else { @@ -1994,8 +1994,8 @@ RPCHelpMan descriptorprocesspsbt() CMutableTransaction mtx; PartiallySignedTransaction psbtx_copy = psbtx; CHECK_NONFATAL(FinalizeAndExtractPSBT(psbtx_copy, mtx)); - CDataStream ssTx_final(SER_NETWORK, PROTOCOL_VERSION); - ssTx_final << mtx; + DataStream ssTx_final; + ssTx_final << TX_WITH_WITNESS(mtx); result.pushKV("hex", HexStr(ssTx_final)); } return result; diff --git a/src/rpc/server.cpp b/src/rpc/server.cpp index ab410ffc05..7adc28f416 100644 --- a/src/rpc/server.cpp +++ b/src/rpc/server.cpp @@ -595,12 +595,9 @@ void RPCRunLater(const std::string& name, std::function<void()> func, int64_t nS deadlineTimers.emplace(name, std::unique_ptr<RPCTimerBase>(timerInterface->NewTimer(func, nSeconds*1000))); } -int RPCSerializationFlags() +bool RPCSerializationWithoutWitness() { - int flag = 0; - if (gArgs.GetIntArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) == 0) - flag |= SERIALIZE_TRANSACTION_NO_WITNESS; - return flag; + return (gArgs.GetIntArg("-rpcserialversion", DEFAULT_RPC_SERIALIZE_VERSION) == 0); } CRPCTable tableRPC; diff --git a/src/rpc/server.h b/src/rpc/server.h index 24658ddb8b..9a49d82570 100644 --- a/src/rpc/server.h +++ b/src/rpc/server.h @@ -183,7 +183,14 @@ void InterruptRPC(); void StopRPC(); std::string JSONRPCExecBatch(const JSONRPCRequest& jreq, const UniValue& vReq); -// Retrieves any serialization flags requested in command line argument -int RPCSerializationFlags(); +// Drop witness when serializing for RPC? +bool RPCSerializationWithoutWitness(); + +template<typename T> +auto RPCTxSerParams(T&& t) +{ + if (RPCSerializationWithoutWitness()) return TX_NO_WITNESS(t); + return TX_WITH_WITNESS(t); +} #endif // BITCOIN_RPC_SERVER_H diff --git a/src/rpc/util.cpp b/src/rpc/util.cpp index 5068d016b6..cf48ee11e7 100644 --- a/src/rpc/util.cpp +++ b/src/rpc/util.cpp @@ -1304,17 +1304,6 @@ std::vector<CScript> EvalDescriptorStringOrObject(const UniValue& scanobject, Fl return ret; } -UniValue GetServicesNames(ServiceFlags services) -{ - UniValue servicesNames(UniValue::VARR); - - for (const auto& flag : serviceFlagsToStr(services)) { - servicesNames.push_back(flag); - } - - return servicesNames; -} - /** Convert a vector of bilingual strings to a UniValue::VARR containing their original untranslated values. */ [[nodiscard]] static UniValue BilingualStringsToUniValue(const std::vector<bilingual_str>& bilingual_strings) { diff --git a/src/rpc/util.h b/src/rpc/util.h index ed14986c49..e2d5ed333c 100644 --- a/src/rpc/util.h +++ b/src/rpc/util.h @@ -9,7 +9,6 @@ #include <consensus/amount.h> #include <node/transaction.h> #include <outputtype.h> -#include <protocol.h> #include <pubkey.h> #include <rpc/protocol.h> #include <rpc/request.h> @@ -60,7 +59,6 @@ extern const std::string UNIX_EPOCH_TIME; extern const std::string EXAMPLE_ADDRESS[2]; class FillableSigningProvider; -class CPubKey; class CScript; struct Sections; @@ -133,9 +131,6 @@ std::pair<int64_t, int64_t> ParseDescriptorRange(const UniValue& value); /** Evaluate a descriptor given as a string, or as a {"desc":...,"range":...} object, with default range of 1000. */ std::vector<CScript> EvalDescriptorStringOrObject(const UniValue& scanobject, FlatSigningProvider& provider, const bool expand_priv = false); -/** Returns, given services flags, a list of humanly readable (known) network services */ -UniValue GetServicesNames(ServiceFlags services); - /** * Serializing JSON objects depends on the outer type. Only arrays and * dictionaries can be nested in json. The top-level outer type is "NONE". diff --git a/src/script/bitcoinconsensus.cpp b/src/script/bitcoinconsensus.cpp index 71005cfb6e..2ca8dac48c 100644 --- a/src/script/bitcoinconsensus.cpp +++ b/src/script/bitcoinconsensus.cpp @@ -16,8 +16,7 @@ namespace { class TxInputStream { public: - TxInputStream(int nVersionIn, const unsigned char *txTo, size_t txToLen) : - m_version(nVersionIn), + TxInputStream(const unsigned char *txTo, size_t txToLen) : m_data(txTo), m_remaining(txToLen) {} @@ -48,9 +47,7 @@ public: return *this; } - int GetVersion() const { return m_version; } private: - const int m_version; const unsigned char* m_data; size_t m_remaining; }; @@ -84,8 +81,8 @@ static int verify_script(const unsigned char *scriptPubKey, unsigned int scriptP } try { - TxInputStream stream(PROTOCOL_VERSION, txTo, txToLen); - CTransaction tx(deserialize, stream); + TxInputStream stream(txTo, txToLen); + CTransaction tx(deserialize, TX_WITH_WITNESS, stream); std::vector<CTxOut> spent_outputs; if (spentOutputs != nullptr) { @@ -102,7 +99,7 @@ static int verify_script(const unsigned char *scriptPubKey, unsigned int scriptP if (nIn >= tx.vin.size()) return set_error(err, bitcoinconsensus_ERR_TX_INDEX); - if (GetSerializeSize(tx, PROTOCOL_VERSION) != txToLen) + if (GetSerializeSize(TX_WITH_WITNESS(tx)) != txToLen) return set_error(err, bitcoinconsensus_ERR_TX_SIZE_MISMATCH); // Regardless of the verification result, the tx did not error. diff --git a/src/span.h b/src/span.h index 7209d21a58..2e8da27cde 100644 --- a/src/span.h +++ b/src/span.h @@ -222,15 +222,32 @@ public: template <typename O> friend class Span; }; +// Return result of calling .data() method on type T. This is used to be able to +// write template deduction guides for the single-parameter Span constructor +// below that will work if the value that is passed has a .data() method, and if +// the data method does not return a void pointer. +// +// It is important to check for the void type specifically below, so the +// deduction guides can be used in SFINAE contexts to check whether objects can +// be converted to spans. If the deduction guides did not explicitly check for +// void, and an object was passed that returned void* from data (like +// std::vector<bool>), the template deduction would succeed, but the Span<void> +// object instantiation would fail, resulting in a hard error, rather than a +// SFINAE error. +// https://stackoverflow.com/questions/68759148/sfinae-to-detect-the-explicitness-of-a-ctad-deduction-guide +// https://stackoverflow.com/questions/16568986/what-happens-when-you-call-data-on-a-stdvectorbool +template<typename T> +using DataResult = std::remove_pointer_t<decltype(std::declval<T&>().data())>; + // Deduction guides for Span // For the pointer/size based and iterator based constructor: template <typename T, typename EndOrSize> Span(T*, EndOrSize) -> Span<T>; // For the array constructor: template <typename T, std::size_t N> Span(T (&)[N]) -> Span<T>; // For the temporaries/rvalue references constructor, only supporting const output. -template <typename T> Span(T&&) -> Span<std::enable_if_t<!std::is_lvalue_reference_v<T>, const std::remove_pointer_t<decltype(std::declval<T&&>().data())>>>; +template <typename T> Span(T&&) -> Span<std::enable_if_t<!std::is_lvalue_reference_v<T> && !std::is_void_v<DataResult<T&&>>, const DataResult<T&&>>>; // For (lvalue) references, supporting mutable output. -template <typename T> Span(T&) -> Span<std::remove_pointer_t<decltype(std::declval<T&>().data())>>; +template <typename T> Span(T&) -> Span<std::enable_if_t<!std::is_void_v<DataResult<T&>>, DataResult<T&>>>; /** Pop the last element off a span, and return a reference to that element. */ template <typename T> diff --git a/src/streams.h b/src/streams.h index d58de5233b..a3f8028da7 100644 --- a/src/streams.h +++ b/src/streams.h @@ -45,45 +45,6 @@ inline void Xor(Span<std::byte> write, Span<const std::byte> key, size_t key_off } } // namespace util -template<typename Stream> -class OverrideStream -{ - Stream* stream; - - const int nVersion; - -public: - OverrideStream(Stream* stream_, int nVersion_) : stream{stream_}, nVersion{nVersion_} {} - - template<typename T> - OverrideStream<Stream>& operator<<(const T& obj) - { - ::Serialize(*this, obj); - return (*this); - } - - template<typename T> - OverrideStream<Stream>& operator>>(T&& obj) - { - ::Unserialize(*this, obj); - return (*this); - } - - void write(Span<const std::byte> src) - { - stream->write(src); - } - - void read(Span<std::byte> dst) - { - stream->read(dst); - } - - int GetVersion() const { return nVersion; } - size_t size() const { return stream->size(); } - void ignore(size_t size) { return stream->ignore(size); } -}; - /* Minimal stream for overwriting and/or appending to an existing byte vector * * The referenced vector will grow as necessary @@ -182,6 +143,11 @@ public: memcpy(dst.data(), m_data.data(), dst.size()); m_data = m_data.subspan(dst.size()); } + + void ignore(size_t n) + { + m_data = m_data.subspan(n); + } }; /** Double ended buffer combining vector and stream-like interfaces. @@ -471,7 +437,7 @@ class AutoFile { protected: std::FILE* m_file; - const std::vector<std::byte> m_xor; + std::vector<std::byte> m_xor; public: explicit AutoFile(std::FILE* file, std::vector<std::byte> data_xor={}) : m_file{file}, m_xor{std::move(data_xor)} {} @@ -511,6 +477,9 @@ public: */ bool IsNull() const { return m_file == nullptr; } + /** Continue with a different XOR key */ + void SetXor(std::vector<std::byte> data_xor) { m_xor = data_xor; } + /** Implementation detail, only used internally. */ std::size_t detail_fread(Span<std::byte> dst); diff --git a/src/test/blockencodings_tests.cpp b/src/test/blockencodings_tests.cpp index 4348a20886..e4ef019daf 100644 --- a/src/test/blockencodings_tests.cpp +++ b/src/test/blockencodings_tests.cpp @@ -51,8 +51,8 @@ static CBlock BuildBlockTestCase() { } // Number of shared use_counts we expect for a tx we haven't touched -// (block + mempool + our copy from the GetSharedTx call) -constexpr long SHARED_TX_OFFSET{3}; +// (block + mempool entry + mempool txns_randomized + our copy from the GetSharedTx call) +constexpr long SHARED_TX_OFFSET{4}; BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) { @@ -62,7 +62,7 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) LOCK2(cs_main, pool.cs); pool.addUnchecked(entry.FromTx(block.vtx[2])); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 0); // Do a simple ShortTxIDs RT { @@ -80,7 +80,7 @@ BOOST_AUTO_TEST_CASE(SimpleRoundTripTest) BOOST_CHECK(!partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 1); size_t poolSize = pool.size(); pool.removeRecursive(*block.vtx[2], MemPoolRemovalReason::REPLACED); @@ -145,7 +145,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) LOCK2(cs_main, pool.cs); pool.addUnchecked(entry.FromTx(block.vtx[2])); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 0); uint256 txhash; @@ -170,7 +170,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) BOOST_CHECK( partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); // +1 because of partialBlock + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 1); // +1 because of partialBlock CBlock block2; { @@ -185,7 +185,7 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) partialBlock.FillBlock(block2, {block.vtx[1]}); // Current implementation doesn't check txn here, but don't require that partialBlock = tmp; } - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 2); // +2 because of partialBlock and block2 + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 2); // +2 because of partialBlock and block2 bool mutated; BOOST_CHECK(block.hashMerkleRoot != BlockMerkleRoot(block2, &mutated)); @@ -196,15 +196,15 @@ BOOST_AUTO_TEST_CASE(NonCoinbasePreforwardRTTest) BOOST_CHECK_EQUAL(block.hashMerkleRoot.ToString(), BlockMerkleRoot(block3, &mutated).ToString()); BOOST_CHECK(!mutated); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[2]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 3); // +2 because of partialBlock and block2 and block3 + BOOST_CHECK_EQUAL(pool.get(block.vtx[2]->GetHash()).use_count(), SHARED_TX_OFFSET + 3); // +2 because of partialBlock and block2 and block3 txhash = block.vtx[2]->GetHash(); block.vtx.clear(); block2.vtx.clear(); block3.vtx.clear(); - BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1 - 1); // + 1 because of partialBlock; -1 because of block. + BOOST_CHECK_EQUAL(pool.get(txhash).use_count(), SHARED_TX_OFFSET + 1 - 1); // + 1 because of partialBlock; -1 because of block. } - BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET - 1); // -1 because of block + BOOST_CHECK_EQUAL(pool.get(txhash).use_count(), SHARED_TX_OFFSET - 1); // -1 because of block } BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) @@ -215,7 +215,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) LOCK2(cs_main, pool.cs); pool.addUnchecked(entry.FromTx(block.vtx[1])); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 0); + BOOST_CHECK_EQUAL(pool.get(block.vtx[1]->GetHash()).use_count(), SHARED_TX_OFFSET + 0); uint256 txhash; @@ -240,7 +240,7 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) BOOST_CHECK( partialBlock.IsTxAvailable(1)); BOOST_CHECK( partialBlock.IsTxAvailable(2)); - BOOST_CHECK_EQUAL(pool.mapTx.find(block.vtx[1]->GetHash())->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1); + BOOST_CHECK_EQUAL(pool.get(block.vtx[1]->GetHash()).use_count(), SHARED_TX_OFFSET + 1); CBlock block2; PartiallyDownloadedBlock partialBlockCopy = partialBlock; @@ -253,9 +253,9 @@ BOOST_AUTO_TEST_CASE(SufficientPreforwardRTTest) txhash = block.vtx[1]->GetHash(); block.vtx.clear(); block2.vtx.clear(); - BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET + 1 - 1); // + 1 because of partialBlock; -1 because of block. + BOOST_CHECK_EQUAL(pool.get(txhash).use_count(), SHARED_TX_OFFSET + 1 - 1); // + 1 because of partialBlock; -1 because of block. } - BOOST_CHECK_EQUAL(pool.mapTx.find(txhash)->GetSharedTx().use_count(), SHARED_TX_OFFSET - 1); // -1 because of block + BOOST_CHECK_EQUAL(pool.get(txhash).use_count(), SHARED_TX_OFFSET - 1); // -1 because of block } BOOST_AUTO_TEST_CASE(EmptyBlockRoundTripTest) diff --git a/src/test/blockmanager_tests.cpp b/src/test/blockmanager_tests.cpp index c6800c498b..e1b31c20f4 100644 --- a/src/test/blockmanager_tests.cpp +++ b/src/test/blockmanager_tests.cpp @@ -49,7 +49,7 @@ BOOST_AUTO_TEST_CASE(blockmanager_find_block_pos) // 8 bytes (for serialization header) + 285 (for serialized genesis block) = 293 // add another 8 bytes for the second block's serialization header and we get 293 + 8 = 301 FlatFilePos actual{blockman.SaveBlockToDisk(params->GenesisBlock(), 1, nullptr)}; - BOOST_CHECK_EQUAL(actual.nPos, BLOCK_SERIALIZATION_HEADER_SIZE + ::GetSerializeSize(params->GenesisBlock(), CLIENT_VERSION) + BLOCK_SERIALIZATION_HEADER_SIZE); + BOOST_CHECK_EQUAL(actual.nPos, BLOCK_SERIALIZATION_HEADER_SIZE + ::GetSerializeSize(TX_WITH_WITNESS(params->GenesisBlock())) + BLOCK_SERIALIZATION_HEADER_SIZE); } BOOST_FIXTURE_TEST_CASE(blockmanager_scan_unlink_already_pruned_files, TestChain100Setup) diff --git a/src/test/bloom_tests.cpp b/src/test/bloom_tests.cpp index 93c0412593..23cbe921ba 100644 --- a/src/test/bloom_tests.cpp +++ b/src/test/bloom_tests.cpp @@ -101,13 +101,13 @@ BOOST_AUTO_TEST_CASE(bloom_match) { // Random real transaction (b4749f017444b051c44dfd2720e88f314ff94f3dd6d56d40ef65854fcd7fff6b) CDataStream stream(ParseHex("01000000010b26e9b7735eb6aabdf358bab62f9816a21ba9ebdb719d5299e88607d722c190000000008b4830450220070aca44506c5cef3a16ed519d7c3c39f8aab192c4e1c90d065f37b8a4af6141022100a8e160b856c2d43d27d8fba71e5aef6405b8643ac4cb7cb3c462aced7f14711a0141046d11fee51b0e60666d5049a9101a72741df480b96ee26488a4d3466b95c9a40ac5eeef87e10a5cd336c19a84565f80fa6c547957b7700ff4dfbdefe76036c339ffffffff021bff3d11000000001976a91404943fdd508053c75000106d3bc6e2754dbcff1988ac2f15de00000000001976a914a266436d2965547608b9e15d9032a7b9d64fa43188ac00000000"), SER_DISK, CLIENT_VERSION); - CTransaction tx(deserialize, stream); + CTransaction tx(deserialize, TX_WITH_WITNESS, stream); // and one which spends it (e2769b09e784f32f62ef849763d4f45b98e07ba658647343b915ff832b110436) unsigned char ch[] = {0x01, 0x00, 0x00, 0x00, 0x01, 0x6b, 0xff, 0x7f, 0xcd, 0x4f, 0x85, 0x65, 0xef, 0x40, 0x6d, 0xd5, 0xd6, 0x3d, 0x4f, 0xf9, 0x4f, 0x31, 0x8f, 0xe8, 0x20, 0x27, 0xfd, 0x4d, 0xc4, 0x51, 0xb0, 0x44, 0x74, 0x01, 0x9f, 0x74, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x49, 0x30, 0x46, 0x02, 0x21, 0x00, 0xda, 0x0d, 0xc6, 0xae, 0xce, 0xfe, 0x1e, 0x06, 0xef, 0xdf, 0x05, 0x77, 0x37, 0x57, 0xde, 0xb1, 0x68, 0x82, 0x09, 0x30, 0xe3, 0xb0, 0xd0, 0x3f, 0x46, 0xf5, 0xfc, 0xf1, 0x50, 0xbf, 0x99, 0x0c, 0x02, 0x21, 0x00, 0xd2, 0x5b, 0x5c, 0x87, 0x04, 0x00, 0x76, 0xe4, 0xf2, 0x53, 0xf8, 0x26, 0x2e, 0x76, 0x3e, 0x2d, 0xd5, 0x1e, 0x7f, 0xf0, 0xbe, 0x15, 0x77, 0x27, 0xc4, 0xbc, 0x42, 0x80, 0x7f, 0x17, 0xbd, 0x39, 0x01, 0x41, 0x04, 0xe6, 0xc2, 0x6e, 0xf6, 0x7d, 0xc6, 0x10, 0xd2, 0xcd, 0x19, 0x24, 0x84, 0x78, 0x9a, 0x6c, 0xf9, 0xae, 0xa9, 0x93, 0x0b, 0x94, 0x4b, 0x7e, 0x2d, 0xb5, 0x34, 0x2b, 0x9d, 0x9e, 0x5b, 0x9f, 0xf7, 0x9a, 0xff, 0x9a, 0x2e, 0xe1, 0x97, 0x8d, 0xd7, 0xfd, 0x01, 0xdf, 0xc5, 0x22, 0xee, 0x02, 0x28, 0x3d, 0x3b, 0x06, 0xa9, 0xd0, 0x3a, 0xcf, 0x80, 0x96, 0x96, 0x8d, 0x7d, 0xbb, 0x0f, 0x91, 0x78, 0xff, 0xff, 0xff, 0xff, 0x02, 0x8b, 0xa7, 0x94, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, 0xa9, 0x14, 0xba, 0xde, 0xec, 0xfd, 0xef, 0x05, 0x07, 0x24, 0x7f, 0xc8, 0xf7, 0x42, 0x41, 0xd7, 0x3b, 0xc0, 0x39, 0x97, 0x2d, 0x7b, 0x88, 0xac, 0x40, 0x94, 0xa8, 0x02, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, 0xa9, 0x14, 0xc1, 0x09, 0x32, 0x48, 0x3f, 0xec, 0x93, 0xed, 0x51, 0xf5, 0xfe, 0x95, 0xe7, 0x25, 0x59, 0xf2, 0xcc, 0x70, 0x43, 0xf9, 0x88, 0xac, 0x00, 0x00, 0x00, 0x00, 0x00}; std::vector<unsigned char> vch(ch, ch + sizeof(ch) -1); CDataStream spendStream(vch, SER_DISK, CLIENT_VERSION); - CTransaction spendingTx(deserialize, spendStream); + CTransaction spendingTx(deserialize, TX_WITH_WITNESS, spendStream); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_ALL); filter.insert(uint256S("0xb4749f017444b051c44dfd2720e88f314ff94f3dd6d56d40ef65854fcd7fff6b")); @@ -213,7 +213,7 @@ BOOST_AUTO_TEST_CASE(merkle_block_2) // With 4 txes CBlock block; CDataStream stream(ParseHex("0100000075616236cc2126035fadb38deb65b9102cc2c41c09cdf29fc051906800000000fe7d5e12ef0ff901f6050211249919b1c0653771832b3a80c66cea42847f0ae1d4d26e49ffff001d00f0a4410401000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0804ffff001d029105ffffffff0100f2052a010000004341046d8709a041d34357697dfcb30a9d05900a6294078012bf3bb09c6f9b525f1d16d5503d7905db1ada9501446ea00728668fc5719aa80be2fdfc8a858a4dbdd4fbac00000000010000000255605dc6f5c3dc148b6da58442b0b2cd422be385eab2ebea4119ee9c268d28350000000049483045022100aa46504baa86df8a33b1192b1b9367b4d729dc41e389f2c04f3e5c7f0559aae702205e82253a54bf5c4f65b7428551554b2045167d6d206dfe6a2e198127d3f7df1501ffffffff55605dc6f5c3dc148b6da58442b0b2cd422be385eab2ebea4119ee9c268d2835010000004847304402202329484c35fa9d6bb32a55a70c0982f606ce0e3634b69006138683bcd12cbb6602200c28feb1e2555c3210f1dddb299738b4ff8bbe9667b68cb8764b5ac17b7adf0001ffffffff0200e1f505000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00180d8f000000004341044a656f065871a353f216ca26cef8dde2f03e8c16202d2e8ad769f02032cb86a5eb5e56842e92e19141d60a01928f8dd2c875a390f67c1f6c94cfc617c0ea45afac0000000001000000025f9a06d3acdceb56be1bfeaa3e8a25e62d182fa24fefe899d1c17f1dad4c2028000000004847304402205d6058484157235b06028c30736c15613a28bdb768ee628094ca8b0030d4d6eb0220328789c9a2ec27ddaec0ad5ef58efded42e6ea17c2e1ce838f3d6913f5e95db601ffffffff5f9a06d3acdceb56be1bfeaa3e8a25e62d182fa24fefe899d1c17f1dad4c2028010000004a493046022100c45af050d3cea806cedd0ab22520c53ebe63b987b8954146cdca42487b84bdd6022100b9b027716a6b59e640da50a864d6dd8a0ef24c76ce62391fa3eabaf4d2886d2d01ffffffff0200e1f505000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00180d8f000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac000000000100000002e2274e5fea1bf29d963914bd301aa63b64daaf8a3e88f119b5046ca5738a0f6b0000000048473044022016e7a727a061ea2254a6c358376aaa617ac537eb836c77d646ebda4c748aac8b0220192ce28bf9f2c06a6467e6531e27648d2b3e2e2bae85159c9242939840295ba501ffffffffe2274e5fea1bf29d963914bd301aa63b64daaf8a3e88f119b5046ca5738a0f6b010000004a493046022100b7a1a755588d4190118936e15cd217d133b0e4a53c3c15924010d5648d8925c9022100aaef031874db2114f2d869ac2de4ae53908fbfea5b2b1862e181626bb9005c9f01ffffffff0200e1f505000000004341044a656f065871a353f216ca26cef8dde2f03e8c16202d2e8ad769f02032cb86a5eb5e56842e92e19141d60a01928f8dd2c875a390f67c1f6c94cfc617c0ea45afac00180d8f000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + stream >> TX_WITH_WITNESS(block); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_ALL); // Match the first transaction @@ -268,7 +268,7 @@ BOOST_AUTO_TEST_CASE(merkle_block_2_with_update_none) // With 4 txes CBlock block; CDataStream stream(ParseHex("0100000075616236cc2126035fadb38deb65b9102cc2c41c09cdf29fc051906800000000fe7d5e12ef0ff901f6050211249919b1c0653771832b3a80c66cea42847f0ae1d4d26e49ffff001d00f0a4410401000000010000000000000000000000000000000000000000000000000000000000000000ffffffff0804ffff001d029105ffffffff0100f2052a010000004341046d8709a041d34357697dfcb30a9d05900a6294078012bf3bb09c6f9b525f1d16d5503d7905db1ada9501446ea00728668fc5719aa80be2fdfc8a858a4dbdd4fbac00000000010000000255605dc6f5c3dc148b6da58442b0b2cd422be385eab2ebea4119ee9c268d28350000000049483045022100aa46504baa86df8a33b1192b1b9367b4d729dc41e389f2c04f3e5c7f0559aae702205e82253a54bf5c4f65b7428551554b2045167d6d206dfe6a2e198127d3f7df1501ffffffff55605dc6f5c3dc148b6da58442b0b2cd422be385eab2ebea4119ee9c268d2835010000004847304402202329484c35fa9d6bb32a55a70c0982f606ce0e3634b69006138683bcd12cbb6602200c28feb1e2555c3210f1dddb299738b4ff8bbe9667b68cb8764b5ac17b7adf0001ffffffff0200e1f505000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00180d8f000000004341044a656f065871a353f216ca26cef8dde2f03e8c16202d2e8ad769f02032cb86a5eb5e56842e92e19141d60a01928f8dd2c875a390f67c1f6c94cfc617c0ea45afac0000000001000000025f9a06d3acdceb56be1bfeaa3e8a25e62d182fa24fefe899d1c17f1dad4c2028000000004847304402205d6058484157235b06028c30736c15613a28bdb768ee628094ca8b0030d4d6eb0220328789c9a2ec27ddaec0ad5ef58efded42e6ea17c2e1ce838f3d6913f5e95db601ffffffff5f9a06d3acdceb56be1bfeaa3e8a25e62d182fa24fefe899d1c17f1dad4c2028010000004a493046022100c45af050d3cea806cedd0ab22520c53ebe63b987b8954146cdca42487b84bdd6022100b9b027716a6b59e640da50a864d6dd8a0ef24c76ce62391fa3eabaf4d2886d2d01ffffffff0200e1f505000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00180d8f000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac000000000100000002e2274e5fea1bf29d963914bd301aa63b64daaf8a3e88f119b5046ca5738a0f6b0000000048473044022016e7a727a061ea2254a6c358376aaa617ac537eb836c77d646ebda4c748aac8b0220192ce28bf9f2c06a6467e6531e27648d2b3e2e2bae85159c9242939840295ba501ffffffffe2274e5fea1bf29d963914bd301aa63b64daaf8a3e88f119b5046ca5738a0f6b010000004a493046022100b7a1a755588d4190118936e15cd217d133b0e4a53c3c15924010d5648d8925c9022100aaef031874db2114f2d869ac2de4ae53908fbfea5b2b1862e181626bb9005c9f01ffffffff0200e1f505000000004341044a656f065871a353f216ca26cef8dde2f03e8c16202d2e8ad769f02032cb86a5eb5e56842e92e19141d60a01928f8dd2c875a390f67c1f6c94cfc617c0ea45afac00180d8f000000004341046a0765b5865641ce08dd39690aade26dfbf5511430ca428a3089261361cef170e3929a68aee3d8d4848b0c5111b0a37b82b86ad559fd2a745b44d8e8d9dfdc0cac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + stream >> TX_WITH_WITNESS(block); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_NONE); // Match the first transaction @@ -320,7 +320,7 @@ BOOST_AUTO_TEST_CASE(merkle_block_3_and_serialize) // With one tx CBlock block; CDataStream stream(ParseHex("0100000079cda856b143d9db2c1caff01d1aecc8630d30625d10e8b4b8b0000000000000b50cc069d6a3e33e3ff84a5c41d9d3febe7c770fdcc96b2c3ff60abe184f196367291b4d4c86041b8fa45d630101000000010000000000000000000000000000000000000000000000000000000000000000ffffffff08044c86041b020a02ffffffff0100f2052a01000000434104ecd3229b0571c3be876feaac0442a9f13c5a572742927af1dc623353ecf8c202225f64868137a18cdd85cbbb4c74fbccfd4f49639cf1bdc94a5672bb15ad5d4cac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + stream >> TX_WITH_WITNESS(block); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_ALL); // Match the only transaction @@ -356,7 +356,7 @@ BOOST_AUTO_TEST_CASE(merkle_block_4) // With 7 txes CBlock block; CDataStream stream(ParseHex("0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f093bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b85290701000000010000000000000000000000000000000000000000000000000000000000000000ffffffff07044c86041b0136ffffffff0100f2052a01000000434104eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c876f2c01ec0f0dd5b2e86e7168cefe0d81113c3807420ce13ad1357231a2252247d97a46a91ac000000000100000001bcad20a6a29827d1424f08989255120bf7f3e9e3cdaaa6bb31b0737fe048724300000000494830450220356e834b046cadc0f8ebb5a8a017b02de59c86305403dad52cd77b55af062ea10221009253cd6c119d4729b77c978e1e2aa19f5ea6e0e52b3f16e32fa608cd5bab753901ffffffff02008d380c010000001976a9142b4b8072ecbba129b6453c63e129e643207249ca88ac0065cd1d000000001976a9141b8dd13b994bcfc787b32aeadf58ccb3615cbd5488ac000000000100000003fdacf9b3eb077412e7a968d2e4f11b9a9dee312d666187ed77ee7d26af16cb0b000000008c493046022100ea1608e70911ca0de5af51ba57ad23b9a51db8d28f82c53563c56a05c20f5a87022100a8bdc8b4a8acc8634c6b420410150775eb7f2474f5615f7fccd65af30f310fbf01410465fdf49e29b06b9a1582287b6279014f834edc317695d125ef623c1cc3aaece245bd69fcad7508666e9c74a49dc9056d5fc14338ef38118dc4afae5fe2c585caffffffff309e1913634ecb50f3c4f83e96e70b2df071b497b8973a3e75429df397b5af83000000004948304502202bdb79c596a9ffc24e96f4386199aba386e9bc7b6071516e2b51dda942b3a1ed022100c53a857e76b724fc14d45311eac5019650d415c3abb5428f3aae16d8e69bec2301ffffffff2089e33491695080c9edc18a428f7d834db5b6d372df13ce2b1b0e0cbcb1e6c10000000049483045022100d4ce67c5896ee251c810ac1ff9ceccd328b497c8f553ab6e08431e7d40bad6b5022033119c0c2b7d792d31f1187779c7bd95aefd93d90a715586d73801d9b47471c601ffffffff0100714460030000001976a914c7b55141d097ea5df7a0ed330cf794376e53ec8d88ac0000000001000000045bf0e214aa4069a3e792ecee1e1bf0c1d397cde8dd08138f4b72a00681743447000000008b48304502200c45de8c4f3e2c1821f2fc878cba97b1e6f8807d94930713aa1c86a67b9bf1e40221008581abfef2e30f957815fc89978423746b2086375ca8ecf359c85c2a5b7c88ad01410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffffd669f7d7958d40fc59d2253d88e0f248e29b599c80bbcec344a83dda5f9aa72c000000008a473044022078124c8beeaa825f9e0b30bff96e564dd859432f2d0cb3b72d3d5d93d38d7e930220691d233b6c0f995be5acb03d70a7f7a65b6bc9bdd426260f38a1346669507a3601410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95fffffffff878af0d93f5229a68166cf051fd372bb7a537232946e0a46f53636b4dafdaa4000000008c493046022100c717d1714551663f69c3c5759bdbb3a0fcd3fab023abc0e522fe6440de35d8290221008d9cbe25bffc44af2b18e81c58eb37293fd7fe1c2e7b46fc37ee8c96c50ab1e201410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff27f2b668859cd7f2f894aa0fd2d9e60963bcd07c88973f425f999b8cbfd7a1e2000000008c493046022100e00847147cbf517bcc2f502f3ddc6d284358d102ed20d47a8aa788a62f0db780022100d17b2d6fa84dcaf1c95d88d7e7c30385aecf415588d749afd3ec81f6022cecd701410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff0100c817a8040000001976a914b6efd80d99179f4f4ff6f4dd0a007d018c385d2188ac000000000100000001834537b2f1ce8ef9373a258e10545ce5a50b758df616cd4356e0032554ebd3c4000000008b483045022100e68f422dd7c34fdce11eeb4509ddae38201773dd62f284e8aa9d96f85099d0b002202243bd399ff96b649a0fad05fa759d6a882f0af8c90cf7632c2840c29070aec20141045e58067e815c2f464c6a2a15f987758374203895710c2d452442e28496ff38ba8f5fd901dc20e29e88477167fe4fc299bf818fd0d9e1632d467b2a3d9503b1aaffffffff0280d7e636030000001976a914f34c3e10eb387efe872acb614c89e78bfca7815d88ac404b4c00000000001976a914a84e272933aaf87e1715d7786c51dfaeb5b65a6f88ac00000000010000000143ac81c8e6f6ef307dfe17f3d906d999e23e0189fda838c5510d850927e03ae7000000008c4930460221009c87c344760a64cb8ae6685a3eec2c1ac1bed5b88c87de51acd0e124f266c16602210082d07c037359c3a257b5c63ebd90f5a5edf97b2ac1c434b08ca998839f346dd40141040ba7e521fa7946d12edbb1d1e95a15c34bd4398195e86433c92b431cd315f455fe30032ede69cad9d1e1ed6c3c4ec0dbfced53438c625462afb792dcb098544bffffffff0240420f00000000001976a9144676d1b820d63ec272f1900d59d43bc6463d96f888ac40420f00000000001976a914648d04341d00d7968b3405c034adc38d4d8fb9bd88ac00000000010000000248cc917501ea5c55f4a8d2009c0567c40cfe037c2e71af017d0a452ff705e3f1000000008b483045022100bf5fdc86dc5f08a5d5c8e43a8c9d5b1ed8c65562e280007b52b133021acd9acc02205e325d613e555f772802bf413d36ba807892ed1a690a77811d3033b3de226e0a01410429fa713b124484cb2bd7b5557b2c0b9df7b2b1fee61825eadc5ae6c37a9920d38bfccdc7dc3cb0c47d7b173dbc9db8d37db0a33ae487982c59c6f8606e9d1791ffffffff41ed70551dd7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d068000000008b4830450221008513ad65187b903aed1102d1d0c47688127658c51106753fed0151ce9c16b80902201432b9ebcb87bd04ceb2de66035fbbaf4bf8b00d1cfe41f1a1f7338f9ad79d210141049d4cf80125bf50be1709f718c07ad15d0fc612b7da1f5570dddc35f2a352f0f27c978b06820edca9ef982c35fda2d255afba340068c5035552368bc7200c1488ffffffff0100093d00000000001976a9148edb68822f1ad580b043c7b3df2e400f8699eb4888ac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + stream >> TX_WITH_WITNESS(block); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_ALL); // Match the last transaction @@ -402,7 +402,7 @@ BOOST_AUTO_TEST_CASE(merkle_block_4_test_p2pubkey_only) // With 7 txes CBlock block; CDataStream stream(ParseHex("0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f093bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b85290701000000010000000000000000000000000000000000000000000000000000000000000000ffffffff07044c86041b0136ffffffff0100f2052a01000000434104eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c876f2c01ec0f0dd5b2e86e7168cefe0d81113c3807420ce13ad1357231a2252247d97a46a91ac000000000100000001bcad20a6a29827d1424f08989255120bf7f3e9e3cdaaa6bb31b0737fe048724300000000494830450220356e834b046cadc0f8ebb5a8a017b02de59c86305403dad52cd77b55af062ea10221009253cd6c119d4729b77c978e1e2aa19f5ea6e0e52b3f16e32fa608cd5bab753901ffffffff02008d380c010000001976a9142b4b8072ecbba129b6453c63e129e643207249ca88ac0065cd1d000000001976a9141b8dd13b994bcfc787b32aeadf58ccb3615cbd5488ac000000000100000003fdacf9b3eb077412e7a968d2e4f11b9a9dee312d666187ed77ee7d26af16cb0b000000008c493046022100ea1608e70911ca0de5af51ba57ad23b9a51db8d28f82c53563c56a05c20f5a87022100a8bdc8b4a8acc8634c6b420410150775eb7f2474f5615f7fccd65af30f310fbf01410465fdf49e29b06b9a1582287b6279014f834edc317695d125ef623c1cc3aaece245bd69fcad7508666e9c74a49dc9056d5fc14338ef38118dc4afae5fe2c585caffffffff309e1913634ecb50f3c4f83e96e70b2df071b497b8973a3e75429df397b5af83000000004948304502202bdb79c596a9ffc24e96f4386199aba386e9bc7b6071516e2b51dda942b3a1ed022100c53a857e76b724fc14d45311eac5019650d415c3abb5428f3aae16d8e69bec2301ffffffff2089e33491695080c9edc18a428f7d834db5b6d372df13ce2b1b0e0cbcb1e6c10000000049483045022100d4ce67c5896ee251c810ac1ff9ceccd328b497c8f553ab6e08431e7d40bad6b5022033119c0c2b7d792d31f1187779c7bd95aefd93d90a715586d73801d9b47471c601ffffffff0100714460030000001976a914c7b55141d097ea5df7a0ed330cf794376e53ec8d88ac0000000001000000045bf0e214aa4069a3e792ecee1e1bf0c1d397cde8dd08138f4b72a00681743447000000008b48304502200c45de8c4f3e2c1821f2fc878cba97b1e6f8807d94930713aa1c86a67b9bf1e40221008581abfef2e30f957815fc89978423746b2086375ca8ecf359c85c2a5b7c88ad01410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffffd669f7d7958d40fc59d2253d88e0f248e29b599c80bbcec344a83dda5f9aa72c000000008a473044022078124c8beeaa825f9e0b30bff96e564dd859432f2d0cb3b72d3d5d93d38d7e930220691d233b6c0f995be5acb03d70a7f7a65b6bc9bdd426260f38a1346669507a3601410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95fffffffff878af0d93f5229a68166cf051fd372bb7a537232946e0a46f53636b4dafdaa4000000008c493046022100c717d1714551663f69c3c5759bdbb3a0fcd3fab023abc0e522fe6440de35d8290221008d9cbe25bffc44af2b18e81c58eb37293fd7fe1c2e7b46fc37ee8c96c50ab1e201410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff27f2b668859cd7f2f894aa0fd2d9e60963bcd07c88973f425f999b8cbfd7a1e2000000008c493046022100e00847147cbf517bcc2f502f3ddc6d284358d102ed20d47a8aa788a62f0db780022100d17b2d6fa84dcaf1c95d88d7e7c30385aecf415588d749afd3ec81f6022cecd701410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff0100c817a8040000001976a914b6efd80d99179f4f4ff6f4dd0a007d018c385d2188ac000000000100000001834537b2f1ce8ef9373a258e10545ce5a50b758df616cd4356e0032554ebd3c4000000008b483045022100e68f422dd7c34fdce11eeb4509ddae38201773dd62f284e8aa9d96f85099d0b002202243bd399ff96b649a0fad05fa759d6a882f0af8c90cf7632c2840c29070aec20141045e58067e815c2f464c6a2a15f987758374203895710c2d452442e28496ff38ba8f5fd901dc20e29e88477167fe4fc299bf818fd0d9e1632d467b2a3d9503b1aaffffffff0280d7e636030000001976a914f34c3e10eb387efe872acb614c89e78bfca7815d88ac404b4c00000000001976a914a84e272933aaf87e1715d7786c51dfaeb5b65a6f88ac00000000010000000143ac81c8e6f6ef307dfe17f3d906d999e23e0189fda838c5510d850927e03ae7000000008c4930460221009c87c344760a64cb8ae6685a3eec2c1ac1bed5b88c87de51acd0e124f266c16602210082d07c037359c3a257b5c63ebd90f5a5edf97b2ac1c434b08ca998839f346dd40141040ba7e521fa7946d12edbb1d1e95a15c34bd4398195e86433c92b431cd315f455fe30032ede69cad9d1e1ed6c3c4ec0dbfced53438c625462afb792dcb098544bffffffff0240420f00000000001976a9144676d1b820d63ec272f1900d59d43bc6463d96f888ac40420f00000000001976a914648d04341d00d7968b3405c034adc38d4d8fb9bd88ac00000000010000000248cc917501ea5c55f4a8d2009c0567c40cfe037c2e71af017d0a452ff705e3f1000000008b483045022100bf5fdc86dc5f08a5d5c8e43a8c9d5b1ed8c65562e280007b52b133021acd9acc02205e325d613e555f772802bf413d36ba807892ed1a690a77811d3033b3de226e0a01410429fa713b124484cb2bd7b5557b2c0b9df7b2b1fee61825eadc5ae6c37a9920d38bfccdc7dc3cb0c47d7b173dbc9db8d37db0a33ae487982c59c6f8606e9d1791ffffffff41ed70551dd7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d068000000008b4830450221008513ad65187b903aed1102d1d0c47688127658c51106753fed0151ce9c16b80902201432b9ebcb87bd04ceb2de66035fbbaf4bf8b00d1cfe41f1a1f7338f9ad79d210141049d4cf80125bf50be1709f718c07ad15d0fc612b7da1f5570dddc35f2a352f0f27c978b06820edca9ef982c35fda2d255afba340068c5035552368bc7200c1488ffffffff0100093d00000000001976a9148edb68822f1ad580b043c7b3df2e400f8699eb4888ac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + stream >> TX_WITH_WITNESS(block); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_P2PUBKEY_ONLY); // Match the generation pubkey @@ -425,7 +425,7 @@ BOOST_AUTO_TEST_CASE(merkle_block_4_test_update_none) // With 7 txes CBlock block; CDataStream stream(ParseHex("0100000082bb869cf3a793432a66e826e05a6fc37469f8efb7421dc880670100000000007f16c5962e8bd963659c793ce370d95f093bc7e367117b3c30c1f8fdd0d9728776381b4d4c86041b554b85290701000000010000000000000000000000000000000000000000000000000000000000000000ffffffff07044c86041b0136ffffffff0100f2052a01000000434104eaafc2314def4ca98ac970241bcab022b9c1e1f4ea423a20f134c876f2c01ec0f0dd5b2e86e7168cefe0d81113c3807420ce13ad1357231a2252247d97a46a91ac000000000100000001bcad20a6a29827d1424f08989255120bf7f3e9e3cdaaa6bb31b0737fe048724300000000494830450220356e834b046cadc0f8ebb5a8a017b02de59c86305403dad52cd77b55af062ea10221009253cd6c119d4729b77c978e1e2aa19f5ea6e0e52b3f16e32fa608cd5bab753901ffffffff02008d380c010000001976a9142b4b8072ecbba129b6453c63e129e643207249ca88ac0065cd1d000000001976a9141b8dd13b994bcfc787b32aeadf58ccb3615cbd5488ac000000000100000003fdacf9b3eb077412e7a968d2e4f11b9a9dee312d666187ed77ee7d26af16cb0b000000008c493046022100ea1608e70911ca0de5af51ba57ad23b9a51db8d28f82c53563c56a05c20f5a87022100a8bdc8b4a8acc8634c6b420410150775eb7f2474f5615f7fccd65af30f310fbf01410465fdf49e29b06b9a1582287b6279014f834edc317695d125ef623c1cc3aaece245bd69fcad7508666e9c74a49dc9056d5fc14338ef38118dc4afae5fe2c585caffffffff309e1913634ecb50f3c4f83e96e70b2df071b497b8973a3e75429df397b5af83000000004948304502202bdb79c596a9ffc24e96f4386199aba386e9bc7b6071516e2b51dda942b3a1ed022100c53a857e76b724fc14d45311eac5019650d415c3abb5428f3aae16d8e69bec2301ffffffff2089e33491695080c9edc18a428f7d834db5b6d372df13ce2b1b0e0cbcb1e6c10000000049483045022100d4ce67c5896ee251c810ac1ff9ceccd328b497c8f553ab6e08431e7d40bad6b5022033119c0c2b7d792d31f1187779c7bd95aefd93d90a715586d73801d9b47471c601ffffffff0100714460030000001976a914c7b55141d097ea5df7a0ed330cf794376e53ec8d88ac0000000001000000045bf0e214aa4069a3e792ecee1e1bf0c1d397cde8dd08138f4b72a00681743447000000008b48304502200c45de8c4f3e2c1821f2fc878cba97b1e6f8807d94930713aa1c86a67b9bf1e40221008581abfef2e30f957815fc89978423746b2086375ca8ecf359c85c2a5b7c88ad01410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffffd669f7d7958d40fc59d2253d88e0f248e29b599c80bbcec344a83dda5f9aa72c000000008a473044022078124c8beeaa825f9e0b30bff96e564dd859432f2d0cb3b72d3d5d93d38d7e930220691d233b6c0f995be5acb03d70a7f7a65b6bc9bdd426260f38a1346669507a3601410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95fffffffff878af0d93f5229a68166cf051fd372bb7a537232946e0a46f53636b4dafdaa4000000008c493046022100c717d1714551663f69c3c5759bdbb3a0fcd3fab023abc0e522fe6440de35d8290221008d9cbe25bffc44af2b18e81c58eb37293fd7fe1c2e7b46fc37ee8c96c50ab1e201410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff27f2b668859cd7f2f894aa0fd2d9e60963bcd07c88973f425f999b8cbfd7a1e2000000008c493046022100e00847147cbf517bcc2f502f3ddc6d284358d102ed20d47a8aa788a62f0db780022100d17b2d6fa84dcaf1c95d88d7e7c30385aecf415588d749afd3ec81f6022cecd701410462bb73f76ca0994fcb8b4271e6fb7561f5c0f9ca0cf6485261c4a0dc894f4ab844c6cdfb97cd0b60ffb5018ffd6238f4d87270efb1d3ae37079b794a92d7ec95ffffffff0100c817a8040000001976a914b6efd80d99179f4f4ff6f4dd0a007d018c385d2188ac000000000100000001834537b2f1ce8ef9373a258e10545ce5a50b758df616cd4356e0032554ebd3c4000000008b483045022100e68f422dd7c34fdce11eeb4509ddae38201773dd62f284e8aa9d96f85099d0b002202243bd399ff96b649a0fad05fa759d6a882f0af8c90cf7632c2840c29070aec20141045e58067e815c2f464c6a2a15f987758374203895710c2d452442e28496ff38ba8f5fd901dc20e29e88477167fe4fc299bf818fd0d9e1632d467b2a3d9503b1aaffffffff0280d7e636030000001976a914f34c3e10eb387efe872acb614c89e78bfca7815d88ac404b4c00000000001976a914a84e272933aaf87e1715d7786c51dfaeb5b65a6f88ac00000000010000000143ac81c8e6f6ef307dfe17f3d906d999e23e0189fda838c5510d850927e03ae7000000008c4930460221009c87c344760a64cb8ae6685a3eec2c1ac1bed5b88c87de51acd0e124f266c16602210082d07c037359c3a257b5c63ebd90f5a5edf97b2ac1c434b08ca998839f346dd40141040ba7e521fa7946d12edbb1d1e95a15c34bd4398195e86433c92b431cd315f455fe30032ede69cad9d1e1ed6c3c4ec0dbfced53438c625462afb792dcb098544bffffffff0240420f00000000001976a9144676d1b820d63ec272f1900d59d43bc6463d96f888ac40420f00000000001976a914648d04341d00d7968b3405c034adc38d4d8fb9bd88ac00000000010000000248cc917501ea5c55f4a8d2009c0567c40cfe037c2e71af017d0a452ff705e3f1000000008b483045022100bf5fdc86dc5f08a5d5c8e43a8c9d5b1ed8c65562e280007b52b133021acd9acc02205e325d613e555f772802bf413d36ba807892ed1a690a77811d3033b3de226e0a01410429fa713b124484cb2bd7b5557b2c0b9df7b2b1fee61825eadc5ae6c37a9920d38bfccdc7dc3cb0c47d7b173dbc9db8d37db0a33ae487982c59c6f8606e9d1791ffffffff41ed70551dd7e841883ab8f0b16bf04176b7d1480e4f0af9f3d4c3595768d068000000008b4830450221008513ad65187b903aed1102d1d0c47688127658c51106753fed0151ce9c16b80902201432b9ebcb87bd04ceb2de66035fbbaf4bf8b00d1cfe41f1a1f7338f9ad79d210141049d4cf80125bf50be1709f718c07ad15d0fc612b7da1f5570dddc35f2a352f0f27c978b06820edca9ef982c35fda2d255afba340068c5035552368bc7200c1488ffffffff0100093d00000000001976a9148edb68822f1ad580b043c7b3df2e400f8699eb4888ac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + stream >> TX_WITH_WITNESS(block); CBloomFilter filter(10, 0.000001, 0, BLOOM_UPDATE_NONE); // Match the generation pubkey diff --git a/src/test/denialofservice_tests.cpp b/src/test/denialofservice_tests.cpp index 6e740a4f53..0fef8c5906 100644 --- a/src/test/denialofservice_tests.cpp +++ b/src/test/denialofservice_tests.cpp @@ -147,9 +147,7 @@ BOOST_AUTO_TEST_CASE(stale_tip_peer_management) constexpr int max_outbound_full_relay = MAX_OUTBOUND_FULL_RELAY_CONNECTIONS; CConnman::Options options; - options.nMaxConnections = DEFAULT_MAX_PEER_CONNECTIONS; - options.m_max_outbound_full_relay = max_outbound_full_relay; - options.nMaxFeeler = MAX_FEELER_CONNECTIONS; + options.m_max_automatic_connections = DEFAULT_MAX_PEER_CONNECTIONS; const auto time_init{GetTime<std::chrono::seconds>()}; SetMockTime(time_init); @@ -248,9 +246,7 @@ BOOST_AUTO_TEST_CASE(block_relay_only_eviction) constexpr int max_outbound_block_relay{MAX_BLOCK_RELAY_ONLY_CONNECTIONS}; constexpr int64_t MINIMUM_CONNECT_TIME{30}; CConnman::Options options; - options.nMaxConnections = DEFAULT_MAX_PEER_CONNECTIONS; - options.m_max_outbound_full_relay = MAX_OUTBOUND_FULL_RELAY_CONNECTIONS; - options.m_max_outbound_block_relay = max_outbound_block_relay; + options.m_max_automatic_connections = DEFAULT_MAX_PEER_CONNECTIONS; connman->Init(options); std::vector<CNode*> vNodes; diff --git a/src/test/fuzz/banman.cpp b/src/test/fuzz/banman.cpp index 3882e0e547..4a040c56de 100644 --- a/src/test/fuzz/banman.cpp +++ b/src/test/fuzz/banman.cpp @@ -63,17 +63,28 @@ FUZZ_TARGET(banman, .init = initialize_banman) // The complexity is O(N^2), where N is the input size, because each call // might call DumpBanlist (or other methods that are at least linear // complexity of the input size). + bool contains_invalid{false}; LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 300) { CallOneOf( fuzzed_data_provider, [&] { - ban_man.Ban(ConsumeNetAddr(fuzzed_data_provider), - ConsumeBanTimeOffset(fuzzed_data_provider), fuzzed_data_provider.ConsumeBool()); + CNetAddr net_addr{ConsumeNetAddr(fuzzed_data_provider)}; + const std::optional<CNetAddr>& addr{LookupHost(net_addr.ToStringAddr(), /*fAllowLookup=*/false)}; + if (addr.has_value() && addr->IsValid()) { + net_addr = *addr; + } else { + contains_invalid = true; + } + ban_man.Ban(net_addr, ConsumeBanTimeOffset(fuzzed_data_provider), fuzzed_data_provider.ConsumeBool()); }, [&] { - ban_man.Ban(ConsumeSubNet(fuzzed_data_provider), - ConsumeBanTimeOffset(fuzzed_data_provider), fuzzed_data_provider.ConsumeBool()); + CSubNet subnet{ConsumeSubNet(fuzzed_data_provider)}; + subnet = LookupSubNet(subnet.ToString()); + if (!subnet.IsValid()) { + contains_invalid = true; + } + ban_man.Ban(subnet, ConsumeBanTimeOffset(fuzzed_data_provider), fuzzed_data_provider.ConsumeBool()); }, [&] { ban_man.ClearBanned(); @@ -109,7 +120,9 @@ FUZZ_TARGET(banman, .init = initialize_banman) BanMan ban_man_read{banlist_file, /*client_interface=*/nullptr, /*default_ban_time=*/0}; banmap_t banmap_read; ban_man_read.GetBanned(banmap_read); - assert(banmap == banmap_read); + if (!contains_invalid) { + assert(banmap == banmap_read); + } } } fs::remove(fs::PathToString(banlist_file + ".json")); diff --git a/src/test/fuzz/block.cpp b/src/test/fuzz/block.cpp index 8c97fba323..09c0ed23d2 100644 --- a/src/test/fuzz/block.cpp +++ b/src/test/fuzz/block.cpp @@ -31,7 +31,7 @@ FUZZ_TARGET(block, .init = initialize_block) int nVersion; ds >> nVersion; ds.SetVersion(nVersion); - ds >> block; + ds >> TX_WITH_WITNESS(block); } catch (const std::ios_base::failure&) { return; } diff --git a/src/test/fuzz/bloom_filter.cpp b/src/test/fuzz/bloom_filter.cpp index 3e303ecc0f..5dbb463a16 100644 --- a/src/test/fuzz/bloom_filter.cpp +++ b/src/test/fuzz/bloom_filter.cpp @@ -10,21 +10,22 @@ #include <uint256.h> #include <cassert> -#include <cstdint> +#include <limits> #include <optional> -#include <string> #include <vector> FUZZ_TARGET(bloom_filter) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + bool good_data{true}; CBloomFilter bloom_filter{ fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(1, 10000000), 1.0 / fuzzed_data_provider.ConsumeIntegralInRange<unsigned int>(1, std::numeric_limits<unsigned int>::max()), fuzzed_data_provider.ConsumeIntegral<unsigned int>(), static_cast<unsigned char>(fuzzed_data_provider.PickValueInArray({BLOOM_UPDATE_NONE, BLOOM_UPDATE_ALL, BLOOM_UPDATE_P2PUBKEY_ONLY, BLOOM_UPDATE_MASK}))}; - LIMITED_WHILE(fuzzed_data_provider.remaining_bytes() > 0, 10000) { + LIMITED_WHILE(good_data && fuzzed_data_provider.remaining_bytes() > 0, 10'000) + { CallOneOf( fuzzed_data_provider, [&] { @@ -37,6 +38,7 @@ FUZZ_TARGET(bloom_filter) [&] { const std::optional<COutPoint> out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); if (!out_point) { + good_data = false; return; } (void)bloom_filter.contains(*out_point); @@ -47,6 +49,7 @@ FUZZ_TARGET(bloom_filter) [&] { const std::optional<uint256> u256 = ConsumeDeserializable<uint256>(fuzzed_data_provider); if (!u256) { + good_data = false; return; } (void)bloom_filter.contains(*u256); @@ -55,8 +58,9 @@ FUZZ_TARGET(bloom_filter) assert(present); }, [&] { - const std::optional<CMutableTransaction> mut_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mut_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!mut_tx) { + good_data = false; return; } const CTransaction tx{*mut_tx}; diff --git a/src/test/fuzz/coins_view.cpp b/src/test/fuzz/coins_view.cpp index b088aa0bd7..8f3e357a84 100644 --- a/src/test/fuzz/coins_view.cpp +++ b/src/test/fuzz/coins_view.cpp @@ -2,26 +2,28 @@ // Distributed under the MIT software license, see the accompanying // file COPYING or http://www.opensource.org/licenses/mit-license.php. -#include <chainparams.h> #include <coins.h> #include <consensus/amount.h> #include <consensus/tx_check.h> #include <consensus/tx_verify.h> #include <consensus/validation.h> -#include <key.h> #include <policy/policy.h> #include <primitives/transaction.h> -#include <pubkey.h> +#include <script/interpreter.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <test/util/setup_common.h> -#include <validation.h> +#include <util/hasher.h> +#include <cassert> #include <cstdint> #include <limits> +#include <memory> #include <optional> +#include <stdexcept> #include <string> +#include <utility> #include <vector> namespace { @@ -44,12 +46,15 @@ void initialize_coins_view() FUZZ_TARGET(coins_view, .init = initialize_coins_view) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + bool good_data{true}; + CCoinsView backend_coins_view; CCoinsViewCache coins_view_cache{&backend_coins_view, /*deterministic=*/true}; COutPoint random_out_point; Coin random_coin; CMutableTransaction random_mutable_transaction; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 10'000) + { CallOneOf( fuzzed_data_provider, [&] { @@ -95,6 +100,7 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) [&] { const std::optional<COutPoint> opt_out_point = ConsumeDeserializable<COutPoint>(fuzzed_data_provider); if (!opt_out_point) { + good_data = false; return; } random_out_point = *opt_out_point; @@ -102,13 +108,15 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) [&] { const std::optional<Coin> opt_coin = ConsumeDeserializable<Coin>(fuzzed_data_provider); if (!opt_coin) { + good_data = false; return; } random_coin = *opt_coin; }, [&] { - const std::optional<CMutableTransaction> opt_mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> opt_mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!opt_mutable_transaction) { + good_data = false; return; } random_mutable_transaction = *opt_mutable_transaction; @@ -116,7 +124,8 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) [&] { CCoinsMapMemoryResource resource; CCoinsMap coins_map{0, SaltedOutpointHasher{/*deterministic=*/true}, CCoinsMap::key_equal{}, &resource}; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 10'000) + { CCoinsCacheEntry coins_cache_entry; coins_cache_entry.flags = fuzzed_data_provider.ConsumeIntegral<unsigned char>(); if (fuzzed_data_provider.ConsumeBool()) { @@ -124,6 +133,7 @@ FUZZ_TARGET(coins_view, .init = initialize_coins_view) } else { const std::optional<Coin> opt_coin = ConsumeDeserializable<Coin>(fuzzed_data_provider); if (!opt_coin) { + good_data = false; return; } coins_cache_entry.coin = *opt_coin; diff --git a/src/test/fuzz/connman.cpp b/src/test/fuzz/connman.cpp index 0dab2a2e97..551e1c526d 100644 --- a/src/test/fuzz/connman.cpp +++ b/src/test/fuzz/connman.cpp @@ -121,7 +121,7 @@ FUZZ_TARGET(connman, .init = initialize_connman) connman.SetTryNewOutboundPeer(fuzzed_data_provider.ConsumeBool()); }); } - (void)connman.GetAddedNodeInfo(); + (void)connman.GetAddedNodeInfo(fuzzed_data_provider.ConsumeBool()); (void)connman.GetExtraFullOutboundCount(); (void)connman.GetLocalServices(); (void)connman.GetMaxOutboundTarget(); diff --git a/src/test/fuzz/deserialize.cpp b/src/test/fuzz/deserialize.cpp index abc230c07d..33c9a93c65 100644 --- a/src/test/fuzz/deserialize.cpp +++ b/src/test/fuzz/deserialize.cpp @@ -217,7 +217,7 @@ FUZZ_TARGET_DESERIALIZE(psbt_output_deserialize, { }) FUZZ_TARGET_DESERIALIZE(block_deserialize, { CBlock block; - DeserializeFromFuzzingInput(buffer, block); + DeserializeFromFuzzingInput(buffer, TX_WITH_WITNESS(block)); }) FUZZ_TARGET_DESERIALIZE(blocklocator_deserialize, { CBlockLocator bl; @@ -225,7 +225,7 @@ FUZZ_TARGET_DESERIALIZE(blocklocator_deserialize, { }) FUZZ_TARGET_DESERIALIZE(blockmerkleroot, { CBlock block; - DeserializeFromFuzzingInput(buffer, block); + DeserializeFromFuzzingInput(buffer, TX_WITH_WITNESS(block)); bool mutated; BlockMerkleRoot(block, &mutated); }) diff --git a/src/test/fuzz/fuzz.h b/src/test/fuzz/fuzz.h index 0534f9bcf1..1f0fa5527a 100644 --- a/src/test/fuzz/fuzz.h +++ b/src/test/fuzz/fuzz.h @@ -14,6 +14,11 @@ /** * Can be used to limit a theoretically unbounded loop. This caps the runtime * to avoid timeouts or OOMs. + * + * This can be used in combination with a check in the condition to confirm + * whether the fuzz engine provided "good" data. If the fuzz input contains + * invalid data, the loop aborts early. This will teach the fuzz engine to look + * for useful data and avoids bloating the fuzz input folder with useless data. */ #define LIMITED_WHILE(condition, limit) \ for (unsigned _count{limit}; (condition) && _count; --_count) diff --git a/src/test/fuzz/merkleblock.cpp b/src/test/fuzz/merkleblock.cpp index 6271367a9c..15e8db7536 100644 --- a/src/test/fuzz/merkleblock.cpp +++ b/src/test/fuzz/merkleblock.cpp @@ -27,7 +27,7 @@ FUZZ_TARGET(merkleblock) }, [&] { CMerkleBlock merkle_block; - const std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider); + const std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider, TX_WITH_WITNESS); CBloomFilter bloom_filter; std::set<uint256> txids; if (opt_block && !opt_block->vtx.empty()) { diff --git a/src/test/fuzz/package_eval.cpp b/src/test/fuzz/package_eval.cpp index 8d316134cc..8658c0b45a 100644 --- a/src/test/fuzz/package_eval.cpp +++ b/src/test/fuzz/package_eval.cpp @@ -40,7 +40,7 @@ void initialize_tx_pool() g_setup = testing_setup.get(); for (int i = 0; i < 2 * COINBASE_MATURITY; ++i) { - COutPoint prevout{MineBlock(g_setup->m_node, P2WSH_OP_TRUE)}; + COutPoint prevout{MineBlock(g_setup->m_node, P2WSH_EMPTY)}; if (i < COINBASE_MATURITY) { // Remember the txids to avoid expensive disk access later on g_outpoints_coinbase_init_mature.push_back(prevout); @@ -195,7 +195,8 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) // Create input const auto sequence = ConsumeSequence(fuzzed_data_provider); const auto script_sig = CScript{}; - const auto script_wit_stack = std::vector<std::vector<uint8_t>>{WITNESS_STACK_ELEM_OP_TRUE}; + const auto script_wit_stack = fuzzed_data_provider.ConsumeBool() ? P2WSH_EMPTY_TRUE_STACK : P2WSH_EMPTY_TWO_STACK; + CTxIn in; in.prevout = outpoint; in.nSequence = sequence; @@ -204,17 +205,30 @@ FUZZ_TARGET(tx_package_eval, .init = initialize_tx_pool) tx_mut.vin.push_back(in); } + + // Duplicate an input + bool dup_input = fuzzed_data_provider.ConsumeBool(); + if (dup_input) { + tx_mut.vin.push_back(tx_mut.vin.back()); + } + + // Refer to a non-existant input + if (fuzzed_data_provider.ConsumeBool()) { + tx_mut.vin.emplace_back(); + } + const auto amount_fee = fuzzed_data_provider.ConsumeIntegralInRange<CAmount>(0, amount_in); const auto amount_out = (amount_in - amount_fee) / num_out; for (int i = 0; i < num_out; ++i) { - tx_mut.vout.emplace_back(amount_out, P2WSH_OP_TRUE); + tx_mut.vout.emplace_back(amount_out, P2WSH_EMPTY); } // TODO vary transaction sizes to catch size-related issues auto tx = MakeTransactionRef(tx_mut); // Restore previously removed outpoints, except in-package outpoints if (!last_tx) { for (const auto& in : tx->vin) { - Assert(outpoints.insert(in.prevout).second); + // It's a fake input, or a new input, or a duplicate + Assert(in == CTxIn() || outpoints.insert(in.prevout).second || dup_input); } // Cache the in-package outpoints being made for (size_t i = 0; i < tx->vout.size(); ++i) { diff --git a/src/test/fuzz/partially_downloaded_block.cpp b/src/test/fuzz/partially_downloaded_block.cpp index ae7a68762e..4a4b46da60 100644 --- a/src/test/fuzz/partially_downloaded_block.cpp +++ b/src/test/fuzz/partially_downloaded_block.cpp @@ -44,7 +44,7 @@ FUZZ_TARGET(partially_downloaded_block, .init = initialize_pdb) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - auto block{ConsumeDeserializable<CBlock>(fuzzed_data_provider)}; + auto block{ConsumeDeserializable<CBlock>(fuzzed_data_provider, TX_WITH_WITNESS)}; if (!block || block->vtx.size() == 0 || block->vtx.size() >= std::numeric_limits<uint16_t>::max()) { return; diff --git a/src/test/fuzz/policy_estimator.cpp b/src/test/fuzz/policy_estimator.cpp index 227ee9d2c4..a6275d2fd2 100644 --- a/src/test/fuzz/policy_estimator.cpp +++ b/src/test/fuzz/policy_estimator.cpp @@ -6,16 +6,14 @@ #include <policy/fees.h> #include <policy/fees_args.h> #include <primitives/transaction.h> +#include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> #include <test/fuzz/fuzz.h> #include <test/fuzz/util.h> #include <test/fuzz/util/mempool.h> #include <test/util/setup_common.h> -#include <txmempool.h> -#include <cstdint> #include <optional> -#include <string> #include <vector> namespace { @@ -31,13 +29,17 @@ void initialize_policy_estimator() FUZZ_TARGET(policy_estimator, .init = initialize_policy_estimator) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); + bool good_data{true}; + CBlockPolicyEstimator block_policy_estimator{FeeestPath(*g_setup->m_node.args), DEFAULT_ACCEPT_STALE_FEE_ESTIMATES}; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 10'000) + { CallOneOf( fuzzed_data_provider, [&] { - const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!mtx) { + good_data = false; return; } const CTransaction tx{*mtx}; @@ -48,9 +50,11 @@ FUZZ_TARGET(policy_estimator, .init = initialize_policy_estimator) }, [&] { std::vector<CTxMemPoolEntry> mempool_entries; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { - const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) + { + const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!mtx) { + good_data = false; break; } const CTransaction tx{*mtx}; diff --git a/src/test/fuzz/primitives_transaction.cpp b/src/test/fuzz/primitives_transaction.cpp index 48815c8910..64f1c8cf8d 100644 --- a/src/test/fuzz/primitives_transaction.cpp +++ b/src/test/fuzz/primitives_transaction.cpp @@ -24,8 +24,8 @@ FUZZ_TARGET(primitives_transaction) const CTxOut tx_out_1{ConsumeMoney(fuzzed_data_provider), script}; const CTxOut tx_out_2{ConsumeMoney(fuzzed_data_provider), ConsumeScript(fuzzed_data_provider)}; assert((tx_out_1 == tx_out_2) != (tx_out_1 != tx_out_2)); - const std::optional<CMutableTransaction> mutable_tx_1 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); - const std::optional<CMutableTransaction> mutable_tx_2 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mutable_tx_1 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); + const std::optional<CMutableTransaction> mutable_tx_2 = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (mutable_tx_1 && mutable_tx_2) { const CTransaction tx_1{*mutable_tx_1}; const CTransaction tx_2{*mutable_tx_2}; diff --git a/src/test/fuzz/rbf.cpp b/src/test/fuzz/rbf.cpp index dbe99029c3..aa6385d12d 100644 --- a/src/test/fuzz/rbf.cpp +++ b/src/test/fuzz/rbf.cpp @@ -33,7 +33,7 @@ FUZZ_TARGET(rbf, .init = initialize_rbf) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); SetMockTime(ConsumeTime(fuzzed_data_provider)); - std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!mtx) { return; } @@ -42,7 +42,7 @@ FUZZ_TARGET(rbf, .init = initialize_rbf) LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 10000) { - const std::optional<CMutableTransaction> another_mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> another_mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!another_mtx) { break; } diff --git a/src/test/fuzz/rpc.cpp b/src/test/fuzz/rpc.cpp index 270cab58e2..97b99ea2a3 100644 --- a/src/test/fuzz/rpc.cpp +++ b/src/test/fuzz/rpc.cpp @@ -3,18 +3,14 @@ // file COPYING or http://www.opensource.org/licenses/mit-license.php. #include <base58.h> -#include <core_io.h> #include <key.h> #include <key_io.h> -#include <node/context.h> #include <primitives/block.h> #include <primitives/transaction.h> #include <psbt.h> -#include <rpc/blockchain.h> #include <rpc/client.h> #include <rpc/request.h> #include <rpc/server.h> -#include <rpc/util.h> #include <span.h> #include <streams.h> #include <test/fuzz/FuzzedDataProvider.h> @@ -22,19 +18,23 @@ #include <test/fuzz/util.h> #include <test/util/setup_common.h> #include <tinyformat.h> +#include <uint256.h> #include <univalue.h> -#include <util/chaintype.h> #include <util/strencodings.h> #include <util/string.h> #include <util/time.h> +#include <algorithm> +#include <cassert> #include <cstdint> +#include <cstdlib> +#include <exception> #include <iostream> #include <memory> #include <optional> #include <stdexcept> -#include <string> #include <vector> +enum class ChainType; namespace { struct RPCFuzzTestingSetup : public TestingSetup { @@ -184,7 +184,7 @@ const std::vector<std::string> RPC_COMMANDS_SAFE_FOR_FUZZING{ "waitfornewblock", }; -std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider, bool& good_data) { const size_t max_string_length = 4096; const size_t max_base58_bytes_length{64}; @@ -249,18 +249,20 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) }, [&] { // hex encoded block - std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider); + std::optional<CBlock> opt_block = ConsumeDeserializable<CBlock>(fuzzed_data_provider, TX_WITH_WITNESS); if (!opt_block) { + good_data = false; return; } CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION}; - data_stream << *opt_block; + data_stream << TX_WITH_WITNESS(*opt_block); r = HexStr(data_stream); }, [&] { // hex encoded block header std::optional<CBlockHeader> opt_block_header = ConsumeDeserializable<CBlockHeader>(fuzzed_data_provider); if (!opt_block_header) { + good_data = false; return; } DataStream data_stream{}; @@ -269,18 +271,21 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) }, [&] { // hex encoded tx - std::optional<CMutableTransaction> opt_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + std::optional<CMutableTransaction> opt_tx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (!opt_tx) { + good_data = false; return; } - CDataStream data_stream{SER_NETWORK, fuzzed_data_provider.ConsumeBool() ? PROTOCOL_VERSION : (PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS)}; - data_stream << *opt_tx; + DataStream data_stream; + auto allow_witness = (fuzzed_data_provider.ConsumeBool() ? TX_WITH_WITNESS : TX_NO_WITNESS); + data_stream << allow_witness(*opt_tx); r = HexStr(data_stream); }, [&] { // base64 encoded psbt std::optional<PartiallySignedTransaction> opt_psbt = ConsumeDeserializable<PartiallySignedTransaction>(fuzzed_data_provider); if (!opt_psbt) { + good_data = false; return; } CDataStream data_stream{SER_NETWORK, PROTOCOL_VERSION}; @@ -291,6 +296,7 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) // base58 encoded key CKey key = ConsumePrivateKey(fuzzed_data_provider); if (!key.IsValid()) { + good_data = false; return; } r = EncodeSecret(key); @@ -299,6 +305,7 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) // hex encoded pubkey CKey key = ConsumePrivateKey(fuzzed_data_provider); if (!key.IsValid()) { + good_data = false; return; } r = HexStr(key.GetPubKey()); @@ -306,18 +313,19 @@ std::string ConsumeScalarRPCArgument(FuzzedDataProvider& fuzzed_data_provider) return r; } -std::string ConsumeArrayRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +std::string ConsumeArrayRPCArgument(FuzzedDataProvider& fuzzed_data_provider, bool& good_data) { std::vector<std::string> scalar_arguments; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) { - scalar_arguments.push_back(ConsumeScalarRPCArgument(fuzzed_data_provider)); + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 100) + { + scalar_arguments.push_back(ConsumeScalarRPCArgument(fuzzed_data_provider, good_data)); } return "[\"" + Join(scalar_arguments, "\",\"") + "\"]"; } -std::string ConsumeRPCArgument(FuzzedDataProvider& fuzzed_data_provider) +std::string ConsumeRPCArgument(FuzzedDataProvider& fuzzed_data_provider, bool& good_data) { - return fuzzed_data_provider.ConsumeBool() ? ConsumeScalarRPCArgument(fuzzed_data_provider) : ConsumeArrayRPCArgument(fuzzed_data_provider); + return fuzzed_data_provider.ConsumeBool() ? ConsumeScalarRPCArgument(fuzzed_data_provider, good_data) : ConsumeArrayRPCArgument(fuzzed_data_provider, good_data); } RPCFuzzTestingSetup* InitializeRPCFuzzTestingSetup() @@ -353,6 +361,7 @@ void initialize_rpc() FUZZ_TARGET(rpc, .init = initialize_rpc) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; + bool good_data{true}; SetMockTime(ConsumeTime(fuzzed_data_provider)); const std::string rpc_command = fuzzed_data_provider.ConsumeRandomLengthString(64); if (!g_limit_to_rpc_command.empty() && rpc_command != g_limit_to_rpc_command) { @@ -363,8 +372,9 @@ FUZZ_TARGET(rpc, .init = initialize_rpc) return; } std::vector<std::string> arguments; - LIMITED_WHILE(fuzzed_data_provider.ConsumeBool(), 100) { - arguments.push_back(ConsumeRPCArgument(fuzzed_data_provider)); + LIMITED_WHILE(good_data && fuzzed_data_provider.ConsumeBool(), 100) + { + arguments.push_back(ConsumeRPCArgument(fuzzed_data_provider, good_data)); } try { rpc_testing_setup->CallRPC(rpc_command, arguments); diff --git a/src/test/fuzz/script_assets_test_minimizer.cpp b/src/test/fuzz/script_assets_test_minimizer.cpp index 66c862a6f9..55a4fbc75c 100644 --- a/src/test/fuzz/script_assets_test_minimizer.cpp +++ b/src/test/fuzz/script_assets_test_minimizer.cpp @@ -54,7 +54,7 @@ CMutableTransaction TxFromHex(const std::string& str) { CMutableTransaction tx; try { - SpanReader{SERIALIZE_TRANSACTION_NO_WITNESS, CheckedParseHex(str)} >> tx; + SpanReader{0, CheckedParseHex(str)} >> TX_NO_WITNESS(tx); } catch (const std::ios_base::failure&) { throw std::runtime_error("Tx deserialization failure"); } diff --git a/src/test/fuzz/script_flags.cpp b/src/test/fuzz/script_flags.cpp index f8594fc233..3b8f5c068d 100644 --- a/src/test/fuzz/script_flags.cpp +++ b/src/test/fuzz/script_flags.cpp @@ -23,7 +23,7 @@ FUZZ_TARGET(script_flags) } try { - const CTransaction tx(deserialize, ds); + const CTransaction tx(deserialize, TX_WITH_WITNESS, ds); unsigned int verify_flags; ds >> verify_flags; diff --git a/src/test/fuzz/script_interpreter.cpp b/src/test/fuzz/script_interpreter.cpp index 5d59771682..5e76443abe 100644 --- a/src/test/fuzz/script_interpreter.cpp +++ b/src/test/fuzz/script_interpreter.cpp @@ -20,13 +20,13 @@ FUZZ_TARGET(script_interpreter) FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); { const CScript script_code = ConsumeScript(fuzzed_data_provider); - const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mtx = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (mtx) { const CTransaction tx_to{*mtx}; const unsigned int in = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); if (in < tx_to.vin.size()) { (void)SignatureHash(script_code, tx_to, in, fuzzed_data_provider.ConsumeIntegral<int>(), ConsumeMoney(fuzzed_data_provider), fuzzed_data_provider.PickValueInArray({SigVersion::BASE, SigVersion::WITNESS_V0}), nullptr); - const std::optional<CMutableTransaction> mtx_precomputed = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mtx_precomputed = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (mtx_precomputed) { const CTransaction tx_precomputed{*mtx_precomputed}; const PrecomputedTransactionData precomputed_transaction_data{tx_precomputed}; diff --git a/src/test/fuzz/script_sigcache.cpp b/src/test/fuzz/script_sigcache.cpp index 486b1e5197..5fdbc9e106 100644 --- a/src/test/fuzz/script_sigcache.cpp +++ b/src/test/fuzz/script_sigcache.cpp @@ -30,7 +30,7 @@ FUZZ_TARGET(script_sigcache, .init = initialize_script_sigcache) { FuzzedDataProvider fuzzed_data_provider(buffer.data(), buffer.size()); - const std::optional<CMutableTransaction> mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); const CTransaction tx{mutable_transaction ? *mutable_transaction : CMutableTransaction{}}; const unsigned int n_in = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); const CAmount amount = ConsumeMoney(fuzzed_data_provider); diff --git a/src/test/fuzz/script_sign.cpp b/src/test/fuzz/script_sign.cpp index 179715b6ea..0944c91c4a 100644 --- a/src/test/fuzz/script_sign.cpp +++ b/src/test/fuzz/script_sign.cpp @@ -86,7 +86,7 @@ FUZZ_TARGET(script_sign, .init = initialize_script_sign) } { - const std::optional<CMutableTransaction> mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> mutable_transaction = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); const std::optional<CTxOut> tx_out = ConsumeDeserializable<CTxOut>(fuzzed_data_provider); const unsigned int n_in = fuzzed_data_provider.ConsumeIntegral<unsigned int>(); if (mutable_transaction && tx_out && mutable_transaction->vin.size() > n_in) { @@ -100,7 +100,7 @@ FUZZ_TARGET(script_sign, .init = initialize_script_sign) if (mutable_transaction) { CTransaction tx_from{*mutable_transaction}; CMutableTransaction tx_to; - const std::optional<CMutableTransaction> opt_tx_to = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider); + const std::optional<CMutableTransaction> opt_tx_to = ConsumeDeserializable<CMutableTransaction>(fuzzed_data_provider, TX_WITH_WITNESS); if (opt_tx_to) { tx_to = *opt_tx_to; } diff --git a/src/test/fuzz/signet.cpp b/src/test/fuzz/signet.cpp index 3ccf5eb6f0..6f1996572e 100644 --- a/src/test/fuzz/signet.cpp +++ b/src/test/fuzz/signet.cpp @@ -25,7 +25,7 @@ void initialize_signet() FUZZ_TARGET(signet, .init = initialize_signet) { FuzzedDataProvider fuzzed_data_provider{buffer.data(), buffer.size()}; - const std::optional<CBlock> block = ConsumeDeserializable<CBlock>(fuzzed_data_provider); + const std::optional<CBlock> block = ConsumeDeserializable<CBlock>(fuzzed_data_provider, TX_WITH_WITNESS); if (!block) { return; } diff --git a/src/test/fuzz/socks5.cpp b/src/test/fuzz/socks5.cpp index 05b8312ab2..af81fcb593 100644 --- a/src/test/fuzz/socks5.cpp +++ b/src/test/fuzz/socks5.cpp @@ -32,7 +32,9 @@ FUZZ_TARGET(socks5, .init = initialize_socks5) ProxyCredentials proxy_credentials; proxy_credentials.username = fuzzed_data_provider.ConsumeRandomLengthString(512); proxy_credentials.password = fuzzed_data_provider.ConsumeRandomLengthString(512); - InterruptSocks5(fuzzed_data_provider.ConsumeBool()); + if (fuzzed_data_provider.ConsumeBool()) { + g_socks5_interrupt(); + } // Set FUZZED_SOCKET_FAKE_LATENCY=1 to exercise recv timeout code paths. This // will slow down fuzzing. g_socks5_recv_timeout = (fuzzed_data_provider.ConsumeBool() && std::getenv("FUZZED_SOCKET_FAKE_LATENCY") != nullptr) ? 1ms : default_socks5_recv_timeout; diff --git a/src/test/fuzz/transaction.cpp b/src/test/fuzz/transaction.cpp index 88c2a334c7..ccaee424fb 100644 --- a/src/test/fuzz/transaction.cpp +++ b/src/test/fuzz/transaction.cpp @@ -40,7 +40,7 @@ FUZZ_TARGET(transaction, .init = initialize_transaction) bool valid_tx = true; const CTransaction tx = [&] { try { - return CTransaction(deserialize, ds); + return CTransaction(deserialize, TX_WITH_WITNESS, ds); } catch (const std::ios_base::failure&) { valid_tx = false; return CTransaction{CMutableTransaction{}}; @@ -53,7 +53,7 @@ FUZZ_TARGET(transaction, .init = initialize_transaction) int nVersion; ds_mtx >> nVersion; ds_mtx.SetVersion(nVersion); - ds_mtx >> mutable_tx; + ds_mtx >> TX_WITH_WITNESS(mutable_tx); } catch (const std::ios_base::failure&) { valid_mutable_tx = false; } diff --git a/src/test/fuzz/tx_pool.cpp b/src/test/fuzz/tx_pool.cpp index 66e537a57b..96095539ec 100644 --- a/src/test/fuzz/tx_pool.cpp +++ b/src/test/fuzz/tx_pool.cpp @@ -154,12 +154,15 @@ void CheckATMPInvariants(const MempoolAcceptResult& res, bool txid_in_mempool, b // It may be already in the mempool since in ATMP cases we don't set MEMPOOL_ENTRY or DIFFERENT_WITNESS Assert(!res.m_state.IsValid()); Assert(res.m_state.IsInvalid()); + + const bool is_reconsiderable{res.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE}; Assert(!res.m_replaced_transactions); Assert(!res.m_vsize); Assert(!res.m_base_fees); - // Unable or unwilling to calculate fees - Assert(!res.m_effective_feerate); - Assert(!res.m_wtxids_fee_calculations); + // Fee information is provided if the failure is TX_RECONSIDERABLE. + // In other cases, validation may be unable or unwilling to calculate the fees. + Assert(res.m_effective_feerate.has_value() == is_reconsiderable); + Assert(res.m_wtxids_fee_calculations.has_value() == is_reconsiderable); Assert(!res.m_other_wtxid); break; } diff --git a/src/test/fuzz/utxo_total_supply.cpp b/src/test/fuzz/utxo_total_supply.cpp index 318797faf2..a8f0f3b7ff 100644 --- a/src/test/fuzz/utxo_total_supply.cpp +++ b/src/test/fuzz/utxo_total_supply.cpp @@ -94,8 +94,8 @@ FUZZ_TARGET(utxo_total_supply) assert(ActiveHeight() == 0); // Get at which height we duplicate the coinbase // Assuming that the fuzzer will mine relatively short chains (less than 200 blocks), we want the duplicate coinbase to be not too high. - // Up to 2000 seems reasonable. - int64_t duplicate_coinbase_height = fuzzed_data_provider.ConsumeIntegralInRange(0, 20 * COINBASE_MATURITY); + // Up to 300 seems reasonable. + int64_t duplicate_coinbase_height = fuzzed_data_provider.ConsumeIntegralInRange(0, 300); // Always pad with OP_0 at the end to avoid bad-cb-length error const CScript duplicate_coinbase_script = CScript() << duplicate_coinbase_height << OP_0; // Mine the first block with this duplicate @@ -121,7 +121,7 @@ FUZZ_TARGET(utxo_total_supply) // Limit to avoid timeout, but enough to cover duplicate_coinbase_height // and CVE-2018-17144. - LIMITED_WHILE(fuzzed_data_provider.remaining_bytes(), 2'000) + LIMITED_WHILE(fuzzed_data_provider.remaining_bytes(), 2'00) { CallOneOf( fuzzed_data_provider, diff --git a/src/test/hash_tests.cpp b/src/test/hash_tests.cpp index 54afcef989..f0d2b9ed72 100644 --- a/src/test/hash_tests.cpp +++ b/src/test/hash_tests.cpp @@ -122,12 +122,12 @@ BOOST_AUTO_TEST_CASE(siphash) (uint64_t(x+4)<<32)|(uint64_t(x+5)<<40)|(uint64_t(x+6)<<48)|(uint64_t(x+7)<<56)); } - CHashWriter ss{CLIENT_VERSION}; + HashWriter ss{}; CMutableTransaction tx; // Note these tests were originally written with tx.nVersion=1 // and the test would be affected by default tx version bumps if not fixed. tx.nVersion = 1; - ss << tx; + ss << TX_WITH_WITNESS(tx); BOOST_CHECK_EQUAL(SipHashUint256(1, 2, ss.GetHash()), 0x79751e980c2a0a35ULL); // Check consistency between CSipHasher and SipHashUint256[Extra]. diff --git a/src/test/mempool_tests.cpp b/src/test/mempool_tests.cpp index db58a0baec..217e4a6d22 100644 --- a/src/test/mempool_tests.cpp +++ b/src/test/mempool_tests.cpp @@ -191,7 +191,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) CheckSort<descendant_score>(pool, sortedOrder); CTxMemPool::setEntries setAncestors; - setAncestors.insert(pool.mapTx.find(tx6.GetHash())); + setAncestors.insert(pool.GetIter(tx6.GetHash()).value()); CMutableTransaction tx7 = CMutableTransaction(); tx7.vin.resize(1); tx7.vin[0].prevout = COutPoint(tx6.GetHash(), 0); @@ -223,7 +223,7 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) tx8.vout.resize(1); tx8.vout[0].scriptPubKey = CScript() << OP_11 << OP_EQUAL; tx8.vout[0].nValue = 10 * COIN; - setAncestors.insert(pool.mapTx.find(tx7.GetHash())); + setAncestors.insert(pool.GetIter(tx7.GetHash()).value()); pool.addUnchecked(entry.Fee(0LL).Time(NodeSeconds{2s}).FromTx(tx8), setAncestors); // Now tx8 should be sorted low, but tx6/tx both high @@ -247,8 +247,8 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) std::vector<std::string> snapshotOrder = sortedOrder; - setAncestors.insert(pool.mapTx.find(tx8.GetHash())); - setAncestors.insert(pool.mapTx.find(tx9.GetHash())); + setAncestors.insert(pool.GetIter(tx8.GetHash()).value()); + setAncestors.insert(pool.GetIter(tx9.GetHash()).value()); /* tx10 depends on tx8 and tx9 and has a high fee*/ CMutableTransaction tx10 = CMutableTransaction(); tx10.vin.resize(2); @@ -291,11 +291,11 @@ BOOST_AUTO_TEST_CASE(MempoolIndexingTest) BOOST_CHECK_EQUAL(pool.size(), 10U); // Now try removing tx10 and verify the sort order returns to normal - pool.removeRecursive(pool.mapTx.find(tx10.GetHash())->GetTx(), REMOVAL_REASON_DUMMY); + pool.removeRecursive(*Assert(pool.get(tx10.GetHash())), REMOVAL_REASON_DUMMY); CheckSort<descendant_score>(pool, snapshotOrder); - pool.removeRecursive(pool.mapTx.find(tx9.GetHash())->GetTx(), REMOVAL_REASON_DUMMY); - pool.removeRecursive(pool.mapTx.find(tx8.GetHash())->GetTx(), REMOVAL_REASON_DUMMY); + pool.removeRecursive(*Assert(pool.get(tx9.GetHash())), REMOVAL_REASON_DUMMY); + pool.removeRecursive(*Assert(pool.get(tx8.GetHash())), REMOVAL_REASON_DUMMY); } BOOST_AUTO_TEST_CASE(MempoolAncestorIndexingTest) diff --git a/src/test/miner_tests.cpp b/src/test/miner_tests.cpp index b4c7cac223..96acbea317 100644 --- a/src/test/miner_tests.cpp +++ b/src/test/miner_tests.cpp @@ -144,7 +144,7 @@ void MinerTestingSetup::TestPackageSelection(const CScript& scriptPubKey, const tx.vout[0].nValue = 5000000000LL - 1000 - 50000; // 0 fee uint256 hashFreeTx = tx.GetHash(); tx_mempool.addUnchecked(entry.Fee(0).FromTx(tx)); - size_t freeTxSize = ::GetSerializeSize(tx, PROTOCOL_VERSION); + size_t freeTxSize = ::GetSerializeSize(TX_WITH_WITNESS(tx)); // Calculate a fee on child transaction that will put the package just // below the block min tx fee (assuming 1 child tx of the same size). diff --git a/src/test/miniminer_tests.cpp b/src/test/miniminer_tests.cpp index 76c079a6e7..311e402e3e 100644 --- a/src/test/miniminer_tests.cpp +++ b/src/test/miniminer_tests.cpp @@ -94,7 +94,7 @@ BOOST_FIXTURE_TEST_CASE(miniminer_negative, TestChain100Setup) const CFeeRate feerate_zero(0); mini_miner_target0.BuildMockTemplate(feerate_zero); // Check the quit condition: - BOOST_CHECK(negative_modified_fees < feerate_zero.GetFee(pool.GetIter(tx_mod_negative->GetHash()).value()->GetTxSize())); + BOOST_CHECK(negative_modified_fees < feerate_zero.GetFee(Assert(pool.GetEntry(tx_mod_negative->GetHash()))->GetTxSize())); BOOST_CHECK(mini_miner_target0.GetMockTemplateTxids().empty()); // With no target feerate, the template includes all transactions, even negative feerate ones. @@ -179,9 +179,9 @@ BOOST_FIXTURE_TEST_CASE(miniminer_1p1c, TestChain100Setup) }; std::map<uint256, TxDimensions> tx_dims; for (const auto& tx : all_transactions) { - const auto it = pool.GetIter(tx->GetHash()).value(); - tx_dims.emplace(tx->GetHash(), TxDimensions{it->GetTxSize(), it->GetModifiedFee(), - CFeeRate(it->GetModifiedFee(), it->GetTxSize())}); + const auto& entry{*Assert(pool.GetEntry(tx->GetHash()))}; + tx_dims.emplace(tx->GetHash(), TxDimensions{entry.GetTxSize(), entry.GetModifiedFee(), + CFeeRate(entry.GetModifiedFee(), entry.GetTxSize())}); } const std::vector<CFeeRate> various_normal_feerates({CFeeRate(0), CFeeRate(500), CFeeRate(999), @@ -325,14 +325,14 @@ BOOST_FIXTURE_TEST_CASE(miniminer_1p1c, TestChain100Setup) const int32_t tx6_vsize{tx_dims.at(tx6->GetHash()).vsize}; const int32_t tx7_vsize{tx_dims.at(tx7->GetHash()).vsize}; - miniminer_info.emplace_back(/*fee_self=*/med_fee,/*fee_ancestor=*/med_fee,/*vsize_self=*/tx0_vsize,/*vsize_ancestor=*/tx0_vsize, tx0); - miniminer_info.emplace_back( med_fee, 2*med_fee, tx1_vsize, tx0_vsize + tx1_vsize, tx1); - miniminer_info.emplace_back( low_fee, low_fee, tx2_vsize, tx2_vsize, tx2); - miniminer_info.emplace_back( high_fee, low_fee + high_fee, tx3_vsize, tx2_vsize + tx3_vsize, tx3); - miniminer_info.emplace_back( low_fee, low_fee, tx4_vsize, tx4_vsize, tx4); - miniminer_info.emplace_back( tx5_mod_fee, low_fee + tx5_mod_fee, tx5_vsize, tx4_vsize + tx5_vsize, tx5); - miniminer_info.emplace_back( high_fee, high_fee, tx6_vsize, tx6_vsize, tx6); - miniminer_info.emplace_back( low_fee, high_fee + low_fee, tx7_vsize, tx6_vsize + tx7_vsize, tx7); + miniminer_info.emplace_back(tx0,/*vsize_self=*/tx0_vsize,/*vsize_ancestor=*/tx0_vsize,/*fee_self=*/med_fee,/*fee_ancestor=*/med_fee); + miniminer_info.emplace_back(tx1, tx1_vsize, tx0_vsize + tx1_vsize, med_fee, 2*med_fee); + miniminer_info.emplace_back(tx2, tx2_vsize, tx2_vsize, low_fee, low_fee); + miniminer_info.emplace_back(tx3, tx3_vsize, tx2_vsize + tx3_vsize, high_fee, low_fee + high_fee); + miniminer_info.emplace_back(tx4, tx4_vsize, tx4_vsize, low_fee, low_fee); + miniminer_info.emplace_back(tx5, tx5_vsize, tx4_vsize + tx5_vsize, tx5_mod_fee, low_fee + tx5_mod_fee); + miniminer_info.emplace_back(tx6, tx6_vsize, tx6_vsize, high_fee, high_fee); + miniminer_info.emplace_back(tx7, tx7_vsize, tx6_vsize + tx7_vsize, low_fee, high_fee + low_fee); } std::map<Txid, std::set<Txid>> descendant_caches; descendant_caches.emplace(tx0->GetHash(), std::set<Txid>{tx0->GetHash(), tx1->GetHash()}); @@ -447,15 +447,15 @@ BOOST_FIXTURE_TEST_CASE(miniminer_overlap, TestChain100Setup) // tx3's feerate is lower than tx2's. same fee, different weight. BOOST_CHECK(tx2_feerate > tx3_feerate); const auto tx3_anc_feerate = CFeeRate(low_fee + med_fee + high_fee + high_fee, tx_vsizes[0] + tx_vsizes[1] + tx_vsizes[2] + tx_vsizes[3]); - const auto tx3_iter = pool.GetIter(tx3->GetHash()); - BOOST_CHECK(tx3_anc_feerate == CFeeRate(tx3_iter.value()->GetModFeesWithAncestors(), tx3_iter.value()->GetSizeWithAncestors())); + const auto& tx3_entry{*Assert(pool.GetEntry(tx3->GetHash()))}; + BOOST_CHECK(tx3_anc_feerate == CFeeRate(tx3_entry.GetModFeesWithAncestors(), tx3_entry.GetSizeWithAncestors())); const auto tx4_feerate = CFeeRate(high_fee, tx_vsizes[4]); const auto tx6_anc_feerate = CFeeRate(high_fee + low_fee + med_fee, tx_vsizes[4] + tx_vsizes[5] + tx_vsizes[6]); - const auto tx6_iter = pool.GetIter(tx6->GetHash()); - BOOST_CHECK(tx6_anc_feerate == CFeeRate(tx6_iter.value()->GetModFeesWithAncestors(), tx6_iter.value()->GetSizeWithAncestors())); + const auto& tx6_entry{*Assert(pool.GetEntry(tx6->GetHash()))}; + BOOST_CHECK(tx6_anc_feerate == CFeeRate(tx6_entry.GetModFeesWithAncestors(), tx6_entry.GetSizeWithAncestors())); const auto tx7_anc_feerate = CFeeRate(high_fee + low_fee + high_fee, tx_vsizes[4] + tx_vsizes[5] + tx_vsizes[7]); - const auto tx7_iter = pool.GetIter(tx7->GetHash()); - BOOST_CHECK(tx7_anc_feerate == CFeeRate(tx7_iter.value()->GetModFeesWithAncestors(), tx7_iter.value()->GetSizeWithAncestors())); + const auto& tx7_entry{*Assert(pool.GetEntry(tx7->GetHash()))}; + BOOST_CHECK(tx7_anc_feerate == CFeeRate(tx7_entry.GetModFeesWithAncestors(), tx7_entry.GetSizeWithAncestors())); BOOST_CHECK(tx4_feerate > tx6_anc_feerate); BOOST_CHECK(tx4_feerate > tx7_anc_feerate); @@ -543,15 +543,14 @@ BOOST_FIXTURE_TEST_CASE(miniminer_overlap, TestChain100Setup) } // Check linearization order std::vector<node::MiniMinerMempoolEntry> miniminer_info; - miniminer_info.emplace_back(/*fee_self=*/low_fee, /*fee_ancestor=*/low_fee,/*vsize_self=*/tx_vsizes[0], /*vsize_ancestor=*/tx_vsizes[0], tx0); - miniminer_info.emplace_back( med_fee, med_fee, tx_vsizes[1], tx_vsizes[1], tx1); - miniminer_info.emplace_back( high_fee, high_fee, tx_vsizes[2], tx_vsizes[2], tx2); - miniminer_info.emplace_back( high_fee, low_fee+med_fee+2*high_fee, tx_vsizes[3], tx_vsizes[0]+tx_vsizes[1]+tx_vsizes[2]+tx_vsizes[3], tx3); - - miniminer_info.emplace_back( high_fee, high_fee, tx_vsizes[4], tx_vsizes[4], tx4); - miniminer_info.emplace_back( low_fee, low_fee + high_fee, tx_vsizes[5], tx_vsizes[4]+tx_vsizes[5], tx5); - miniminer_info.emplace_back( med_fee, high_fee+low_fee+med_fee, tx_vsizes[6], tx_vsizes[4]+tx_vsizes[5]+tx_vsizes[6], tx6); - miniminer_info.emplace_back( high_fee, high_fee+low_fee+high_fee, tx_vsizes[7], tx_vsizes[4]+tx_vsizes[5]+tx_vsizes[7], tx7); + miniminer_info.emplace_back(tx0,/*vsize_self=*/tx_vsizes[0], /*vsize_ancestor=*/tx_vsizes[0], /*fee_self=*/low_fee, /*fee_ancestor=*/low_fee); + miniminer_info.emplace_back(tx1, tx_vsizes[1], tx_vsizes[1], med_fee, med_fee); + miniminer_info.emplace_back(tx2, tx_vsizes[2], tx_vsizes[2], high_fee, high_fee); + miniminer_info.emplace_back(tx3, tx_vsizes[3], tx_vsizes[0]+tx_vsizes[1]+tx_vsizes[2]+tx_vsizes[3], high_fee, low_fee+med_fee+2*high_fee); + miniminer_info.emplace_back(tx4, tx_vsizes[4], tx_vsizes[4], high_fee, high_fee); + miniminer_info.emplace_back(tx5, tx_vsizes[5], tx_vsizes[4]+tx_vsizes[5], low_fee, low_fee + high_fee); + miniminer_info.emplace_back(tx6, tx_vsizes[6], tx_vsizes[4]+tx_vsizes[5]+tx_vsizes[6], med_fee, high_fee+low_fee+med_fee); + miniminer_info.emplace_back(tx7, tx_vsizes[7], tx_vsizes[4]+tx_vsizes[5]+tx_vsizes[7], high_fee, high_fee+low_fee+high_fee); std::map<Txid, std::set<Txid>> descendant_caches; descendant_caches.emplace(tx0->GetHash(), std::set<Txid>{tx0->GetHash(), tx3->GetHash()}); @@ -647,7 +646,7 @@ BOOST_FIXTURE_TEST_CASE(manual_ctor, TestChain100Setup) CTxMemPool& pool = *Assert(m_node.mempool); LOCK2(cs_main, pool.cs); { - // 3 pairs of fee-bumping grandparent + parent, plus 1 low-feerate child. + // 3 pairs of grandparent + fee-bumping parent, plus 1 low-feerate child. // 0 fee + high fee auto grandparent_zero_fee = make_tx({{m_coinbase_txns.at(0)->GetHash(), 0}}, 1); auto parent_high_feerate = make_tx({{grandparent_zero_fee->GetHash(), 0}}, 1); @@ -665,13 +664,13 @@ BOOST_FIXTURE_TEST_CASE(manual_ctor, TestChain100Setup) const int64_t child_vsize{1000}; std::vector<node::MiniMinerMempoolEntry> miniminer_info; - miniminer_info.emplace_back(/*fee_self=*/0, /*fee_ancestor=*/0,/*vsize_self=*/tx_vsize,/*vsize_ancestor=*/tx_vsize, grandparent_zero_fee); - miniminer_info.emplace_back( high_fee, high_fee, tx_vsize, 2*tx_vsize, parent_high_feerate); - miniminer_info.emplace_back( 2*low_fee, 2*low_fee, tx_vsize, tx_vsize, grandparent_double_low_feerate); - miniminer_info.emplace_back( med_fee, 2*low_fee+med_fee, tx_vsize, 2*tx_vsize, parent_med_feerate); - miniminer_info.emplace_back( low_fee, low_fee, tx_vsize, tx_vsize, grandparent_low_feerate); - miniminer_info.emplace_back( 2*low_fee, 3*low_fee, tx_vsize, 2*tx_vsize, parent_double_low_feerate); - miniminer_info.emplace_back( low_fee, high_fee+med_fee+6*low_fee, child_vsize, 6*tx_vsize+child_vsize, child); + miniminer_info.emplace_back(grandparent_zero_fee, /*vsize_self=*/tx_vsize,/*vsize_ancestor=*/tx_vsize, /*fee_self=*/0,/*fee_ancestor=*/0); + miniminer_info.emplace_back(parent_high_feerate, tx_vsize, 2*tx_vsize, high_fee, high_fee); + miniminer_info.emplace_back(grandparent_double_low_feerate, tx_vsize, tx_vsize, 2*low_fee, 2*low_fee); + miniminer_info.emplace_back(parent_med_feerate, tx_vsize, 2*tx_vsize, med_fee, 2*low_fee+med_fee); + miniminer_info.emplace_back(grandparent_low_feerate, tx_vsize, tx_vsize, low_fee, low_fee); + miniminer_info.emplace_back(parent_double_low_feerate, tx_vsize, 2*tx_vsize, 2*low_fee, 3*low_fee); + miniminer_info.emplace_back(child, child_vsize, 6*tx_vsize+child_vsize, low_fee, high_fee+med_fee+6*low_fee); std::map<Txid, std::set<Txid>> descendant_caches; descendant_caches.emplace(grandparent_zero_fee->GetHash(), std::set<Txid>{grandparent_zero_fee->GetHash(), parent_high_feerate->GetHash(), child->GetHash()}); descendant_caches.emplace(grandparent_low_feerate->GetHash(), std::set<Txid>{grandparent_low_feerate->GetHash(), parent_double_low_feerate->GetHash(), child->GetHash()}); @@ -693,7 +692,7 @@ BOOST_FIXTURE_TEST_CASE(manual_ctor, TestChain100Setup) BOOST_CHECK_EQUAL(sequences.at(grandparent_double_low_feerate->GetHash()), 1); BOOST_CHECK_EQUAL(sequences.at(parent_med_feerate->GetHash()), 1); - // CPFP low + med + // CPFP low + double low BOOST_CHECK_EQUAL(sequences.at(grandparent_low_feerate->GetHash()), 2); BOOST_CHECK_EQUAL(sequences.at(parent_double_low_feerate->GetHash()), 2); diff --git a/src/test/net_peer_connection_tests.cpp b/src/test/net_peer_connection_tests.cpp new file mode 100644 index 0000000000..3d3f296d82 --- /dev/null +++ b/src/test/net_peer_connection_tests.cpp @@ -0,0 +1,147 @@ +// Copyright (c) 2023-present The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <chainparams.h> +#include <compat/compat.h> +#include <net.h> +#include <net_processing.h> +#include <netaddress.h> +#include <netbase.h> +#include <netgroup.h> +#include <node/connection_types.h> +#include <protocol.h> +#include <random.h> +#include <test/util/logging.h> +#include <test/util/net.h> +#include <test/util/random.h> +#include <test/util/setup_common.h> +#include <tinyformat.h> +#include <util/chaintype.h> +#include <version.h> + +#include <algorithm> +#include <cstdint> +#include <memory> +#include <optional> +#include <string> +#include <vector> + +#include <boost/test/unit_test.hpp> + +struct LogIPsTestingSetup : public TestingSetup { + LogIPsTestingSetup() + : TestingSetup{ChainType::MAIN, /*extra_args=*/{"-logips"}} {} +}; + +BOOST_FIXTURE_TEST_SUITE(net_peer_connection_tests, LogIPsTestingSetup) + +static CService ip(uint32_t i) +{ + struct in_addr s; + s.s_addr = i; + return CService{CNetAddr{s}, Params().GetDefaultPort()}; +} + +/** Create a peer and connect to it. If the optional `address` (IP/CJDNS only) isn't passed, a random address is created. */ +static void AddPeer(NodeId& id, std::vector<CNode*>& nodes, PeerManager& peerman, ConnmanTestMsg& connman, ConnectionType conn_type, bool onion_peer = false, std::optional<std::string> address = std::nullopt) +{ + CAddress addr{}; + + if (address.has_value()) { + addr = CAddress{MaybeFlipIPv6toCJDNS(LookupNumeric(address.value(), Params().GetDefaultPort())), NODE_NONE}; + } else if (onion_peer) { + auto tor_addr{g_insecure_rand_ctx.randbytes(ADDR_TORV3_SIZE)}; + BOOST_REQUIRE(addr.SetSpecial(OnionToString(tor_addr))); + } + + while (!addr.IsLocal() && !addr.IsRoutable()) { + addr = CAddress{ip(g_insecure_rand_ctx.randbits(32)), NODE_NONE}; + } + + BOOST_REQUIRE(addr.IsValid()); + + const bool inbound_onion{onion_peer && conn_type == ConnectionType::INBOUND}; + + nodes.emplace_back(new CNode{++id, + /*sock=*/nullptr, + addr, + /*nKeyedNetGroupIn=*/0, + /*nLocalHostNonceIn=*/0, + CAddress{}, + /*addrNameIn=*/"", + conn_type, + /*inbound_onion=*/inbound_onion}); + CNode& node = *nodes.back(); + node.SetCommonVersion(PROTOCOL_VERSION); + + peerman.InitializeNode(node, ServiceFlags(NODE_NETWORK | NODE_WITNESS)); + node.fSuccessfullyConnected = true; + + connman.AddTestNode(node); +} + +BOOST_AUTO_TEST_CASE(test_addnode_getaddednodeinfo_and_connection_detection) +{ + auto connman = std::make_unique<ConnmanTestMsg>(0x1337, 0x1337, *m_node.addrman, *m_node.netgroupman, Params()); + auto peerman = PeerManager::make(*connman, *m_node.addrman, nullptr, *m_node.chainman, *m_node.mempool, {}); + NodeId id{0}; + std::vector<CNode*> nodes; + + // Connect a localhost peer. + { + ASSERT_DEBUG_LOG("Added connection to 127.0.0.1:8333 peer=1"); + AddPeer(id, nodes, *peerman, *connman, ConnectionType::MANUAL, /*onion_peer=*/false, /*address=*/"127.0.0.1"); + BOOST_REQUIRE(nodes.back() != nullptr); + } + + // Call ConnectNode(), which is also called by RPC addnode onetry, for a localhost + // address that resolves to multiple IPs, including that of the connected peer. + // The connection attempt should consistently fail due to the check in ConnectNode(). + for (int i = 0; i < 10; ++i) { + ASSERT_DEBUG_LOG("Not opening a connection to localhost, already connected to 127.0.0.1:8333"); + BOOST_CHECK(!connman->ConnectNodePublic(*peerman, "localhost", ConnectionType::MANUAL)); + } + + // Add 3 more peer connections. + AddPeer(id, nodes, *peerman, *connman, ConnectionType::OUTBOUND_FULL_RELAY); + AddPeer(id, nodes, *peerman, *connman, ConnectionType::BLOCK_RELAY, /*onion_peer=*/true); + AddPeer(id, nodes, *peerman, *connman, ConnectionType::INBOUND); + + BOOST_TEST_MESSAGE("Call AddNode() for all the peers"); + for (auto node : connman->TestNodes()) { + BOOST_CHECK(connman->AddNode({/*m_added_node=*/node->addr.ToStringAddrPort(), /*m_use_v2transport=*/true})); + BOOST_TEST_MESSAGE(strprintf("peer id=%s addr=%s", node->GetId(), node->addr.ToStringAddrPort())); + } + + BOOST_TEST_MESSAGE("\nCall AddNode() with 2 addrs resolving to existing localhost addnode entry; neither should be added"); + BOOST_CHECK(!connman->AddNode({/*m_added_node=*/"127.0.0.1", /*m_use_v2transport=*/true})); + BOOST_CHECK(!connman->AddNode({/*m_added_node=*/"127.1", /*m_use_v2transport=*/true})); + + BOOST_TEST_MESSAGE("\nExpect GetAddedNodeInfo to return expected number of peers with `include_connected` true/false"); + BOOST_CHECK_EQUAL(connman->GetAddedNodeInfo(/*include_connected=*/true).size(), nodes.size()); + BOOST_CHECK(connman->GetAddedNodeInfo(/*include_connected=*/false).empty()); + + BOOST_TEST_MESSAGE("\nPrint GetAddedNodeInfo contents:"); + for (const auto& info : connman->GetAddedNodeInfo(/*include_connected=*/true)) { + BOOST_TEST_MESSAGE(strprintf("\nadded node: %s", info.m_params.m_added_node)); + BOOST_TEST_MESSAGE(strprintf("connected: %s", info.fConnected)); + if (info.fConnected) { + BOOST_TEST_MESSAGE(strprintf("IP address: %s", info.resolvedAddress.ToStringAddrPort())); + BOOST_TEST_MESSAGE(strprintf("direction: %s", info.fInbound ? "inbound" : "outbound")); + } + } + + BOOST_TEST_MESSAGE("\nCheck that all connected peers are correctly detected as connected"); + for (auto node : connman->TestNodes()) { + BOOST_CHECK(connman->AlreadyConnectedPublic(node->addr)); + } + + // Clean up + for (auto node : connman->TestNodes()) { + peerman->FinalizeNode(*node); + } + connman->ClearTestNodes(); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/script_tests.cpp b/src/test/script_tests.cpp index 6e0c962601..874f3e51f7 100644 --- a/src/test/script_tests.cpp +++ b/src/test/script_tests.cpp @@ -141,8 +141,8 @@ void DoTest(const CScript& scriptPubKey, const CScript& scriptSig, const CScript } #if defined(HAVE_CONSENSUS_LIB) - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << tx2; + DataStream stream; + stream << TX_WITH_WITNESS(tx2); uint32_t libconsensus_flags{flags & bitcoinconsensus_SCRIPT_FLAGS_VERIFY_ALL}; if (libconsensus_flags == flags) { int expectedSuccessCode = expect ? 1 : 0; @@ -1470,7 +1470,7 @@ BOOST_AUTO_TEST_CASE(script_HasValidOps) static CMutableTransaction TxFromHex(const std::string& str) { CMutableTransaction tx; - SpanReader{SERIALIZE_TRANSACTION_NO_WITNESS, ParseHex(str)} >> tx; + SpanReader{0, ParseHex(str)} >> TX_NO_WITNESS(tx); return tx; } @@ -1513,8 +1513,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_returns_true) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << spendTx; + DataStream stream; + stream << TX_WITH_WITNESS(spendTx); bitcoinconsensus_error err; int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); @@ -1536,8 +1536,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_index_err) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << spendTx; + DataStream stream; + stream << TX_WITH_WITNESS(spendTx); bitcoinconsensus_error err; int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); @@ -1559,8 +1559,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_size) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << spendTx; + DataStream stream; + stream << TX_WITH_WITNESS(spendTx); bitcoinconsensus_error err; int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size() * 2, nIn, libconsensus_flags, &err); @@ -1582,7 +1582,7 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_tx_serialization) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); + DataStream stream; stream << 0xffffffff; bitcoinconsensus_error err; @@ -1605,8 +1605,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_amount_required_err) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << spendTx; + DataStream stream; + stream << TX_WITH_WITNESS(spendTx); bitcoinconsensus_error err; int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); @@ -1628,8 +1628,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_invalid_flags) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << spendTx; + DataStream stream; + stream << TX_WITH_WITNESS(spendTx); bitcoinconsensus_error err; int result = bitcoinconsensus_verify_script(scriptPubKey.data(), scriptPubKey.size(), UCharCast(stream.data()), stream.size(), nIn, libconsensus_flags, &err); @@ -1651,8 +1651,8 @@ BOOST_AUTO_TEST_CASE(bitcoinconsensus_verify_script_spent_outputs_required_err) CTransaction creditTx{BuildCreditingTransaction(scriptPubKey, 1)}; CTransaction spendTx{BuildSpendingTransaction(scriptSig, wit, creditTx)}; - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << spendTx; + DataStream stream; + stream << TX_WITH_WITNESS(spendTx); bitcoinconsensus_error err; int result{bitcoinconsensus_verify_script_with_spent_outputs(scriptPubKey.data(), scriptPubKey.size(), creditTx.vout[0].nValue, UCharCast(stream.data()), stream.size(), nullptr, 0, nIn, libconsensus_flags, &err)}; @@ -1718,8 +1718,8 @@ static void AssetTest(const UniValue& test) CachingTransactionSignatureChecker txcheck(&tx, idx, prevouts[idx].nValue, true, txdata); #if defined(HAVE_CONSENSUS_LIB) - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << tx; + DataStream stream; + stream << TX_WITH_WITNESS(tx); std::vector<UTXO> utxos; utxos.resize(prevouts.size()); for (size_t i = 0; i < prevouts.size(); i++) { @@ -1752,8 +1752,8 @@ static void AssetTest(const UniValue& test) CachingTransactionSignatureChecker txcheck(&tx, idx, prevouts[idx].nValue, true, txdata); #if defined(HAVE_CONSENSUS_LIB) - CDataStream stream(SER_NETWORK, PROTOCOL_VERSION); - stream << tx; + DataStream stream; + stream << TX_WITH_WITNESS(tx); std::vector<UTXO> utxos; utxos.resize(prevouts.size()); for (size_t i = 0; i < prevouts.size(); i++) { @@ -1816,7 +1816,7 @@ BOOST_AUTO_TEST_CASE(bip341_keypath_test_vectors) for (const auto& vec : vectors.getValues()) { auto txhex = ParseHex(vec["given"]["rawUnsignedTx"].get_str()); CMutableTransaction tx; - SpanReader{PROTOCOL_VERSION, txhex} >> tx; + SpanReader{PROTOCOL_VERSION, txhex} >> TX_WITH_WITNESS(tx); std::vector<CTxOut> utxos; for (const auto& utxo_spent : vec["given"]["utxosSpent"].getValues()) { auto script_bytes = ParseHex(utxo_spent["scriptPubKey"].get_str()); diff --git a/src/test/serialize_tests.cpp b/src/test/serialize_tests.cpp index 49827be559..6d5a14ad9e 100644 --- a/src/test/serialize_tests.cpp +++ b/src/test/serialize_tests.cpp @@ -36,7 +36,7 @@ public: READWRITE(obj.boolval); READWRITE(obj.stringval); READWRITE(obj.charstrval); - READWRITE(obj.txval); + READWRITE(TX_WITH_WITNESS(obj.txval)); } bool operator==(const CSerializeMethodsTestSingle& rhs) const @@ -56,7 +56,7 @@ public: SERIALIZE_METHODS(CSerializeMethodsTestMany, obj) { - READWRITE(obj.intval, obj.boolval, obj.stringval, obj.charstrval, obj.txval); + READWRITE(obj.intval, obj.boolval, obj.stringval, obj.charstrval, TX_WITH_WITNESS(obj.txval)); } }; @@ -85,6 +85,8 @@ BOOST_AUTO_TEST_CASE(sizes) BOOST_CHECK_EQUAL(GetSerializeSize(int64_t(0), 0), 8U); BOOST_CHECK_EQUAL(GetSerializeSize(uint64_t(0), 0), 8U); BOOST_CHECK_EQUAL(GetSerializeSize(bool(0), 0), 1U); + BOOST_CHECK_EQUAL(GetSerializeSize(std::array<uint8_t, 1>{0}, 0), 1U); + BOOST_CHECK_EQUAL(GetSerializeSize(std::array<uint8_t, 2>{0, 0}, 0), 2U); } BOOST_AUTO_TEST_CASE(varints) @@ -179,6 +181,16 @@ BOOST_AUTO_TEST_CASE(vector_bool) BOOST_CHECK((HashWriter{} << vec1).GetHash() == (HashWriter{} << vec2).GetHash()); } +BOOST_AUTO_TEST_CASE(array) +{ + std::array<uint8_t, 32> array1{1, 0, 0, 1, 1, 1, 0, 0, 0, 0, 1, 0, 0, 1, 1, 0, 0, 0, 1, 1, 1, 1, 0, 1, 0, 0, 1}; + DataStream ds; + ds << array1; + std::array<uint8_t, 32> array2; + ds >> array2; + BOOST_CHECK(array1 == array2); +} + BOOST_AUTO_TEST_CASE(noncanonical) { // Write some non-canonical CompactSize encodings, and @@ -228,7 +240,7 @@ BOOST_AUTO_TEST_CASE(class_methods) CSerializeMethodsTestMany methodtest2(intval, boolval, stringval, charstrval, tx_ref); CSerializeMethodsTestSingle methodtest3; CSerializeMethodsTestMany methodtest4; - CDataStream ss(SER_DISK, PROTOCOL_VERSION); + DataStream ss; BOOST_CHECK(methodtest1 == methodtest2); ss << methodtest1; ss >> methodtest4; @@ -238,8 +250,8 @@ BOOST_AUTO_TEST_CASE(class_methods) BOOST_CHECK(methodtest2 == methodtest3); BOOST_CHECK(methodtest3 == methodtest4); - CDataStream ss2{SER_DISK, PROTOCOL_VERSION}; - ss2 << intval << boolval << stringval << charstrval << txval; + DataStream ss2; + ss2 << intval << boolval << stringval << charstrval << TX_WITH_WITNESS(txval); ss2 >> methodtest3; BOOST_CHECK(methodtest3 == methodtest4); { diff --git a/src/test/sighash_tests.cpp b/src/test/sighash_tests.cpp index 9653edd84b..bc2e3ca45e 100644 --- a/src/test/sighash_tests.cpp +++ b/src/test/sighash_tests.cpp @@ -78,8 +78,8 @@ uint256 static SignatureHashOld(CScript scriptCode, const CTransaction& txTo, un } // Serialize and hash - CHashWriter ss{SERIALIZE_TRANSACTION_NO_WITNESS}; - ss << txTmp << nHashType; + HashWriter ss{}; + ss << TX_NO_WITNESS(txTmp) << nHashType; return ss.GetHash(); } @@ -138,8 +138,8 @@ BOOST_AUTO_TEST_CASE(sighash_test) sho = SignatureHashOld(scriptCode, CTransaction(txTo), nIn, nHashType); sh = SignatureHash(scriptCode, txTo, nIn, nHashType, 0, SigVersion::BASE); #if defined(PRINT_SIGHASH_JSON) - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION); - ss << txTo; + DataStream ss; + ss << TX_WITH_WITNESS(txTo); std::cout << "\t[\"" ; std::cout << HexStr(ss) << "\", \""; @@ -188,8 +188,8 @@ BOOST_AUTO_TEST_CASE(sighash_from_data) nHashType = test[3].getInt<int>(); sigHashHex = test[4].get_str(); - CDataStream stream(ParseHex(raw_tx), SER_NETWORK, PROTOCOL_VERSION); - stream >> tx; + DataStream stream(ParseHex(raw_tx)); + stream >> TX_WITH_WITNESS(tx); TxValidationState state; BOOST_CHECK_MESSAGE(CheckTransaction(*tx, state), strTest); diff --git a/src/test/span_tests.cpp b/src/test/span_tests.cpp new file mode 100644 index 0000000000..f6cac10b09 --- /dev/null +++ b/src/test/span_tests.cpp @@ -0,0 +1,73 @@ +// Copyright (c) 2023 The Bitcoin Core developers +// Distributed under the MIT software license, see the accompanying +// file COPYING or http://www.opensource.org/licenses/mit-license.php. + +#include <span.h> + +#include <boost/test/unit_test.hpp> +#include <array> +#include <set> +#include <vector> + +namespace { +struct Ignore +{ + template<typename T> Ignore(T&&) {} +}; +template<typename T> +bool Spannable(T&& value, decltype(Span{value})* enable = nullptr) +{ + return true; +} +bool Spannable(Ignore) +{ + return false; +} + +#if defined(__clang__) +# pragma clang diagnostic push +# pragma clang diagnostic ignored "-Wunneeded-member-function" +# pragma clang diagnostic ignored "-Wunused-member-function" +#endif +struct SpannableYes +{ + int* data(); + size_t size(); +}; +struct SpannableNo +{ + void* data(); + size_t size(); +}; +#if defined(__clang__) +# pragma clang diagnostic pop +#endif +} // namespace + +BOOST_AUTO_TEST_SUITE(span_tests) + +// Make sure template Span template deduction guides accurately enable calls to +// Span constructor overloads that work, and disable calls to constructor overloads that +// don't work. This makes it is possible to use the Span constructor in a SFINAE +// contexts like in the Spannable function above to detect whether types are or +// aren't compatible with Spans at compile time. +// +// Previously there was a bug where writing a SFINAE check for vector<bool> was +// not possible, because in libstdc++ vector<bool> has a data() memeber +// returning void*, and the Span template guide ignored the data() return value, +// so the template substitution would succeed, but the constructor would fail, +// resulting in a fatal compile error, rather than a SFINAE error that could be +// handled. +BOOST_AUTO_TEST_CASE(span_constructor_sfinae) +{ + BOOST_CHECK(Spannable(std::vector<int>{})); + BOOST_CHECK(!Spannable(std::set<int>{})); + BOOST_CHECK(!Spannable(std::vector<bool>{})); + BOOST_CHECK(Spannable(std::array<int, 3>{})); + BOOST_CHECK(Spannable(Span<int>{})); + BOOST_CHECK(Spannable("char array")); + BOOST_CHECK(Spannable(SpannableYes{})); + BOOST_CHECK(!Spannable(SpannableNo{})); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/transaction_tests.cpp b/src/test/transaction_tests.cpp index a4c0db8aea..220b61f7e7 100644 --- a/src/test/transaction_tests.cpp +++ b/src/test/transaction_tests.cpp @@ -233,8 +233,8 @@ BOOST_AUTO_TEST_CASE(tx_valid) } std::string transaction = test[1].get_str(); - CDataStream stream(ParseHex(transaction), SER_NETWORK, PROTOCOL_VERSION); - CTransaction tx(deserialize, stream); + DataStream stream(ParseHex(transaction)); + CTransaction tx(deserialize, TX_WITH_WITNESS, stream); TxValidationState state; BOOST_CHECK_MESSAGE(CheckTransaction(tx, state), strTest); @@ -321,8 +321,8 @@ BOOST_AUTO_TEST_CASE(tx_invalid) } std::string transaction = test[1].get_str(); - CDataStream stream(ParseHex(transaction), SER_NETWORK, PROTOCOL_VERSION ); - CTransaction tx(deserialize, stream); + DataStream stream(ParseHex(transaction)); + CTransaction tx(deserialize, TX_WITH_WITNESS, stream); TxValidationState state; if (!CheckTransaction(tx, state) || state.IsInvalid()) { @@ -371,9 +371,9 @@ BOOST_AUTO_TEST_CASE(basic_transaction_tests) // Random real transaction (e2769b09e784f32f62ef849763d4f45b98e07ba658647343b915ff832b110436) unsigned char ch[] = {0x01, 0x00, 0x00, 0x00, 0x01, 0x6b, 0xff, 0x7f, 0xcd, 0x4f, 0x85, 0x65, 0xef, 0x40, 0x6d, 0xd5, 0xd6, 0x3d, 0x4f, 0xf9, 0x4f, 0x31, 0x8f, 0xe8, 0x20, 0x27, 0xfd, 0x4d, 0xc4, 0x51, 0xb0, 0x44, 0x74, 0x01, 0x9f, 0x74, 0xb4, 0x00, 0x00, 0x00, 0x00, 0x8c, 0x49, 0x30, 0x46, 0x02, 0x21, 0x00, 0xda, 0x0d, 0xc6, 0xae, 0xce, 0xfe, 0x1e, 0x06, 0xef, 0xdf, 0x05, 0x77, 0x37, 0x57, 0xde, 0xb1, 0x68, 0x82, 0x09, 0x30, 0xe3, 0xb0, 0xd0, 0x3f, 0x46, 0xf5, 0xfc, 0xf1, 0x50, 0xbf, 0x99, 0x0c, 0x02, 0x21, 0x00, 0xd2, 0x5b, 0x5c, 0x87, 0x04, 0x00, 0x76, 0xe4, 0xf2, 0x53, 0xf8, 0x26, 0x2e, 0x76, 0x3e, 0x2d, 0xd5, 0x1e, 0x7f, 0xf0, 0xbe, 0x15, 0x77, 0x27, 0xc4, 0xbc, 0x42, 0x80, 0x7f, 0x17, 0xbd, 0x39, 0x01, 0x41, 0x04, 0xe6, 0xc2, 0x6e, 0xf6, 0x7d, 0xc6, 0x10, 0xd2, 0xcd, 0x19, 0x24, 0x84, 0x78, 0x9a, 0x6c, 0xf9, 0xae, 0xa9, 0x93, 0x0b, 0x94, 0x4b, 0x7e, 0x2d, 0xb5, 0x34, 0x2b, 0x9d, 0x9e, 0x5b, 0x9f, 0xf7, 0x9a, 0xff, 0x9a, 0x2e, 0xe1, 0x97, 0x8d, 0xd7, 0xfd, 0x01, 0xdf, 0xc5, 0x22, 0xee, 0x02, 0x28, 0x3d, 0x3b, 0x06, 0xa9, 0xd0, 0x3a, 0xcf, 0x80, 0x96, 0x96, 0x8d, 0x7d, 0xbb, 0x0f, 0x91, 0x78, 0xff, 0xff, 0xff, 0xff, 0x02, 0x8b, 0xa7, 0x94, 0x0e, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, 0xa9, 0x14, 0xba, 0xde, 0xec, 0xfd, 0xef, 0x05, 0x07, 0x24, 0x7f, 0xc8, 0xf7, 0x42, 0x41, 0xd7, 0x3b, 0xc0, 0x39, 0x97, 0x2d, 0x7b, 0x88, 0xac, 0x40, 0x94, 0xa8, 0x02, 0x00, 0x00, 0x00, 0x00, 0x19, 0x76, 0xa9, 0x14, 0xc1, 0x09, 0x32, 0x48, 0x3f, 0xec, 0x93, 0xed, 0x51, 0xf5, 0xfe, 0x95, 0xe7, 0x25, 0x59, 0xf2, 0xcc, 0x70, 0x43, 0xf9, 0x88, 0xac, 0x00, 0x00, 0x00, 0x00, 0x00}; std::vector<unsigned char> vch(ch, ch + sizeof(ch) -1); - CDataStream stream(vch, SER_DISK, CLIENT_VERSION); + DataStream stream(vch); CMutableTransaction tx; - stream >> tx; + stream >> TX_WITH_WITNESS(tx); TxValidationState state; BOOST_CHECK_MESSAGE(CheckTransaction(CTransaction(tx), state) && state.IsValid(), "Simple deserialized transaction should be valid."); @@ -418,9 +418,9 @@ static void CreateCreditAndSpend(const FillableSigningProvider& keystore, const outputm.vout.resize(1); outputm.vout[0].nValue = 1; outputm.vout[0].scriptPubKey = outscript; - CDataStream ssout(SER_NETWORK, PROTOCOL_VERSION); - ssout << outputm; - ssout >> output; + DataStream ssout; + ssout << TX_WITH_WITNESS(outputm); + ssout >> TX_WITH_WITNESS(output); assert(output->vin.size() == 1); assert(output->vin[0] == outputm.vin[0]); assert(output->vout.size() == 1); @@ -437,9 +437,9 @@ static void CreateCreditAndSpend(const FillableSigningProvider& keystore, const SignatureData empty; bool ret = SignSignature(keystore, *output, inputm, 0, SIGHASH_ALL, empty); assert(ret == success); - CDataStream ssin(SER_NETWORK, PROTOCOL_VERSION); - ssin << inputm; - ssin >> input; + DataStream ssin; + ssin << TX_WITH_WITNESS(inputm); + ssin >> TX_WITH_WITNESS(input); assert(input.vin.size() == 1); assert(input.vin[0] == inputm.vin[0]); assert(input.vout.size() == 1); @@ -524,9 +524,9 @@ BOOST_AUTO_TEST_CASE(test_big_witness_transaction) assert(hashSigned); } - CDataStream ssout(SER_NETWORK, PROTOCOL_VERSION); - ssout << mtx; - CTransaction tx(deserialize, ssout); + DataStream ssout; + ssout << TX_WITH_WITNESS(mtx); + CTransaction tx(deserialize, TX_WITH_WITNESS, ssout); // check all inputs concurrently, with the cache PrecomputedTransactionData txdata(tx); diff --git a/src/test/txpackage_tests.cpp b/src/test/txpackage_tests.cpp index debefa7f93..84c9ecc3d1 100644 --- a/src/test/txpackage_tests.cpp +++ b/src/test/txpackage_tests.cpp @@ -11,6 +11,7 @@ #include <test/util/random.h> #include <test/util/script.h> #include <test/util/setup_common.h> +#include <test/util/txmempool.h> #include <validation.h> #include <boost/test/unit_test.hpp> @@ -132,36 +133,35 @@ BOOST_FIXTURE_TEST_CASE(package_validation_tests, TestChain100Setup) /*output_destination=*/child_locking_script, /*output_amount=*/CAmount(48 * COIN), /*submit=*/false); CTransactionRef tx_child = MakeTransactionRef(mtx_child); - const auto result_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, {tx_parent, tx_child}, /*test_accept=*/true); - BOOST_CHECK_MESSAGE(result_parent_child.m_state.IsValid(), - "Package validation unexpectedly failed: " << result_parent_child.m_state.GetRejectReason()); - BOOST_CHECK(result_parent_child.m_tx_results.size() == 2); - auto it_parent = result_parent_child.m_tx_results.find(tx_parent->GetWitnessHash()); - auto it_child = result_parent_child.m_tx_results.find(tx_child->GetWitnessHash()); - BOOST_CHECK(it_parent != result_parent_child.m_tx_results.end()); - BOOST_CHECK_MESSAGE(it_parent->second.m_state.IsValid(), - "Package validation unexpectedly failed: " << it_parent->second.m_state.GetRejectReason()); - BOOST_CHECK(it_parent->second.m_effective_feerate.value().GetFee(GetVirtualTransactionSize(*tx_parent)) == COIN); - BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().size(), 1); - BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().front(), tx_parent->GetWitnessHash()); - BOOST_CHECK(it_child != result_parent_child.m_tx_results.end()); - BOOST_CHECK_MESSAGE(it_child->second.m_state.IsValid(), - "Package validation unexpectedly failed: " << it_child->second.m_state.GetRejectReason()); - BOOST_CHECK(it_child->second.m_effective_feerate.value().GetFee(GetVirtualTransactionSize(*tx_child)) == COIN); - BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().size(), 1); - BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().front(), tx_child->GetWitnessHash()); + Package package_parent_child{tx_parent, tx_child}; + const auto result_parent_child = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_parent_child, /*test_accept=*/true); + if (auto err_parent_child{CheckPackageMempoolAcceptResult(package_parent_child, result_parent_child, /*expect_valid=*/true, nullptr)}) { + BOOST_ERROR(err_parent_child.value()); + } else { + auto it_parent = result_parent_child.m_tx_results.find(tx_parent->GetWitnessHash()); + auto it_child = result_parent_child.m_tx_results.find(tx_child->GetWitnessHash()); + + BOOST_CHECK(it_parent->second.m_effective_feerate.value().GetFee(GetVirtualTransactionSize(*tx_parent)) == COIN); + BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().size(), 1); + BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().front(), tx_parent->GetWitnessHash()); + BOOST_CHECK(it_child->second.m_effective_feerate.value().GetFee(GetVirtualTransactionSize(*tx_child)) == COIN); + BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().size(), 1); + BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().front(), tx_child->GetWitnessHash()); + } // A single, giant transaction submitted through ProcessNewPackage fails on single tx policy. CTransactionRef giant_ptx = create_placeholder_tx(999, 999); BOOST_CHECK(GetVirtualTransactionSize(*giant_ptx) > DEFAULT_ANCESTOR_SIZE_LIMIT_KVB * 1000); - auto result_single_large = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, {giant_ptx}, /*test_accept=*/true); - BOOST_CHECK(result_single_large.m_state.IsInvalid()); - BOOST_CHECK_EQUAL(result_single_large.m_state.GetResult(), PackageValidationResult::PCKG_TX); - BOOST_CHECK_EQUAL(result_single_large.m_state.GetRejectReason(), "transaction failed"); - BOOST_CHECK(result_single_large.m_tx_results.size() == 1); - auto it_giant_tx = result_single_large.m_tx_results.find(giant_ptx->GetWitnessHash()); - BOOST_CHECK(it_giant_tx != result_single_large.m_tx_results.end()); - BOOST_CHECK_EQUAL(it_giant_tx->second.m_state.GetRejectReason(), "tx-size"); + Package package_single_giant{giant_ptx}; + auto result_single_large = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_single_giant, /*test_accept=*/true); + if (auto err_single_large{CheckPackageMempoolAcceptResult(package_single_giant, result_single_large, /*expect_valid=*/false, nullptr)}) { + BOOST_ERROR(err_single_large.value()); + } else { + BOOST_CHECK_EQUAL(result_single_large.m_state.GetResult(), PackageValidationResult::PCKG_TX); + BOOST_CHECK_EQUAL(result_single_large.m_state.GetRejectReason(), "transaction failed"); + auto it_giant_tx = result_single_large.m_tx_results.find(giant_ptx->GetWitnessHash()); + BOOST_CHECK_EQUAL(it_giant_tx->second.m_state.GetRejectReason(), "tx-size"); + } // Check that mempool size hasn't changed. BOOST_CHECK_EQUAL(m_node.mempool->size(), initialPoolSize); @@ -281,6 +281,7 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) } auto result_unrelated_submit = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_unrelated, /*test_accept=*/false); + // We don't expect m_tx_results for each transaction when basic sanity checks haven't passed. BOOST_CHECK(result_unrelated_submit.m_state.IsInvalid()); BOOST_CHECK_EQUAL(result_unrelated_submit.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); BOOST_CHECK_EQUAL(result_unrelated_submit.m_state.GetRejectReason(), "package-not-child-with-parents"); @@ -336,20 +337,20 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) CMutableTransaction mtx_parent_invalid{mtx_parent}; mtx_parent_invalid.vin[0].scriptWitness = bad_witness; CTransactionRef tx_parent_invalid = MakeTransactionRef(mtx_parent_invalid); + Package package_invalid_parent{tx_parent_invalid, tx_child}; auto result_quit_early = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {tx_parent_invalid, tx_child}, /*test_accept=*/ false); - BOOST_CHECK(result_quit_early.m_state.IsInvalid()); + package_invalid_parent, /*test_accept=*/ false); + if (auto err_parent_invalid{CheckPackageMempoolAcceptResult(package_invalid_parent, result_quit_early, /*expect_valid=*/false, m_node.mempool.get())}) { + BOOST_ERROR(err_parent_invalid.value()); + } else { + auto it_parent = result_quit_early.m_tx_results.find(tx_parent_invalid->GetWitnessHash()); + auto it_child = result_quit_early.m_tx_results.find(tx_child->GetWitnessHash()); + BOOST_CHECK_EQUAL(it_parent->second.m_state.GetResult(), TxValidationResult::TX_WITNESS_MUTATED); + BOOST_CHECK_EQUAL(it_parent->second.m_state.GetRejectReason(), "bad-witness-nonstandard"); + BOOST_CHECK_EQUAL(it_child->second.m_state.GetResult(), TxValidationResult::TX_MISSING_INPUTS); + BOOST_CHECK_EQUAL(it_child->second.m_state.GetRejectReason(), "bad-txns-inputs-missingorspent"); + } BOOST_CHECK_EQUAL(result_quit_early.m_state.GetResult(), PackageValidationResult::PCKG_TX); - BOOST_CHECK(!result_quit_early.m_tx_results.empty()); - BOOST_CHECK_EQUAL(result_quit_early.m_tx_results.size(), 2); - auto it_parent = result_quit_early.m_tx_results.find(tx_parent_invalid->GetWitnessHash()); - auto it_child = result_quit_early.m_tx_results.find(tx_child->GetWitnessHash()); - BOOST_CHECK(it_parent != result_quit_early.m_tx_results.end()); - BOOST_CHECK(it_child != result_quit_early.m_tx_results.end()); - BOOST_CHECK_EQUAL(it_parent->second.m_state.GetResult(), TxValidationResult::TX_WITNESS_MUTATED); - BOOST_CHECK_EQUAL(it_parent->second.m_state.GetRejectReason(), "bad-witness-nonstandard"); - BOOST_CHECK_EQUAL(it_child->second.m_state.GetResult(), TxValidationResult::TX_MISSING_INPUTS); - BOOST_CHECK_EQUAL(it_child->second.m_state.GetRejectReason(), "bad-txns-inputs-missingorspent"); } // Child with missing parent. @@ -381,36 +382,27 @@ BOOST_FIXTURE_TEST_CASE(package_submission_tests, TestChain100Setup) BOOST_CHECK(it_parent->second.m_effective_feerate == CFeeRate(1 * COIN, GetVirtualTransactionSize(*tx_parent))); BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().size(), 1); BOOST_CHECK_EQUAL(it_parent->second.m_wtxids_fee_calculations.value().front(), tx_parent->GetWitnessHash()); - BOOST_CHECK(it_child != submit_parent_child.m_tx_results.end()); - BOOST_CHECK(it_child->second.m_state.IsValid()); BOOST_CHECK(it_child->second.m_effective_feerate == CFeeRate(1 * COIN, GetVirtualTransactionSize(*tx_child))); BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().size(), 1); BOOST_CHECK_EQUAL(it_child->second.m_wtxids_fee_calculations.value().front(), tx_child->GetWitnessHash()); BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_parent->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_child->GetHash()))); } // Already-in-mempool transactions should be detected and de-duplicated. { const auto submit_deduped = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_parent_child, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_deduped.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_deduped.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_deduped.m_tx_results.size(), package_parent_child.size()); - auto it_parent_deduped = submit_deduped.m_tx_results.find(tx_parent->GetWitnessHash()); - auto it_child_deduped = submit_deduped.m_tx_results.find(tx_child->GetWitnessHash()); - BOOST_CHECK(it_parent_deduped != submit_deduped.m_tx_results.end()); - BOOST_CHECK(it_parent_deduped->second.m_state.IsValid()); - BOOST_CHECK(it_parent_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); - BOOST_CHECK(it_child_deduped != submit_deduped.m_tx_results.end()); - BOOST_CHECK(it_child_deduped->second.m_state.IsValid()); - BOOST_CHECK(it_child_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + if (auto err_deduped{CheckPackageMempoolAcceptResult(package_parent_child, submit_deduped, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_deduped.value()); + } else { + auto it_parent_deduped = submit_deduped.m_tx_results.find(tx_parent->GetWitnessHash()); + auto it_child_deduped = submit_deduped.m_tx_results.find(tx_child->GetWitnessHash()); + BOOST_CHECK(it_parent_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK(it_child_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + } BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_parent->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_child->GetHash()))); } } @@ -470,51 +462,39 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // Try submitting Package1{parent, child1} and Package2{parent, child2} where the children are // same-txid-different-witness. { + Package package_parent_child1{ptx_parent, ptx_child1}; const auto submit_witness1 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_parent, ptx_child1}, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_witness1.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_witness1.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_witness1.m_tx_results.size(), 2); - auto it_parent1 = submit_witness1.m_tx_results.find(ptx_parent->GetWitnessHash()); - auto it_child1 = submit_witness1.m_tx_results.find(ptx_child1->GetWitnessHash()); - BOOST_CHECK(it_parent1 != submit_witness1.m_tx_results.end()); - BOOST_CHECK_MESSAGE(it_parent1->second.m_state.IsValid(), - "Transaction unexpectedly failed: " << it_parent1->second.m_state.GetRejectReason()); - BOOST_CHECK(it_child1 != submit_witness1.m_tx_results.end()); - BOOST_CHECK_MESSAGE(it_child1->second.m_state.IsValid(), - "Transaction unexpectedly failed: " << it_child1->second.m_state.GetRejectReason()); - - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_child1->GetHash()))); + package_parent_child1, /*test_accept=*/false); + if (auto err_witness1{CheckPackageMempoolAcceptResult(package_parent_child1, submit_witness1, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_witness1.value()); + } // Child2 would have been validated individually. + Package package_parent_child2{ptx_parent, ptx_child2}; const auto submit_witness2 = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_parent, ptx_child2}, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_witness2.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_witness2.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_witness2.m_tx_results.size(), 2); - auto it_parent2_deduped = submit_witness2.m_tx_results.find(ptx_parent->GetWitnessHash()); - auto it_child2 = submit_witness2.m_tx_results.find(ptx_child2->GetWitnessHash()); - BOOST_CHECK(it_parent2_deduped != submit_witness2.m_tx_results.end()); - BOOST_CHECK(it_parent2_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); - BOOST_CHECK(it_child2 != submit_witness2.m_tx_results.end()); - BOOST_CHECK(it_child2->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); - BOOST_CHECK_EQUAL(ptx_child1->GetWitnessHash(), it_child2->second.m_other_wtxid.value()); - - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_child2->GetHash()))); - BOOST_CHECK(!m_node.mempool->exists(GenTxid::Wtxid(ptx_child2->GetWitnessHash()))); + package_parent_child2, /*test_accept=*/false); + if (auto err_witness2{CheckPackageMempoolAcceptResult(package_parent_child2, submit_witness2, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_witness2.value()); + } else { + auto it_parent2_deduped = submit_witness2.m_tx_results.find(ptx_parent->GetWitnessHash()); + auto it_child2 = submit_witness2.m_tx_results.find(ptx_child2->GetWitnessHash()); + BOOST_CHECK(it_parent2_deduped->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK(it_child2->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); + BOOST_CHECK_EQUAL(ptx_child1->GetWitnessHash(), it_child2->second.m_other_wtxid.value()); + } // Deduplication should work when wtxid != txid. Submit package with the already-in-mempool // transactions again, which should not fail. const auto submit_segwit_dedup = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_parent, ptx_child1}, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_segwit_dedup.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_segwit_dedup.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_segwit_dedup.m_tx_results.size(), 2); - auto it_parent_dup = submit_segwit_dedup.m_tx_results.find(ptx_parent->GetWitnessHash()); - auto it_child_dup = submit_segwit_dedup.m_tx_results.find(ptx_child1->GetWitnessHash()); - BOOST_CHECK(it_parent_dup->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); - BOOST_CHECK(it_child_dup->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + package_parent_child1, /*test_accept=*/false); + if (auto err_segwit_dedup{CheckPackageMempoolAcceptResult(package_parent_child1, submit_segwit_dedup, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_segwit_dedup.value()); + } else { + auto it_parent_dup = submit_segwit_dedup.m_tx_results.find(ptx_parent->GetWitnessHash()); + auto it_child_dup = submit_segwit_dedup.m_tx_results.find(ptx_child1->GetWitnessHash()); + BOOST_CHECK(it_parent_dup->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK(it_child_dup->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + } } // Try submitting Package1{child2, grandchild} where child2 is same-txid-different-witness as @@ -535,21 +515,17 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // We already submitted child1 above. { + Package package_child2_grandchild{ptx_child2, ptx_grandchild}; const auto submit_spend_ignored = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, - {ptx_child2, ptx_grandchild}, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_spend_ignored.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_spend_ignored.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_spend_ignored.m_tx_results.size(), 2); - auto it_child2_ignored = submit_spend_ignored.m_tx_results.find(ptx_child2->GetWitnessHash()); - auto it_grandchild = submit_spend_ignored.m_tx_results.find(ptx_grandchild->GetWitnessHash()); - BOOST_CHECK(it_child2_ignored != submit_spend_ignored.m_tx_results.end()); - BOOST_CHECK(it_child2_ignored->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); - BOOST_CHECK(it_grandchild != submit_spend_ignored.m_tx_results.end()); - BOOST_CHECK(it_grandchild->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_child2->GetHash()))); - BOOST_CHECK(!m_node.mempool->exists(GenTxid::Wtxid(ptx_child2->GetWitnessHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Wtxid(ptx_grandchild->GetWitnessHash()))); + package_child2_grandchild, /*test_accept=*/false); + if (auto err_spend_ignored{CheckPackageMempoolAcceptResult(package_child2_grandchild, submit_spend_ignored, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_spend_ignored.value()); + } else { + auto it_child2_ignored = submit_spend_ignored.m_tx_results.find(ptx_child2->GetWitnessHash()); + auto it_grandchild = submit_spend_ignored.m_tx_results.find(ptx_grandchild->GetWitnessHash()); + BOOST_CHECK(it_child2_ignored->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); + BOOST_CHECK(it_grandchild->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + } } // A package Package{parent1, parent2, parent3, child} where the parents are a mixture of @@ -641,36 +617,28 @@ BOOST_FIXTURE_TEST_CASE(package_witness_swap_tests, TestChain100Setup) // child should be accepted { const auto mixed_result = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_mixed, false); - BOOST_CHECK_MESSAGE(mixed_result.m_state.IsValid(), mixed_result.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(mixed_result.m_tx_results.size(), package_mixed.size()); - auto it_parent1 = mixed_result.m_tx_results.find(ptx_parent1->GetWitnessHash()); - auto it_parent2 = mixed_result.m_tx_results.find(ptx_parent2_v1->GetWitnessHash()); - auto it_parent3 = mixed_result.m_tx_results.find(ptx_parent3->GetWitnessHash()); - auto it_child = mixed_result.m_tx_results.find(ptx_mixed_child->GetWitnessHash()); - BOOST_CHECK(it_parent1 != mixed_result.m_tx_results.end()); - BOOST_CHECK(it_parent2 != mixed_result.m_tx_results.end()); - BOOST_CHECK(it_parent3 != mixed_result.m_tx_results.end()); - BOOST_CHECK(it_child != mixed_result.m_tx_results.end()); - - BOOST_CHECK(it_parent1->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); - BOOST_CHECK(it_parent2->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); - BOOST_CHECK(it_parent3->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK_EQUAL(ptx_parent2_v2->GetWitnessHash(), it_parent2->second.m_other_wtxid.value()); - - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent1->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent2_v1->GetHash()))); - BOOST_CHECK(!m_node.mempool->exists(GenTxid::Wtxid(ptx_parent2_v1->GetWitnessHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_parent3->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(ptx_mixed_child->GetHash()))); - - // package feerate should include parent3 and child. It should not include parent1 or parent2_v1. - const CFeeRate expected_feerate(1 * COIN, GetVirtualTransactionSize(*ptx_parent3) + GetVirtualTransactionSize(*ptx_mixed_child)); - BOOST_CHECK(it_parent3->second.m_effective_feerate.value() == expected_feerate); - BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); - std::vector<uint256> expected_wtxids({ptx_parent3->GetWitnessHash(), ptx_mixed_child->GetWitnessHash()}); - BOOST_CHECK(it_parent3->second.m_wtxids_fee_calculations.value() == expected_wtxids); - BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); + if (auto err_mixed{CheckPackageMempoolAcceptResult(package_mixed, mixed_result, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_mixed.value()); + } else { + auto it_parent1 = mixed_result.m_tx_results.find(ptx_parent1->GetWitnessHash()); + auto it_parent2 = mixed_result.m_tx_results.find(ptx_parent2_v1->GetWitnessHash()); + auto it_parent3 = mixed_result.m_tx_results.find(ptx_parent3->GetWitnessHash()); + auto it_child = mixed_result.m_tx_results.find(ptx_mixed_child->GetWitnessHash()); + + BOOST_CHECK(it_parent1->second.m_result_type == MempoolAcceptResult::ResultType::MEMPOOL_ENTRY); + BOOST_CHECK(it_parent2->second.m_result_type == MempoolAcceptResult::ResultType::DIFFERENT_WITNESS); + BOOST_CHECK(it_parent3->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK_EQUAL(ptx_parent2_v2->GetWitnessHash(), it_parent2->second.m_other_wtxid.value()); + + // package feerate should include parent3 and child. It should not include parent1 or parent2_v1. + const CFeeRate expected_feerate(1 * COIN, GetVirtualTransactionSize(*ptx_parent3) + GetVirtualTransactionSize(*ptx_mixed_child)); + BOOST_CHECK(it_parent3->second.m_effective_feerate.value() == expected_feerate); + BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); + std::vector<Wtxid> expected_wtxids({ptx_parent3->GetWitnessHash(), ptx_mixed_child->GetWitnessHash()}); + BOOST_CHECK(it_parent3->second.m_wtxids_fee_calculations.value() == expected_wtxids); + BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); + } } } @@ -715,15 +683,17 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_cpfp_deprio = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_cpfp, /*test_accept=*/ false); - BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_state.GetResult(), PackageValidationResult::PCKG_TX); - BOOST_CHECK(submit_cpfp_deprio.m_state.IsInvalid()); - BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_tx_results.find(tx_parent->GetWitnessHash())->second.m_state.GetResult(), - TxValidationResult::TX_MEMPOOL_POLICY); - BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_tx_results.find(tx_child->GetWitnessHash())->second.m_state.GetResult(), - TxValidationResult::TX_MISSING_INPUTS); - BOOST_CHECK(submit_cpfp_deprio.m_tx_results.find(tx_parent->GetWitnessHash())->second.m_state.GetRejectReason() == "min relay fee not met"); - BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - const CFeeRate expected_feerate(0, GetVirtualTransactionSize(*tx_parent) + GetVirtualTransactionSize(*tx_child)); + if (auto err_cpfp_deprio{CheckPackageMempoolAcceptResult(package_cpfp, submit_cpfp_deprio, /*expect_valid=*/false, m_node.mempool.get())}) { + BOOST_ERROR(err_cpfp_deprio.value()); + } else { + BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_state.GetResult(), PackageValidationResult::PCKG_TX); + BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_tx_results.find(tx_parent->GetWitnessHash())->second.m_state.GetResult(), + TxValidationResult::TX_MEMPOOL_POLICY); + BOOST_CHECK_EQUAL(submit_cpfp_deprio.m_tx_results.find(tx_child->GetWitnessHash())->second.m_state.GetResult(), + TxValidationResult::TX_MISSING_INPUTS); + BOOST_CHECK(submit_cpfp_deprio.m_tx_results.find(tx_parent->GetWitnessHash())->second.m_state.GetRejectReason() == "min relay fee not met"); + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); + } } // Clear the prioritisation of the parent transaction. @@ -735,31 +705,27 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_cpfp = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_cpfp, /*test_accept=*/ false); + if (auto err_cpfp{CheckPackageMempoolAcceptResult(package_cpfp, submit_cpfp, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_cpfp.value()); + } else { + auto it_parent = submit_cpfp.m_tx_results.find(tx_parent->GetWitnessHash()); + auto it_child = submit_cpfp.m_tx_results.find(tx_child->GetWitnessHash()); + BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_parent->second.m_base_fees.value() == coinbase_value - parent_value); + BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_child->second.m_base_fees.value() == COIN); + + const CFeeRate expected_feerate(coinbase_value - child_value, + GetVirtualTransactionSize(*tx_parent) + GetVirtualTransactionSize(*tx_child)); + BOOST_CHECK(it_parent->second.m_effective_feerate.value() == expected_feerate); + BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); + std::vector<Wtxid> expected_wtxids({tx_parent->GetWitnessHash(), tx_child->GetWitnessHash()}); + BOOST_CHECK(it_parent->second.m_wtxids_fee_calculations.value() == expected_wtxids); + BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); + BOOST_CHECK(expected_feerate.GetFeePerK() > 1000); + } expected_pool_size += 2; - BOOST_CHECK_MESSAGE(submit_cpfp.m_state.IsValid(), - "Package validation unexpectedly failed: " << submit_cpfp.m_state.GetRejectReason()); - BOOST_CHECK_EQUAL(submit_cpfp.m_tx_results.size(), package_cpfp.size()); - auto it_parent = submit_cpfp.m_tx_results.find(tx_parent->GetWitnessHash()); - auto it_child = submit_cpfp.m_tx_results.find(tx_child->GetWitnessHash()); - BOOST_CHECK(it_parent != submit_cpfp.m_tx_results.end()); - BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_parent->second.m_base_fees.value() == coinbase_value - parent_value); - BOOST_CHECK(it_child != submit_cpfp.m_tx_results.end()); - BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_child->second.m_base_fees.value() == COIN); - BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_parent->GetHash()))); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_child->GetHash()))); - - const CFeeRate expected_feerate(coinbase_value - child_value, - GetVirtualTransactionSize(*tx_parent) + GetVirtualTransactionSize(*tx_child)); - BOOST_CHECK(it_parent->second.m_effective_feerate.value() == expected_feerate); - BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); - std::vector<uint256> expected_wtxids({tx_parent->GetWitnessHash(), tx_child->GetWitnessHash()}); - BOOST_CHECK(it_parent->second.m_wtxids_fee_calculations.value() == expected_wtxids); - BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); - BOOST_CHECK(expected_feerate.GetFeePerK() > 1000); } // Just because we allow low-fee parents doesn't mean we allow low-feerate packages. @@ -785,15 +751,28 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) package_still_too_low.push_back(tx_child_cheap); BOOST_CHECK(m_node.mempool->GetMinFee().GetFee(GetVirtualTransactionSize(*tx_child_cheap)) <= child_fee); BOOST_CHECK(m_node.mempool->GetMinFee().GetFee(GetVirtualTransactionSize(*tx_parent_cheap) + GetVirtualTransactionSize(*tx_child_cheap)) > parent_fee + child_fee); + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - // Cheap package should fail with package-fee-too-low. + // Cheap package should fail for being too low fee. { - BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_package_too_low = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_still_too_low, /*test_accept=*/false); - BOOST_CHECK_MESSAGE(submit_package_too_low.m_state.IsInvalid(), "Package validation unexpectedly succeeded"); - BOOST_CHECK_EQUAL(submit_package_too_low.m_state.GetResult(), PackageValidationResult::PCKG_POLICY); - BOOST_CHECK_EQUAL(submit_package_too_low.m_state.GetRejectReason(), "package-fee-too-low"); + if (auto err_package_too_low{CheckPackageMempoolAcceptResult(package_still_too_low, submit_package_too_low, /*expect_valid=*/false, m_node.mempool.get())}) { + BOOST_ERROR(err_package_too_low.value()); + } else { + // Individual feerate of parent is too low. + BOOST_CHECK_EQUAL(submit_package_too_low.m_tx_results.at(tx_parent_cheap->GetWitnessHash()).m_state.GetResult(), + TxValidationResult::TX_RECONSIDERABLE); + BOOST_CHECK(submit_package_too_low.m_tx_results.at(tx_parent_cheap->GetWitnessHash()).m_effective_feerate.value() == + CFeeRate(parent_fee, GetVirtualTransactionSize(*tx_parent_cheap))); + // Package feerate of parent + child is too low. + BOOST_CHECK_EQUAL(submit_package_too_low.m_tx_results.at(tx_child_cheap->GetWitnessHash()).m_state.GetResult(), + TxValidationResult::TX_RECONSIDERABLE); + BOOST_CHECK(submit_package_too_low.m_tx_results.at(tx_child_cheap->GetWitnessHash()).m_effective_feerate.value() == + CFeeRate(parent_fee + child_fee, GetVirtualTransactionSize(*tx_parent_cheap) + GetVirtualTransactionSize(*tx_child_cheap))); + } + BOOST_CHECK_EQUAL(submit_package_too_low.m_state.GetResult(), PackageValidationResult::PCKG_TX); + BOOST_CHECK_EQUAL(submit_package_too_low.m_state.GetRejectReason(), "transaction failed"); BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); } @@ -804,25 +783,26 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) { const auto submit_prioritised_package = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_still_too_low, /*test_accept=*/false); + if (auto err_prioritised{CheckPackageMempoolAcceptResult(package_still_too_low, submit_prioritised_package, /*expect_valid=*/true, m_node.mempool.get())}) { + BOOST_ERROR(err_prioritised.value()); + } else { + const CFeeRate expected_feerate(1 * COIN + parent_fee + child_fee, + GetVirtualTransactionSize(*tx_parent_cheap) + GetVirtualTransactionSize(*tx_child_cheap)); + BOOST_CHECK_EQUAL(submit_prioritised_package.m_tx_results.size(), package_still_too_low.size()); + auto it_parent = submit_prioritised_package.m_tx_results.find(tx_parent_cheap->GetWitnessHash()); + auto it_child = submit_prioritised_package.m_tx_results.find(tx_child_cheap->GetWitnessHash()); + BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_parent->second.m_base_fees.value() == parent_fee); + BOOST_CHECK(it_parent->second.m_effective_feerate.value() == expected_feerate); + BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_child->second.m_base_fees.value() == child_fee); + BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); + std::vector<Wtxid> expected_wtxids({tx_parent_cheap->GetWitnessHash(), tx_child_cheap->GetWitnessHash()}); + BOOST_CHECK(it_parent->second.m_wtxids_fee_calculations.value() == expected_wtxids); + BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); + } expected_pool_size += 2; - BOOST_CHECK_MESSAGE(submit_prioritised_package.m_state.IsValid(), - "Package validation unexpectedly failed" << submit_prioritised_package.m_state.GetRejectReason()); - const CFeeRate expected_feerate(1 * COIN + parent_fee + child_fee, - GetVirtualTransactionSize(*tx_parent_cheap) + GetVirtualTransactionSize(*tx_child_cheap)); - BOOST_CHECK_EQUAL(submit_prioritised_package.m_tx_results.size(), package_still_too_low.size()); - auto it_parent = submit_prioritised_package.m_tx_results.find(tx_parent_cheap->GetWitnessHash()); - auto it_child = submit_prioritised_package.m_tx_results.find(tx_child_cheap->GetWitnessHash()); - BOOST_CHECK(it_parent != submit_prioritised_package.m_tx_results.end()); - BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_parent->second.m_base_fees.value() == parent_fee); - BOOST_CHECK(it_parent->second.m_effective_feerate.value() == expected_feerate); - BOOST_CHECK(it_child != submit_prioritised_package.m_tx_results.end()); - BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_child->second.m_base_fees.value() == child_fee); - BOOST_CHECK(it_child->second.m_effective_feerate.value() == expected_feerate); - std::vector<uint256> expected_wtxids({tx_parent_cheap->GetWitnessHash(), tx_child_cheap->GetWitnessHash()}); - BOOST_CHECK(it_parent->second.m_wtxids_fee_calculations.value() == expected_wtxids); - BOOST_CHECK(it_child->second.m_wtxids_fee_calculations.value() == expected_wtxids); + BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); } // Package feerate is calculated without topology in mind; it's just aggregating fees and sizes. @@ -851,31 +831,27 @@ BOOST_FIXTURE_TEST_CASE(package_cpfp_tests, TestChain100Setup) BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); const auto submit_rich_parent = ProcessNewPackage(m_node.chainman->ActiveChainstate(), *m_node.mempool, package_rich_parent, /*test_accept=*/false); + if (auto err_rich_parent{CheckPackageMempoolAcceptResult(package_rich_parent, submit_rich_parent, /*expect_valid=*/false, m_node.mempool.get())}) { + BOOST_ERROR(err_rich_parent.value()); + } else { + // The child would have been validated on its own and failed. + BOOST_CHECK_EQUAL(submit_rich_parent.m_state.GetResult(), PackageValidationResult::PCKG_TX); + BOOST_CHECK_EQUAL(submit_rich_parent.m_state.GetRejectReason(), "transaction failed"); + + auto it_parent = submit_rich_parent.m_tx_results.find(tx_parent_rich->GetWitnessHash()); + auto it_child = submit_rich_parent.m_tx_results.find(tx_child_poor->GetWitnessHash()); + BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); + BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::INVALID); + BOOST_CHECK(it_parent->second.m_state.GetRejectReason() == ""); + BOOST_CHECK_MESSAGE(it_parent->second.m_base_fees.value() == high_parent_fee, + strprintf("rich parent: expected fee %s, got %s", high_parent_fee, it_parent->second.m_base_fees.value())); + BOOST_CHECK(it_parent->second.m_effective_feerate == CFeeRate(high_parent_fee, GetVirtualTransactionSize(*tx_parent_rich))); + BOOST_CHECK_EQUAL(it_child->second.m_result_type, MempoolAcceptResult::ResultType::INVALID); + BOOST_CHECK_EQUAL(it_child->second.m_state.GetResult(), TxValidationResult::TX_MEMPOOL_POLICY); + BOOST_CHECK(it_child->second.m_state.GetRejectReason() == "min relay fee not met"); + } expected_pool_size += 1; - BOOST_CHECK_MESSAGE(submit_rich_parent.m_state.IsInvalid(), "Package validation unexpectedly succeeded"); - - // The child would have been validated on its own and failed. - BOOST_CHECK_EQUAL(submit_rich_parent.m_state.GetResult(), PackageValidationResult::PCKG_TX); - BOOST_CHECK_EQUAL(submit_rich_parent.m_state.GetRejectReason(), "transaction failed"); - - auto it_parent = submit_rich_parent.m_tx_results.find(tx_parent_rich->GetWitnessHash()); - auto it_child = submit_rich_parent.m_tx_results.find(tx_child_poor->GetWitnessHash()); - BOOST_CHECK(it_parent != submit_rich_parent.m_tx_results.end()); - BOOST_CHECK(it_child != submit_rich_parent.m_tx_results.end()); - BOOST_CHECK(it_parent->second.m_result_type == MempoolAcceptResult::ResultType::VALID); - BOOST_CHECK(it_child->second.m_result_type == MempoolAcceptResult::ResultType::INVALID); - BOOST_CHECK(it_parent->second.m_state.GetRejectReason() == ""); - BOOST_CHECK_MESSAGE(it_parent->second.m_base_fees.value() == high_parent_fee, - strprintf("rich parent: expected fee %s, got %s", high_parent_fee, it_parent->second.m_base_fees.value())); - BOOST_CHECK(it_parent->second.m_effective_feerate == CFeeRate(high_parent_fee, GetVirtualTransactionSize(*tx_parent_rich))); - BOOST_CHECK(it_child != submit_rich_parent.m_tx_results.end()); - BOOST_CHECK_EQUAL(it_child->second.m_result_type, MempoolAcceptResult::ResultType::INVALID); - BOOST_CHECK_EQUAL(it_child->second.m_state.GetResult(), TxValidationResult::TX_MEMPOOL_POLICY); - BOOST_CHECK(it_child->second.m_state.GetRejectReason() == "min relay fee not met"); - BOOST_CHECK_EQUAL(m_node.mempool->size(), expected_pool_size); - BOOST_CHECK(m_node.mempool->exists(GenTxid::Txid(tx_parent_rich->GetHash()))); - BOOST_CHECK(!m_node.mempool->exists(GenTxid::Txid(tx_child_poor->GetHash()))); } } BOOST_AUTO_TEST_SUITE_END() diff --git a/src/test/util/net.cpp b/src/test/util/net.cpp index bf5a653090..e0404e33ed 100644 --- a/src/test/util/net.cpp +++ b/src/test/util/net.cpp @@ -4,11 +4,15 @@ #include <test/util/net.h> -#include <chainparams.h> -#include <node/eviction.h> #include <net.h> #include <net_processing.h> +#include <netaddress.h> #include <netmessagemaker.h> +#include <node/connection_types.h> +#include <node/eviction.h> +#include <protocol.h> +#include <random.h> +#include <serialize.h> #include <span.h> #include <vector> @@ -98,6 +102,17 @@ bool ConnmanTestMsg::ReceiveMsgFrom(CNode& node, CSerializedNetMsg&& ser_msg) co return complete; } +CNode* ConnmanTestMsg::ConnectNodePublic(PeerManager& peerman, const char* pszDest, ConnectionType conn_type) +{ + CNode* node = ConnectNode(CAddress{}, pszDest, /*fCountFailure=*/false, conn_type, /*use_v2transport=*/true); + if (!node) return nullptr; + node->SetCommonVersion(PROTOCOL_VERSION); + peerman.InitializeNode(*node, ServiceFlags(NODE_NETWORK | NODE_WITNESS)); + node->fSuccessfullyConnected = true; + AddTestNode(*node); + return node; +} + std::vector<NodeEvictionCandidate> GetRandomNodeEvictionCandidates(int n_candidates, FastRandomContext& random_context) { std::vector<NodeEvictionCandidate> candidates; diff --git a/src/test/util/net.h b/src/test/util/net.h index 497292542b..59c4ddb4b1 100644 --- a/src/test/util/net.h +++ b/src/test/util/net.h @@ -6,16 +6,30 @@ #define BITCOIN_TEST_UTIL_NET_H #include <compat/compat.h> -#include <node/eviction.h> -#include <netaddress.h> #include <net.h> +#include <net_permissions.h> +#include <net_processing.h> +#include <netaddress.h> +#include <node/connection_types.h> +#include <node/eviction.h> +#include <sync.h> #include <util/sock.h> +#include <algorithm> #include <array> #include <cassert> +#include <chrono> +#include <cstdint> #include <cstring> #include <memory> #include <string> +#include <unordered_map> +#include <vector> + +class FastRandomContext; + +template <typename C> +class Span; struct ConnmanTestMsg : public CConnman { using CConnman::CConnman; @@ -25,6 +39,12 @@ struct ConnmanTestMsg : public CConnman { m_peer_connect_timeout = timeout; } + std::vector<CNode*> TestNodes() + { + LOCK(m_nodes_mutex); + return m_nodes; + } + void AddTestNode(CNode& node) { LOCK(m_nodes_mutex); @@ -56,6 +76,11 @@ struct ConnmanTestMsg : public CConnman { bool ReceiveMsgFrom(CNode& node, CSerializedNetMsg&& ser_msg) const; void FlushSendBuffer(CNode& node) const; + + bool AlreadyConnectedPublic(const CAddress& addr) { return AlreadyConnectedToAddress(addr); }; + + CNode* ConnectNodePublic(PeerManager& peerman, const char* pszDest, ConnectionType conn_type) + EXCLUSIVE_LOCKS_REQUIRED(!m_unused_i2p_sessions_mutex); }; constexpr ServiceFlags ALL_SERVICE_FLAGS[]{ diff --git a/src/test/util/script.h b/src/test/util/script.h index 428b3e10b3..96c4d55e5a 100644 --- a/src/test/util/script.h +++ b/src/test/util/script.h @@ -18,6 +18,18 @@ static const CScript P2WSH_OP_TRUE{ return hash; }())}; +static const std::vector<uint8_t> EMPTY{}; +static const CScript P2WSH_EMPTY{ + CScript{} + << OP_0 + << ToByteVector([] { + uint256 hash; + CSHA256().Write(EMPTY.data(), EMPTY.size()).Finalize(hash.begin()); + return hash; + }())}; +static const std::vector<std::vector<uint8_t>> P2WSH_EMPTY_TRUE_STACK{{static_cast<uint8_t>(OP_TRUE)}, {}}; +static const std::vector<std::vector<uint8_t>> P2WSH_EMPTY_TWO_STACK{{static_cast<uint8_t>(OP_2)}, {}}; + /** Flags that are not forbidden by an assert in script validation */ bool IsValidFlagCombination(unsigned flags); diff --git a/src/test/util/setup_common.cpp b/src/test/util/setup_common.cpp index 6ae94a8101..9979b75444 100644 --- a/src/test/util/setup_common.cpp +++ b/src/test/util/setup_common.cpp @@ -147,6 +147,7 @@ BasicTestingSetup::BasicTestingSetup(const ChainType chainType, const std::vecto BasicTestingSetup::~BasicTestingSetup() { + m_node.kernel.reset(); SetMockTime(0s); // Reset mocktime for following tests LogInstance().DisconnectTestLogger(); fs::remove_all(m_path_root); @@ -205,8 +206,9 @@ ChainTestingSetup::~ChainTestingSetup() m_node.netgroupman.reset(); m_node.args = nullptr; m_node.mempool.reset(); - m_node.scheduler.reset(); + m_node.fee_estimator.reset(); m_node.chainman.reset(); + m_node.scheduler.reset(); } void ChainTestingSetup::LoadVerifyActivateChainstate() @@ -520,6 +522,6 @@ CBlock getBlock13b8a() { CBlock block; CDataStream stream(ParseHex("0100000090f0a9f110702f808219ebea1173056042a714bad51b916cb6800000000000005275289558f51c9966699404ae2294730c3c9f9bda53523ce50e9b95e558da2fdb261b4d4c86041b1ab1bf930901000000010000000000000000000000000000000000000000000000000000000000000000ffffffff07044c86041b0146ffffffff0100f2052a01000000434104e18f7afbe4721580e81e8414fc8c24d7cfacf254bb5c7b949450c3e997c2dc1242487a8169507b631eb3771f2b425483fb13102c4eb5d858eef260fe70fbfae0ac00000000010000000196608ccbafa16abada902780da4dc35dafd7af05fa0da08cf833575f8cf9e836000000004a493046022100dab24889213caf43ae6adc41cf1c9396c08240c199f5225acf45416330fd7dbd022100fe37900e0644bf574493a07fc5edba06dbc07c311b947520c2d514bc5725dcb401ffffffff0100f2052a010000001976a914f15d1921f52e4007b146dfa60f369ed2fc393ce288ac000000000100000001fb766c1288458c2bafcfec81e48b24d98ec706de6b8af7c4e3c29419bfacb56d000000008c493046022100f268ba165ce0ad2e6d93f089cfcd3785de5c963bb5ea6b8c1b23f1ce3e517b9f022100da7c0f21adc6c401887f2bfd1922f11d76159cbc597fbd756a23dcbb00f4d7290141042b4e8625a96127826915a5b109852636ad0da753c9e1d5606a50480cd0c40f1f8b8d898235e571fe9357d9ec842bc4bba1827daaf4de06d71844d0057707966affffffff0280969800000000001976a9146963907531db72d0ed1a0cfb471ccb63923446f388ac80d6e34c000000001976a914f0688ba1c0d1ce182c7af6741e02658c7d4dfcd388ac000000000100000002c40297f730dd7b5a99567eb8d27b78758f607507c52292d02d4031895b52f2ff010000008b483045022100f7edfd4b0aac404e5bab4fd3889e0c6c41aa8d0e6fa122316f68eddd0a65013902205b09cc8b2d56e1cd1f7f2fafd60a129ed94504c4ac7bdc67b56fe67512658b3e014104732012cb962afa90d31b25d8fb0e32c94e513ab7a17805c14ca4c3423e18b4fb5d0e676841733cb83abaf975845c9f6f2a8097b7d04f4908b18368d6fc2d68ecffffffffca5065ff9617cbcba45eb23726df6498a9b9cafed4f54cbab9d227b0035ddefb000000008a473044022068010362a13c7f9919fa832b2dee4e788f61f6f5d344a7c2a0da6ae740605658022006d1af525b9a14a35c003b78b72bd59738cd676f845d1ff3fc25049e01003614014104732012cb962afa90d31b25d8fb0e32c94e513ab7a17805c14ca4c3423e18b4fb5d0e676841733cb83abaf975845c9f6f2a8097b7d04f4908b18368d6fc2d68ecffffffff01001ec4110200000043410469ab4181eceb28985b9b4e895c13fa5e68d85761b7eee311db5addef76fa8621865134a221bd01f28ec9999ee3e021e60766e9d1f3458c115fb28650605f11c9ac000000000100000001cdaf2f758e91c514655e2dc50633d1e4c84989f8aa90a0dbc883f0d23ed5c2fa010000008b48304502207ab51be6f12a1962ba0aaaf24a20e0b69b27a94fac5adf45aa7d2d18ffd9236102210086ae728b370e5329eead9accd880d0cb070aea0c96255fae6c4f1ddcce1fd56e014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff02404b4c00000000001976a9142b6ba7c9d796b75eef7942fc9288edd37c32f5c388ac002d3101000000001976a9141befba0cdc1ad56529371864d9f6cb042faa06b588ac000000000100000001b4a47603e71b61bc3326efd90111bf02d2f549b067f4c4a8fa183b57a0f800cb010000008a4730440220177c37f9a505c3f1a1f0ce2da777c339bd8339ffa02c7cb41f0a5804f473c9230220585b25a2ee80eb59292e52b987dad92acb0c64eced92ed9ee105ad153cdb12d001410443bd44f683467e549dae7d20d1d79cbdb6df985c6e9c029c8d0c6cb46cc1a4d3cf7923c5021b27f7a0b562ada113bc85d5fda5a1b41e87fe6e8802817cf69996ffffffff0280651406000000001976a9145505614859643ab7b547cd7f1f5e7e2a12322d3788ac00aa0271000000001976a914ea4720a7a52fc166c55ff2298e07baf70ae67e1b88ac00000000010000000586c62cd602d219bb60edb14a3e204de0705176f9022fe49a538054fb14abb49e010000008c493046022100f2bc2aba2534becbdf062eb993853a42bbbc282083d0daf9b4b585bd401aa8c9022100b1d7fd7ee0b95600db8535bbf331b19eed8d961f7a8e54159c53675d5f69df8c014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff03ad0e58ccdac3df9dc28a218bcf6f1997b0a93306faaa4b3a28ae83447b2179010000008b483045022100be12b2937179da88599e27bb31c3525097a07cdb52422d165b3ca2f2020ffcf702200971b51f853a53d644ebae9ec8f3512e442b1bcb6c315a5b491d119d10624c83014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff2acfcab629bbc8685792603762c921580030ba144af553d271716a95089e107b010000008b483045022100fa579a840ac258871365dd48cd7552f96c8eea69bd00d84f05b283a0dab311e102207e3c0ee9234814cfbb1b659b83671618f45abc1326b9edcc77d552a4f2a805c0014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffffdcdc6023bbc9944a658ddc588e61eacb737ddf0a3cd24f113b5a8634c517fcd2000000008b4830450221008d6df731df5d32267954bd7d2dda2302b74c6c2a6aa5c0ca64ecbabc1af03c75022010e55c571d65da7701ae2da1956c442df81bbf076cdbac25133f99d98a9ed34c014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffffe15557cd5ce258f479dfd6dc6514edf6d7ed5b21fcfa4a038fd69f06b83ac76e010000008b483045022023b3e0ab071eb11de2eb1cc3a67261b866f86bf6867d4558165f7c8c8aca2d86022100dc6e1f53a91de3efe8f63512850811f26284b62f850c70ca73ed5de8771fb451014104462e76fd4067b3a0aa42070082dcb0bf2f388b6495cf33d789904f07d0f55c40fbd4b82963c69b3dc31895d0c772c812b1d5fbcade15312ef1c0e8ebbb12dcd4ffffffff01404b4c00000000001976a9142b6ba7c9d796b75eef7942fc9288edd37c32f5c388ac00000000010000000166d7577163c932b4f9690ca6a80b6e4eb001f0a2fa9023df5595602aae96ed8d000000008a4730440220262b42546302dfb654a229cefc86432b89628ff259dc87edd1154535b16a67e102207b4634c020a97c3e7bbd0d4d19da6aa2269ad9dded4026e896b213d73ca4b63f014104979b82d02226b3a4597523845754d44f13639e3bf2df5e82c6aab2bdc79687368b01b1ab8b19875ae3c90d661a3d0a33161dab29934edeb36aa01976be3baf8affffffff02404b4c00000000001976a9144854e695a02af0aeacb823ccbc272134561e0a1688ac40420f00000000001976a914abee93376d6b37b5c2940655a6fcaf1c8e74237988ac0000000001000000014e3f8ef2e91349a9059cb4f01e54ab2597c1387161d3da89919f7ea6acdbb371010000008c49304602210081f3183471a5ca22307c0800226f3ef9c353069e0773ac76bb580654d56aa523022100d4c56465bdc069060846f4fbf2f6b20520b2a80b08b168b31e66ddb9c694e240014104976c79848e18251612f8940875b2b08d06e6dc73b9840e8860c066b7e87432c477e9a59a453e71e6d76d5fe34058b800a098fc1740ce3012e8fc8a00c96af966ffffffff02c0e1e400000000001976a9144134e75a6fcb6042034aab5e18570cf1f844f54788ac404b4c00000000001976a9142b6ba7c9d796b75eef7942fc9288edd37c32f5c388ac00000000"), SER_NETWORK, PROTOCOL_VERSION); - stream >> block; + stream >> TX_WITH_WITNESS(block); return block; } diff --git a/src/test/util/txmempool.cpp b/src/test/util/txmempool.cpp index 147e589deb..6d3bb01be8 100644 --- a/src/test/util/txmempool.cpp +++ b/src/test/util/txmempool.cpp @@ -89,11 +89,14 @@ std::optional<std::string> CheckPackageMempoolAcceptResult(const Package& txns, } // m_effective_feerate and m_wtxids_fee_calculations should exist iff the result was valid - if (atmp_result.m_effective_feerate.has_value() != valid) { + // or if the failure was TX_RECONSIDERABLE + const bool valid_or_reconsiderable{atmp_result.m_result_type == MempoolAcceptResult::ResultType::VALID || + atmp_result.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE}; + if (atmp_result.m_effective_feerate.has_value() != valid_or_reconsiderable) { return strprintf("tx %s result should %shave m_effective_feerate", wtxid.ToString(), valid ? "" : "not "); } - if (atmp_result.m_wtxids_fee_calculations.has_value() != valid) { + if (atmp_result.m_wtxids_fee_calculations.has_value() != valid_or_reconsiderable) { return strprintf("tx %s result should %shave m_effective_feerate", wtxid.ToString(), valid ? "" : "not "); } diff --git a/src/test/validation_block_tests.cpp b/src/test/validation_block_tests.cpp index 64cb5522eb..35e5c6a037 100644 --- a/src/test/validation_block_tests.cpp +++ b/src/test/validation_block_tests.cpp @@ -283,8 +283,7 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) // Check that all txs are in the pool { - LOCK(m_node.mempool->cs); - BOOST_CHECK_EQUAL(m_node.mempool->mapTx.size(), txs.size()); + BOOST_CHECK_EQUAL(m_node.mempool->size(), txs.size()); } // Run a thread that simulates an RPC caller that is polling while @@ -295,7 +294,7 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) // not some intermediate amount. while (true) { LOCK(m_node.mempool->cs); - if (m_node.mempool->mapTx.size() == 0) { + if (m_node.mempool->size() == 0) { // We are done with the reorg break; } @@ -304,7 +303,7 @@ BOOST_AUTO_TEST_CASE(mempool_locks_reorg) // be atomic. So the caller assumes that the returned mempool // is consistent. That is, it has all txs that were there // before the reorg. - assert(m_node.mempool->mapTx.size() == txs.size()); + assert(m_node.mempool->size() == txs.size()); continue; } LOCK(cs_main); diff --git a/src/txmempool.cpp b/src/txmempool.cpp index 461662ad93..e057d7ece1 100644 --- a/src/txmempool.cpp +++ b/src/txmempool.cpp @@ -412,6 +412,7 @@ CTxMemPool::CTxMemPool(const Options& opts) m_max_datacarrier_bytes{opts.max_datacarrier_bytes}, m_require_standard{opts.require_standard}, m_full_rbf{opts.full_rbf}, + m_persist_v1_dat{opts.persist_v1_dat}, m_limits{opts.limits} { } @@ -480,8 +481,8 @@ void CTxMemPool::addUnchecked(const CTxMemPoolEntry &entry, setEntries &setAnces minerPolicyEstimator->processTransaction(entry, validFeeEstimate); } - vTxHashes.emplace_back(tx.GetWitnessHash(), newit); - newit->vTxHashesIdx = vTxHashes.size() - 1; + txns_randomized.emplace_back(newit->GetSharedTx()); + newit->idx_randomized = txns_randomized.size() - 1; TRACE3(mempool, added, entry.GetTx().GetHash().data(), @@ -517,14 +518,16 @@ void CTxMemPool::removeUnchecked(txiter it, MemPoolRemovalReason reason) RemoveUnbroadcastTx(hash, true /* add logging because unchecked */ ); - if (vTxHashes.size() > 1) { - vTxHashes[it->vTxHashesIdx] = std::move(vTxHashes.back()); - vTxHashes[it->vTxHashesIdx].second->vTxHashesIdx = it->vTxHashesIdx; - vTxHashes.pop_back(); - if (vTxHashes.size() * 2 < vTxHashes.capacity()) - vTxHashes.shrink_to_fit(); + if (txns_randomized.size() > 1) { + // Update idx_randomized of the to-be-moved entry. + Assert(GetEntry(txns_randomized.back()->GetHash()))->idx_randomized = it->idx_randomized; + // Remove entry from txns_randomized by replacing it with the back and deleting the back. + txns_randomized[it->idx_randomized] = std::move(txns_randomized.back()); + txns_randomized.pop_back(); + if (txns_randomized.size() * 2 < txns_randomized.capacity()) + txns_randomized.shrink_to_fit(); } else - vTxHashes.clear(); + txns_randomized.clear(); totalTxSize -= it->GetTxSize(); m_total_fee -= it->GetFee(); @@ -836,6 +839,18 @@ static TxMempoolInfo GetInfo(CTxMemPool::indexed_transaction_set::const_iterator return TxMempoolInfo{it->GetSharedTx(), it->GetTime(), it->GetFee(), it->GetTxSize(), it->GetModifiedFee() - it->GetFee()}; } +std::vector<CTxMemPoolEntryRef> CTxMemPool::entryAll() const +{ + AssertLockHeld(cs); + + std::vector<CTxMemPoolEntryRef> ret; + ret.reserve(mapTx.size()); + for (const auto& it : GetSortedDepthAndScore()) { + ret.emplace_back(*it); + } + return ret; +} + std::vector<TxMempoolInfo> CTxMemPool::infoAll() const { LOCK(cs); @@ -850,6 +865,13 @@ std::vector<TxMempoolInfo> CTxMemPool::infoAll() const return ret; } +const CTxMemPoolEntry* CTxMemPool::GetEntry(const Txid& txid) const +{ + AssertLockHeld(cs); + const auto i = mapTx.find(txid); + return i == mapTx.end() ? nullptr : &(*i); +} + CTransactionRef CTxMemPool::get(const uint256& hash) const { LOCK(cs); @@ -1033,7 +1055,7 @@ void CCoinsViewMemPool::Reset() size_t CTxMemPool::DynamicMemoryUsage() const { LOCK(cs); // Estimate the overhead of mapTx to be 15 pointers + an allocation, as no exact formula for boost::multi_index_contained is implemented. - return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 15 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(vTxHashes) + cachedInnerUsage; + return memusage::MallocUsage(sizeof(CTxMemPoolEntry) + 15 * sizeof(void*)) * mapTx.size() + memusage::DynamicUsage(mapNextTx) + memusage::DynamicUsage(mapDeltas) + memusage::DynamicUsage(txns_randomized) + cachedInnerUsage; } void CTxMemPool::RemoveUnbroadcastTx(const uint256& txid, const bool unchecked) { diff --git a/src/txmempool.h b/src/txmempool.h index 374c212a44..aed6acd9da 100644 --- a/src/txmempool.h +++ b/src/txmempool.h @@ -392,7 +392,7 @@ public: indexed_transaction_set mapTx GUARDED_BY(cs); using txiter = indexed_transaction_set::nth_index<0>::type::const_iterator; - std::vector<std::pair<uint256, txiter>> vTxHashes GUARDED_BY(cs); //!< All tx witness hashes/entries in mapTx, in random order + std::vector<CTransactionRef> txns_randomized GUARDED_BY(cs); //!< All transactions in mapTx, in random order typedef std::set<txiter, CompareIteratorByHash> setEntries; @@ -446,6 +446,7 @@ public: const std::optional<unsigned> m_max_datacarrier_bytes; const bool m_require_standard; const bool m_full_rbf; + const bool m_persist_v1_dat; const Limits m_limits; @@ -684,6 +685,8 @@ public: return (mapTx.count(gtxid.GetHash()) != 0); } + const CTxMemPoolEntry* GetEntry(const Txid& txid) const LIFETIMEBOUND EXCLUSIVE_LOCKS_REQUIRED(cs); + CTransactionRef get(const uint256& hash) const; txiter get_iter_from_wtxid(const uint256& wtxid) const EXCLUSIVE_LOCKS_REQUIRED(cs) { @@ -695,6 +698,7 @@ public: /** Returns info for a transaction if its entry_sequence < last_sequence */ TxMempoolInfo info_for_relay(const GenTxid& gtxid, uint64_t last_sequence) const; + std::vector<CTxMemPoolEntryRef> entryAll() const EXCLUSIVE_LOCKS_REQUIRED(cs); std::vector<TxMempoolInfo> infoAll() const; size_t DynamicMemoryUsage() const; diff --git a/src/util/fs.h b/src/util/fs.h index 8f79f6cba6..7e2803b6aa 100644 --- a/src/util/fs.h +++ b/src/util/fs.h @@ -184,6 +184,7 @@ static inline path PathFromString(const std::string& string) * already exists or is a symlink to an existing directory. * This is a temporary workaround for an issue in libstdc++ that has been fixed * upstream [PR101510]. + * https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101510 */ static inline bool create_directories(const std::filesystem::path& p) { diff --git a/src/util/fs_helpers.cpp b/src/util/fs_helpers.cpp index 2a9eb3502e..8aa7493aa8 100644 --- a/src/util/fs_helpers.cpp +++ b/src/util/fs_helpers.cpp @@ -11,13 +11,11 @@ #include <logging.h> #include <sync.h> -#include <tinyformat.h> #include <util/fs.h> #include <util/getuniquepath.h> #include <util/syserror.h> #include <cerrno> -#include <filesystem> #include <fstream> #include <map> #include <memory> @@ -263,7 +261,7 @@ bool RenameOver(fs::path src, fs::path dest) { #ifdef __MINGW64__ // This is a workaround for a bug in libstdc++ which - // implements std::filesystem::rename with _wrename function. + // implements fs::rename with _wrename function. // This bug has been fixed in upstream: // - GCC 10.3: 8dd1c1085587c9f8a21bb5e588dfe1e8cdbba79e // - GCC 11.1: 1dfd95f0a0ca1d9e6cbc00e6cbfd1fa20a98f312 diff --git a/src/util/sock.cpp b/src/util/sock.cpp index d16dc56aa3..e896b87160 100644 --- a/src/util/sock.cpp +++ b/src/util/sock.cpp @@ -242,7 +242,7 @@ bool Sock::WaitMany(std::chrono::milliseconds timeout, EventsPerSock& events_per #endif /* USE_POLL */ } -void Sock::SendComplete(const std::string& data, +void Sock::SendComplete(Span<const unsigned char> data, std::chrono::milliseconds timeout, CThreadInterrupt& interrupt) const { @@ -283,6 +283,13 @@ void Sock::SendComplete(const std::string& data, } } +void Sock::SendComplete(Span<const char> data, + std::chrono::milliseconds timeout, + CThreadInterrupt& interrupt) const +{ + SendComplete(MakeUCharSpan(data), timeout, interrupt); +} + std::string Sock::RecvUntilTerminator(uint8_t terminator, std::chrono::milliseconds timeout, CThreadInterrupt& interrupt, diff --git a/src/util/sock.h b/src/util/sock.h index d78e01929b..65e7ffc165 100644 --- a/src/util/sock.h +++ b/src/util/sock.h @@ -228,7 +228,14 @@ public: * @throws std::runtime_error if the operation cannot be completed. In this case only some of * the data will be written to the socket. */ - virtual void SendComplete(const std::string& data, + virtual void SendComplete(Span<const unsigned char> data, + std::chrono::milliseconds timeout, + CThreadInterrupt& interrupt) const; + + /** + * Convenience method, equivalent to `SendComplete(MakeUCharSpan(data), timeout, interrupt)`. + */ + virtual void SendComplete(Span<const char> data, std::chrono::milliseconds timeout, CThreadInterrupt& interrupt) const; diff --git a/src/validation.cpp b/src/validation.cpp index 8d7e366125..ed72a7c97a 100644 --- a/src/validation.cpp +++ b/src/validation.cpp @@ -353,7 +353,7 @@ void Chainstate::MaybeUpdateMempoolForReorg( const std::optional<LockPoints> new_lock_points{CalculateLockPointsAtTip(m_chain.Tip(), view_mempool, tx)}; if (new_lock_points.has_value() && CheckSequenceLocksAtTip(m_chain.Tip(), *new_lock_points)) { // Now update the mempool entry lockpoints as well. - m_mempool->mapTx.modify(it, [&new_lock_points](CTxMemPoolEntry& e) { e.UpdateLockPoints(*new_lock_points); }); + it->UpdateLockPoints(*new_lock_points); } else { return true; } @@ -362,9 +362,7 @@ void Chainstate::MaybeUpdateMempoolForReorg( // If the transaction spends any coinbase outputs, it must be mature. if (it->GetSpendsCoinbase()) { for (const CTxIn& txin : tx.vin) { - auto it2 = m_mempool->mapTx.find(txin.prevout.hash); - if (it2 != m_mempool->mapTx.end()) - continue; + if (m_mempool->exists(GenTxid::Txid(txin.prevout.hash))) continue; const Coin& coin{CoinsTip().AccessCoin(txin.prevout)}; assert(!coin.IsSpent()); const auto mempool_spend_height{m_chain.Tip()->nHeight + 1}; @@ -670,11 +668,11 @@ private: AssertLockHeld(m_pool.cs); CAmount mempoolRejectFee = m_pool.GetMinFee().GetFee(package_size); if (mempoolRejectFee > 0 && package_fee < mempoolRejectFee) { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool min fee not met", strprintf("%d < %d", package_fee, mempoolRejectFee)); + return state.Invalid(TxValidationResult::TX_RECONSIDERABLE, "mempool min fee not met", strprintf("%d < %d", package_fee, mempoolRejectFee)); } if (package_fee < m_pool.m_min_relay_feerate.GetFee(package_size)) { - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "min relay fee not met", + return state.Invalid(TxValidationResult::TX_RECONSIDERABLE, "min relay fee not met", strprintf("%d < %d", package_fee, m_pool.m_min_relay_feerate.GetFee(package_size))); } return true; @@ -724,7 +722,7 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) } // Transactions smaller than 65 non-witness bytes are not relayed to mitigate CVE-2017-12842. - if (::GetSerializeSize(tx, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) < MIN_STANDARD_TX_NONWITNESS_SIZE) + if (::GetSerializeSize(TX_NO_WITNESS(tx)) < MIN_STANDARD_TX_NONWITNESS_SIZE) return state.Invalid(TxValidationResult::TX_NOT_STANDARD, "tx-size-small"); // Only accept nLockTime-using transactions that can be mined in the next @@ -867,6 +865,8 @@ bool MemPoolAccept::PreChecks(ATMPArgs& args, Workspace& ws) // method of ensuring the tx remains bumped. For example, the fee-bumping child could disappear // due to a replacement. if (!bypass_limits && ws.m_modified_fees < m_pool.m_min_relay_feerate.GetFee(ws.m_vsize)) { + // Even though this is a fee-related failure, this result is TX_MEMPOOL_POLICY, not + // TX_RECONSIDERABLE, because it cannot be bypassed using package validation. return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "min relay fee not met", strprintf("%d < %d", ws.m_modified_fees, m_pool.m_min_relay_feerate.GetFee(ws.m_vsize))); } @@ -981,6 +981,9 @@ bool MemPoolAccept::ReplacementChecks(Workspace& ws) // descendant transaction of a direct conflict to pay a higher feerate than the transaction that // might replace them, under these rules. if (const auto err_string{PaysMoreThanConflicts(ws.m_iters_conflicting, newFeeRate, hash)}) { + // Even though this is a fee-related failure, this result is TX_MEMPOOL_POLICY, not + // TX_RECONSIDERABLE, because it cannot be bypassed using package validation. + // This must be changed if package RBF is enabled. return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", *err_string); } @@ -1002,6 +1005,9 @@ bool MemPoolAccept::ReplacementChecks(Workspace& ws) } if (const auto err_string{PaysForRBF(ws.m_conflicting_fees, ws.m_modified_fees, ws.m_vsize, m_pool.m_incremental_relay_feerate, hash)}) { + // Even though this is a fee-related failure, this result is TX_MEMPOOL_POLICY, not + // TX_RECONSIDERABLE, because it cannot be bypassed using package validation. + // This must be changed if package RBF is enabled. return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "insufficient fee", *err_string); } return true; @@ -1139,7 +1145,8 @@ bool MemPoolAccept::Finalize(const ATMPArgs& args, Workspace& ws) if (!args.m_package_submission && !bypass_limits) { LimitMempoolSize(m_pool, m_active_chainstate.CoinsTip()); if (!m_pool.exists(GenTxid::Txid(hash))) - return state.Invalid(TxValidationResult::TX_MEMPOOL_POLICY, "mempool full"); + // The tx no longer meets our (new) mempool minimum feerate but could be reconsidered in a package. + return state.Invalid(TxValidationResult::TX_RECONSIDERABLE, "mempool full"); } return true; } @@ -1201,7 +1208,7 @@ bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& } } - std::vector<uint256> all_package_wtxids; + std::vector<Wtxid> all_package_wtxids; all_package_wtxids.reserve(workspaces.size()); std::transform(workspaces.cbegin(), workspaces.cend(), std::back_inserter(all_package_wtxids), [](const auto& ws) { return ws.m_ptx->GetWitnessHash(); }); @@ -1211,7 +1218,7 @@ bool MemPoolAccept::SubmitPackage(const ATMPArgs& args, std::vector<Workspace>& const auto effective_feerate = args.m_package_feerates ? ws.m_package_feerate : CFeeRate{ws.m_modified_fees, static_cast<uint32_t>(ws.m_vsize)}; const auto effective_feerate_wtxids = args.m_package_feerates ? all_package_wtxids : - std::vector<uint256>({ws.m_ptx->GetWitnessHash()}); + std::vector<Wtxid>{ws.m_ptx->GetWitnessHash()}; results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, ws.m_base_fees, effective_feerate, effective_feerate_wtxids)); @@ -1226,8 +1233,15 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef LOCK(m_pool.cs); // mempool "read lock" (held through GetMainSignals().TransactionAddedToMempool()) Workspace ws(ptx); + const std::vector<Wtxid> single_wtxid{ws.m_ptx->GetWitnessHash()}; - if (!PreChecks(args, ws)) return MempoolAcceptResult::Failure(ws.m_state); + if (!PreChecks(args, ws)) { + if (ws.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE) { + // Failed for fee reasons. Provide the effective feerate and which tx was included. + return MempoolAcceptResult::FeeFailure(ws.m_state, CFeeRate(ws.m_modified_fees, ws.m_vsize), single_wtxid); + } + return MempoolAcceptResult::Failure(ws.m_state); + } if (m_rbf && !ReplacementChecks(ws)) return MempoolAcceptResult::Failure(ws.m_state); @@ -1238,14 +1252,18 @@ MempoolAcceptResult MemPoolAccept::AcceptSingleTransaction(const CTransactionRef if (!ConsensusScriptChecks(args, ws)) return MempoolAcceptResult::Failure(ws.m_state); const CFeeRate effective_feerate{ws.m_modified_fees, static_cast<uint32_t>(ws.m_vsize)}; - const std::vector<uint256> single_wtxid{ws.m_ptx->GetWitnessHash()}; // Tx was accepted, but not added if (args.m_test_accept) { return MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, ws.m_base_fees, effective_feerate, single_wtxid); } - if (!Finalize(args, ws)) return MempoolAcceptResult::Failure(ws.m_state); + if (!Finalize(args, ws)) { + // The only possible failure reason is fee-related (mempool full). + // Failed for fee reasons. Provide the effective feerate and which txns were included. + Assume(ws.m_state.GetResult() == TxValidationResult::TX_RECONSIDERABLE); + return MempoolAcceptResult::FeeFailure(ws.m_state, CFeeRate(ws.m_modified_fees, ws.m_vsize), {ws.m_ptx->GetWitnessHash()}); + } GetMainSignals().TransactionAddedToMempool(ptx, m_pool.GetAndIncrementSequence()); @@ -1299,11 +1317,16 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: const auto m_total_modified_fees = std::accumulate(workspaces.cbegin(), workspaces.cend(), CAmount{0}, [](CAmount sum, auto& ws) { return sum + ws.m_modified_fees; }); const CFeeRate package_feerate(m_total_modified_fees, m_total_vsize); + std::vector<Wtxid> all_package_wtxids; + all_package_wtxids.reserve(workspaces.size()); + std::transform(workspaces.cbegin(), workspaces.cend(), std::back_inserter(all_package_wtxids), + [](const auto& ws) { return ws.m_ptx->GetWitnessHash(); }); TxValidationState placeholder_state; if (args.m_package_feerates && !CheckFeeRate(m_total_vsize, m_total_modified_fees, placeholder_state)) { - package_state.Invalid(PackageValidationResult::PCKG_POLICY, "package-fee-too-low"); - return PackageMempoolAcceptResult(package_state, {}); + package_state.Invalid(PackageValidationResult::PCKG_TX, "transaction failed"); + return PackageMempoolAcceptResult(package_state, {{workspaces.back().m_ptx->GetWitnessHash(), + MempoolAcceptResult::FeeFailure(placeholder_state, CFeeRate(m_total_modified_fees, m_total_vsize), all_package_wtxids)}}); } // Apply package mempool ancestor/descendant limits. Skip if there is only one transaction, @@ -1314,10 +1337,6 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: return PackageMempoolAcceptResult(package_state, std::move(results)); } - std::vector<uint256> all_package_wtxids; - all_package_wtxids.reserve(workspaces.size()); - std::transform(workspaces.cbegin(), workspaces.cend(), std::back_inserter(all_package_wtxids), - [](const auto& ws) { return ws.m_ptx->GetWitnessHash(); }); for (Workspace& ws : workspaces) { ws.m_package_feerate = package_feerate; if (!PolicyScriptChecks(args, ws)) { @@ -1330,7 +1349,7 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptMultipleTransactions(const std:: const auto effective_feerate = args.m_package_feerates ? ws.m_package_feerate : CFeeRate{ws.m_modified_fees, static_cast<uint32_t>(ws.m_vsize)}; const auto effective_feerate_wtxids = args.m_package_feerates ? all_package_wtxids : - std::vector<uint256>{ws.m_ptx->GetWitnessHash()}; + std::vector<Wtxid>{ws.m_ptx->GetWitnessHash()}; results.emplace(ws.m_ptx->GetWitnessHash(), MempoolAcceptResult::Success(std::move(ws.m_replaced_transactions), ws.m_vsize, ws.m_base_fees, effective_feerate, @@ -1485,9 +1504,8 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, // transactions that are already in the mempool, and only call AcceptMultipleTransactions() with // the new transactions. This ensures we don't double-count transaction counts and sizes when // checking ancestor/descendant limits, or double-count transaction fees for fee-related policy. - auto iter = m_pool.GetIter(txid); - assert(iter != std::nullopt); - results_final.emplace(wtxid, MempoolAcceptResult::MempoolTx(iter.value()->GetTxSize(), iter.value()->GetFee())); + const auto& entry{*Assert(m_pool.GetEntry(txid))}; + results_final.emplace(wtxid, MempoolAcceptResult::MempoolTx(entry.GetTxSize(), entry.GetFee())); } else if (m_pool.exists(GenTxid::Txid(txid))) { // Transaction with the same non-witness data but different witness (same txid, // different wtxid) already exists in the mempool. @@ -1496,10 +1514,9 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, // transaction for the mempool one. Note that we are ignoring the validity of the // package transaction passed in. // TODO: allow witness replacement in packages. - auto iter = m_pool.GetIter(txid); - assert(iter != std::nullopt); + const auto& entry{*Assert(m_pool.GetEntry(txid))}; // Provide the wtxid of the mempool tx so that the caller can look it up in the mempool. - results_final.emplace(wtxid, MempoolAcceptResult::MempoolTxDifferentWitness(iter.value()->GetTx().GetWitnessHash())); + results_final.emplace(wtxid, MempoolAcceptResult::MempoolTxDifferentWitness(entry.GetTx().GetWitnessHash())); } else { // Transaction does not already exist in the mempool. // Try submitting the transaction on its own. @@ -1510,7 +1527,7 @@ PackageMempoolAcceptResult MemPoolAccept::AcceptPackage(const Package& package, // in package validation, because its fees should only be "used" once. assert(m_pool.exists(GenTxid::Wtxid(wtxid))); results_final.emplace(wtxid, single_res); - } else if (single_res.m_state.GetResult() != TxValidationResult::TX_MEMPOOL_POLICY && + } else if (single_res.m_state.GetResult() != TxValidationResult::TX_RECONSIDERABLE && single_res.m_state.GetResult() != TxValidationResult::TX_MISSING_INPUTS) { // Package validation policy only differs from individual policy in its evaluation // of feerate. For example, if a transaction fails here due to violation of a @@ -3668,7 +3685,7 @@ bool CheckBlock(const CBlock& block, BlockValidationState& state, const Consensu // checks that use witness data may be performed here. // Size limits - if (block.vtx.empty() || block.vtx.size() * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT || ::GetSerializeSize(block, PROTOCOL_VERSION | SERIALIZE_TRANSACTION_NO_WITNESS) * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT) + if (block.vtx.empty() || block.vtx.size() * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT || ::GetSerializeSize(TX_NO_WITNESS(block)) * WITNESS_SCALE_FACTOR > MAX_BLOCK_WEIGHT) return state.Invalid(BlockValidationResult::BLOCK_CONSENSUS, "bad-blk-length", "size limits failed"); // First transaction must be coinbase, the rest must not be @@ -4707,7 +4724,7 @@ void ChainstateManager::LoadExternalBlockFile( // This block can be processed immediately; rewind to its start, read and deserialize it. blkdat.SetPos(nBlockPos); pblock = std::make_shared<CBlock>(); - blkdat >> *pblock; + blkdat >> TX_WITH_WITNESS(*pblock); nRewind = blkdat.GetPos(); BlockValidationState state; diff --git a/src/validation.h b/src/validation.h index 7ce60da634..e669ec46d5 100644 --- a/src/validation.h +++ b/src/validation.h @@ -114,7 +114,27 @@ double GuessVerificationProgress(const ChainTxData& data, const CBlockIndex* pin void PruneBlockFilesManual(Chainstate& active_chainstate, int nManualPruneHeight); /** -* Validation result for a single transaction mempool acceptance. +* Validation result for a transaction evaluated by MemPoolAccept (single or package). +* Here are the expected fields and properties of a result depending on its ResultType, applicable to +* results returned from package evaluation: +*+---------------------------+----------------+-------------------+------------------+----------------+-------------------+ +*| Field or property | VALID | INVALID | MEMPOOL_ENTRY | DIFFERENT_WITNESS | +*| | |--------------------------------------| | | +*| | | TX_RECONSIDERABLE | Other | | | +*+---------------------------+----------------+-------------------+------------------+----------------+-------------------+ +*| txid in mempool? | yes | no | no* | yes | yes | +*| wtxid in mempool? | yes | no | no* | yes | no | +*| m_state | yes, IsValid() | yes, IsInvalid() | yes, IsInvalid() | yes, IsValid() | yes, IsValid() | +*| m_replaced_transactions | yes | no | no | no | no | +*| m_vsize | yes | no | no | yes | no | +*| m_base_fees | yes | no | no | yes | no | +*| m_effective_feerate | yes | yes | no | no | no | +*| m_wtxids_fee_calculations | yes | yes | no | no | no | +*| m_other_wtxid | no | no | no | no | yes | +*+---------------------------+----------------+-------------------+------------------+----------------+-------------------+ +* (*) Individual transaction acceptance doesn't return MEMPOOL_ENTRY and DIFFERENT_WITNESS. It returns +* INVALID, with the errors txn-already-in-mempool and txn-same-nonwitness-data-in-mempool +* respectively. In those cases, the txid or wtxid may be in the mempool for a TX_CONFLICT. */ struct MempoolAcceptResult { /** Used to indicate the results of mempool validation. */ @@ -130,7 +150,6 @@ struct MempoolAcceptResult { /** Contains information about why the transaction failed. */ const TxValidationState m_state; - // The following fields are only present when m_result_type = ResultType::VALID or MEMPOOL_ENTRY /** Mempool transactions replaced by the tx. */ const std::optional<std::list<CTransactionRef>> m_replaced_transactions; /** Virtual size as used by the mempool, calculated using serialized size and sigops. */ @@ -141,7 +160,6 @@ struct MempoolAcceptResult { * using prioritisetransaction (i.e. modified fees). If this transaction was submitted as a * package, this is the package feerate, which may also include its descendants and/or * ancestors (see m_wtxids_fee_calculations below). - * Only present when m_result_type = ResultType::VALID. */ const std::optional<CFeeRate> m_effective_feerate; /** Contains the wtxids of the transactions used for fee-related checks. Includes this @@ -149,9 +167,8 @@ struct MempoolAcceptResult { * package. This is not necessarily equivalent to the list of transactions passed to * ProcessNewPackage(). * Only present when m_result_type = ResultType::VALID. */ - const std::optional<std::vector<uint256>> m_wtxids_fee_calculations; + const std::optional<std::vector<Wtxid>> m_wtxids_fee_calculations; - // The following field is only present when m_result_type = ResultType::DIFFERENT_WITNESS /** The wtxid of the transaction in the mempool which has the same txid but different witness. */ const std::optional<uint256> m_other_wtxid; @@ -159,11 +176,17 @@ struct MempoolAcceptResult { return MempoolAcceptResult(state); } + static MempoolAcceptResult FeeFailure(TxValidationState state, + CFeeRate effective_feerate, + const std::vector<Wtxid>& wtxids_fee_calculations) { + return MempoolAcceptResult(state, effective_feerate, wtxids_fee_calculations); + } + static MempoolAcceptResult Success(std::list<CTransactionRef>&& replaced_txns, int64_t vsize, CAmount fees, CFeeRate effective_feerate, - const std::vector<uint256>& wtxids_fee_calculations) { + const std::vector<Wtxid>& wtxids_fee_calculations) { return MempoolAcceptResult(std::move(replaced_txns), vsize, fees, effective_feerate, wtxids_fee_calculations); } @@ -189,7 +212,7 @@ private: int64_t vsize, CAmount fees, CFeeRate effective_feerate, - const std::vector<uint256>& wtxids_fee_calculations) + const std::vector<Wtxid>& wtxids_fee_calculations) : m_result_type(ResultType::VALID), m_replaced_transactions(std::move(replaced_txns)), m_vsize{vsize}, @@ -197,6 +220,15 @@ private: m_effective_feerate(effective_feerate), m_wtxids_fee_calculations(wtxids_fee_calculations) {} + /** Constructor for fee-related failure case */ + explicit MempoolAcceptResult(TxValidationState state, + CFeeRate effective_feerate, + const std::vector<Wtxid>& wtxids_fee_calculations) + : m_result_type(ResultType::INVALID), + m_state(state), + m_effective_feerate(effective_feerate), + m_wtxids_fee_calculations(wtxids_fee_calculations) {} + /** Constructor for already-in-mempool case. It wouldn't replace any transactions. */ explicit MempoolAcceptResult(int64_t vsize, CAmount fees) : m_result_type(ResultType::MEMPOOL_ENTRY), m_vsize{vsize}, m_base_fees(fees) {} diff --git a/src/version.h b/src/version.h index 204df3758b..c2ebeecbfb 100644 --- a/src/version.h +++ b/src/version.h @@ -35,7 +35,4 @@ static const int INVALID_CB_NO_BAN_VERSION = 70015; //! "wtxidrelay" command for wtxid-based relay starts with this version static const int WTXID_RELAY_VERSION = 70016; -// Make sure that none of the values above collide with -// `SERIALIZE_TRANSACTION_NO_WITNESS`. - #endif // BITCOIN_VERSION_H diff --git a/src/wallet/context.h b/src/wallet/context.h index 57a6ed77f7..58b9924fb4 100644 --- a/src/wallet/context.h +++ b/src/wallet/context.h @@ -13,6 +13,7 @@ #include <vector> class ArgsManager; +class CScheduler; namespace interfaces { class Chain; class Wallet; @@ -34,6 +35,7 @@ using LoadWalletFn = std::function<void(std::unique_ptr<interfaces::Wallet> wall //! behavior. struct WalletContext { interfaces::Chain* chain{nullptr}; + CScheduler* scheduler{nullptr}; ArgsManager* args{nullptr}; // Currently a raw pointer because the memory is not managed by this struct // It is unsafe to lock this after locking a CWallet::cs_wallet mutex because // this could introduce inconsistent lock ordering and cause deadlocks. diff --git a/src/wallet/feebumper.cpp b/src/wallet/feebumper.cpp index cafa6c70a1..d9a08310a8 100644 --- a/src/wallet/feebumper.cpp +++ b/src/wallet/feebumper.cpp @@ -86,7 +86,7 @@ static feebumper::Result CheckFeeRate(const CWallet& wallet, const CMutableTrans reused_inputs.push_back(txin.prevout); } - std::optional<CAmount> combined_bump_fee = wallet.chain().CalculateCombinedBumpFee(reused_inputs, newFeerate); + std::optional<CAmount> combined_bump_fee = wallet.chain().calculateCombinedBumpFee(reused_inputs, newFeerate); if (!combined_bump_fee.has_value()) { errors.push_back(strprintf(Untranslated("Failed to calculate bump fees, because unconfirmed UTXOs depend on enormous cluster of unconfirmed transactions."))); } diff --git a/src/wallet/init.cpp b/src/wallet/init.cpp index 0d0a8650ac..088343458c 100644 --- a/src/wallet/init.cpp +++ b/src/wallet/init.cpp @@ -95,8 +95,6 @@ void WalletInit::AddWalletOptions(ArgsManager& argsman) const argsman.AddArg("-walletrejectlongchains", strprintf("Wallet will not create transactions that violate mempool chain limits (default: %u)", DEFAULT_WALLET_REJECT_LONG_CHAINS), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); argsman.AddArg("-walletcrosschain", strprintf("Allow reusing wallet files across chains (default: %u)", DEFAULT_WALLETCROSSCHAIN), ArgsManager::ALLOW_ANY | ArgsManager::DEBUG_ONLY, OptionsCategory::WALLET_DEBUG_TEST); - - argsman.AddHiddenArgs({"-zapwallettxes"}); } bool WalletInit::ParameterInteraction() const @@ -118,10 +116,6 @@ bool WalletInit::ParameterInteraction() const LogPrintf("%s: parameter interaction: -blocksonly=1 -> setting -walletbroadcast=0\n", __func__); } - if (gArgs.IsArgSet("-zapwallettxes")) { - return InitError(Untranslated("-zapwallettxes has been removed. If you are attempting to remove a stuck transaction from your wallet, please use abandontransaction instead.")); - } - return true; } diff --git a/src/wallet/interfaces.cpp b/src/wallet/interfaces.cpp index c9419be0ab..4ca5e29229 100644 --- a/src/wallet/interfaces.cpp +++ b/src/wallet/interfaces.cpp @@ -11,6 +11,7 @@ #include <policy/fees.h> #include <primitives/transaction.h> #include <rpc/server.h> +#include <scheduler.h> #include <support/allocators/secure.h> #include <sync.h> #include <uint256.h> @@ -212,7 +213,7 @@ public: } return true; } - std::vector<WalletAddress> getAddresses() const override + std::vector<WalletAddress> getAddresses() override { LOCK(m_wallet->cs_wallet); std::vector<WalletAddress> result; @@ -585,10 +586,15 @@ public: } bool verify() override { return VerifyWallets(m_context); } bool load() override { return LoadWallets(m_context); } - void start(CScheduler& scheduler) override { return StartWallets(m_context, scheduler); } + void start(CScheduler& scheduler) override + { + m_context.scheduler = &scheduler; + return StartWallets(m_context); + } void flush() override { return FlushWallets(m_context); } void stop() override { return StopWallets(m_context); } void setMockTime(int64_t time) override { return SetMockTime(time); } + void schedulerMockForward(std::chrono::seconds delta) override { Assert(m_context.scheduler)->MockForward(delta); } //! WalletLoader methods util::Result<std::unique_ptr<Wallet>> createWallet(const std::string& name, const SecureString& passphrase, uint64_t wallet_creation_flags, std::vector<bilingual_str>& warnings) override diff --git a/src/wallet/load.cpp b/src/wallet/load.cpp index 4cdfadbee2..8b78a670e4 100644 --- a/src/wallet/load.cpp +++ b/src/wallet/load.cpp @@ -141,7 +141,7 @@ bool LoadWallets(WalletContext& context) } } -void StartWallets(WalletContext& context, CScheduler& scheduler) +void StartWallets(WalletContext& context) { for (const std::shared_ptr<CWallet>& pwallet : GetWallets(context)) { pwallet->postInitProcess(); @@ -149,9 +149,9 @@ void StartWallets(WalletContext& context, CScheduler& scheduler) // Schedule periodic wallet flushes and tx rebroadcasts if (context.args->GetBoolArg("-flushwallet", DEFAULT_FLUSHWALLET)) { - scheduler.scheduleEvery([&context] { MaybeCompactWalletDB(context); }, std::chrono::milliseconds{500}); + context.scheduler->scheduleEvery([&context] { MaybeCompactWalletDB(context); }, 500ms); } - scheduler.scheduleEvery([&context] { MaybeResendWalletTxs(context); }, 1min); + context.scheduler->scheduleEvery([&context] { MaybeResendWalletTxs(context); }, 1min); } void FlushWallets(WalletContext& context) diff --git a/src/wallet/load.h b/src/wallet/load.h index 0882f7f8ad..c079cad955 100644 --- a/src/wallet/load.h +++ b/src/wallet/load.h @@ -26,7 +26,7 @@ bool VerifyWallets(WalletContext& context); bool LoadWallets(WalletContext& context); //! Complete startup of wallets. -void StartWallets(WalletContext& context, CScheduler& scheduler); +void StartWallets(WalletContext& context); //! Flush all wallets in preparation for shutdown. void FlushWallets(WalletContext& context); diff --git a/src/wallet/rpc/spend.cpp b/src/wallet/rpc/spend.cpp index 6b96fc4e49..0f7a882034 100644 --- a/src/wallet/rpc/spend.cpp +++ b/src/wallet/rpc/spend.cpp @@ -1617,8 +1617,8 @@ RPCHelpMan walletprocesspsbt() CMutableTransaction mtx; // Returns true if complete, which we already think it is. CHECK_NONFATAL(FinalizeAndExtractPSBT(psbtx, mtx)); - CDataStream ssTx_final(SER_NETWORK, PROTOCOL_VERSION); - ssTx_final << mtx; + DataStream ssTx_final; + ssTx_final << TX_WITH_WITNESS(mtx); result.pushKV("hex", HexStr(ssTx_final)); } diff --git a/src/wallet/rpc/transactions.cpp b/src/wallet/rpc/transactions.cpp index c34391e6e8..cd7ed461f0 100644 --- a/src/wallet/rpc/transactions.cpp +++ b/src/wallet/rpc/transactions.cpp @@ -783,7 +783,7 @@ RPCHelpMan gettransaction() ListTransactions(*pwallet, wtx, 0, false, details, filter, /*filter_label=*/std::nullopt); entry.pushKV("details", details); - std::string strHex = EncodeHexTx(*wtx.tx, pwallet->chain().rpcSerializationFlags()); + std::string strHex = EncodeHexTx(*wtx.tx, pwallet->chain().rpcSerializationWithoutWitness()); entry.pushKV("hex", strHex); if (verbose) { diff --git a/src/wallet/spend.cpp b/src/wallet/spend.cpp index ed621cad61..b573062d32 100644 --- a/src/wallet/spend.cpp +++ b/src/wallet/spend.cpp @@ -259,7 +259,7 @@ util::Result<PreSelectedInputs> FetchSelectedInputs(const CWallet& wallet, const { PreSelectedInputs result; const bool can_grind_r = wallet.CanGrindR(); - std::map<COutPoint, CAmount> map_of_bump_fees = wallet.chain().CalculateIndividualBumpFees(coin_control.ListSelected(), coin_selection_params.m_effective_feerate); + std::map<COutPoint, CAmount> map_of_bump_fees = wallet.chain().calculateIndividualBumpFees(coin_control.ListSelected(), coin_selection_params.m_effective_feerate); for (const COutPoint& outpoint : coin_control.ListSelected()) { int input_bytes = -1; CTxOut txout; @@ -453,7 +453,7 @@ CoinsResult AvailableCoins(const CWallet& wallet, } if (feerate.has_value()) { - std::map<COutPoint, CAmount> map_of_bump_fees = wallet.chain().CalculateIndividualBumpFees(outpoints, feerate.value()); + std::map<COutPoint, CAmount> map_of_bump_fees = wallet.chain().calculateIndividualBumpFees(outpoints, feerate.value()); for (auto& [_, outputs] : result.coins) { for (auto& output : outputs) { @@ -725,7 +725,7 @@ util::Result<SelectionResult> ChooseSelectionResult(interfaces::Chain& chain, co outpoints.push_back(coin->outpoint); summed_bump_fees += coin->ancestor_bump_fees; } - std::optional<CAmount> combined_bump_fee = chain.CalculateCombinedBumpFee(outpoints, coin_selection_params.m_effective_feerate); + std::optional<CAmount> combined_bump_fee = chain.calculateCombinedBumpFee(outpoints, coin_selection_params.m_effective_feerate); if (!combined_bump_fee.has_value()) { return util::Error{_("Failed to calculate bump fees, because unconfirmed UTXOs depend on enormous cluster of unconfirmed transactions.")}; } diff --git a/src/wallet/test/psbt_wallet_tests.cpp b/src/wallet/test/psbt_wallet_tests.cpp index 9510f28282..b46f361363 100644 --- a/src/wallet/test/psbt_wallet_tests.cpp +++ b/src/wallet/test/psbt_wallet_tests.cpp @@ -34,12 +34,12 @@ BOOST_AUTO_TEST_CASE(psbt_updater_test) // Create prevtxs and add to wallet CDataStream s_prev_tx1(ParseHex("0200000000010158e87a21b56daf0c23be8e7070456c336f7cbaa5c8757924f545887bb2abdd7501000000171600145f275f436b09a8cc9a2eb2a2f528485c68a56323feffffff02d8231f1b0100000017a914aed962d6654f9a2b36608eb9d64d2b260db4f1118700c2eb0b0000000017a914b7f5faf40e3d40a5a459b1db3535f2b72fa921e88702483045022100a22edcc6e5bc511af4cc4ae0de0fcd75c7e04d8c1c3a8aa9d820ed4b967384ec02200642963597b9b1bc22c75e9f3e117284a962188bf5e8a74c895089046a20ad770121035509a48eb623e10aace8bfd0212fdb8a8e5af3c94b0b133b95e114cab89e4f7965000000"), SER_NETWORK, PROTOCOL_VERSION); CTransactionRef prev_tx1; - s_prev_tx1 >> prev_tx1; + s_prev_tx1 >> TX_WITH_WITNESS(prev_tx1); m_wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(prev_tx1->GetHash()), std::forward_as_tuple(prev_tx1, TxStateInactive{})); CDataStream s_prev_tx2(ParseHex("0200000001aad73931018bd25f84ae400b68848be09db706eac2ac18298babee71ab656f8b0000000048473044022058f6fc7c6a33e1b31548d481c826c015bd30135aad42cd67790dab66d2ad243b02204a1ced2604c6735b6393e5b41691dd78b00f0c5942fb9f751856faa938157dba01feffffff0280f0fa020000000017a9140fb9463421696b82c833af241c78c17ddbde493487d0f20a270100000017a91429ca74f8a08f81999428185c97b5d852e4063f618765000000"), SER_NETWORK, PROTOCOL_VERSION); CTransactionRef prev_tx2; - s_prev_tx2 >> prev_tx2; + s_prev_tx2 >> TX_WITH_WITNESS(prev_tx2); m_wallet.mapWallet.emplace(std::piecewise_construct, std::forward_as_tuple(prev_tx2->GetHash()), std::forward_as_tuple(prev_tx2, TxStateInactive{})); // Import descriptors for keys and scripts diff --git a/src/wallet/transaction.cpp b/src/wallet/transaction.cpp index 4f78fe7520..6777257e53 100644 --- a/src/wallet/transaction.cpp +++ b/src/wallet/transaction.cpp @@ -4,6 +4,10 @@ #include <wallet/transaction.h> +#include <interfaces/chain.h> + +using interfaces::FoundBlock; + namespace wallet { bool CWalletTx::IsEquivalentTo(const CWalletTx& _tx) const { @@ -25,6 +29,27 @@ int64_t CWalletTx::GetTxTime() const return n ? n : nTimeReceived; } +void CWalletTx::updateState(interfaces::Chain& chain) +{ + bool active; + auto lookup_block = [&](const uint256& hash, int& height, TxState& state) { + // If tx block (or conflicting block) was reorged out of chain + // while the wallet was shutdown, change tx status to UNCONFIRMED + // and reset block height, hash, and index. ABANDONED tx don't have + // associated blocks and don't need to be updated. The case where a + // transaction was reorged out while online and then reconfirmed + // while offline is covered by the rescan logic. + if (!chain.findBlock(hash, FoundBlock().inActiveChain(active).height(height)) || !active) { + state = TxStateInactive{}; + } + }; + if (auto* conf = state<TxStateConfirmed>()) { + lookup_block(conf->confirmed_block_hash, conf->confirmed_block_height, m_state); + } else if (auto* conf = state<TxStateConflicted>()) { + lookup_block(conf->conflicting_block_hash, conf->conflicting_block_height, m_state); + } +} + void CWalletTx::CopyFrom(const CWalletTx& _tx) { *this = _tx; diff --git a/src/wallet/transaction.h b/src/wallet/transaction.h index db858fa5ba..ddeb931112 100644 --- a/src/wallet/transaction.h +++ b/src/wallet/transaction.h @@ -22,6 +22,10 @@ #include <variant> #include <vector> +namespace interfaces { +class Chain; +} // namespace interfaces + namespace wallet { //! State of transaction confirmed in a block. struct TxStateConfirmed { @@ -161,7 +165,7 @@ public: std::vector<uint256> vMerkleBranch; int nIndex; - s >> tx >> hashBlock >> vMerkleBranch >> nIndex; + s >> TX_WITH_WITNESS(tx) >> hashBlock >> vMerkleBranch >> nIndex; } }; @@ -272,7 +276,7 @@ public: bool dummy_bool = false; //!< Used to be fSpent uint256 serializedHash = TxStateSerializedBlockHash(m_state); int serializedIndex = TxStateSerializedIndex(m_state); - s << tx << serializedHash << dummy_vector1 << serializedIndex << dummy_vector2 << mapValueCopy << vOrderForm << fTimeReceivedIsTxTime << nTimeReceived << fFromMe << dummy_bool; + s << TX_WITH_WITNESS(tx) << serializedHash << dummy_vector1 << serializedIndex << dummy_vector2 << mapValueCopy << vOrderForm << fTimeReceivedIsTxTime << nTimeReceived << fFromMe << dummy_bool; } template<typename Stream> @@ -285,7 +289,7 @@ public: bool dummy_bool; //! Used to be fSpent uint256 serialized_block_hash; int serializedIndex; - s >> tx >> serialized_block_hash >> dummy_vector1 >> serializedIndex >> dummy_vector2 >> mapValue >> vOrderForm >> fTimeReceivedIsTxTime >> nTimeReceived >> fFromMe >> dummy_bool; + s >> TX_WITH_WITNESS(tx) >> serialized_block_hash >> dummy_vector1 >> serializedIndex >> dummy_vector2 >> mapValue >> vOrderForm >> fTimeReceivedIsTxTime >> nTimeReceived >> fFromMe >> dummy_bool; m_state = TxStateInterpretSerialized({serialized_block_hash, serializedIndex}); @@ -326,6 +330,10 @@ public: template<typename T> const T* state() const { return std::get_if<T>(&m_state); } template<typename T> T* state() { return std::get_if<T>(&m_state); } + //! Update transaction state when attaching to a chain, filling in heights + //! of conflicted and confirmed blocks + void updateState(interfaces::Chain& chain); + bool isAbandoned() const { return state<TxStateInactive>() && state<TxStateInactive>()->abandoned; } bool isConflicted() const { return state<TxStateConflicted>(); } bool isInactive() const { return state<TxStateInactive>(); } diff --git a/src/wallet/wallet.cpp b/src/wallet/wallet.cpp index 162d7f9ec7..ecf18fbe78 100644 --- a/src/wallet/wallet.cpp +++ b/src/wallet/wallet.cpp @@ -1184,23 +1184,7 @@ bool CWallet::LoadToWallet(const uint256& hash, const UpdateWalletTxFn& fill_wtx // If wallet doesn't have a chain (e.g when using bitcoin-wallet tool), // don't bother to update txn. if (HaveChain()) { - bool active; - auto lookup_block = [&](const uint256& hash, int& height, TxState& state) { - // If tx block (or conflicting block) was reorged out of chain - // while the wallet was shutdown, change tx status to UNCONFIRMED - // and reset block height, hash, and index. ABANDONED tx don't have - // associated blocks and don't need to be updated. The case where a - // transaction was reorged out while online and then reconfirmed - // while offline is covered by the rescan logic. - if (!chain().findBlock(hash, FoundBlock().inActiveChain(active).height(height)) || !active) { - state = TxStateInactive{}; - } - }; - if (auto* conf = wtx.state<TxStateConfirmed>()) { - lookup_block(conf->confirmed_block_hash, conf->confirmed_block_height, wtx.m_state); - } else if (auto* conf = wtx.state<TxStateConflicted>()) { - lookup_block(conf->conflicting_block_hash, conf->conflicting_block_height, wtx.m_state); - } + wtx.updateState(chain()); } if (/* insertion took place */ ins.second) { wtx.m_it_wtxOrdered = wtxOrdered.insert(std::make_pair(wtx.nOrderPos, &wtx)); @@ -3319,8 +3303,10 @@ int CWallet::GetTxDepthInMainChain(const CWalletTx& wtx) const { AssertLockHeld(cs_wallet); if (auto* conf = wtx.state<TxStateConfirmed>()) { + assert(conf->confirmed_block_height >= 0); return GetLastBlockHeight() - conf->confirmed_block_height + 1; } else if (auto* conf = wtx.state<TxStateConflicted>()) { + assert(conf->conflicting_block_height >= 0); return -1 * (GetLastBlockHeight() - conf->conflicting_block_height + 1); } else { return 0; diff --git a/src/wallet/wallet.h b/src/wallet/wallet.h index 85c91ef15a..bc45010200 100644 --- a/src/wallet/wallet.h +++ b/src/wallet/wallet.h @@ -503,6 +503,13 @@ public: * <0 : conflicts with a transaction this deep in the blockchain * 0 : in memory pool, waiting to be included in a block * >=1 : this many blocks deep in the main chain + * + * Preconditions: it is only valid to call this function when the wallet is + * online and the block index is loaded. So this cannot be called by + * bitcoin-wallet tool code or by wallet migration code. If this is called + * without the wallet being online, it won't be able able to determine the + * the height of the last block processed, or the heights of blocks + * referenced in transaction, and might cause assert failures. */ int GetTxDepthInMainChain(const CWalletTx& wtx) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet); bool IsTxInMainChain(const CWalletTx& wtx) const EXCLUSIVE_LOCKS_REQUIRED(cs_wallet) diff --git a/src/zmq/zmqpublishnotifier.cpp b/src/zmq/zmqpublishnotifier.cpp index 1241431523..a4fb915d71 100644 --- a/src/zmq/zmqpublishnotifier.cpp +++ b/src/zmq/zmqpublishnotifier.cpp @@ -244,14 +244,14 @@ bool CZMQPublishRawBlockNotifier::NotifyBlock(const CBlockIndex *pindex) { LogPrint(BCLog::ZMQ, "Publish rawblock %s to %s\n", pindex->GetBlockHash().GetHex(), this->address); - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); + DataStream ss; CBlock block; if (!m_get_block_by_index(block, *pindex)) { zmqError("Can't read block from disk"); return false; } - ss << block; + ss << RPCTxSerParams(block); return SendZmqMessage(MSG_RAWBLOCK, &(*ss.begin()), ss.size()); } @@ -260,8 +260,8 @@ bool CZMQPublishRawTransactionNotifier::NotifyTransaction(const CTransaction &tr { uint256 hash = transaction.GetHash(); LogPrint(BCLog::ZMQ, "Publish rawtx %s to %s\n", hash.GetHex(), this->address); - CDataStream ss(SER_NETWORK, PROTOCOL_VERSION | RPCSerializationFlags()); - ss << transaction; + DataStream ss; + ss << RPCTxSerParams(transaction); return SendZmqMessage(MSG_RAWTX, &(*ss.begin()), ss.size()); } |